From 834167644c28b565e0c72eb8ea9880b14cac581d Mon Sep 17 00:00:00 2001 From: Jeff Ahrenholz Date: Fri, 30 Jun 2017 09:07:16 -0700 Subject: [PATCH 01/83] globally replace mailing list URLs and dev email update to: https://publists.nrl.navy.mil/mailman/listinfo/core-users https://publists.nrl.navy.mil/mailman/listinfo/core-dev dev email: core-dev@nrl.navy.mil --- README.rst | 4 ++-- configure.ac | 2 +- daemon/core/bsd/netgraph.py | 2 +- daemon/core/bsd/nodes.py | 2 +- daemon/core/bsd/vnet.py | 2 +- daemon/core/bsd/vnode.py | 2 +- daemon/ns3/setup.py | 2 +- daemon/setup.py | 2 +- daemon/src/setup.py | 2 +- doc/man/core-cleanup.1 | 2 +- doc/man/core-daemon.1 | 2 +- doc/man/core-gui.1 | 2 +- doc/man/core-xen-cleanup.1 | 2 +- doc/man/coresendmsg.1 | 2 +- doc/man/netns.1 | 2 +- doc/man/vcmd.1 | 2 +- doc/man/vnoded.1 | 2 +- gui/initgui.tcl | 2 +- packaging/deb.mk | 2 +- packaging/deb/control | 2 +- 20 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 116671c9..32645021 100644 --- a/README.rst +++ b/README.rst @@ -78,7 +78,7 @@ If you have questions, comments, or trouble, please use the CORE mailing lists: - `core-dev`_ for bugs, compile errors, and other development issues -.. _core-users: https://pf.itd.nrl.navy.mil/mailman/listinfo/core-users -.. _core-dev: https://pf.itd.nrl.navy.mil/mailman/listinfo/core-dev +.. _core-users: https://publists.nrl.navy.mil/mailman/listinfo/core-users +.. _core-dev: https://publists.nrl.navy.mil/mailman/listinfo/core-dev diff --git a/configure.ac b/configure.ac index a81fa8e7..e7f87331 100644 --- a/configure.ac +++ b/configure.ac @@ -12,7 +12,7 @@ # # this defines the CORE version number, must be static for AC_INIT # -AC_INIT(core, m4_esyscmd_s([./revision.sh 4.8]), core-dev@pf.itd.nrl.navy.mil) +AC_INIT(core, m4_esyscmd_s([./revision.sh 4.8]), core-dev@nrl.navy.mil) VERSION=$PACKAGE_VERSION CORE_VERSION=$PACKAGE_VERSION CORE_VERSION_DATE=m4_esyscmd_s([./revision.sh -d]) diff --git a/daemon/core/bsd/netgraph.py b/daemon/core/bsd/netgraph.py index 3cb9bffb..55bde8f0 100644 --- a/daemon/core/bsd/netgraph.py +++ b/daemon/core/bsd/netgraph.py @@ -3,7 +3,7 @@ # Copyright (c)2010-2012 the Boeing Company. # See the LICENSE file included in this distribution. # -# authors: core-dev@pf.itd.nrl.navy.mil +# authors: core-dev@nrl.navy.mil # ''' netgraph.py: Netgraph helper functions; for now these are wrappers around diff --git a/daemon/core/bsd/nodes.py b/daemon/core/bsd/nodes.py index 2eea4815..998b47ae 100644 --- a/daemon/core/bsd/nodes.py +++ b/daemon/core/bsd/nodes.py @@ -3,7 +3,7 @@ # Copyright (c)2010-2013 the Boeing Company. # See the LICENSE file included in this distribution. # -# author: core-dev@pf.itd.nrl.navy.mil +# author: core-dev@nrl.navy.mil # ''' diff --git a/daemon/core/bsd/vnet.py b/daemon/core/bsd/vnet.py index a92eb849..17b91f9d 100644 --- a/daemon/core/bsd/vnet.py +++ b/daemon/core/bsd/vnet.py @@ -3,7 +3,7 @@ # Copyright (c)2010-2012 the Boeing Company. # See the LICENSE file included in this distribution. # -# authors: core-dev@pf.itd.nrl.navy.mil +# authors: core-dev@nrl.navy.mil # ''' vnet.py: NetgraphNet and NetgraphPipeNet classes that implement virtual networks diff --git a/daemon/core/bsd/vnode.py b/daemon/core/bsd/vnode.py index 9f723d21..d833bf36 100644 --- a/daemon/core/bsd/vnode.py +++ b/daemon/core/bsd/vnode.py @@ -3,7 +3,7 @@ # Copyright (c)2010-2012 the Boeing Company. # See the LICENSE file included in this distribution. # -# authors: core-dev@pf.itd.nrl.navy.mil +# authors: core-dev@nrl.navy.mil # ''' vnode.py: SimpleJailNode and JailNode classes that implement the FreeBSD diff --git a/daemon/ns3/setup.py b/daemon/ns3/setup.py index 0794de42..5740c7f6 100644 --- a/daemon/ns3/setup.py +++ b/daemon/ns3/setup.py @@ -13,7 +13,7 @@ setup(name = "corens3-python", description = "Python ns-3 components of CORE", url = "http://www.nrl.navy.mil/itd/ncs/products/core", author = "Boeing Research & Technology", - author_email = "core-dev@pf.itd.nrl.navy.mil", + author_email = "core-dev@nrl.navy.mil", license = "GPLv2", long_description="Python scripts and modules for building virtual " \ "simulated networks.") diff --git a/daemon/setup.py b/daemon/setup.py index fd033e23..3f01316d 100644 --- a/daemon/setup.py +++ b/daemon/setup.py @@ -22,7 +22,7 @@ setup(name = "core-python", description = "Python components of CORE", url = "http://www.nrl.navy.mil/itd/ncs/products/core", author = "Boeing Research & Technology", - author_email = "core-dev@pf.itd.nrl.navy.mil", + author_email = "core-dev@nrl.navy.mil", license = "BSD", long_description="Python scripts and modules for building virtual " \ "emulated networks.") diff --git a/daemon/src/setup.py b/daemon/src/setup.py index 89de2535..60f5c724 100644 --- a/daemon/src/setup.py +++ b/daemon/src/setup.py @@ -23,7 +23,7 @@ setup(name = "core-python-netns", ext_modules = [netns, vcmd], url = "http://www.nrl.navy.mil/itd/ncs/products/core", author = "Boeing Research & Technology", - author_email = "core-dev@pf.itd.nrl.navy.mil", + author_email = "core-dev@nrl.navy.mil", license = "BSD", long_description="Extension modules and utilities to support virtual " \ "nodes using Linux network namespaces") diff --git a/doc/man/core-cleanup.1 b/doc/man/core-cleanup.1 index a14928ef..e5f0942a 100644 --- a/doc/man/core-cleanup.1 +++ b/doc/man/core-cleanup.1 @@ -25,6 +25,6 @@ remove the core-daemon.log file .BR vnoded(1) .SH BUGS Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/doc/man/core-daemon.1 b/doc/man/core-daemon.1 index 285ab85a..75f40def 100644 --- a/doc/man/core-daemon.1 +++ b/doc/man/core-daemon.1 @@ -48,5 +48,5 @@ enable debug logging; default = False .BR vnoded(1) .SH BUGS Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/doc/man/core-gui.1 b/doc/man/core-gui.1 index 07e7982c..84202e6e 100644 --- a/doc/man/core-gui.1 +++ b/doc/man/core-gui.1 @@ -40,5 +40,5 @@ With no parameters, starts the GUI in edit mode with a blank canvas. .BR vnoded(1) .SH BUGS Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/doc/man/core-xen-cleanup.1 b/doc/man/core-xen-cleanup.1 index 98d74cc0..44bf8a18 100644 --- a/doc/man/core-xen-cleanup.1 +++ b/doc/man/core-xen-cleanup.1 @@ -23,6 +23,6 @@ also kill the Python daemon .SH BUGS Warning! This script will remove logical volumes that match the name "/dev/vg*/c*-n*-" on all volume groups. Use with care. Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/doc/man/coresendmsg.1 b/doc/man/coresendmsg.1 index 029ff6fa..4c69fcd4 100644 --- a/doc/man/coresendmsg.1 +++ b/doc/man/coresendmsg.1 @@ -82,4 +82,4 @@ coresendmsg \-H .BR vnoded(1) .SH BUGS Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/doc/man/netns.1 b/doc/man/netns.1 index aa8ce5a9..ad509bab 100644 --- a/doc/man/netns.1 +++ b/doc/man/netns.1 @@ -26,5 +26,5 @@ wait for command to complete (useful for interactive commands) .BR vnoded(1) .SH BUGS Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/doc/man/vcmd.1 b/doc/man/vcmd.1 index 1c22b348..1909264b 100644 --- a/doc/man/vcmd.1 +++ b/doc/man/vcmd.1 @@ -38,5 +38,5 @@ control channel name (e.g. '/tmp/pycore.45647/n3') .BR vnoded(1), .SH BUGS Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/doc/man/vnoded.1 b/doc/man/vnoded.1 index 35b18c2c..b452bcc8 100644 --- a/doc/man/vnoded.1 +++ b/doc/man/vnoded.1 @@ -40,5 +40,5 @@ establish the specified for receiving control commands .BR vcmd(1), .SH BUGS Report bugs to -.BI core-dev@pf.itd.nrl.navy.mil. +.BI core-dev@nrl.navy.mil. diff --git a/gui/initgui.tcl b/gui/initgui.tcl index ed1a428b..2179aec0 100644 --- a/gui/initgui.tcl +++ b/gui/initgui.tcl @@ -604,7 +604,7 @@ menu .menubar.help -tearoff 0 .menubar.help add command -label "CORE website (www)" -command \ "_launchBrowser http://www.nrl.navy.mil/itd/ncs/products/core" .menubar.help add command -label "Mailing list (www)" -command \ - "_launchBrowser http://pf.itd.nrl.navy.mil/mailman/listinfo/core-users" + "_launchBrowser https://publists.nrl.navy.mil/mailman/listinfo/core-users" .menubar.help add command -label "About" -command popupAbout # diff --git a/packaging/deb.mk b/packaging/deb.mk index 14116c12..13e83015 100644 --- a/packaging/deb.mk +++ b/packaging/deb.mk @@ -20,7 +20,7 @@ build: changelog changelog: debian echo "core ($(CORE_VERSION)-1) unstable; urgency=low" > $(COREBUILD)/debian/changelog.generated echo " * interim package generated from source" >> $(COREBUILD)/debian/changelog.generated - echo " -- CORE Developers $$(date -R)" >> $(COREBUILD)/debian/changelog.generated + echo " -- CORE Developers $$(date -R)" >> $(COREBUILD)/debian/changelog.generated cd $(COREBUILD)/debian && \ { test ! -L changelog && mv -f changelog changelog.save; } && \ { test "$$(readlink changelog)" = "changelog.generated" || \ diff --git a/packaging/deb/control b/packaging/deb/control index 15caa7a3..e3e5a77d 100644 --- a/packaging/deb/control +++ b/packaging/deb/control @@ -1,7 +1,7 @@ Source: core Section: net Priority: optional -Maintainer: CORE Developers +Maintainer: CORE Developers Standards-Version: 3.8.4 Build-Depends: debhelper (>= 9), cdbs, dh-autoreconf, autoconf, automake, gcc, libev-dev, make, python-dev, libreadline-dev, bridge-utils, ebtables, iproute2 | iproute, imagemagick, pkg-config, help2man # python-sphinx From 7167f841d30792837436eed55e1f88185ee76365 Mon Sep 17 00:00:00 2001 From: Igal Date: Mon, 7 Aug 2017 22:56:30 -0400 Subject: [PATCH 02/83] broke commands to separate lines I couldn't find a way to do it without the extra line break for the `.rst` file format. --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 32645021..cc41df24 100644 --- a/README.rst +++ b/README.rst @@ -34,8 +34,11 @@ Building CORE To build this software you should use: ./bootstrap.sh + ./configure + make + sudo make install Here is what is installed with 'make install': From 86523c75d780a0f14d0e5093c6e3c9f053c3752c Mon Sep 17 00:00:00 2001 From: Igal Date: Sat, 19 Aug 2017 10:26:10 -0700 Subject: [PATCH 03/83] removed cflag -Werror This solves #128 gcc7 has a compatibility problem with python and issues a warning as described at https://bugzilla.redhat.com/show_bug.cgi?id=1473425 Setting the `-Werror` flag treats all warnings as errors, which fails the build. IMHO it is better to remove the flag from regular builds. Advanced users can add it if they choose to. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e7f87331..e35c41e6 100644 --- a/configure.ac +++ b/configure.ac @@ -115,7 +115,7 @@ AC_SUBST(CORE_STATE_DIR) # default compiler flags # _GNU_SOURCE is defined to get c99 defines for lrint() -CFLAGS="$CFLAGS -O3 -Werror -Wall -D_GNU_SOURCE" +CFLAGS="$CFLAGS -O3 -Wall -D_GNU_SOURCE" # debug flags #CFLAGS="$CFLAGS -g -Werror -Wall -D_GNU_SOURCE" From 585d62ebb941d6d41d15f934c6ba6c03410f7889 Mon Sep 17 00:00:00 2001 From: Igal Date: Tue, 22 Aug 2017 23:13:54 -0700 Subject: [PATCH 04/83] added -Wno-int-in-bool-context instead of removing -Werror --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e35c41e6..1b5590ae 100644 --- a/configure.ac +++ b/configure.ac @@ -115,7 +115,7 @@ AC_SUBST(CORE_STATE_DIR) # default compiler flags # _GNU_SOURCE is defined to get c99 defines for lrint() -CFLAGS="$CFLAGS -O3 -Wall -D_GNU_SOURCE" +CFLAGS="$CFLAGS -O3 -Werror -Wall -Wno-int-in-bool-context -D_GNU_SOURCE" # debug flags #CFLAGS="$CFLAGS -g -Werror -Wall -D_GNU_SOURCE" From a5ae485fa6b786ad7db63973d15f0fe12290d8e7 Mon Sep 17 00:00:00 2001 From: Gabriel Somlo Date: Mon, 9 Oct 2017 16:18:46 -0400 Subject: [PATCH 05/83] SimpleLxcNode: Don't umount directories before killing vnoded A node's private mounts are currently removed before killing vnoded, which makes them unavailable during container service shutdown. Any such service accessing the filesystem for atexit() cleanup (e.g., rsyslogd), will do so on the host filesystem instead, very likely causing unintended damage. For example, the default behavior of rsyslogd is to remove its listening socket (/dev/log, or /run/systemd/journal/dev-log) at shutdown from its atexit() handler. If the node's private '/dev' or '/run/systemd/journal' mount has already been removed, the host-side /dev/log or /run/systemd/journal/dev-log sockets will be removed instead! Since non-persistent (mount) namespaces are automatically destroyed by the kernel when the last process referencing them is killed, we should simply rely on that behavior instead of explicitly (and prematurely) unmounting a node's private directories during shutdown. Signed-off-by: Gabriel Somlo --- daemon/core/netns/vnode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/core/netns/vnode.py b/daemon/core/netns/vnode.py index 3f50d028..e7819a83 100644 --- a/daemon/core/netns/vnode.py +++ b/daemon/core/netns/vnode.py @@ -120,7 +120,8 @@ class SimpleLxcNode(PyCoreNode): # unmount all targets while self._mounts: source, target = self._mounts.pop(-1) - self.umount(target) + # Mount namespaces automatically removed when last process exits! + #self.umount(target) # shutdown all interfaces for netif in self.netifs(): From 6d3d17f470be5a361ce14f9aab92920e9f057b09 Mon Sep 17 00:00:00 2001 From: Gabriel Somlo Date: Wed, 11 Oct 2017 14:54:27 -0400 Subject: [PATCH 06/83] SimpleLxcNode: Remove unused umount method The umount method was used to remove private mount points before tearing down a node. Since non-persistend mount namespaces are automatically cleaned up by he kernel, this method is now unused. Signed-off-by: Gabriel Somlo --- daemon/core/netns/vnode.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/daemon/core/netns/vnode.py b/daemon/core/netns/vnode.py index e7819a83..a50ebb07 100644 --- a/daemon/core/netns/vnode.py +++ b/daemon/core/netns/vnode.py @@ -117,11 +117,9 @@ class SimpleLxcNode(PyCoreNode): if not self.up: return - # unmount all targets - while self._mounts: - source, target = self._mounts.pop(-1) - # Mount namespaces automatically removed when last process exits! - #self.umount(target) + # unmount all targets (NOTE: non-persistent mount namespaces are + # removed by the kernel when last referencing process is killed) + self._mounts = [] # shutdown all interfaces for netif in self.netifs(): @@ -251,19 +249,6 @@ class SimpleLxcNode(PyCoreNode): except IOError: logger.exception("mounting failed for %s at %s", source, target) - def umount(self, target): - """ - Unmount a target directory. - - :param str target: target directory to unmount - :return: nothing - """ - logger.info("unmounting: %s", target) - try: - self.cmd([constants.UMOUNT_BIN, "-n", "-l", target]) - except IOError: - logger.exception("unmounting failed for %s" % target) - def newifindex(self): """ Retrieve a new interface index. From 5901f2e1deab1a0bd69441c254189ace9356ab9b Mon Sep 17 00:00:00 2001 From: Gabriel Somlo Date: Mon, 20 Nov 2017 16:42:48 -0500 Subject: [PATCH 07/83] daemon: use "require" to make eggs available for import Since version 5.0, python modules (e.g. core, corens3, and netns) are installed as .egg bundles, and are not available for importing by default, unless explicitly we use "pkg-resources.require()". Signed-off-by: Gabriel Somlo --- daemon/sbin/core-daemon | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon/sbin/core-daemon b/daemon/sbin/core-daemon index 59fbb861..0e615147 100755 --- a/daemon/sbin/core-daemon +++ b/daemon/sbin/core-daemon @@ -25,6 +25,9 @@ import sys import threading import time +from pkg_resources import require +require("core_python", "corens3_python", "core_python_netns") + from core import constants from core import corehandlers from core import coreserver From f8e941a2b0986b511be3dddc0a212d82ec1dbb4e Mon Sep 17 00:00:00 2001 From: Gabriel Somlo Date: Mon, 20 Nov 2017 17:07:50 -0500 Subject: [PATCH 08/83] daemon: add ${pyexecdir} to $PYTHONPATH in daemon/src/Makefile On x86_64, the "core_python_netns" module is installed into ${pyexecdir}, a.k.a. %{python_sitearch}, a.k.a. "/usr/lib64/python2.7/site-packages". Adding ${pyexecdir} to $PYTHONPATH will prevent the "install" target in "daemon/src/Makefile" from failing. Signed-off-by: Gabriel Somlo --- daemon/src/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/src/Makefile.am b/daemon/src/Makefile.am index 0cec0814..ddafc601 100644 --- a/daemon/src/Makefile.am +++ b/daemon/src/Makefile.am @@ -40,7 +40,7 @@ libnetns.a: install-exec-local: $(MKDIR_P) ${DESTDIR}/${pythondir} $(MKDIR_P) ${DESTDIR}/${pyexecdir} - SBINDIR=${DESTDIR}/@SBINDIR@ PYTHONPATH=${DESTDIR}/${pythondir} $(PYTHON) $(SETUPPY) $(SETUPPYFLAGS) install \ + SBINDIR=${DESTDIR}/@SBINDIR@ PYTHONPATH=${DESTDIR}/${pythondir}:${DESTDIR}/${pyexecdir} $(PYTHON) $(SETUPPY) $(SETUPPYFLAGS) install \ --prefix=${DESTDIR}/${pyprefix} \ --install-purelib=${DESTDIR}/${pythondir} \ --install-platlib=${DESTDIR}/${pyexecdir} \ From 54d79270d53410844cc25fa538ddcc281bf30658 Mon Sep 17 00:00:00 2001 From: Gabriel Somlo Date: Mon, 20 Nov 2017 17:19:59 -0500 Subject: [PATCH 09/83] packaging: update RPM spec file to match 5.0 release Building compliant RPM packages for Fedora/RHEL/Centos requires all dependencies to be satisfied via the distribution specific packaging infrastructure (i.e. rpm + yum|dnf). To ensure that packages correctly pull in their dependencies (i.e., without relying on side-loaded software that just happens to be installed on the host where they're being built), the best practice recommended is to build them in "mock". E.g., rpmbuild -bs core.spec mock -r fedora-rawhide-x86_64 core-*.src.rpm This patch modifies the enclosed spec file to correctly list all other RPM packages required to build and run core-* RPMs. Note that at the time of this writing the "python-logzero" package is pending release into the Fedora (26 and newer) repositories: https://bugzilla.redhat.com/show_bug.cgi?id=1514100 Signed-off-by: Gabriel Somlo --- packaging/rpm/core.spec.in | 96 +++++--------------------------------- 1 file changed, 11 insertions(+), 85 deletions(-) diff --git a/packaging/rpm/core.spec.in b/packaging/rpm/core.spec.in index 1ccbbc86..b68b11de 100644 --- a/packaging/rpm/core.spec.in +++ b/packaging/rpm/core.spec.in @@ -24,6 +24,7 @@ building virtual networks using Linux network namespace containers and bridging. Summary: Common Open Research Emulator daemon back-end Group: System Tools Requires: bash bridge-utils ebtables iproute libev python net-tools +Requires: python2-logzero python-enum34 %if 0%{?el6} Requires: procps %else @@ -36,6 +37,8 @@ Requires: kernel-modules-extra Requires: iproute-tc %endif BuildRequires: make automake autoconf libev-devel python-devel bridge-utils ebtables iproute net-tools ImageMagick help2man +BuildRequires: python2-pytest-runner python2-sphinx +BuildRequires: python2-logzero python-enum34 %if 0%{?el6} BuildRequires: procps %else @@ -77,6 +80,10 @@ make -j4 %install rm -rf $RPM_BUILD_ROOT make DESTDIR=$RPM_BUILD_ROOT install +rm -f $RPM_BUILD_ROOT/%{python_sitelib}/site.py* \ + $RPM_BUILD_ROOT/%{python_sitearch}/site.py* \ + $RPM_BUILD_ROOT/%{python_sitelib}/easy-install.pth \ + $RPM_BUILD_ROOT/%{python_sitearch}/easy-install.pth %clean rm -rf $RPM_BUILD_ROOT @@ -314,91 +321,8 @@ fi %doc %{_mandir}/man1/vnoded.1.gz /etc/logrotate.d/core-daemon /etc/systemd/system/core-daemon.service -%{python_sitearch}/core_python_netns-1.0-py%{python_version}.egg-info -%{python_sitearch}/netns.so -%{python_sitearch}/vcmd.so -%dir %{python_sitelib}/core -%{python_sitelib}/core/sdt.py* -%{python_sitelib}/core/service.py* -%{python_sitelib}/core/coreserver.py* -%dir %{python_sitelib}/core/addons -%{python_sitelib}/core/addons/__init__.py* -%dir %{python_sitelib}/core/api -%{python_sitelib}/core/api/coreapi.py* -%{python_sitelib}/core/api/data.py* -%{python_sitelib}/core/api/__init__.py* -%{python_sitelib}/core/broker.py* -%dir %{python_sitelib}/core/bsd -%{python_sitelib}/core/bsd/__init__.py* -%{python_sitelib}/core/bsd/netgraph.py* -%{python_sitelib}/core/bsd/nodes.py* -%{python_sitelib}/core/bsd/vnet.py* -%{python_sitelib}/core/bsd/vnode.py* -%{python_sitelib}/core/conf.py* -%{python_sitelib}/core/constants.py* -%{python_sitelib}/core/coreobj.py* -%dir %{python_sitelib}/core/emane -%{python_sitelib}/core/emane/bypass.py* -%{python_sitelib}/core/emane/commeffect.py* -%{python_sitelib}/core/emane/emane.py* -%{python_sitelib}/core/emane/ieee80211abg.py* -%{python_sitelib}/core/emane/__init__.py* -%{python_sitelib}/core/emane/nodes.py* -%{python_sitelib}/core/emane/rfpipe.py* -%{python_sitelib}/core/emane/tdma.py* -%{python_sitelib}/core/emane/universal.py* -%{python_sitelib}/core/__init__.py* -%{python_sitelib}/core/location.py* -%dir %{python_sitelib}/core/misc -%{python_sitelib}/core/misc/event.py* -%{python_sitelib}/core/misc/__init__.py* -%{python_sitelib}/core/misc/ipaddr.py* -%{python_sitelib}/core/misc/LatLongUTMconversion.py* -%{python_sitelib}/core/misc/quagga.py* -%{python_sitelib}/core/misc/utils.py* -%{python_sitelib}/core/misc/utm.py* -%{python_sitelib}/core/misc/xmldeployment.py* -%{python_sitelib}/core/misc/xmlparser.py* -%{python_sitelib}/core/misc/xmlparser0.py* -%{python_sitelib}/core/misc/xmlparser1.py* -%{python_sitelib}/core/misc/xmlsession.py* -%{python_sitelib}/core/misc/xmlutils.py* -%{python_sitelib}/core/misc/xmlwriter.py* -%{python_sitelib}/core/misc/xmlwriter0.py* -%{python_sitelib}/core/misc/xmlwriter1.py* -%{python_sitelib}/core/mobility.py* -%dir %{python_sitelib}/core/netns -%{python_sitelib}/core/netns/__init__.py* -%{python_sitelib}/core/netns/nodes.py* -%{python_sitelib}/core/netns/vif.py* -%{python_sitelib}/core/netns/vnet.py* -%{python_sitelib}/core/netns/vnodeclient.py* -%{python_sitelib}/core/netns/vnode.py* -%dir %{python_sitelib}/corens3 -%{python_sitelib}/corens3/constants.py* -%{python_sitelib}/corens3/__init__.py* -%{python_sitelib}/corens3/obj.py* -%{python_sitelib}/corens3_python-@COREDPY_VERSION@-py%{python_version}.egg-info -%dir %{python_sitelib}/core/phys -%{python_sitelib}/core/phys/__init__.py* -%{python_sitelib}/core/phys/pnodes.py* -%{python_sitelib}/core_python-@COREDPY_VERSION@-py%{python_version}.egg-info -%dir %{python_sitelib}/core/services -%{python_sitelib}/core/services/bird.py* -%{python_sitelib}/core/services/__init__.py* -%{python_sitelib}/core/services/dockersvc.py* -%{python_sitelib}/core/services/nrl.py* -%{python_sitelib}/core/services/quagga.py* -%{python_sitelib}/core/services/security.py* -%{python_sitelib}/core/services/startup.py* -%{python_sitelib}/core/services/ucarp.py* -%{python_sitelib}/core/services/utility.py* -%{python_sitelib}/core/services/xorp.py* -%{python_sitelib}/core/session.py* -%dir %{python_sitelib}/core/xen -%{python_sitelib}/core/xen/__init__.py* -%{python_sitelib}/core/xen/xenconfig.py* -%{python_sitelib}/core/xen/xen.py* +%{python_sitearch}/* +%{python_sitelib}/* %{_sbindir}/core-cleanup %{_sbindir}/core-daemon %{_sbindir}/core-manage @@ -409,6 +333,8 @@ fi %{_sbindir}/vnoded %changelog +* Mon Nov 20 2017 Gabriel Somlo - 5.0 +- update RPM spec file to match 5.0 release * Fri Sep 01 2017 CORE Developers - 5.0 - Added Ryu SD and Open vSwitch services, code cleanup and refactoring * Fri Jun 5 2015 CORE Developers - 4.8 From e8ad324d3afab574f1c6f442c0a2e5bc53eca778 Mon Sep 17 00:00:00 2001 From: Gabriel Somlo Date: Mon, 12 Feb 2018 17:26:35 -0500 Subject: [PATCH 10/83] gui: exec.tcl: cache local IP address across calls to getMyIP Modify getMyIP to cache the local machine's IP address, and return it without further accessing of the underlying resolver libraries during subsequent invocations. getMyIP is called roughly once a second from within monitor_loop when refreshing CPU utilization in the bottom-right corner of the GUI window. Other, dedicated CPU usage windows might also call getMyIP at regular intervals. With systemd commit cda458a54 (between v232 and v233), the implementation of gethostbyname4_r() was updated to a more intransigently standard-compliant error reporting convention, which in turn causes glibc to retry in a more labor intensive way (see https://github.com/systemd/systemd/pull/5359). Under certain circumstances depending on the local hostname and IP configuration, the glibc/systemd back-end resolver routines triggered by getMyIP's call to [socket ...] and [fconfigure ...] end up taking long enough to noticeably slow down refreshing the main core-gui window, to the point where interaction with the GUI becomes difficult. Signed-off-by: Gabriel Somlo --- gui/exec.tcl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/gui/exec.tcl b/gui/exec.tcl index 1e5eec76..62711603 100644 --- a/gui/exec.tcl +++ b/gui/exec.tcl @@ -782,14 +782,17 @@ proc manageCPUwindow {xpos ypos start} { } proc getMyIP { } { - if { [catch {set theServer [socket -server none -myaddr \ - [info hostname] 0]} ] } { - return "127.0.0.1" + variable myIP + if { ![info exists myIP] } { + if { [catch {set theServer [socket -server none \ + -myaddr [info hostname] 0]} ] } { + set myIP "127.0.0.1" + } else { + set myIP [lindex [fconfigure $theServer -sockname] 0] + close $theServer + } } - set myIP [lindex [fconfigure $theServer -sockname] 0] - close $theServer return $myIP - } # display all values stored in cpu usage history for each server From edc1a9202237a785e55f291180243ad696c0ece9 Mon Sep 17 00:00:00 2001 From: TinCanTech Date: Sat, 24 Feb 2018 18:44:44 +0000 Subject: [PATCH 11/83] Update core.conf Fix typ0s --- daemon/data/core.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/daemon/data/core.conf b/daemon/data/core.conf index 8ee2a532..6e000361 100644 --- a/daemon/data/core.conf +++ b/daemon/data/core.conf @@ -29,12 +29,12 @@ quagga_sbin_search = "/usr/local/sbin /usr/sbin /usr/lib/quagga" # # # uncomment and edit to establish a distributed control backchannel -#controlnet = core1:172.16.1.0/24 core:172.16.2.0/24 core3:172.16.3.0/24 core4 :172.16.4.0/24 core5:172.16.5.0/24 +#controlnet = core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 core4:172.16.4.0/24 core5:172.16.5.0/24 # uncomment and edit to establish distributed auxiliary control channels. -#controlnet1 = core1:172.17.1.0/24 core:172.17.2.0/24 core3:172.17.3.0/24 core4 :172.17.4.0/24 core5:172.17.5.0/24 -#controlnet2 = core1:172.18.1.0/24 core:172.18.2.0/24 core3:172.18.3.0/24 core4 :172.18.4.0/24 core5:172.18.5.0/24 -#controlnet3 = core1:172.19.1.0/24 core:172.19.2.0/24 core3:172.19.3.0/24 core4 :172.19.4.0/24 core5:172.19.5.0/24 +#controlnet1 = core1:172.17.1.0/24 core2:172.17.2.0/24 core3:172.17.3.0/24 core4:172.17.4.0/24 core5:172.17.5.0/24 +#controlnet2 = core1:172.18.1.0/24 core2:172.18.2.0/24 core3:172.18.3.0/24 core4:172.18.4.0/24 core5:172.18.5.0/24 +#controlnet3 = core1:172.19.1.0/24 core2:172.19.2.0/24 core3:172.19.3.0/24 core4:172.19.4.0/24 core5:172.19.5.0/24 # uncomment and edit to assign host interfaces to auxilary control channels # for use in connecting with other servers in a distributed environments. From d3c78c3a7a6830b5c6f0f33c6f312d0fd0ef1134 Mon Sep 17 00:00:00 2001 From: kevlar Date: Thu, 1 Mar 2018 17:21:20 -0800 Subject: [PATCH 12/83] Added jenkinsfile describing build and test pipeline --- Jenkinsfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..835cb2e3 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,18 @@ +pipeline { + agent any + stages { + stage('build') { + steps { + sh './bootstrap.sh' + sh './configure --prefix=/tmp/core_build' + sh 'make' + sh 'make install' + } + } + stage('test') { + steps { + sh 'pytest' + } + } + } +} \ No newline at end of file From a4ed96897995907fa0c340be4a492f8c3f9f7f32 Mon Sep 17 00:00:00 2001 From: kevlar Date: Tue, 6 Mar 2018 09:06:30 -0800 Subject: [PATCH 13/83] removed custom prefix from jenkinsfile configure --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 835cb2e3..be9fe8ef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ pipeline { stage('build') { steps { sh './bootstrap.sh' - sh './configure --prefix=/tmp/core_build' + sh './configure' sh 'make' sh 'make install' } From 14ad62d334444412f520e176714af3eed2ab8f70 Mon Sep 17 00:00:00 2001 From: kevlar Date: Tue, 6 Mar 2018 12:55:12 -0800 Subject: [PATCH 14/83] Set make install to run with sudo --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index be9fe8ef..c0c5a299 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ pipeline { sh './bootstrap.sh' sh './configure' sh 'make' - sh 'make install' + sh 'sudo make install' } } stage('test') { From 2b42434818507dba0d5a19893d65541641cd9317 Mon Sep 17 00:00:00 2001 From: kevlar Date: Wed, 7 Mar 2018 09:55:44 -0800 Subject: [PATCH 15/83] added install dependencies step --- Jenkinsfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c0c5a299..245c12a1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,13 @@ pipeline { agent any stages { - stage('build') { + stage('install dependencies') { + steps { + sh 'sudo apt-get install libev-dev bridge-utils ebtables libtk-img bash iproute python tcl8.5 tk8.5 autoconf automake gcc libev-dev make python-dev libreadline-dev pkg-config imagemagick help2man python-sphinx python-setuptools python-pip' + sh 'sudo pip install mock pytest pytest-runner' + } + } + stage('build core') { steps { sh './bootstrap.sh' sh './configure' @@ -9,9 +15,9 @@ pipeline { sh 'sudo make install' } } - stage('test') { + stage('test core') { steps { - sh 'pytest' + sh 'pytest daemon/tests' } } } From ca43eed5bab95a9811d3211abb0c819a2d9b388a Mon Sep 17 00:00:00 2001 From: kevlar Date: Wed, 7 Mar 2018 09:58:11 -0800 Subject: [PATCH 16/83] removed dependencies --- Jenkinsfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 245c12a1..e729f568 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,12 +1,6 @@ pipeline { agent any stages { - stage('install dependencies') { - steps { - sh 'sudo apt-get install libev-dev bridge-utils ebtables libtk-img bash iproute python tcl8.5 tk8.5 autoconf automake gcc libev-dev make python-dev libreadline-dev pkg-config imagemagick help2man python-sphinx python-setuptools python-pip' - sh 'sudo pip install mock pytest pytest-runner' - } - } stage('build core') { steps { sh './bootstrap.sh' From 9b01898e257c41334a001c59b07cb4af246c0441 Mon Sep 17 00:00:00 2001 From: kevlar Date: Wed, 7 Mar 2018 10:00:50 -0800 Subject: [PATCH 17/83] broke out testing to individual tests --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e729f568..e70e3dc6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,9 @@ pipeline { } stage('test core') { steps { - sh 'pytest daemon/tests' + sh 'pytest daemon/tests/test_core.py' + sh 'pytest daemon/tests/test_gui.py' + sh 'pytest daemon/tests/test_emane.py' } } } From 02538522a4ec7b2964e51ffab89d96959585020c Mon Sep 17 00:00:00 2001 From: Tom Goff Date: Wed, 21 Mar 2018 15:25:06 -0400 Subject: [PATCH 18/83] daemon: Add support for EMANE 1.2.1. --- daemon/core/emane/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon/core/emane/__init__.py b/daemon/core/emane/__init__.py index 892feb6e..c96ee574 100644 --- a/daemon/core/emane/__init__.py +++ b/daemon/core/emane/__init__.py @@ -10,6 +10,7 @@ EMANE091 = 91 EMANE092 = 92 EMANE093 = 93 EMANE101 = 101 +EMANE121 = 121 VERSION = None VERSIONSTR = None @@ -44,6 +45,8 @@ def emane_version(): VERSION = EMANE093 elif result.startswith("1.0.1"): VERSION = EMANE101 + elif result.startswith("1.2.1"): + VERSION = EMANE121 VERSIONSTR = result.strip() From 67a78828c3166be7fe2cfaa4aad58a9ce8c37376 Mon Sep 17 00:00:00 2001 From: stuartmarsden Date: Sun, 20 May 2018 22:21:19 +0400 Subject: [PATCH 19/83] update_link had wrong arg order --- daemon/core/emulator/coreemu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 6712227c..8fa6ffb9 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -378,7 +378,7 @@ class EmuSession(Session): if node_two: node_two.lock.release() - def update_link(self, node_one_id, node_two_id, link_options, interface_one_id=None, interface_two_id=None): + def update_link(self, node_one_id, node_two_id, interface_one_id=None, interface_two_id=None, link_options=LinkOptions()): """ Update link information between nodes. From aa6b83f29bc51c6740947c0b7d7393ec8e0712c5 Mon Sep 17 00:00:00 2001 From: stuartmarsden Date: Sun, 20 May 2018 23:14:29 +0400 Subject: [PATCH 20/83] fix netem 0% loss and duplicate issue --- daemon/core/netns/openvswitch.py | 4 ++-- daemon/core/netns/vnet.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/daemon/core/netns/openvswitch.py b/daemon/core/netns/openvswitch.py index 341ab92a..46ba2f2c 100644 --- a/daemon/core/netns/openvswitch.py +++ b/daemon/core/netns/openvswitch.py @@ -248,10 +248,10 @@ class OvsNet(PyCoreNet): if jitter is not None: netem += ["%sus" % jitter, "25%"] - if loss is not None: + if loss is not None and loss > 0: netem += ["loss", "%s%%" % min(loss, 100)] - if duplicate is not None: + if duplicate is not None and duplicate > 0: netem += ["duplicate", "%s%%" % min(duplicate, 100)] if delay <= 0 and jitter <= 0 and loss <= 0 and duplicate <= 0: diff --git a/daemon/core/netns/vnet.py b/daemon/core/netns/vnet.py index 545ca648..7a517d08 100644 --- a/daemon/core/netns/vnet.py +++ b/daemon/core/netns/vnet.py @@ -475,9 +475,9 @@ class LxBrNet(PyCoreNet): else: netem += ["%sus" % jitter, "25%"] - if loss is not None: + if loss is not None and loss > 0: netem += ["loss", "%s%%" % min(loss, 100)] - if duplicate is not None: + if duplicate is not None and duplicate > 0: netem += ["duplicate", "%s%%" % min(duplicate, 100)] if delay <= 0 and jitter <= 0 and loss <= 0 and duplicate <= 0: # possibly remove netem if it exists and parent queue wasn't removed From cd4e79d64ffb96312ccafa8a9fa823f90a667d9e Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 22 May 2018 20:46:34 -0700 Subject: [PATCH 21/83] Delete README.rst --- README.rst | 100 ----------------------------------------------------- 1 file changed, 100 deletions(-) delete mode 100644 README.rst diff --git a/README.rst b/README.rst deleted file mode 100644 index b2f4101a..00000000 --- a/README.rst +++ /dev/null @@ -1,100 +0,0 @@ -==== -CORE -==== - -CORE: Common Open Research Emulator - -Copyright (c)2005-2017 the Boeing Company. - -See the LICENSE file included in this distribution. - -About -===== - -CORE is a tool for emulating networks using a GUI or Python scripts. The CORE -project site (1) is a good source of introductory information, with a manual, -screenshots, and demos about this software. The GitHub project (2) hosts the -source repos, wiki, and bug tracker. There is a deprecated -Google Code page (3) with the old wiki, blog, bug tracker, and quickstart guide. - -1. http://www.nrl.navy.mil/itd/ncs/products/core - -2. https://github.com/coreemu/core - -3. http://code.google.com/p/coreemu/ - -4. `Official Documentation`_ - -.. _Official Documentation: https://downloads.pf.itd.nrl.navy.mil/docs/core/core-html/index.html - - -Building CORE -============= - -To build this software you should use: - - ./bootstrap.sh - - ./configure - - make - - sudo make install - -Note: You may need to pass the proxy settings to sudo make install: - sudo make install HTTP_PROXY= - -Here is what is installed with 'make install': - - /usr/local/bin/core-gui - /usr/local/sbin/core-daemon - /usr/local/sbin/[vcmd, vnoded, coresendmsg, core-cleanup.sh] - /usr/local/lib/core/* - /usr/local/share/core/* - /usr/local/lib/python2.6/dist-packages/core/* - /usr/local/lib/python2.6/dist-packages/[netns,vcmd].so - /etc/core/* - /etc/init.d/core - -See the manual for the software required for building CORE. - -Building Documentation -====================== - -Being able to build documentation depends on help2man being installed. - -Once that has been done you can run the following commands: - - ./bootstrap.sh - ./configure - make html - -Running CORE -============ - -First start the CORE services: - - sudo /etc/init.d/core-daemon start - -This automatically runs the core-daemon program. -Assuming the GUI is in your PATH, run the CORE GUI by typing the following: - - core-gui - -This launches the CORE GUI. You do not need to run the GUI as root. - - -Support -======= - -If you have questions, comments, or trouble, please use the CORE mailing lists: - -- `core-users`_ for general comments and questions - -- `core-dev`_ for bugs, compile errors, and other development issues - - -.. _core-users: https://publists.nrl.navy.mil/mailman/listinfo/core-users -.. _core-dev: https://publists.nrl.navy.mil/mailman/listinfo/core-dev - - From b5df7a85f9e1e46fc1657029be97cb841389a58d Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 22 May 2018 20:56:24 -0700 Subject: [PATCH 22/83] Update Changelog --- Changelog | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Changelog b/Changelog index 8ae480ae..dd034ab6 100644 --- a/Changelog +++ b/Changelog @@ -1,17 +1,34 @@ -2018-XX-XX CORE 5.1 +2018-05-22 CORE 5.1 * DAEMON: + - removed and cleared out code that is either legacy or no longer supported (Xen, BSD, Kernel patching, RPM/DEB specific files) - default nodes are now set in the node map - moved ns3 and netns directories to the top of the repo - changes to make use of fpm as the tool for building packages - removed usage of logzero to avoid dependency issues for built packages - removed daemon addons directory + - added CoreEmu to core.emulator.coreemu to help begin serving as the basis for a more formal API for scripting and creating new external APIs out of + - cleaned up logging, moved more logging to DEBUG from INFO, tried to mold INFO message to be more simple and informative + - EMANE 1.0.1-1.21 supported + - updates to leverage EMANE python bindings for dynamically parsing phy/mac manifest files + - example custom EMANE model lives under /usr/share/core/examples/myemane/examplemodel.py + - EMANE TDMA model now supports an option to start a TDMA schedule when running + - fixed issues with coresendmsg script due to code refactoring + - added make target for generating documentation "make doc" + - Python 2.7+ is now required + - ns3 is no longer bundled by default, but will be produced as a separate package for installation + * GUI + - updated broken help links in GUI Help->About + * Packaging + - fixed PYTHON_PATH to PYTHONPATH in sysv script + - added make command to leverage FPM as the tool for creating deb/rpm packages going forward, there is documentation within README.md to try it out * TEST: - fixed some broken tests + - new test cases based on CoreEmu usage * BUGFIXES: - #142 - duplication of custom services - #136 - sphinx-apidoc command not found - #137 - make command fails when using distclean - + 2017-09-01 CORE 5.0 * DEVELOPMENT: - support for editorconfig to help standardize development across IDEs, from the defined configuration file From 80fb71455de39a9f81416ee77d64b9dd0cb2aa8a Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 23 May 2018 10:05:43 -0700 Subject: [PATCH 23/83] Update configure.ac stop $HOME from expanding during make --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index d40bd31f..8c522601 100644 --- a/configure.ac +++ b/configure.ac @@ -36,7 +36,7 @@ AC_ARG_WITH([guiconfdir], [AS_HELP_STRING([--with-guiconfdir=dir], [specify GUI configuration directory])], [CORE_GUI_CONF_DIR="$with_guiconfdir"], - [CORE_GUI_CONF_DIR="\${HOME}/.core"]) + [CORE_GUI_CONF_DIR="\$\${HOME}/.core"]) AC_SUBST(CORE_GUI_CONF_DIR) AC_ARG_ENABLE([gui], [AS_HELP_STRING([--enable-gui[=ARG]], From ec04f457e03c4ae86f51f08f87c0ce5467e882f7 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 24 May 2018 16:28:26 -0700 Subject: [PATCH 24/83] Update corehandlers.py fixes #162 --- daemon/core/corehandlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 362f4d54..0d41d095 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -958,7 +958,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): :return: reply messages """ if message.flags & MessageFlags.ADD.value: - node_num = message.get_tlv(FileTlvs.NUMBER.value) + node_num = message.get_tlv(FileTlvs.NODE.value) file_name = message.get_tlv(FileTlvs.NAME.value) file_type = message.get_tlv(FileTlvs.TYPE.value) source_name = message.get_tlv(FileTlvs.SOURCE_NAME.value) From 17f874e25f3a359c5528af70415c6f9aea3e69e6 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 25 May 2018 08:45:36 -0700 Subject: [PATCH 25/83] removed unused packaging files --- packaging/deb.mk | 56 ------ packaging/deb/control | 28 --- packaging/rpm/core.spec.in | 362 ------------------------------------- 3 files changed, 446 deletions(-) delete mode 100644 packaging/deb.mk delete mode 100644 packaging/deb/control delete mode 100644 packaging/rpm/core.spec.in diff --git a/packaging/deb.mk b/packaging/deb.mk deleted file mode 100644 index 13e83015..00000000 --- a/packaging/deb.mk +++ /dev/null @@ -1,56 +0,0 @@ -DEBBUILD = .debbuild - -CORE_VERSION = $(shell cat .version 2> /dev/null) - -COREBUILD = $(DEBBUILD)/core-$(CORE_VERSION) - -.PHONY: all -all: clean .version build - -.PHONY: clean -clean: - rm -rf $(DEBBUILD) - -.PHONY: build -build: changelog - cd $(COREBUILD) && dpkg-buildpackage -b -us -uc - @printf "\ndebian packages built in $(DEBBUILD)\n\n" - -.PHONY: changelog -changelog: debian - echo "core ($(CORE_VERSION)-1) unstable; urgency=low" > $(COREBUILD)/debian/changelog.generated - echo " * interim package generated from source" >> $(COREBUILD)/debian/changelog.generated - echo " -- CORE Developers $$(date -R)" >> $(COREBUILD)/debian/changelog.generated - cd $(COREBUILD)/debian && \ - { test ! -L changelog && mv -f changelog changelog.save; } && \ - { test "$$(readlink changelog)" = "changelog.generated" || \ - ln -sf changelog.generated changelog; } - -.PHONY: debian -debian: corebuild - cd $(COREBUILD) && ln -s packaging/deb debian - -.PHONY: corebuild -corebuild: $(DEBBUILD) dist - tar -C $(DEBBUILD) -xzf core-$(CORE_VERSION).tar.gz - -.PHONY: dist -dist: Makefile - $(MAKE) dist - -Makefile: configure - ./configure - -configure: bootstrap.sh - ./bootstrap.sh - -bootstrap.sh: - @printf "\nERROR: make must be called from the top-level directory:\n" - @printf " make -f packaging/$(lastword $(MAKEFILE_LIST))\n\n" - @false - -.version: Makefile - $(MAKE) $@ - -$(DEBBUILD): - mkdir -p $@ diff --git a/packaging/deb/control b/packaging/deb/control deleted file mode 100644 index e3e5a77d..00000000 --- a/packaging/deb/control +++ /dev/null @@ -1,28 +0,0 @@ -Source: core -Section: net -Priority: optional -Maintainer: CORE Developers -Standards-Version: 3.8.4 -Build-Depends: debhelper (>= 9), cdbs, dh-autoreconf, autoconf, automake, gcc, libev-dev, make, python-dev, libreadline-dev, bridge-utils, ebtables, iproute2 | iproute, imagemagick, pkg-config, help2man -# python-sphinx -Homepage: http://www.nrl.navy.mil/itd/ncs/products/core - -Package: core-daemon -Architecture: any -Depends: bash (>=3.0), bridge-utils, ebtables, iproute2 | iproute, libev4, python (>=2.6), dpkg (>=1.15.4), procps, ${shlibs:Depends} -Recommends: quagga -Description: Emulate virtual networks in a box. - The Common Open Research Emulator provides Python modules for building virtual - networks using Linux network namespace containers and bridging. This is the - daemon package containing the backend Python modules and core-daemon. - -Package: core-gui -Architecture: all -Depends: bash (>=3.0), tcl (>= 8.5), tk (>= 8.5), xterm -Recommends: libtk-img -Description: Emulate virtual networks in a box. - The Common Open Research Emulator provides Python modules for building virtual - networks using Linux network namespace containers and bridging. This is the - GUI package containing a canvas-based Tcl/Tk GUI for easily drawing virtual - network topologies. - diff --git a/packaging/rpm/core.spec.in b/packaging/rpm/core.spec.in deleted file mode 100644 index b68b11de..00000000 --- a/packaging/rpm/core.spec.in +++ /dev/null @@ -1,362 +0,0 @@ -%define version @PACKAGE_VERSION@ -%define lib_version @GENERIC_RELEASE@ -%define python_version %(%{__python} -c "import sys; print '%s.%s' % (sys.version_info[0], sys.version_info[1])")%{nil} - -%if 0%{?fedora} >= 17 -%define with_kernel_modules_extra 1 -%else -%define with_kernel_modules_extra 0 -%endif - -Name: core -Summary: Common Open Research Emulator for use with network namespaces -License: BSD -Prefix: /usr -Release: 1%{?dist} -Source: core-%{version}.tar.gz -URL: http://www.nrl.navy.mil/itd/ncs/products/core -Version: %{version} -%description -The Common Open Research Emulator provides Python modules and a GUI for -building virtual networks using Linux network namespace containers and bridging. - -%package daemon -Summary: Common Open Research Emulator daemon back-end -Group: System Tools -Requires: bash bridge-utils ebtables iproute libev python net-tools -Requires: python2-logzero python-enum34 -%if 0%{?el6} -Requires: procps -%else -Requires: procps-ng -%endif -%if %{with_kernel_modules_extra} -Requires: kernel-modules-extra -%endif -%if 0%{?fedora} >= 25 -Requires: iproute-tc -%endif -BuildRequires: make automake autoconf libev-devel python-devel bridge-utils ebtables iproute net-tools ImageMagick help2man -BuildRequires: python2-pytest-runner python2-sphinx -BuildRequires: python2-logzero python-enum34 -%if 0%{?el6} -BuildRequires: procps -%else -BuildRequires: procps-ng -%endif -%if 0%{?fedora} >= 25 -BuildRequires: iproute-tc -%endif -Provides: core-daemon -# python-sphinx -%description daemon -The Common Open Research Emulator provides Python modules for building virtual -networks using Linux network namespace containers and bridging. - -%package gui -Summary: Common Open Research Emulator GUI front-end -Group: System Tools -Requires: tcl tk xterm -BuildArch: noarch -BuildRequires: make automake autoconf -Provides: core-gui -%description gui -The Common Open Research Emulator canvas-based Tcl/Tk GUI for easily drawing -virtual network topologies. - -%prep - -%setup -q - -%build - -./bootstrap.sh -# not using --disable-gui/--disable-daemon, because RPM expects both to be -# installed by this build process -# assume Fedora, using systemd startup script -CFLAGS="-fno-strict-aliasing $RPM_OPT_FLAGS" %configure --with-startup=systemd -make -j4 - -%install -rm -rf $RPM_BUILD_ROOT -make DESTDIR=$RPM_BUILD_ROOT install -rm -f $RPM_BUILD_ROOT/%{python_sitelib}/site.py* \ - $RPM_BUILD_ROOT/%{python_sitearch}/site.py* \ - $RPM_BUILD_ROOT/%{python_sitelib}/easy-install.pth \ - $RPM_BUILD_ROOT/%{python_sitearch}/easy-install.pth - -%clean -rm -rf $RPM_BUILD_ROOT - -%post - -%post daemon -# don't run EMANE with realtime option under Fedora -sed -i 's/emane_realtime = True/emane_realtime = False/' /etc/core/core.conf - -%preun daemon -if [ "$1" -eq 0 ]; then - systemctl stop core-daemon.service > /dev/null 2>&1 || true - - if [ -x @SBINDIR@/core-cleanup ]; then - @SBINDIR@/core-cleanup > /dev/null 2>&1 || true - fi -fi - -%postun - -%files gui -%{_bindir}/core-gui -%dir @CORE_LIB_DIR@ -%dir @CORE_LIB_DIR@/addons -@CORE_LIB_DIR@/addons/ipsecservice.tcl -@CORE_LIB_DIR@/annotations.tcl -@CORE_LIB_DIR@/api.tcl -@CORE_LIB_DIR@/canvas.tcl -@CORE_LIB_DIR@/cfgparse.tcl -@CORE_LIB_DIR@/core-bsd-cleanup.sh -@CORE_LIB_DIR@/core.tcl -@CORE_LIB_DIR@/debug.tcl -@CORE_LIB_DIR@/editor.tcl -@CORE_LIB_DIR@/exceptions.tcl -@CORE_LIB_DIR@/exec.tcl -@CORE_LIB_DIR@/filemgmt.tcl -@CORE_LIB_DIR@/gpgui.tcl -@CORE_LIB_DIR@/graph_partitioning.tcl -@CORE_LIB_DIR@/help.tcl -%{_datadir}/applications/core-gui.desktop -%{_datadir}/pixmaps/core-gui.xpm -%dir %{_datadir}/%{name} -%dir %{_datadir}/%{name}/icons -%dir %{_datadir}/%{name}/icons/normal -%{_datadir}/%{name}/icons/normal/antenna.gif -%{_datadir}/%{name}/icons/normal/ap.gif -%{_datadir}/%{name}/icons/normal/core-icon.png -%{_datadir}/%{name}/icons/normal/core-icon.xbm -%{_datadir}/%{name}/icons/normal/core-logo-275x75.gif -%{_datadir}/%{name}/icons/normal/document-properties.gif -%{_datadir}/%{name}/icons/normal/gps-diagram.xbm -%{_datadir}/%{name}/icons/normal/host.gif -%{_datadir}/%{name}/icons/normal/hub.gif -%{_datadir}/%{name}/icons/normal/lanswitch.gif -%{_datadir}/%{name}/icons/normal/mdr.gif -%{_datadir}/%{name}/icons/normal/oval.gif -%{_datadir}/%{name}/icons/normal/pc.gif -%{_datadir}/%{name}/icons/normal/rj45.gif -%{_datadir}/%{name}/icons/normal/router_black.gif -%{_datadir}/%{name}/icons/normal/router.gif -%{_datadir}/%{name}/icons/normal/router_green.gif -%{_datadir}/%{name}/icons/normal/router_purple.gif -%{_datadir}/%{name}/icons/normal/router_red.gif -%{_datadir}/%{name}/icons/normal/router_yellow.gif -%{_datadir}/%{name}/icons/normal/simple.xbm -%{_datadir}/%{name}/icons/normal/text.gif -%{_datadir}/%{name}/icons/normal/thumb-unknown.gif -%{_datadir}/%{name}/icons/normal/tunnel.gif -%{_datadir}/%{name}/icons/normal/wlan.gif -%{_datadir}/%{name}/icons/normal/xen.gif -%dir %{_datadir}/%{name}/icons/svg -%{_datadir}/%{name}/icons/svg/ap.svg -%{_datadir}/%{name}/icons/svg/cel.svg -%{_datadir}/%{name}/icons/svg/hub.svg -%{_datadir}/%{name}/icons/svg/lanswitch.svg -%{_datadir}/%{name}/icons/svg/mdr.svg -%{_datadir}/%{name}/icons/svg/otr.svg -%{_datadir}/%{name}/icons/svg/rj45.svg -%{_datadir}/%{name}/icons/svg/router_black.svg -%{_datadir}/%{name}/icons/svg/router_green.svg -%{_datadir}/%{name}/icons/svg/router_purple.svg -%{_datadir}/%{name}/icons/svg/router_red.svg -%{_datadir}/%{name}/icons/svg/router.svg -%{_datadir}/%{name}/icons/svg/router_yellow.svg -%{_datadir}/%{name}/icons/svg/start.svg -%{_datadir}/%{name}/icons/svg/tunnel.svg -%{_datadir}/%{name}/icons/svg/vlan.svg -%{_datadir}/%{name}/icons/svg/xen.svg -%dir %{_datadir}/%{name}/icons/tiny -%{_datadir}/%{name}/icons/tiny/ap.gif -%{_datadir}/%{name}/icons/tiny/arrow.down.gif -%{_datadir}/%{name}/icons/tiny/arrow.gif -%{_datadir}/%{name}/icons/tiny/arrow.up.gif -%{_datadir}/%{name}/icons/tiny/blank.gif -%{_datadir}/%{name}/icons/tiny/button.play.gif -%{_datadir}/%{name}/icons/tiny/button.stop.gif -%{_datadir}/%{name}/icons/tiny/cel.gif -%{_datadir}/%{name}/icons/tiny/delete.gif -%{_datadir}/%{name}/icons/tiny/document-new.gif -%{_datadir}/%{name}/icons/tiny/document-properties.gif -%{_datadir}/%{name}/icons/tiny/document-save.gif -%{_datadir}/%{name}/icons/tiny/edit-delete.gif -%{_datadir}/%{name}/icons/tiny/eraser.gif -%{_datadir}/%{name}/icons/tiny/fileopen.gif -%{_datadir}/%{name}/icons/tiny/folder.gif -%{_datadir}/%{name}/icons/tiny/host.gif -%{_datadir}/%{name}/icons/tiny/hub.gif -%{_datadir}/%{name}/icons/tiny/lanswitch.gif -%{_datadir}/%{name}/icons/tiny/link.gif -%{_datadir}/%{name}/icons/tiny/marker.gif -%{_datadir}/%{name}/icons/tiny/mdr.gif -%{_datadir}/%{name}/icons/tiny/mobility.gif -%{_datadir}/%{name}/icons/tiny/moboff.gif -%{_datadir}/%{name}/icons/tiny/observe.gif -%{_datadir}/%{name}/icons/tiny/oval.gif -%{_datadir}/%{name}/icons/tiny/pc.gif -%{_datadir}/%{name}/icons/tiny/ping.gif -%{_datadir}/%{name}/icons/tiny/plot.gif -%{_datadir}/%{name}/icons/tiny/rectangle.gif -%{_datadir}/%{name}/icons/tiny/rj45.gif -%{_datadir}/%{name}/icons/tiny/router_black.gif -%{_datadir}/%{name}/icons/tiny/router.gif -%{_datadir}/%{name}/icons/tiny/router_green.gif -%{_datadir}/%{name}/icons/tiny/router_purple.gif -%{_datadir}/%{name}/icons/tiny/router_red.gif -%{_datadir}/%{name}/icons/tiny/router_yellow.gif -%{_datadir}/%{name}/icons/tiny/run.gif -%{_datadir}/%{name}/icons/tiny/script_pause.gif -%{_datadir}/%{name}/icons/tiny/script_play.gif -%{_datadir}/%{name}/icons/tiny/script_stop.gif -%{_datadir}/%{name}/icons/tiny/select.gif -%{_datadir}/%{name}/icons/tiny/start.gif -%{_datadir}/%{name}/icons/tiny/stock_connect.gif -%{_datadir}/%{name}/icons/tiny/stock_disconnect.gif -%{_datadir}/%{name}/icons/tiny/stop.gif -%{_datadir}/%{name}/icons/tiny/text.gif -%{_datadir}/%{name}/icons/tiny/trace.gif -%{_datadir}/%{name}/icons/tiny/tunnel.gif -%{_datadir}/%{name}/icons/tiny/twonode.gif -%{_datadir}/%{name}/icons/tiny/view-refresh.gif -%{_datadir}/%{name}/icons/tiny/wlan.gif -%{_datadir}/%{name}/icons/tiny/xen.gif -@CORE_LIB_DIR@/initgui.tcl -@CORE_LIB_DIR@/ipv4.tcl -@CORE_LIB_DIR@/ipv6.tcl -@CORE_LIB_DIR@/linkcfg.tcl -@CORE_LIB_DIR@/mobility.tcl -@CORE_LIB_DIR@/nodecfg.tcl -@CORE_LIB_DIR@/nodes.tcl -@CORE_LIB_DIR@/ns2imunes.tcl -@CORE_LIB_DIR@/plugins.tcl -@CORE_LIB_DIR@/services.tcl -@CORE_LIB_DIR@/tooltips.tcl -@CORE_LIB_DIR@/topogen.tcl -@CORE_LIB_DIR@/traffic.tcl -@CORE_LIB_DIR@/util.tcl -@CORE_LIB_DIR@/version.tcl -@CORE_LIB_DIR@/widget.tcl -@CORE_LIB_DIR@/wlanscript.tcl -@CORE_LIB_DIR@/wlan.tcl -%dir %{_datadir}/%{name}/examples -%dir %{_datadir}/%{name}/examples/configs -%{_datadir}/%{name}/examples/configs/sample10-kitchen-sink.imn -%{_datadir}/%{name}/examples/configs/sample1-bg.gif -%{_datadir}/%{name}/examples/configs/sample1.imn -%{_datadir}/%{name}/examples/configs/sample1.scen -%{_datadir}/%{name}/examples/configs/sample2-ssh.imn -%{_datadir}/%{name}/examples/configs/sample3-bgp.imn -%{_datadir}/%{name}/examples/configs/sample4-bg.jpg -%{_datadir}/%{name}/examples/configs/sample4-nrlsmf.imn -%{_datadir}/%{name}/examples/configs/sample4.scen -%{_datadir}/%{name}/examples/configs/sample5-mgen.imn -%{_datadir}/%{name}/examples/configs/sample6-emane-rfpipe.imn -%{_datadir}/%{name}/examples/configs/sample7-emane-ieee80211abg.imn -%{_datadir}/%{name}/examples/configs/sample8-ipsec-service.imn -%{_datadir}/%{name}/examples/configs/sample9-vpn.imn -%doc %{_mandir}/man1/core-gui.1.gz - -%files daemon -%config @CORE_CONF_DIR@/core.conf -%config @CORE_CONF_DIR@/perflogserver.conf -%config @CORE_CONF_DIR@/xen.conf -%dir %{_datadir}/%{name} -%dir %{_datadir}/%{name}/examples -%{_datadir}/%{name}/examples/controlnet_updown -%dir %{_datadir}/%{name}/examples/corens3 -%{_datadir}/%{name}/examples/corens3/ns3lte.py* -%{_datadir}/%{name}/examples/corens3/ns3wifi.py* -%{_datadir}/%{name}/examples/corens3/ns3wifirandomwalk.py* -%{_datadir}/%{name}/examples/corens3/ns3wimax.py* -%{_datadir}/%{name}/examples/emanemanifest2core.py* -%{_datadir}/%{name}/examples/emanemodel2core.py* -%{_datadir}/%{name}/examples/findcore.py* -%dir %{_datadir}/%{name}/examples/hooks -%{_datadir}/%{name}/examples/hooks/configuration_hook.sh -%{_datadir}/%{name}/examples/hooks/datacollect_hook.sh -%{_datadir}/%{name}/examples/hooks/perflogserver.py* -%{_datadir}/%{name}/examples/hooks/perflogstart.sh -%{_datadir}/%{name}/examples/hooks/perflogstop.sh -%{_datadir}/%{name}/examples/hooks/sessiondatacollect.sh -%{_datadir}/%{name}/examples/hooks/timesyncstart.sh -%{_datadir}/%{name}/examples/hooks/timesyncstop.sh -%dir %{_datadir}/%{name}/examples/myservices -%{_datadir}/%{name}/examples/myservices/__init__.py* -%{_datadir}/%{name}/examples/myservices/README.txt -%{_datadir}/%{name}/examples/myservices/sample.py* -%dir %{_datadir}/%{name}/examples/netns -%{_datadir}/%{name}/examples/netns/basicrange.py* -%{_datadir}/%{name}/examples/netns/daemonnodes.py* -%{_datadir}/%{name}/examples/netns/distributed.py* -%{_datadir}/%{name}/examples/netns/emane80211.py* -%{_datadir}/%{name}/examples/netns/howmanynodes.py* -%{_datadir}/%{name}/examples/netns/iperf-performance-chain.py* -%{_datadir}/%{name}/examples/netns/iperf-performance.sh -%{_datadir}/%{name}/examples/netns/ospfmanetmdrtest.py* -%{_datadir}/%{name}/examples/netns/switch.py* -%{_datadir}/%{name}/examples/netns/switchtest.py* -%{_datadir}/%{name}/examples/netns/twonodes.sh -%{_datadir}/%{name}/examples/netns/wlanemanetests.py* -%{_datadir}/%{name}/examples/netns/wlantest.py* -%dir %{_datadir}/%{name}/examples/services -%{_datadir}/%{name}/examples/services/sampleFirewall -%{_datadir}/%{name}/examples/services/sampleIPsec -%{_datadir}/%{name}/examples/services/sampleVPNClient -%{_datadir}/%{name}/examples/services/sampleVPNServer -%{_datadir}/%{name}/examples/stopsession.py* -%doc %{_mandir}/man1/core-cleanup.1.gz -%doc %{_mandir}/man1/core-daemon.1.gz -%doc %{_mandir}/man1/core-manage.1.gz -%doc %{_mandir}/man1/coresendmsg.1.gz -%doc %{_mandir}/man1/core-xen-cleanup.1.gz -%doc %{_mandir}/man1/netns.1.gz -%doc %{_mandir}/man1/vcmd.1.gz -%doc %{_mandir}/man1/vnoded.1.gz -/etc/logrotate.d/core-daemon -/etc/systemd/system/core-daemon.service -%{python_sitearch}/* -%{python_sitelib}/* -%{_sbindir}/core-cleanup -%{_sbindir}/core-daemon -%{_sbindir}/core-manage -%{_sbindir}/coresendmsg -%{_sbindir}/core-xen-cleanup -%{_sbindir}/netns -%{_sbindir}/vcmd -%{_sbindir}/vnoded - -%changelog -* Mon Nov 20 2017 Gabriel Somlo - 5.0 -- update RPM spec file to match 5.0 release -* Fri Sep 01 2017 CORE Developers - 5.0 -- Added Ryu SD and Open vSwitch services, code cleanup and refactoring -* Fri Jun 5 2015 CORE Developers - 4.8 -- Support for NRL Network Modeling Framework (NMF) XML representation, bugfixes -* Wed Aug 6 2014 Jeff Ahrenholz - 4.7 -- EMANE 0.9.1, asymmetric links, bugfixes -* Thu Aug 22 2013 Jeff Ahrenholz - 4.6 -- cored now core-daemon, core now core-gui for CORE 4.6 release -* Wed Apr 3 2013 Jeff Ahrenholz - 4.5 -- split into gui and daemon RPMs for CORE 4.5 release -* Tue Sep 4 2012 Jeff Ahrenholz - 4.4 -- update files list for CORE 4.4 release, removed info file -* Tue Feb 7 2012 Jeff Ahrenholz - 4.3 -- update files list for CORE 4.3 release, freshen dependencies -* Tue Aug 16 2011 Jeff Ahrenholz - 4.2 -- update for CORE 4.2 release; use dir variables, more arch independent -* Mon Dec 13 2010 Jeff Ahrenholz - 4.1 -- update for CORE 4.1 release; added calls to ldconfig and removal of pyc files -* Wed Aug 4 2010 Jeff Ahrenholz - 4.0 -- update for CORE 4.0 release for Python and network namespaces -* Thu Sep 10 2009 Jeff Ahrenholz - 3.5 -- update for CORE 3.5 release to include init script -* Fri May 29 2009 Jeff Ahrenholz - 3.4 -- initial spec file for CORE 3.4 release - From 9f9dfbf6c2af825c8b69f76ff688756d3cbed374 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 30 May 2018 21:20:52 -0700 Subject: [PATCH 26/83] Update coreemu.py set umask 0 to replicate previous behavior --- daemon/core/emulator/coreemu.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 8fa6ffb9..980b6074 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -849,6 +849,9 @@ class CoreEmu(object): :param dict config: configuration options """ + # set umask 0 + os.umask(0) + # configuration self.config = config From 2ede43e3aedaaf365a142a1857b7fbbdd6612cbf Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 6 Jun 2018 14:51:45 -0700 Subject: [PATCH 27/83] initial commit with things working for the most part --- daemon/core/broker.py | 62 +--- daemon/core/conf.py | 567 +++++++---------------------- daemon/core/corehandlers.py | 413 ++++++++++++++++++++- daemon/core/coreobj.py | 4 +- daemon/core/emane/bypass.py | 17 +- daemon/core/emane/commeffect.py | 27 +- daemon/core/emane/emanemanager.py | 240 ++++++------ daemon/core/emane/emanemanifest.py | 20 +- daemon/core/emane/emanemodel.py | 97 ++--- daemon/core/emane/nodes.py | 15 +- daemon/core/emane/tdma.py | 14 +- daemon/core/emulator/coreemu.py | 7 +- daemon/core/location.py | 37 +- daemon/core/mobility.py | 277 +++++++------- daemon/core/netns/nodes.py | 10 +- daemon/core/sdt.py | 14 +- daemon/core/service.py | 154 +------- daemon/core/session.py | 402 ++++++-------------- daemon/core/xml/xmlparser0.py | 2 + daemon/core/xml/xmlparser1.py | 8 +- daemon/core/xml/xmlwriter1.py | 28 +- 21 files changed, 1018 insertions(+), 1397 deletions(-) diff --git a/daemon/core/broker.py b/daemon/core/broker.py index 8b25e751..7ef14266 100644 --- a/daemon/core/broker.py +++ b/daemon/core/broker.py @@ -11,7 +11,6 @@ import threading from core import logger from core.api import coreapi -from core.conf import ConfigurableManager from core.coreobj import PyCoreNet from core.coreobj import PyCoreNode from core.enumerations import ConfigDataTypes @@ -81,7 +80,7 @@ class CoreDistributedServer(object): self.sock = None -class CoreBroker(ConfigurableManager): +class CoreBroker(object): """ Helps with brokering messages between CORE daemon servers. """ @@ -100,7 +99,7 @@ class CoreBroker(ConfigurableManager): :return: nothing """ - ConfigurableManager.__init__(self) + # ConfigurableManager.__init__(self) self.session = session self.session_clients = [] self.session_id_master = None @@ -611,62 +610,6 @@ class CoreBroker(ConfigurableManager): """ self.physical_nodes.add(nodenum) - def configure_reset(self, config_data): - """ - Ignore reset messages, because node delete responses may still - arrive and require the use of nodecounts. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: nothing - """ - return None - - def configure_values(self, config_data): - """ - Receive configuration message with a list of server:host:port - combinations that we"ll need to connect with. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: nothing - """ - values = config_data.data_values - session_id = config_data.session - - if values is None: - logger.info("emulation server data missing") - return None - values = values.split("|") - - # string of "server:ip:port,server:ip:port,..." - server_strings = values[0] - server_list = server_strings.split(",") - - for server in server_list: - server_items = server.split(":") - (name, host, port) = server_items[:3] - - if host == "": - host = None - - if port == "": - port = None - else: - port = int(port) - - if session_id is not None: - # receive session ID and my IP from master - self.session_id_master = int(session_id.split("|")[0]) - self.myip = host - host = None - port = None - - # this connects to the server immediately; maybe we should wait - # or spin off a new "client" thread here - self.addserver(name, host, port) - self.setupserver(name) - - return None - def handle_message(self, message): """ Handle an API message. Determine whether this needs to be handled @@ -733,6 +676,7 @@ class CoreBroker(ConfigurableManager): if server is None: logger.warn("ignoring unknown server: %s", servername) return + if server.sock is None or server.host is None or server.port is None: logger.info("ignoring disconnected server: %s", servername) return diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 335db819..fae75d6c 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -2,384 +2,24 @@ Common support for configurable CORE objects. """ -import string +from collections import OrderedDict from core import logger from core.data import ConfigData -from core.enumerations import ConfigDataTypes -from core.enumerations import ConfigFlags -class ConfigurableManager(object): - """ - A generic class for managing Configurables. This class can register - with a session to receive Config Messages for setting some parameters - for itself or for the Configurables that it manages. - """ - # name corresponds to configuration object field - name = "" - - # type corresponds with register message types - config_type = None - - def __init__(self): - """ - Creates a ConfigurableManager instance. - """ - # configurable key=values, indexed by node number - self.configs = {} - - # TODO: fix the need for this and isolate to the mobility class that wants it - self._modelclsmap = {} - - def configure(self, session, config_data): - """ - Handle configure messages. The configuration message sent to a - ConfigurableManager usually is used to: - 1. Request a list of Configurables (request flag) - 2. Reset manager and clear configs (reset flag) - 3. Send values that configure the manager or one of its Configurables - - :param core.session.Session session: CORE session object - :param ConfigData config_data: configuration data for carrying out a configuration - :return: response messages - """ - - if config_data.type == ConfigFlags.REQUEST.value: - return self.configure_request(config_data) - elif config_data.type == ConfigFlags.RESET.value: - return self.configure_reset(config_data) - else: - return self.configure_values(config_data) - - def configure_request(self, config_data): - """ - Request configuration data. - - :param ConfigData config_data: configuration data for carrying out a configuration - :return: nothing - """ - return None - - def configure_reset(self, config_data): - """ - By default, resets this manager to clear configs. - - :param ConfigData config_data: configuration data for carrying out a configuration - :return: reset response messages, or None - """ - return self.reset() - - def configure_values(self, config_data): - """ - Values have been sent to this manager. - - :param ConfigData config_data: configuration data for carrying out a configuration - :return: nothing - """ - return None - - def configure_values_keyvalues(self, config_data, target, keys): - """ - Helper that can be used for configure_values for parsing in - 'key=value' strings from a values field. The key name must be - in the keys list, and target.key=value is set. - - :param ConfigData config_data: configuration data for carrying out a configuration - :param target: target to set attribute values on - :param keys: list of keys to verify validity - :return: nothing - """ - values = config_data.data_values - - if values is None: - return None - - kvs = values.split('|') - for kv in kvs: - try: - key, value = kv.split('=', 1) - if value is not None and not value.strip(): - value = None - except ValueError: - # value only - key = keys[kvs.index(kv)] - value = kv - if key not in keys: - raise ValueError("invalid key: %s" % key) - if value is not None: - setattr(target, key, value) - - return None - - def reset(self): - """ - Reset functionality for the configurable class. - - :return: nothing - """ - return None - - def setconfig(self, nodenum, conftype, values): - """ - Add configuration values for a node to a dictionary; values are - usually received from a Configuration Message, and may refer to a - node for which no object exists yet - - :param int nodenum: node id - :param conftype: configuration types - :param values: configuration values - :return: nothing - """ - logger.info("setting config for node(%s): %s - %s", nodenum, conftype, values) - conflist = [] - if nodenum in self.configs: - oldlist = self.configs[nodenum] - found = False - for t, v in oldlist: - if t == conftype: - # replace existing config - found = True - conflist.append((conftype, values)) - else: - conflist.append((t, v)) - if not found: - conflist.append((conftype, values)) - else: - conflist.append((conftype, values)) - self.configs[nodenum] = conflist - - def getconfig(self, nodenum, conftype, defaultvalues): - """ - Get configuration values for a node; if the values don't exist in - our dictionary then return the default values supplied - - :param int nodenum: node id - :param conftype: configuration type - :param defaultvalues: default values - :return: configuration type and default values - :type: tuple - """ - logger.info("getting config for node(%s): %s - default(%s)", - nodenum, conftype, defaultvalues) - if nodenum in self.configs: - # return configured values - conflist = self.configs[nodenum] - for t, v in conflist: - if conftype is None or t == conftype: - return t, v - # return default values provided (may be None) - return conftype, defaultvalues - - def getallconfigs(self, use_clsmap=True): - """ - Return (nodenum, conftype, values) tuples for all stored configs. - Used when reconnecting to a session. - - :param bool use_clsmap: should a class map be used, default to True - :return: list of all configurations - :rtype: list - """ - r = [] - for nodenum in self.configs: - for t, v in self.configs[nodenum]: - if use_clsmap: - t = self._modelclsmap[t] - r.append((nodenum, t, v)) - return r - - def clearconfig(self, nodenum): - """ - remove configuration values for the specified node; - when nodenum is None, remove all configuration values - - :param int nodenum: node id - :return: nothing - """ - if nodenum is None: - self.configs = {} - return - if nodenum in self.configs: - self.configs.pop(nodenum) - - def setconfig_keyvalues(self, nodenum, conftype, keyvalues): - """ - Key values list of tuples for a node. - - :param int nodenum: node id - :param conftype: configuration type - :param keyvalues: key valyes - :return: nothing - """ - if conftype not in self._modelclsmap: - logger.warn("unknown model type '%s'", conftype) - return - model = self._modelclsmap[conftype] - keys = model.getnames() - # defaults are merged with supplied values here - values = list(model.getdefaultvalues()) - for key, value in keyvalues: - if key not in keys: - logger.warn("Skipping unknown configuration key for %s: '%s'", conftype, key) - continue - i = keys.index(key) - values[i] = value - self.setconfig(nodenum, conftype, values) - - def getmodels(self, n): - """ - Return a list of model classes and values for a net if one has been - configured. This is invoked when exporting a session to XML. - This assumes self.configs contains an iterable of (model-names, values) - and a self._modelclsmapdict exists. - - :param n: network node to get models for - :return: list of model and values tuples for the network node - :rtype: list - """ - r = [] - if n.objid in self.configs: - v = self.configs[n.objid] - for model in v: - cls = self._modelclsmap[model[0]] - vals = model[1] - r.append((cls, vals)) - return r - - -class Configurable(object): - """ - A generic class for managing configuration parameters. - Parameters are sent via Configuration Messages, which allow the GUI - to build dynamic dialogs depending on what is being configured. - """ - name = "" - # Configuration items: - # ('name', 'type', 'default', 'possible-value-list', 'caption') - config_matrix = [] - config_groups = None - bitmap = None - - def __init__(self, session=None, object_id=None): - """ - Creates a Configurable instance. - - :param core.session.Session session: session for this configurable - :param object_id: - """ - self.session = session - self.object_id = object_id - - def reset(self): - """ - Reset method. - - :return: nothing - """ - pass - - def register(self): - """ - Register method. - - :return: nothing - """ - pass +class ConfigShim(object): + @classmethod + def str_to_dict(cls, key_values): + key_values = key_values.split("|") + values = OrderedDict() + for key_value in key_values: + key, value = key_value.split("=", 1) + values[key] = value + return values @classmethod - def getdefaultvalues(cls): - """ - Retrieve default values from configuration matrix. - - :return: tuple of default values - :rtype: tuple - """ - return tuple(map(lambda x: x[2], cls.config_matrix)) - - @classmethod - def getnames(cls): - """ - Retrieve name values from configuration matrix. - - :return: tuple of name values - :rtype: tuple - """ - return tuple(map(lambda x: x[0], cls.config_matrix)) - - @classmethod - def configure(cls, manager, config_data): - """ - Handle configuration messages for this object. - - :param ConfigurableManager manager: configuration manager - :param config_data: configuration data - :return: configuration data object - :rtype: ConfigData - """ - reply = None - node_id = config_data.node - object_name = config_data.object - config_type = config_data.type - interface_id = config_data.interface_number - values_str = config_data.data_values - - if interface_id is not None: - node_id = node_id * 1000 + interface_id - - logger.debug("received configure message for %s nodenum:%s", cls.name, str(node_id)) - if config_type == ConfigFlags.REQUEST.value: - logger.info("replying to configure request for %s model", cls.name) - # when object name is "all", the reply to this request may be None - # if this node has not been configured for this model; otherwise we - # reply with the defaults for this model - if object_name == "all": - defaults = None - typeflags = ConfigFlags.UPDATE.value - else: - defaults = cls.getdefaultvalues() - typeflags = ConfigFlags.NONE.value - values = manager.getconfig(node_id, cls.name, defaults)[1] - if values is None: - logger.warn("no active configuration for node (%s), ignoring request") - # node has no active config for this model (don't send defaults) - return None - # reply with config options - reply = cls.config_data(0, node_id, typeflags, values) - elif config_type == ConfigFlags.RESET.value: - if object_name == "all": - manager.clearconfig(node_id) - # elif conftype == coreapi.CONF_TYPE_FLAGS_UPDATE: - else: - # store the configuration values for later use, when the node - # object has been created - if object_name is None: - logger.info("no configuration object for node %s", node_id) - return None - defaults = cls.getdefaultvalues() - if values_str is None: - # use default or preconfigured values - values = manager.getconfig(node_id, cls.name, defaults)[1] - else: - # use new values supplied from the conf message - values = values_str.split('|') - # determine new or old style config - new = cls.haskeyvalues(values) - if new: - new_values = list(defaults) - keys = cls.getnames() - for v in values: - key, value = v.split('=', 1) - try: - new_values[keys.index(key)] = value - except ValueError: - logger.info("warning: ignoring invalid key '%s'" % key) - values = new_values - manager.setconfig(node_id, object_name, values) - - return reply - - @classmethod - def config_data(cls, flags, node_id, type_flags, values): + def config_data(cls, flags, node_id, type_flags, configurable_options, config): """ Convert this class to a Config API message. Some TLVs are defined by the class, but node number, conf type flags, and values must @@ -388,106 +28,143 @@ class Configurable(object): :param flags: message flags :param int node_id: node id :param type_flags: type flags - :param values: values + :param ConfigurableOptions configurable_options: options to create config data for + :param dict config: configuration values for options :return: configuration data object :rtype: ConfigData """ - keys = cls.getnames() - keyvalues = map(lambda a, b: "%s=%s" % (a, b), keys, values) - values_str = string.join(keyvalues, '|') - datatypes = tuple(map(lambda x: x[1], cls.config_matrix)) - captions = reduce(lambda a, b: a + '|' + b, map(lambda x: x[4], cls.config_matrix)) - possible_valuess = reduce(lambda a, b: a + '|' + b, map(lambda x: x[3], cls.config_matrix)) + key_values = None + captions = None + data_types = [] + possible_values = [] + logger.debug("configurable: %s", configurable_options) + logger.debug("configuration options: %s", configurable_options.configurations) + logger.debug("configuration data: %s", config) + for configuration in configurable_options.configurations(): + if not captions: + captions = configuration.label + else: + captions += "|%s" % configuration.label + + data_types.append(configuration.type.value) + + options = ",".join(configuration.options) + possible_values.append(options) + + _id = configuration.id + config_value = config.get(_id, configuration.default) + key_value = "%s=%s" % (_id, config_value) + if not key_values: + key_values = key_value + else: + key_values += "|%s" % key_value return ConfigData( message_type=flags, node=node_id, - object=cls.name, + object=configurable_options.name, type=type_flags, - data_types=datatypes, - data_values=values_str, + data_types=tuple(data_types), + data_values=key_values, captions=captions, - possible_values=possible_valuess, - bitmap=cls.bitmap, - groups=cls.config_groups + possible_values="|".join(possible_values), + bitmap=configurable_options.bitmap, + groups=configurable_options.config_groups() ) - @staticmethod - def booltooffon(value): - """ - Convenience helper turns bool into on (True) or off (False) string. - :param str value: value to retrieve on/off value for - :return: on or off string - :rtype: str - """ - if value == "1" or value == "true" or value == "on": - return "on" - else: - return "off" +class Configuration(object): + def __init__(self, _id, _type, label, default="", options=None): + self.id = _id + self.type = _type + self.default = default + if not options: + options = [] + self.options = options + if not label: + label = _id + self.label = label - @staticmethod - def offontobool(value): - """ - Convenience helper for converting an on/off string to a integer. + def __str__(self): + return "%s(id=%s, type=%s, default=%s, options=%s)" % ( + self.__class__.__name__, self.id, self.type, self.default, self.options) - :param str value: on/off string - :return: on/off integer value - :rtype: int - """ - if type(value) == str: - if value.lower() == "on": - return 1 - elif value.lower() == "off": - return 0 - return value + +class ConfigurableOptions(object): + # unique name to receive configuration changes + name = None + bitmap = None @classmethod - def valueof(cls, name, values): + def configurations(cls): """ - Helper to return a value by the name defined in confmatrix. - Checks if it is boolean + Returns configuration options supported by this class. - :param str name: name to get value of - :param values: values to get value from - :return: value for name + :return: list of configuration options + :rtype: list[Configuration] """ - i = cls.getnames().index(name) - if cls.config_matrix[i][1] == ConfigDataTypes.BOOL.value and values[i] != "": - return cls.booltooffon(values[i]) - else: - return values[i] + return [] - @staticmethod - def haskeyvalues(values): + @classmethod + def config_groups(cls): """ - Helper to check for list of key=value pairs versus a plain old - list of values. Returns True if all elements are "key=value". + String formatted to specify configuration groupings, using list index positions. - :param values: items to check for key/value pairs - :return: True if all values are key/value pairs, False otherwise - :rtype: bool + Example: + "Group1:start-stop|Group2:start-stop" + + :return: config groups + :rtype: str """ - if len(values) == 0: - return False - for v in values: - if "=" not in v: - return False - return True + return None - def getkeyvaluelist(self): + @classmethod + def default_values(cls): """ - Helper to return a list of (key, value) tuples. Keys come from - configuration matrix and values are instance attributes. + Retrieves default values for configurations. - :return: tuples of key value pairs - :rtype: list + :return: mapping of configuration options that can also be iterated in order of definition + :rtype: OrderedDict """ - key_values = [] + return OrderedDict([(config.id, config.default) for config in cls.configurations()]) - for name in self.getnames(): - if hasattr(self, name): - value = getattr(self, name) - key_values.append((name, value)) - return key_values +class NewConfigurableManager(object): + _default_node = -1 + _default_type = "default" + + def __init__(self): + self._configuration_maps = {} + + def nodes(self): + return [node_id for node_id in self._configuration_maps.iterkeys() if node_id != self._default_node] + + def config_reset(self, node_id=None): + if not node_id: + self._configuration_maps.clear() + elif node_id in self._configuration_maps: + self._configuration_maps.pop(node_id) + + def set_config(self, _id, value, node_id=_default_node, config_type=_default_type): + logger.debug("setting config for node(%s) type(%s): %s=%s", node_id, config_type, _id, value) + node_type_map = self.get_configs(node_id, config_type) + node_type_map[_id] = value + + def set_configs(self, config, node_id=_default_node, config_type=_default_type): + logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) + node_type_map = self.get_configs(node_id, config_type) + node_type_map.clear() + node_type_map.update(config) + + def get_config(self, _id, node_id=_default_node, config_type=_default_type): + logger.debug("getting config for node(%s) type(%s): %s", node_id, config_type, _id) + node_type_map = self.get_configs(node_id, config_type) + return node_type_map.get(_id) + + def get_configs(self, node_id=_default_node, config_type=_default_type): + logger.debug("getting configs for node(%s) type(%s)", node_id, config_type) + node_map = self._configuration_maps.setdefault(node_id, {}) + return node_map.setdefault(config_type, {}) + + def get_config_types(self, node_id=_default_node): + return self._configuration_maps.get(node_id, {}) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 0d41d095..141ec3e2 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -10,15 +10,17 @@ import shutil import sys import threading import time +from itertools import repeat from core import logger from core.api import coreapi +from core.conf import ConfigShim from core.data import ConfigData from core.data import EventData from core.emulator.emudata import InterfaceData from core.emulator.emudata import LinkOptions from core.emulator.emudata import NodeOptions -from core.enumerations import ConfigTlvs +from core.enumerations import ConfigTlvs, ConfigFlags, ConfigDataTypes from core.enumerations import EventTlvs from core.enumerations import EventTypes from core.enumerations import ExceptionTlvs @@ -35,6 +37,8 @@ from core.enumerations import SessionTlvs from core.misc import nodeutils from core.misc import structutils from core.misc import utils +from core.mobility import BasicRangeModel, Ns2ScriptedMobility +from core.service import ServiceManager class CoreHandler(SocketServer.BaseRequestHandler): @@ -407,12 +411,19 @@ class CoreHandler(SocketServer.BaseRequestHandler): tlv_data = "" tlv_data += coreapi.CoreRegisterTlv.pack(RegisterTlvs.EXECUTE_SERVER.value, "core-daemon") tlv_data += coreapi.CoreRegisterTlv.pack(RegisterTlvs.EMULATION_SERVER.value, "core-daemon") - - # get config objects for session - for name in self.session.config_objects: - config_type, callback = self.session.config_objects[name] - # type must be in coreapi.reg_tlvs - tlv_data += coreapi.CoreRegisterTlv.pack(config_type, name) + tlv_data += coreapi.CoreRegisterTlv.pack(self.session.broker.config_type, self.session.broker.name) + tlv_data += coreapi.CoreRegisterTlv.pack(self.session.location.config_type, self.session.location.name) + tlv_data += coreapi.CoreRegisterTlv.pack(self.session.mobility.config_type, self.session.mobility.name) + for model_name in self.session.mobility.mobility_models(): + model_class = self.session.mobility.get_model_class(model_name) + tlv_data += coreapi.CoreRegisterTlv.pack(model_class.config_type, model_class.name) + tlv_data += coreapi.CoreRegisterTlv.pack(self.session.services.config_type, self.session.services.name) + tlv_data += coreapi.CoreRegisterTlv.pack(self.session.emane.config_type, self.session.emane.name) + for model_name in self.session.emane.emane_models(): + model_class = self.session.emane.get_model_class(model_name) + tlv_data += coreapi.CoreRegisterTlv.pack(model_class.config_type, model_class.name) + tlv_data += coreapi.CoreRegisterTlv.pack(self.session.options.config_type, self.session.options.name) + tlv_data += coreapi.CoreRegisterTlv.pack(self.session.metadata.config_type, self.session.metadata.name) return coreapi.CoreRegMessage.pack(MessageFlags.ADD.value, tlv_data) @@ -941,15 +952,399 @@ class CoreHandler(SocketServer.BaseRequestHandler): opaque=message.get_tlv(ConfigTlvs.OPAQUE.value) ) logger.debug("configuration message for %s node %s", config_data.object, config_data.node) + message_type = ConfigFlags(config_data.type) - # dispatch to any registered callback for this object type - replies = self.session.config_object(config_data) + replies = [] + + # handle session configuration + if config_data.object == "all": + replies = self.handle_config_all(message_type, config_data) + elif config_data.object == self.session.options.name: + replies = self.handle_config_session(message_type, config_data) + elif config_data.object == self.session.location.name: + self.handle_config_location(message_type, config_data) + elif config_data.object == self.session.metadata.name: + replies = self.handle_config_metadata(message_type, config_data) + elif config_data.object == self.session.broker.name: + self.handle_config_broker(message_type, config_data) + elif config_data.object == self.session.services.name: + replies = self.handle_config_services(message_type, config_data) + elif config_data.object == self.session.mobility.name: + self.handle_config_mobility(message_type, config_data) + elif config_data.object in [BasicRangeModel.name, Ns2ScriptedMobility.name]: + replies = self.handle_config_mobility_models(message_type, config_data) + elif config_data.object == self.session.emane.name: + replies = self.handle_config_emane(message_type, config_data) + elif config_data.object in self.session.emane.emane_models(): + replies = self.handle_config_emane_models(message_type, config_data) + else: + raise Exception("no handler for configuration: %s", config_data.object) for reply in replies: self.handle_broadcast_config(reply) return [] + def handle_config_all(self, message_type, config_data): + replies = [] + + if message_type == ConfigFlags.RESET: + node_id = config_data.node + self.session.location.reset() + self.session.services.reset() + self.session.mobility.reset() + self.session.mobility.config_reset(node_id) + self.session.emane.config_reset(node_id) + else: + raise Exception("cant handle config all: %s" % message_type) + + return replies + + def handle_config_session(self, message_type, config_data): + replies = [] + if message_type == ConfigFlags.REQUEST: + type_flags = ConfigFlags.NONE.value + config = self.session.options.get_configs() + config_response = ConfigShim.config_data(0, None, type_flags, self.session.options, config) + replies.append(config_response) + elif message_type != ConfigFlags.RESET and config_data.data_values: + values = ConfigShim.str_to_dict(config_data.data_values) + for key, value in values.iteritems(): + self.session.options.set_config(key, value) + return replies + + def handle_config_location(self, message_type, config_data): + if message_type == ConfigFlags.RESET: + self.session.location.reset() + else: + if not config_data.data_values: + logger.warn("location data missing") + else: + values = config_data.data_values.split("|") + + # Cartesian coordinate reference point + refx, refy = map(lambda x: float(x), values[0:2]) + refz = 0.0 + lat, lon, alt = map(lambda x: float(x), values[2:5]) + # xyz point + self.session.location.refxyz = (refx, refy, refz) + # geographic reference point + self.session.location.setrefgeo(lat, lon, alt) + self.session.location.refscale = float(values[5]) + logger.info("location configured: %s = %s scale=%s", self.session.location.refxyz, + self.session.location.refgeo, self.session.location.refscale) + logger.info("location configured: UTM%s", self.session.location.refutm) + + def handle_config_metadata(self, message_type, config_data): + replies = [] + if message_type == ConfigFlags.REQUEST: + node_id = config_data.node + data_values = "|".join(["%s=%s" % item for item in self.session.metadata.get_configs().iteritems()]) + data_types = tuple(ConfigDataTypes.STRING.value for _ in self.session.metadata.get_configs()) + config_response = ConfigData( + message_type=0, + node=node_id, + object=self.session.metadata.name, + type=ConfigFlags.NONE.value, + data_types=data_types, + data_values=data_values + ) + replies.append(config_response) + elif message_type != ConfigFlags.RESET and config_data.data_values: + values = ConfigShim.str_to_dict(config_data.data_values) + for key, value in values.iteritems(): + self.session.metadata.set_config(key, value) + return replies + + def handle_config_broker(self, message_type, config_data): + if message_type not in [ConfigFlags.REQUEST, ConfigFlags.RESET]: + session_id = config_data.session + if not config_data.data_values: + logger.info("emulation server data missing") + else: + values = config_data.data_values.split("|") + + # string of "server:ip:port,server:ip:port,..." + server_strings = values[0] + server_list = server_strings.split(",") + + for server in server_list: + server_items = server.split(":") + name, host, port = server_items[:3] + + if host == "": + host = None + + if port == "": + port = None + else: + port = int(port) + + if session_id is not None: + # receive session ID and my IP from master + self.session.broker.session_id_master = int(session_id.split("|")[0]) + self.session.broker.myip = host + host = None + port = None + + # this connects to the server immediately; maybe we should wait + # or spin off a new "client" thread here + self.session.broker.addserver(name, host, port) + self.session.broker.setupserver(name) + + def handle_config_services(self, message_type, config_data): + replies = [] + node_id = config_data.node + opaque = config_data.opaque + + if message_type == ConfigFlags.REQUEST: + session_id = config_data.session + opaque = config_data.opaque + + logger.debug("configuration request: node(%s) session(%s) opaque(%s)", node_id, session_id, opaque) + + # send back a list of available services + if opaque is None: + type_flag = ConfigFlags.NONE.value + data_types = tuple(repeat(ConfigDataTypes.BOOL.value, len(ServiceManager.services))) + values = "|".join(repeat('0', len(ServiceManager.services))) + names = map(lambda x: x._name, ServiceManager.services) + captions = "|".join(names) + possible_values = "" + for s in ServiceManager.services: + if s._custom_needed: + possible_values += '1' + possible_values += '|' + groups = self.session.services.buildgroups(ServiceManager.services) + # send back the properties for this service + else: + if not node_id: + return replies + + node = self.session.get_object(node_id) + if node is None: + logger.warn("Request to configure service for unknown node %s", node_id) + return replies + servicesstring = opaque.split(':') + services, unknown = self.session.services.servicesfromopaque(opaque, node.objid) + for u in unknown: + logger.warn("Request for unknown service '%s'" % u) + + if not services: + return replies + + if len(servicesstring) == 3: + # a file request: e.g. "service:zebra:quagga.conf" + file_data = self.session.services.getservicefile(services, node, servicesstring[2]) + self.session.broadcast_file(file_data) + # short circuit this request early to avoid returning response below + return replies + + # the first service in the list is the one being configured + svc = services[0] + # send back: + # dirs, configs, startindex, startup, shutdown, metadata, config + type_flag = ConfigFlags.UPDATE.value + data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(svc.keys))) + values = svc.tovaluelist(node, services) + captions = None + possible_values = None + groups = None + + config_response = ConfigData( + message_type=0, + node=node_id, + object=self.session.services.name, + type=type_flag, + data_types=data_types, + data_values=values, + captions=captions, + possible_values=possible_values, + groups=groups, + session=session_id, + opaque=opaque + ) + replies.append(config_response) + elif message_type == ConfigFlags.RESET: + self.session.services.reset() + else: + data_types = config_data.data_types + values = config_data.data_values + + error_message = "services config message that I don't know how to handle" + if values is None: + logger.error(error_message) + else: + if opaque is None: + values = values.split('|') + # store default services for a node type in self.defaultservices[] + if data_types is None or data_types[0] != ConfigDataTypes.STRING.value: + logger.info(error_message) + return None + key = values.pop(0) + self.session.services.defaultservices[key] = values + logger.debug("default services for type %s set to %s", key, values) + elif node_id: + # store service customized config in self.customservices[] + services, unknown = self.session.services.servicesfromopaque(opaque, node_id) + for u in unknown: + logger.warn("Request for unknown service '%s'" % u) + + if services: + svc = services[0] + values = ConfigShim.str_to_dict(values) + self.session.services.setcustomservice(node_id, svc, values) + + return replies + + def handle_config_mobility(self, message_type, _): + if message_type == ConfigFlags.RESET: + self.session.mobility.reset() + + def handle_config_mobility_models(self, message_type, config_data): + replies = [] + node_id = config_data.node + object_name = config_data.object + interface_id = config_data.interface_number + values_str = config_data.data_values + + if interface_id is not None: + node_id = node_id * 1000 + interface_id + + logger.debug("received configure message for %s nodenum: %s", object_name, node_id) + if message_type == ConfigFlags.REQUEST: + logger.info("replying to configure request for model: %s", object_name) + if object_name == "all": + typeflags = ConfigFlags.UPDATE.value + else: + typeflags = ConfigFlags.NONE.value + + model_class = self.session.mobility.get_model_class(object_name) + if not model_class: + logger.warn("model class does not exist: %s", object_name) + return [] + + config = self.session.mobility.get_configs(node_id, object_name) + if not config: + config = model_class.default_values() + + config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) + replies.append(config_response) + elif message_type == ConfigFlags.RESET: + if object_name == "all": + self.session.mobility.config_reset(node_id) + else: + # store the configuration values for later use, when the node + if not object_name: + logger.warn("no configuration object for node: %s", node_id) + return [] + + model_class = self.session.mobility.get_model_class(object_name) + if not model_class: + logger.warn("model class does not exist: %s", object_name) + return [] + + if values_str: + config = ConfigShim.str_to_dict(values_str) + else: + config = model_class.default_values() + + self.session.mobility.set_configs(config, node_id, object_name) + + return replies + + def handle_config_emane(self, message_type, config_data): + replies = [] + node_id = config_data.node + object_name = config_data.object + config_type = config_data.type + interface_id = config_data.interface_number + values_str = config_data.data_values + + if interface_id is not None: + node_id = node_id * 1000 + interface_id + + logger.debug("received configure message for %s nodenum: %s", object_name, node_id) + if message_type == ConfigFlags.REQUEST: + logger.info("replying to configure request for %s model", object_name) + if object_name == "all": + typeflags = ConfigFlags.UPDATE.value + else: + typeflags = ConfigFlags.NONE.value + config = self.session.emane.get_configs() + config_response = ConfigShim.config_data(0, node_id, typeflags, self.session.emane.emane_config, config) + replies.append(config_response) + elif config_type == ConfigFlags.RESET.value: + if object_name == "all": + self.session.emane.config_reset(node_id) + else: + if not object_name: + logger.info("no configuration object for node %s", node_id) + return [] + + if values_str: + config = ConfigShim.str_to_dict(values_str) + self.session.emane.set_configs(config) + + # extra logic to start slave Emane object after nemid has been configured from the master + if message_type == ConfigFlags.UPDATE and self.session.master is False: + # instantiation was previously delayed by setup returning Emane.NOT_READY + self.session.instantiate() + + return replies + + def handle_config_emane_models(self, message_type, config_data): + replies = [] + node_id = config_data.node + object_name = config_data.object + interface_id = config_data.interface_number + values_str = config_data.data_values + + if interface_id is not None: + node_id = node_id * 1000 + interface_id + + logger.debug("received configure message for %s nodenum: %s", object_name, node_id) + if message_type == ConfigFlags.REQUEST: + logger.info("replying to configure request for model: %s", object_name) + if object_name == "all": + typeflags = ConfigFlags.UPDATE.value + else: + typeflags = ConfigFlags.NONE.value + + model_class = self.session.emane.get_model_class(object_name) + if not model_class: + logger.warn("model class does not exist: %s", object_name) + return [] + + config = self.session.emane.get_configs(node_id, object_name) + if not config: + config = model_class.default_values() + + config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) + replies.append(config_response) + elif message_type == ConfigFlags.RESET: + if object_name == "all": + self.session.emane.config_reset(node_id) + else: + # store the configuration values for later use, when the node + if not object_name: + logger.warn("no configuration object for node: %s", node_id) + return [] + + model_class = self.session.emane.get_model_class(object_name) + if not model_class: + logger.warn("model class does not exist: %s", object_name) + return [] + + if values_str: + config = ConfigShim.str_to_dict(values_str) + else: + config = model_class.default_values() + + self.session.emane.set_configs(config, node_id, object_name) + + return replies + def handle_file_message(self, message): """ File Message handler diff --git a/daemon/core/coreobj.py b/daemon/core/coreobj.py index 742f0e89..badd22ce 100644 --- a/daemon/core/coreobj.py +++ b/daemon/core/coreobj.py @@ -305,8 +305,8 @@ class PyCoreNode(PyCoreObj): :return: nothing """ - preserve = getattr(self.session.options, "preservedir", None) - if preserve == "1": + preserve = self.session.options.get_config("preservedir") == "1" + if preserve: return if self.tmpnodedir: diff --git a/daemon/core/emane/bypass.py b/daemon/core/emane/bypass.py index 2e487e9e..8f60f238 100644 --- a/daemon/core/emane/bypass.py +++ b/daemon/core/emane/bypass.py @@ -1,7 +1,7 @@ """ EMANE Bypass model for CORE """ - +from core.conf import Configuration from core.emane import emanemodel from core.enumerations import ConfigDataTypes @@ -15,13 +15,20 @@ class EmaneBypassModel(emanemodel.EmaneModel): # mac definitions mac_library = "bypassmaclayer" mac_config = [ - ("none", ConfigDataTypes.BOOL.value, "0", "True,False", - "There are no parameters for the bypass model."), + Configuration( + _id="none", + _type=ConfigDataTypes.BOOL, + default="0", + options=["True", "False"], + label="There are no parameters for the bypass model." + ) ] # phy definitions phy_library = "bypassphylayer" phy_config = [] - # override gui display tabs - config_groups_override = "Bypass Parameters:1-1" + # override config groups + @classmethod + def config_groups(cls): + return "Bypass Parameters:1-1" diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index cc976f6c..4b737832 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -35,8 +35,13 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): shim_defaults = {} config_shim = emanemanifest.parse(shim_xml, shim_defaults) - config_groups_override = "CommEffect SHIM Parameters:1-%d" % len(config_shim) - config_matrix_override = config_shim + @classmethod + def configurations(cls): + return cls.config_shim + + @classmethod + def config_groups(cls): + return "CommEffect SHIM Parameters:1-%d" % len(cls.configurations()) def build_xml_files(self, emane_manager, interface): """ @@ -49,8 +54,9 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): :param interface: interface for the emane node :return: nothing """ - values = emane_manager.getifcconfig(self.object_id, self.name, self.getdefaultvalues(), interface) - if values is None: + default_values = self.default_values() + config = emane_manager.getifcconfig(self.object_id, self.name, default_values, interface) + if not config: return # retrieve xml names @@ -67,23 +73,22 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): nem_element.appendChild(shim_xml) emane_manager.xmlwrite(nem_document, nem_name) - names = self.getnames() - shim_names = list(names) - shim_names.remove("filterfile") - shim_document = emane_manager.xmldoc("shim") shim_element = shim_document.getElementsByTagName("shim").pop() shim_element.setAttribute("name", "%s SHIM" % self.name) shim_element.setAttribute("library", self.shim_library) # append all shim options (except filterfile) to shimdoc - for name in shim_names: - value = self.valueof(name, values) + for configuration in self.config_shim: + name = configuration.id + if name == "filterfile": + continue + value = config[name] param = emane_manager.xmlparam(shim_document, name, value) shim_element.appendChild(param) # empty filterfile is not allowed - ff = self.valueof("filterfile", values) + ff = config["filterfile"] if ff.strip() != "": shim_element.appendChild(emane_manager.xmlparam(shim_document, "filterfile", ff)) emane_manager.xmlwrite(shim_document, shim_name) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 2463a619..567d84d6 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -10,7 +10,9 @@ from core import CoreCommandError from core import constants from core import logger from core.api import coreapi -from core.conf import ConfigurableManager +from core.conf import ConfigShim +from core.conf import Configuration +from core.conf import NewConfigurableManager from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel @@ -50,7 +52,7 @@ EMANE_MODELS = [ ] -class EmaneManager(ConfigurableManager): +class EmaneManager(NewConfigurableManager): """ EMANE controller object. Lives in a Session instance and is used for building EMANE config files from all of the EmaneNode objects in this @@ -70,7 +72,7 @@ class EmaneManager(ConfigurableManager): :param core.session.Session session: session this manager is tied to :return: nothing """ - ConfigurableManager.__init__(self) + super(EmaneManager, self).__init__() self.session = session self._emane_nodes = {} self._emane_node_lock = threading.Lock() @@ -84,16 +86,22 @@ class EmaneManager(ConfigurableManager): # model for global EMANE configuration options self.emane_config = EmaneGlobalModel(session, None) + self.set_configs(self.emane_config.default_values()) + session.broker.handlers.add(self.handledistributed) self.service = None self.event_device = None - self._modelclsmap = { - self.emane_config.name: self.emane_config - } + self._modelclsmap = {} self.service = None self.emane_check() + def emane_models(self): + return self._modelclsmap.keys() + + def get_model_class(self, model_name): + return self._modelclsmap[model_name] + def emane_check(self): """ Check if emane is installed and load models. @@ -138,9 +146,8 @@ class EmaneManager(ConfigurableManager): return # Get the control network to be used for events - values = self.getconfig(None, "emane", self.emane_config.getdefaultvalues())[1] - group, port = self.emane_config.valueof("eventservicegroup", values).split(":") - self.event_device = self.emane_config.valueof("eventservicedevice", values) + group, port = self.get_config("eventservicegroup").split(":") + self.event_device = self.get_config("eventservicedevice") eventnetidx = self.session.get_control_net_index(self.event_device) if eventnetidx < 0: logger.error("invalid emane event service device provided: %s", self.event_device) @@ -170,7 +177,6 @@ class EmaneManager(ConfigurableManager): for emane_model in emane_models: logger.info("loading emane model: %s", emane_model.__name__) self._modelclsmap[emane_model.name] = emane_model - self.session.add_config_object(emane_model.name, emane_model.config_type, emane_model.configure_emane) def add_node(self, emane_node): """ @@ -196,26 +202,31 @@ class EmaneManager(ConfigurableManager): nodes.add(netif.node) return nodes - def getmodels(self, n): + def getmodels(self, node): """ - Used with XML export; see ConfigurableManager.getmodels() + Used with XML export. """ - r = ConfigurableManager.getmodels(self, n) - # EMANE global params are stored with first EMANE node (if non-default - # values are configured) - sorted_ids = sorted(self.configs.keys()) - if None in self.configs and len(sorted_ids) > 1 and n.objid == sorted_ids[1]: - v = self.configs[None] - for model in v: - cls = self._modelclsmap[model[0]] - vals = model[1] - r.append((cls, vals)) - return r + configs = self.get_config_types(node.objid) + models = [] + for model_name, config in configs.iteritems(): + model_class = self._modelclsmap[model_name] + models.append((model_class, config)) + logger.debug("emane models: %s", models) + return models - def getifcconfig(self, nodenum, conftype, defaultvalues, ifc): + def getifcconfig(self, node_id, config_type, default_values, ifc): + """ + Retrieve interface configuration or node configuration if not provided. + + :param int node_id: node id + :param str config_type: configuration type + :param dict default_values: default configuration values + :param ifc: node interface + :return: + """ # use the network-wide config values or interface(NEM)-specific values? if ifc is None: - return self.getconfig(nodenum, conftype, defaultvalues)[1] + return self.get_configs(node_id, config_type) or default_values else: # don"t use default values when interface config is the same as net # note here that using ifc.node.objid as key allows for only one type @@ -229,16 +240,19 @@ class EmaneManager(ConfigurableManager): if ifc.netindex is not None: key += ifc.netindex - values = self.getconfig(key, conftype, None)[1] - if not values: - values = self.getconfig(ifc.node.objid, conftype, None)[1] + # try retrieve interface specific configuration + config = self.get_configs(key, config_type) - if not values and ifc.transport_type == "raw": + # otherwise retrieve the interfaces node configuration + if not config: + config = self.get_configs(ifc.node.objid, config_type) + + if not config and ifc.transport_type == "raw": # with EMANE 0.9.2+, we need an extra NEM XML from # model.buildnemxmlfiles(), so defaults are returned here - values = self.getconfig(nodenum, conftype, defaultvalues)[1] + config = self.get_configs(node_id, config_type) or default_values - return values + return config def setup(self): """ @@ -264,9 +278,7 @@ class EmaneManager(ConfigurableManager): # - needs to be configured before checkdistributed() for distributed # - needs to exist when eventservice binds to it (initeventservice) if self.session.master: - values = self.getconfig(None, self.emane_config.name, self.emane_config.getdefaultvalues())[1] - logger.debug("emane config default values: %s", values) - otadev = self.emane_config.valueof("otamanagerdevice", values) + otadev = self.get_config("otamanagerdevice") netidx = self.session.get_control_net_index(otadev) logger.debug("emane ota manager device: index(%s) otadev(%s)", netidx, otadev) if netidx < 0: @@ -275,7 +287,7 @@ class EmaneManager(ConfigurableManager): ctrlnet = self.session.add_remove_control_net(net_index=netidx, remove=False, conf_required=False) self.distributedctrlnet(ctrlnet) - eventdev = self.emane_config.valueof("eventservicedevice", values) + eventdev = self.get_config("eventservicedevice") logger.debug("emane event service device: eventdev(%s)", eventdev) if eventdev != otadev: netidx = self.session.get_control_net_index(eventdev) @@ -288,10 +300,11 @@ class EmaneManager(ConfigurableManager): self.distributedctrlnet(ctrlnet) if self.checkdistributed(): - # we are slave, but haven"t received a platformid yet - cfgval = self.getconfig(None, self.emane_config.name, self.emane_config.getdefaultvalues())[1] - i = self.emane_config.getnames().index("platform_id_start") - if cfgval[i] == self.emane_config.getdefaultvalues()[i]: + # we are slave, but haven't received a platformid yet + platform_id_start = "platform_id_start" + default_values = self.emane_config.default_values() + value = self.get_config(platform_id_start) + if value == default_values[platform_id_start]: return EmaneManager.NOT_READY self.setnodemodels() @@ -359,7 +372,7 @@ class EmaneManager(ConfigurableManager): with self._emane_node_lock: self._emane_nodes.clear() - # don"t clear self._ifccounts here; NEM counts are needed for buildxml + # don't clear self._ifccounts here; NEM counts are needed for buildxml self.platformport = self.session.get_config_item_int("emane_platform_port", 8100) self.transformport = self.session.get_config_item_int("emane_transform_port", 8200) @@ -416,20 +429,16 @@ class EmaneManager(ConfigurableManager): if not master: return True - cfgval = self.getconfig(None, self.emane_config.name, self.emane_config.getdefaultvalues())[1] - values = list(cfgval) - nemcount = 0 with self._emane_node_lock: for key in self._emane_nodes: emane_node = self._emane_nodes[key] nemcount += emane_node.numnetif() - nemid = int(self.emane_config.valueof("nem_id_start", values)) + nemid = int(self.get_config("nem_id_start")) nemid += nemcount - platformid = int(self.emane_config.valueof("platform_id_start", values)) - names = list(self.emane_config.getnames()) + platformid = int(self.get_config("platform_id_start")) # build an ordered list of servers so platform ID is deterministic servers = [] @@ -448,9 +457,10 @@ class EmaneManager(ConfigurableManager): platformid += 1 typeflags = ConfigFlags.UPDATE.value - values[names.index("platform_id_start")] = str(platformid) - values[names.index("nem_id_start")] = str(nemid) - msg = EmaneGlobalModel.config_data(flags=0, node_id=None, type_flags=typeflags, values=values) + self.set_config("platform_id_start", str(platformid)) + self.set_config("nem_id_start", str(nemid)) + msg = ConfigShim.config_data(0, None, typeflags, self.emane_config, self.get_configs()) + # TODO: this needs to be converted into a sendable TLV message server.sock.send(msg) # increment nemid for next server by number of interfaces with self._ifccountslock: @@ -489,8 +499,9 @@ class EmaneManager(ConfigurableManager): if len(servers) < 2: return - prefix = session.config.get("controlnet") - prefix = getattr(session.options, "controlnet", prefix) + prefix = session.options.get_config("controlnet") + if not prefix: + prefix = session.config.get("controlnet") prefixes = prefix.split() # normal Config messaging will distribute controlnets if len(prefixes) >= len(servers): @@ -557,24 +568,17 @@ class EmaneManager(ConfigurableManager): for key in self._emane_nodes: self.setnodemodel(key) - def setnodemodel(self, key): - logger.debug("setting emane node model: %s", key) - emane_node = self._emane_nodes[key] - if key not in self.configs: - logger.debug("no emane node model configuration, leaving") + def setnodemodel(self, node_id): + logger.debug("setting emane models for node: %s", node_id) + node_config_types = self.get_config_types(node_id) + if not node_config_types: + logger.debug("no emane node model configuration, leaving: %s", node_id) return False - for t, v in self.configs[key]: - logger.debug("configuration: key(%s) value(%s)", t, v) - if t is None: - continue - if t == self.emane_config.name: - continue - - # only use the first valid EmaneModel - # convert model name to class (e.g. emane_rfpipe -> EmaneRfPipe) - cls = self._modelclsmap[t] - emane_node.setmodel(cls, v) + emane_node = self._emane_nodes[node_id] + for model_class, config in self.getmodels(emane_node): + logger.debug("setting emane model(%s) config(%s)", model_class, config) + emane_node.setmodel(model_class, config) return True # no model has been configured for this EmaneNode @@ -588,8 +592,8 @@ class EmaneManager(ConfigurableManager): emane_node = None netif = None - for key in self._emane_nodes: - emane_node = self._emane_nodes[key] + for node_id in self._emane_nodes: + emane_node = self._emane_nodes[node_id] netif = emane_node.getnemnetif(nemid) if netif is not None: break @@ -607,7 +611,7 @@ class EmaneManager(ConfigurableManager): count += len(emane_node.netifs()) return count - def newplatformxmldoc(self, values, otadev=None, eventdev=None): + def newplatformxmldoc(self, otadev=None, eventdev=None): """ Start a new platform XML file. Use global EMANE config values as keys. Override OTA manager and event service devices if @@ -615,21 +619,20 @@ class EmaneManager(ConfigurableManager): """ doc = self.xmldoc("platform") plat = doc.getElementsByTagName("platform").pop() - names = list(self.emane_config.getnames()) - platform_names = names[:len(self.emane_config.emulator_config)] - platform_names.remove("platform_id_start") - platform_values = list(values) + if otadev: - i = platform_names.index("otamanagerdevice") - platform_values[i] = otadev + self.set_config("otamanagerdevice", otadev) if eventdev: - i = platform_names.index("eventservicedevice") - platform_values[i] = eventdev + self.set_config("eventservicedevice", eventdev) # append all platform options (except starting id) to doc - for name in platform_names: - value = self.emane_config.valueof(name, platform_values) + for configuration in self.emane_config.emulator_config: + name = configuration.id + if name == "platform_id_start": + continue + + value = self.get_config(name) param = self.xmlparam(doc, name, value) plat.appendChild(param) @@ -639,8 +642,7 @@ class EmaneManager(ConfigurableManager): """ Build a platform.xml file now that all nodes are configured. """ - values = self.getconfig(None, "emane", self.emane_config.getdefaultvalues())[1] - nemid = int(self.emane_config.valueof("nem_id_start", values)) + nemid = int(self.get_config("nem_id_start")) platformxmls = {} # assume self._objslock is already held here @@ -660,7 +662,7 @@ class EmaneManager(ConfigurableManager): eventdev = None if key not in platformxmls: - platformxmls[key] = self.newplatformxmldoc(values, otadev, eventdev) + platformxmls[key] = self.newplatformxmldoc(otadev, eventdev) doc = platformxmls[key] plat = doc.getElementsByTagName("platform").pop() @@ -713,13 +715,11 @@ class EmaneManager(ConfigurableManager): Build the libemaneeventservice.xml file if event service options were changed in the global config. """ - defaults = self.emane_config.getdefaultvalues() - values = self.getconfig(None, "emane", self.emane_config.getdefaultvalues())[1] need_xml = False - keys = ("eventservicegroup", "eventservicedevice") - for k in keys: - a = self.emane_config.valueof(k, defaults) - b = self.emane_config.valueof(k, values) + default_values = self.emane_config.default_values() + for name in ["eventservicegroup", "eventservicedevice"]: + a = default_values[name] + b = self.get_config(name) if a != b: need_xml = True @@ -729,12 +729,12 @@ class EmaneManager(ConfigurableManager): return try: - group, port = self.emane_config.valueof("eventservicegroup", values).split(":") + group, port = self.get_config("eventservicegroup").split(":") except ValueError: logger.exception("invalid eventservicegroup in EMANE config") return - dev = self.emane_config.valueof("eventservicedevice", values) + dev = self.get_config("eventservicedevice") doc = self.xmldoc("emaneeventmsgsvc") es = doc.getElementsByTagName("emaneeventmsgsvc").pop() kvs = (("group", group), ("port", port), ("device", dev), ("mcloop", "1"), ("ttl", "32")) @@ -761,13 +761,12 @@ class EmaneManager(ConfigurableManager): if realtime: emanecmd += "-r", - values = self.getconfig(None, "emane", self.emane_config.getdefaultvalues())[1] - otagroup, otaport = self.emane_config.valueof("otamanagergroup", values).split(":") - otadev = self.emane_config.valueof("otamanagerdevice", values) + otagroup, otaport = self.get_config("otamanagergroup").split(":") + otadev = self.get_config("otamanagerdevice") otanetidx = self.session.get_control_net_index(otadev) - eventgroup, eventport = self.emane_config.valueof("eventservicegroup", values).split(":") - eventdev = self.emane_config.valueof("eventservicedevice", values) + eventgroup, eventport = self.get_config("eventservicegroup").split(":") + eventdev = self.get_config("eventservicedevice") eventservicenetidx = self.session.get_control_net_index(eventdev) run_emane_on_host = False @@ -799,8 +798,7 @@ class EmaneManager(ConfigurableManager): node.check_cmd(args) # start emane - args = emanecmd + ["-f", os.path.join(path, "emane%d.log" % n), - os.path.join(path, "platform%d.xml" % n)] + args = emanecmd + ["-f", os.path.join(path, "emane%d.log" % n), os.path.join(path, "platform%d.xml" % n)] output = node.check_cmd(args) logger.info("node(%s) emane daemon running: %s", node.name, args) logger.info("node(%s) emane daemon output: %s", node.name, output) @@ -855,23 +853,6 @@ class EmaneManager(ConfigurableManager): emane_node = self._emane_nodes[key] emane_node.deinstallnetifs() - def configure(self, session, config_data): - """ - Handle configuration messages for global EMANE config. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - """ - r = self.emane_config.configure_emane(session, config_data) - - # extra logic to start slave Emane object after nemid has been configured from the master - config_type = config_data.type - if config_type == ConfigFlags.UPDATE.value and self.session.master is False: - # instantiation was previously delayed by self.setup() - # returning Emane.NOT_READY - self.session.instantiate() - - return r - def doeventmonitor(self): """ Returns boolean whether or not EMANE events will be monitored. @@ -900,11 +881,10 @@ class EmaneManager(ConfigurableManager): return if self.service is None: - errmsg = "Warning: EMANE events will not be generated " \ - "because the emaneeventservice\n binding was " \ - "unable to load " \ - "(install the python-emaneeventservice bindings)" - logger.error(errmsg) + logger.error("Warning: EMANE events will not be generated " + "because the emaneeventservice\n binding was " + "unable to load " + "(install the python-emaneeventservice bindings)") return self.doeventloop = True self.eventmonthread = threading.Thread(target=self.eventmonitorloop) @@ -1019,6 +999,7 @@ class EmaneGlobalModel(EmaneModel): """ Global EMANE configuration options. """ + _DEFAULT_DEV = "ctrl0" name = "emane" @@ -1033,19 +1014,26 @@ class EmaneGlobalModel(EmaneModel): emulator_config = emanemanifest.parse(emulator_xml, emulator_defaults) emulator_config.insert( 0, - ("platform_id_start", ConfigDataTypes.INT32.value, "1", "", "Starting Platform ID (core)") + Configuration(_id="platform_id_start", _type=ConfigDataTypes.INT32, default="1", + label="Starting Platform ID (core)") ) nem_config = [ - ("nem_id_start", ConfigDataTypes.INT32.value, "1", "", "Starting NEM ID (core)"), + Configuration(_id="nem_id_start", _type=ConfigDataTypes.INT32, default="1", + label="Starting NEM ID (core)") ] - config_matrix_override = emulator_config + nem_config - config_groups_override = "Platform Attributes:1-%d|NEM Parameters:%d-%d" % ( - len(emulator_config), len(emulator_config) + 1, len(config_matrix_override)) + @classmethod + def configurations(cls): + return cls.emulator_config + cls.nem_config + + @classmethod + def config_groups(cls): + return "Platform Attributes:1-%d|NEM Parameters:%d-%d" % ( + len(cls.emulator_config), len(cls.emulator_config) + 1, len(cls.configurations())) def __init__(self, session, object_id=None): - EmaneModel.__init__(self, session, object_id) + super(EmaneGlobalModel, self).__init__(session, object_id) def build_xml_files(self, emane_manager, interface): """ diff --git a/daemon/core/emane/emanemanifest.py b/daemon/core/emane/emanemanifest.py index 15bdb535..ed27f445 100644 --- a/daemon/core/emane/emanemanifest.py +++ b/daemon/core/emane/emanemanifest.py @@ -1,4 +1,5 @@ from core import logger +from core.conf import Configuration from core.enumerations import ConfigDataTypes manifest = None @@ -23,7 +24,7 @@ def _type_value(config_type): config_type = "FLOAT" elif config_type == "INETADDR": config_type = "STRING" - return ConfigDataTypes[config_type].value + return ConfigDataTypes[config_type] def _get_possible(config_type, config_regex): @@ -36,14 +37,13 @@ def _get_possible(config_type, config_regex): :rtype: str """ if config_type == "bool": - return "On,Off" + return ["On", "Off"] if config_type == "string" and config_regex: possible = config_regex[2:-2] - possible = possible.replace("|", ",") - return possible + return possible.split("|") - return "" + return [] def _get_default(config_type_name, config_value): @@ -116,7 +116,13 @@ def parse(manifest_path, defaults): if config_name.endswith("uri"): config_descriptions = "%s file" % config_descriptions - config_tuple = (config_name, config_type_value, config_default, possible, config_descriptions) - configurations.append(config_tuple) + configuration = Configuration( + _id=config_name, + _type=config_type_value, + default=config_default, + options=possible, + label=config_descriptions + ) + configurations.append(configuration) return configurations diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 3c80977f..b02468fd 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -34,51 +34,12 @@ def value_to_params(doc, name, value): return xmlutils.add_param_list_to_parent(doc, parent=None, name=name, values=values) -class EmaneModelMetaClass(type): - """ - Hack into making class level properties to streamline emane model creation, until the Configurable class is - removed or refactored. - """ - - @property - def config_matrix(cls): - """ - Convenience method for creating the config matrix, allow for a custom override. - - :param EmaneModel cls: emane class - :return: config matrix value - :rtype: list - """ - if cls.config_matrix_override: - return cls.config_matrix_override - else: - return cls.mac_config + cls.phy_config - - @property - def config_groups(cls): - """ - Convenience method for creating the config groups, allow for a custom override. - - :param EmaneModel cls: emane class - :return: config groups value - :rtype: str - """ - if cls.config_groups_override: - return cls.config_groups_override - else: - mac_len = len(cls.mac_config) - config_len = len(cls.config_matrix) - return "MAC Parameters:1-%d|PHY Parameters:%d-%d" % (mac_len, mac_len + 1, config_len) - - class EmaneModel(WirelessModel): """ EMANE models inherit from this parent class, which takes care of handling configuration messages based on the list of configurable parameters. Helper functions also live here. """ - __metaclass__ = EmaneModelMetaClass - # default mac configuration settings mac_library = None mac_xml = None @@ -97,10 +58,20 @@ class EmaneModel(WirelessModel): config_ignore = set() config_groups_override = None - config_matrix_override = None + configurations_override = None + + @classmethod + def configurations(cls): + return cls.mac_config + cls.phy_config + + @classmethod + def config_groups(cls): + mac_len = len(cls.mac_config) + config_len = len(cls.configurations()) + return "MAC Parameters:1-%d|PHY Parameters:%d-%d" % (mac_len, mac_len + 1, config_len) def __init__(self, session, object_id=None): - WirelessModel.__init__(self, session, object_id) + super(EmaneModel, self).__init__(session, object_id) def build_xml_files(self, emane_manager, interface): """ @@ -111,8 +82,8 @@ class EmaneModel(WirelessModel): :return: nothing """ # retrieve configuration values - values = emane_manager.getifcconfig(self.object_id, self.name, self.getdefaultvalues(), interface) - if values is None: + config = emane_manager.getifcconfig(self.object_id, self.name, self.default_values(), interface) + if not config: return # create document and write to disk @@ -122,12 +93,12 @@ class EmaneModel(WirelessModel): # create mac document and write to disk mac_name = self.mac_name(interface) - mac_document = self.create_mac_doc(emane_manager, values) + mac_document = self.create_mac_doc(emane_manager, config) emane_manager.xmlwrite(mac_document, mac_name) # create phy document and write to disk phy_name = self.phy_name(interface) - phy_document = self.create_phy_doc(emane_manager, values) + phy_document = self.create_phy_doc(emane_manager, config) emane_manager.xmlwrite(phy_document, phy_name) def create_nem_doc(self, emane_manager, interface): @@ -157,18 +128,15 @@ class EmaneModel(WirelessModel): return nem_document - def create_mac_doc(self, emane_manager, values): + def create_mac_doc(self, emane_manager, config): """ Create the mac xml document. :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager - :param tuple values: all current configuration values, mac + phy + :param dict config: all current configuration values, mac + phy :return: nem document :rtype: xml.dom.minidom.Document """ - names = list(self.getnames()) - mac_names = names[:len(self.mac_config)] - mac_document = emane_manager.xmldoc("mac") mac_element = mac_document.getElementsByTagName("mac").pop() mac_element.setAttribute("name", "%s MAC" % self.name) @@ -177,13 +145,14 @@ class EmaneModel(WirelessModel): raise ValueError("must define emane model library") mac_element.setAttribute("library", self.mac_library) - for name in mac_names: + for mac_configuration in self.mac_config: # ignore custom configurations + name = mac_configuration.id if name in self.config_ignore: continue # check if value is a multi param - value = self.valueof(name, values) + value = config[name] param = value_to_params(mac_document, name, value) if not param: param = emane_manager.xmlparam(mac_document, name, value) @@ -192,18 +161,15 @@ class EmaneModel(WirelessModel): return mac_document - def create_phy_doc(self, emane_manager, values): + def create_phy_doc(self, emane_manager, config): """ Create the phy xml document. :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager - :param tuple values: all current configuration values, mac + phy + :param dict config: all current configuration values, mac + phy :return: nem document :rtype: xml.dom.minidom.Document """ - names = list(self.getnames()) - phy_names = names[len(self.mac_config):] - phy_document = emane_manager.xmldoc("phy") phy_element = phy_document.getElementsByTagName("phy").pop() phy_element.setAttribute("name", "%s PHY" % self.name) @@ -212,13 +178,14 @@ class EmaneModel(WirelessModel): phy_element.setAttribute("library", self.phy_library) # append all phy options - for name in phy_names: + for phy_configuration in self.phy_config: # ignore custom configurations + name = phy_configuration.id if name in self.config_ignore: continue # check if value is a multi param - value = self.valueof(name, values) + value = config[name] param = value_to_params(phy_document, name, value) if not param: param = emane_manager.xmlparam(phy_document, name, value) @@ -227,16 +194,6 @@ class EmaneModel(WirelessModel): return phy_document - @classmethod - def configure_emane(cls, session, config_data): - """ - Handle configuration messages for configuring an emane model. - - :param core.session.Session session: session to configure emane - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - """ - return cls.configure(session.emane, config_data) - def post_startup(self, emane_manager): """ Logic to execute after the emane manager is finished with startup. @@ -317,7 +274,7 @@ class EmaneModel(WirelessModel): if interface: node_id = interface.node.objid - if emane_manager.getifcconfig(node_id, self.name, None, interface) is not None: + if emane_manager.getifcconfig(node_id, self.name, {}, interface): name = interface.localname.replace(".", "_") return "%s%s" % (name, self.name) diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index bfe3e742..0dbf0721 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -174,14 +174,9 @@ class EmaneNode(EmaneNet): trans.setAttribute("library", "trans%s" % transport_type.lower()) trans.appendChild(emane.xmlparam(transdoc, "bitrate", "0")) - flowcontrol = False - names = self.model.getnames() - values = emane.getconfig(self.objid, self.model.name, self.model.getdefaultvalues())[1] - - if "flowcontrolenable" in names and values: - i = names.index("flowcontrolenable") - if self.model.booltooffon(values[i]) == "on": - flowcontrol = True + config = emane.get_configs(self.objid, self.model.name) + logger.debug("transport xml config: %s", config) + flowcontrol = config.get("flowcontrolenable", "0") == "1" if "virtual" in transport_type.lower(): if os.path.exists("/dev/net/tun_flowctl"): @@ -193,11 +188,11 @@ class EmaneNode(EmaneNet): emane.xmlwrite(transdoc, self.transportxmlname(transport_type.lower())) - def transportxmlname(self, type): + def transportxmlname(self, _type): """ Return the string name for the Transport XML file, e.g. 'n3transvirtual.xml' """ - return "n%strans%s.xml" % (self.objid, type) + return "n%strans%s.xml" % (self.objid, _type) def installnetifs(self, do_netns=True): """ diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index f3c43541..5d43a692 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -6,6 +6,7 @@ import os from core import constants from core import logger +from core.conf import Configuration from core.emane import emanemanifest from core.emane import emanemodel from core.enumerations import ConfigDataTypes @@ -29,7 +30,12 @@ class EmaneTdmaModel(emanemodel.EmaneModel): default_schedule = os.path.join(constants.CORE_DATA_DIR, "examples", "tdma", "schedule.xml") mac_config.insert( 0, - (schedule_name, ConfigDataTypes.STRING.value, default_schedule, "", "TDMA schedule file (core)") + Configuration( + _id=schedule_name, + _type=ConfigDataTypes.STRING, + default=default_schedule, + label="TDMA schedule file (core)" + ) ) config_ignore = {schedule_name} @@ -41,10 +47,10 @@ class EmaneTdmaModel(emanemodel.EmaneModel): :return: nothing """ # get configured schedule - values = emane_manager.getconfig(self.object_id, self.name, self.getdefaultvalues())[1] - if values is None: + config = emane_manager.get_configs() + if not config: return - schedule = self.valueof(self.schedule_name, values) + schedule = config[self.schedule_name] event_device = emane_manager.event_device diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 980b6074..ff814e4d 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -378,7 +378,8 @@ class EmuSession(Session): if node_two: node_two.lock.release() - def update_link(self, node_one_id, node_two_id, interface_one_id=None, interface_two_id=None, link_options=LinkOptions()): + def update_link(self, node_one_id, node_two_id, interface_one_id=None, interface_two_id=None, + link_options=LinkOptions()): """ Update link information between nodes. @@ -480,7 +481,7 @@ class EmuSession(Session): # set node start based on current session state, override and check when rj45 start = self.state > EventTypes.DEFINITION_STATE.value - enable_rj45 = getattr(self.options, "enablerj45", "0") == "1" + enable_rj45 = self.options.get_config("enablerj45") == "1" if _type == NodeTypes.RJ45 and not enable_rj45: start = False @@ -851,7 +852,7 @@ class CoreEmu(object): """ # set umask 0 os.umask(0) - + # configuration self.config = config diff --git a/daemon/core/location.py b/daemon/core/location.py index b7d1bd1a..6525d10e 100644 --- a/daemon/core/location.py +++ b/daemon/core/location.py @@ -6,19 +6,15 @@ https://pypi.python.org/pypi/utm (version 0.3.0). """ from core import logger -from core.conf import ConfigurableManager from core.enumerations import RegisterTlvs from core.misc import utm -class CoreLocation(ConfigurableManager): +class CoreLocation(object): """ Member of session class for handling global location data. This keeps track of a latitude/longitude/altitude reference point and scale in order to convert between X,Y and geo coordinates. - - TODO: this could be updated to use more generic - Configurable/ConfigurableManager code like other Session objects """ name = "location" config_type = RegisterTlvs.UTILITY.value @@ -29,7 +25,7 @@ class CoreLocation(ConfigurableManager): :return: nothing """ - ConfigurableManager.__init__(self) + # ConfigurableManager.__init__(self) self.reset() self.zonemap = {} self.refxyz = (0.0, 0.0, 0.0) @@ -52,35 +48,6 @@ class CoreLocation(ConfigurableManager): # cached distance to refpt in other zones self.zoneshifts = {} - def configure_values(self, config_data): - """ - Receive configuration message for setting the reference point - and scale. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: nothing - """ - values = config_data.data_values - - if values is None: - logger.warn("location data missing") - return None - values = values.split('|') - - # Cartesian coordinate reference point - refx, refy = map(lambda x: float(x), values[0:2]) - refz = 0.0 - self.refxyz = (refx, refy, refz) - # Geographic reference point - lat, lon, alt = map(lambda x: float(x), values[2:5]) - self.setrefgeo(lat, lon, alt) - self.refscale = float(values[5]) - logger.info("location configured: (%.2f,%.2f,%.2f) = (%.5f,%.5f,%.5f) scale=%.2f" % - (self.refxyz[0], self.refxyz[1], self.refxyz[2], self.refgeo[0], - self.refgeo[1], self.refgeo[2], self.refscale)) - logger.info("location configured: UTM(%.5f,%.5f,%.5f)" % - (self.refutm[1], self.refutm[2], self.refutm[3])) - def px2m(self, val): """ Convert the specified value in pixels to meters using the diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 73facd8a..442eaeef 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -9,10 +9,12 @@ import threading import time from core import logger -from core.conf import Configurable -from core.conf import ConfigurableManager +from core.conf import ConfigurableOptions +from core.conf import Configuration +from core.conf import NewConfigurableManager from core.coreobj import PyCoreNode -from core.data import EventData, LinkData +from core.data import EventData +from core.data import LinkData from core.enumerations import ConfigDataTypes from core.enumerations import EventTypes from core.enumerations import LinkTypes @@ -24,7 +26,7 @@ from core.misc import utils from core.misc.ipaddress import IpAddress -class MobilityManager(ConfigurableManager): +class MobilityManager(NewConfigurableManager): """ Member of session class for handling configuration data for mobility and range models. @@ -38,20 +40,42 @@ class MobilityManager(ConfigurableManager): :param core.session.Session session: session this manager is tied to """ - ConfigurableManager.__init__(self) + super(MobilityManager, self).__init__() self.session = session - # configurations for basic range, indexed by WLAN node number, are - # stored in self.configs + # configurations for basic range, indexed by WLAN node number, are stored in configurations # mapping from model names to their classes self._modelclsmap = { BasicRangeModel.name: BasicRangeModel, Ns2ScriptedMobility.name: Ns2ScriptedMobility } + # dummy node objects for tracking position of nodes on other servers self.phys = {} self.physnets = {} self.session.broker.handlers.add(self.physnodehandlelink) + def mobility_models(self): + return self._modelclsmap.keys() + + def get_model_class(self, model_name): + return self._modelclsmap[model_name] + + def getmodels(self, node): + """ + Return a list of model classes and values for a net if one has been + configured. This is invoked when exporting a session to XML. + + :param node: network node to get models for + :return: list of model and values tuples for the network node + :rtype: list + """ + configs = self.get_config_types(node.objid) + models = [] + for model_name, config in configs.iteritems(): + model_class = self._modelclsmap[model_name] + models.append((model_class, config)) + return models + def startup(self, node_ids=None): """ Session is transitioning from instantiation to runtime state. @@ -61,7 +85,7 @@ class MobilityManager(ConfigurableManager): :return: nothing """ if node_ids is None: - node_ids = self.configs.keys() + node_ids = self.nodes() for node_id in node_ids: logger.info("checking mobility startup for node: %s", node_id) @@ -69,22 +93,22 @@ class MobilityManager(ConfigurableManager): try: node = self.session.get_object(node_id) except KeyError: - logger.warn("skipping mobility configuration for unknown node %d." % node_id) + logger.warn("skipping mobility configuration for unknown node: %s", node_id) continue - if node_id not in self.configs: - logger.warn("missing mobility configuration for node %d." % node_id) + node_configs = self.get_config_types(node_id) + if not node_configs: + logger.warn("missing mobility configuration for node: %s", node_id) continue - v = self.configs[node_id] - - for model in v: + for model_name in node_configs.iterkeys(): try: - logger.info("setting mobility model to node: %s", model) - cls = self._modelclsmap[model[0]] - node.setmodel(cls, model[1]) + clazz = self._modelclsmap[model_name] + model_config = self.get_config(node_id, model_name) + logger.info("setting mobility model(%s) to node: %s", model_name, model_config) + node.setmodel(clazz, model_config) except KeyError: - logger.warn("skipping mobility configuration for unknown model '%s'" % model[0]) + logger.warn("skipping mobility configuration for unknown model: %s", model_name) continue if self.session.master: @@ -99,26 +123,32 @@ class MobilityManager(ConfigurableManager): :return: nothing """ - self.clearconfig(nodenum=None) + self.config_reset() - def setconfig(self, node_id, config_type, values): + def set_configs(self, config, node_id=None, config_type=None): """ - Normal setconfig() with check for run-time updates for WLANs. + Adds a check for run-time updates for WLANs after providing normal set configs. + :param dict config: configuration value :param int node_id: node id - :param config_type: configuration type - :param values: configuration value + :param str config_type: configuration type :return: nothing """ - super(MobilityManager, self).setconfig(node_id, config_type, values) + if not node_id or not config_type: + raise ValueError("mobility manager invalid node id or config type: %s - %s", node_id, config_type) + + super(MobilityManager, self).set_configs(config, node_id, config_type) + if self.session is None: return + if self.session.state == EventTypes.RUNTIME_STATE.value: try: node = self.session.get_object(node_id) - node.updatemodel(config_type, values) + # TODO: this need to be updated + node.updatemodel(config_type, config) except KeyError: - logger.exception("Skipping mobility configuration for unknown node %d.", node_id) + logger.exception("skipping mobility configuration for unknown node %s", node_id) def handleevent(self, event_data): """ @@ -206,13 +236,13 @@ class MobilityManager(ConfigurableManager): :param list moved_netifs: moved network interfaces :return: nothing """ - for nodenum in self.configs: + for node_id in self.nodes(): try: - n = self.session.get_object(nodenum) + node = self.session.get_object(node_id) except KeyError: continue - if n.model: - n.model.update(moved, moved_netifs) + if node.model: + node.model.update(moved, moved_netifs) def addphys(self, netnum, node): """ @@ -222,12 +252,12 @@ class MobilityManager(ConfigurableManager): :param core.coreobj.PyCoreNode node: node to add physical network to :return: nothing """ - nodenum = node.objid - self.phys[nodenum] = node + node_id = node.objid + self.phys[node_id] = node if netnum not in self.physnets: - self.physnets[netnum] = [nodenum, ] + self.physnets[netnum] = [node_id, ] else: - self.physnets[netnum].append(nodenum) + self.physnets[netnum].append(node_id) # TODO: remove need for handling old style message def physnodehandlelink(self, message): @@ -247,8 +277,7 @@ class MobilityManager(ConfigurableManager): return if nn[1] in self.session.broker.physical_nodes: # record the fact that this PhysicalNode is linked to a net - dummy = PyCoreNode(session=self.session, objid=nn[1], - name="n%d" % nn[1], start=False) + dummy = PyCoreNode(session=self.session, objid=nn[1], name="n%d" % nn[1], start=False) self.addphys(nn[0], dummy) # TODO: remove need to handling old style messages @@ -291,7 +320,7 @@ class MobilityManager(ConfigurableManager): netif.poshook(netif, x, y, z) -class WirelessModel(Configurable): +class WirelessModel(ConfigurableOptions): """ Base class used by EMANE models and the basic range model. Used for managing arbitrary configuration parameters. @@ -308,9 +337,8 @@ class WirelessModel(Configurable): :param int object_id: object id :param values: values """ - Configurable.__init__(self, session, object_id) - # 'values' can be retrieved from a ConfigurableManager, or used here - # during initialization, depending on the model. + self.session = session + self.object_id = object_id def all_link_data(self, flags): """ @@ -333,12 +361,12 @@ class WirelessModel(Configurable): """ raise NotImplementedError - def updateconfig(self, values): + def updateconfig(self, config): """ For run-time updates of model config. Returns True when position callback and set link parameters should be invoked. - :param values: value to update + :param dict config: value to update :return: False :rtype: bool """ @@ -353,23 +381,20 @@ class BasicRangeModel(WirelessModel): """ name = "basic_range" - # configuration parameters are - # ( 'name', 'type', 'default', 'possible-value-list', 'caption') - config_matrix = [ - ("range", ConfigDataTypes.UINT32.value, '275', - '', 'wireless range (pixels)'), - ("bandwidth", ConfigDataTypes.UINT32.value, '54000', - '', 'bandwidth (bps)'), - ("jitter", ConfigDataTypes.FLOAT.value, '0.0', - '', 'transmission jitter (usec)'), - ("delay", ConfigDataTypes.FLOAT.value, '5000.0', - '', 'transmission delay (usec)'), - ("error", ConfigDataTypes.FLOAT.value, '0.0', - '', 'error rate (%)'), - ] + @classmethod + def configurations(cls): + return [ + Configuration(_id="range", _type=ConfigDataTypes.UINT32, default="275", label="wireless range (pixels)"), + Configuration(_id="bandwidth", _type=ConfigDataTypes.UINT32, default="54000", label="bandwidth (bps)"), + Configuration(_id="jitter", _type=ConfigDataTypes.FLOAT, default="0.0", label="transmission jitter (usec)"), + Configuration(_id="delay", _type=ConfigDataTypes.FLOAT, default="5000.0", + label="transmission delay (usec)"), + Configuration(_id="error", _type=ConfigDataTypes.FLOAT, default="0.0", label="error rate (%)") + ] - # value groupings - config_groups = "Basic Range Parameters:1-%d" % len(config_matrix) + @classmethod + def config_groups(cls): + return "Basic Range Parameters:1-%d" % len(cls.configurations()) def __init__(self, session, object_id, values=None): """ @@ -377,57 +402,49 @@ class BasicRangeModel(WirelessModel): :param core.session.Session session: related core session :param int object_id: object id - :param values: values + :param dict values: values """ super(BasicRangeModel, self).__init__(session=session, object_id=object_id) + self.session = session self.wlan = session.get_object(object_id) self._netifs = {} self._netifslock = threading.Lock() - if values is None: - values = session.mobility.getconfig(object_id, self.name, self.getdefaultvalues())[1] - self.range = float(self.valueof("range", values)) - logger.info("Basic range model configured for WLAN %d using range %d", object_id, self.range) - self.valuestolinkparams(values) + if not values: + values = self.default_values() - # link parameters + # TODO: can this be handled in a better spot + self.session.mobility.set_configs(values, node_id=object_id, config_type=self.name) + + self.range = None self.bw = None self.delay = None self.loss = None self.jitter = None - def valuestolinkparams(self, values): + self.values_from_config(values) + + def values_from_config(self, config): """ Values to convert to link parameters. - :param values: values to convert + :param dict config: values to convert :return: nothing """ - self.bw = int(self.valueof("bandwidth", values)) + self.range = float(config["range"]) + logger.info("Basic range model configured for WLAN %d using range %d", self.wlan.objid, self.range) + self.bw = int(config["bandwidth"]) if self.bw == 0.0: self.bw = None - self.delay = float(self.valueof("delay", values)) + self.delay = float(config["delay"]) if self.delay == 0.0: self.delay = None - self.loss = float(self.valueof("error", values)) + self.loss = float(config["error"]) if self.loss == 0.0: self.loss = None - self.jitter = float(self.valueof("jitter", values)) + self.jitter = float(config["jitter"]) if self.jitter == 0.0: self.jitter = None - @classmethod - def configure_mob(cls, session, config_data): - """ - Handle configuration messages for setting up a model. - Pass the MobilityManager object as the manager object. - - :param core.session.Session session: current session calling function - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: configuration data - :rtype: core.data.ConfigData - """ - return cls.configure(session.mobility, config_data) - def setlinkparams(self): """ Apply link parameters to all interfaces. This is invoked from @@ -435,8 +452,7 @@ class BasicRangeModel(WirelessModel): """ with self._netifslock: for netif in self._netifs: - self.wlan.linkconfig(netif, bw=self.bw, delay=self.delay, - loss=self.loss, duplicate=None, + self.wlan.linkconfig(netif, bw=self.bw, delay=self.delay, loss=self.loss, duplicate=None, jitter=self.jitter) def get_position(self, netif): @@ -461,7 +477,6 @@ class BasicRangeModel(WirelessModel): :param z: z position :return: nothing """ - # print "set_position(%s, x=%s, y=%s, z=%s)" % (netif.localname, x, y, z) self._netifslock.acquire() self._netifs[netif] = (x, y, z) if x is None or y is None: @@ -487,7 +502,7 @@ class BasicRangeModel(WirelessModel): with self._netifslock: while len(moved_netifs): netif = moved_netifs.pop() - (nx, ny, nz) = netif.node.getposition() + nx, ny, nz = netif.node.getposition() if netif in self._netifs: self._netifs[netif] = (nx, ny, nz) for netif2 in self._netifs: @@ -557,18 +572,17 @@ class BasicRangeModel(WirelessModel): c = p1[2] - p2[2] return math.hypot(math.hypot(a, b), c) - def updateconfig(self, values): + def updateconfig(self, config): """ Configuration has changed during runtime. MobilityManager.setconfig() -> WlanNode.updatemodel() -> WirelessModel.updateconfig() - :param values: values to update configuration + :param dict config: values to update configuration :return: was update successful :rtype: bool """ - self.valuestolinkparams(values) - self.range = float(self.valueof("range", values)) + self.values_from_config(config) return True def create_link_data(self, interface1, interface2, message_type): @@ -581,7 +595,6 @@ class BasicRangeModel(WirelessModel): :return: link data :rtype: LinkData """ - return LinkData( message_type=message_type, node1_id=interface1.node.objid, @@ -678,6 +691,7 @@ class WayPointMobility(WirelessModel): :return: """ super(WayPointMobility, self).__init__(session=session, object_id=object_id, values=values) + self.state = self.STATE_STOPPED self.queue = [] self.queue_copy = [] @@ -705,7 +719,6 @@ class WayPointMobility(WirelessModel): self.lasttime = time.time() now = self.lasttime - self.timezero dt = self.lasttime - t - # print "runround(now=%.2f, dt=%.2f)" % (now, dt) # keep current waypoints up-to-date self.updatepoints(now) @@ -741,7 +754,6 @@ class WayPointMobility(WirelessModel): moved_netifs.append(netif) # calculate all ranges after moving nodes; this saves calculations - # self.wlan.model.update(moved) self.session.mobility.updatewlans(moved, moved_netifs) # TODO: check session state @@ -806,7 +818,6 @@ class WayPointMobility(WirelessModel): self.endtime = self.lasttime - self.timezero del self.points[node.objid] return False - # print "node %s dx,dy= <%s, %d>" % (node.name, dx, dy) if (x1 + dx) < 0.0: dx = 0.0 - x1 if (y1 + dy) < 0.0: @@ -826,7 +837,7 @@ class WayPointMobility(WirelessModel): node = netif.node if node.objid not in self.initial: continue - (x, y, z) = self.initial[node.objid].coords + x, y, z = self.initial[node.objid].coords self.setnodeposition(node, x, y, z) moved.append(node) moved_netifs.append(netif) @@ -845,7 +856,6 @@ class WayPointMobility(WirelessModel): :param speed: speed :return: nothing """ - # print "addwaypoint: %s %s %s,%s,%s %s" % (time, nodenum, x, y, z, speed) wp = WayPoint(time, nodenum, coords=(x, y, z), speed=speed) heapq.heappush(self.queue, wp) @@ -976,25 +986,22 @@ class Ns2ScriptedMobility(WayPointMobility): """ name = "ns2script" - config_matrix = [ - ("file", ConfigDataTypes.STRING.value, '', - '', 'mobility script file'), - ("refresh_ms", ConfigDataTypes.UINT32.value, '50', - '', 'refresh time (ms)'), - ("loop", ConfigDataTypes.BOOL.value, '1', - 'On,Off', 'loop'), - ("autostart", ConfigDataTypes.STRING.value, '', - '', 'auto-start seconds (0.0 for runtime)'), - ("map", ConfigDataTypes.STRING.value, '', - '', 'node mapping (optional, e.g. 0:1,1:2,2:3)'), - ("script_start", ConfigDataTypes.STRING.value, '', - '', 'script file to run upon start'), - ("script_pause", ConfigDataTypes.STRING.value, '', - '', 'script file to run upon pause'), - ("script_stop", ConfigDataTypes.STRING.value, '', - '', 'script file to run upon stop'), - ] - config_groups = "ns-2 Mobility Script Parameters:1-%d" % len(config_matrix) + @classmethod + def configurations(cls): + return [ + Configuration(_id="file", _type=ConfigDataTypes.STRING, label="mobility script file"), + Configuration(_id="refresh_ms", _type=ConfigDataTypes.UINT32, default="50", label="mobility script file"), + Configuration(_id="loop", _type=ConfigDataTypes.BOOL, default="1", options=["On", "Off"], label="loop"), + Configuration(_id="autostart", _type=ConfigDataTypes.STRING, label="auto-start seconds (0.0 for runtime)"), + Configuration(_id="map", _type=ConfigDataTypes.STRING, label="node mapping (optional, e.g. 0:1,1:2,2:3)"), + Configuration(_id="script_start", _type=ConfigDataTypes.STRING, label="script file to run upon start"), + Configuration(_id="script_pause", _type=ConfigDataTypes.STRING, label="script file to run upon pause"), + Configuration(_id="script_stop", _type=ConfigDataTypes.STRING, label="script file to run upon stop") + ] + + @classmethod + def config_groups(cls): + return "ns-2 Mobility Script Parameters:1-%d" % len(cls.configurations()) def __init__(self, session, object_id, values=None): """ @@ -1007,32 +1014,24 @@ class Ns2ScriptedMobility(WayPointMobility): super(Ns2ScriptedMobility, self).__init__(session=session, object_id=object_id, values=values) self._netifs = {} self._netifslock = threading.Lock() - if values is None: - values = session.mobility.getconfig(object_id, self.name, self.getdefaultvalues())[1] - self.file = self.valueof("file", values) - self.refresh_ms = int(self.valueof("refresh_ms", values)) - self.loop = self.valueof("loop", values).lower() == "on" - self.autostart = self.valueof("autostart", values) - self.parsemap(self.valueof("map", values)) - self.script_start = self.valueof("script_start", values) - self.script_pause = self.valueof("script_pause", values) - self.script_stop = self.valueof("script_stop", values) + + if not values: + values = self.default_values() + self.session.mobility.set_configs(values, node_id=object_id, config_type=self.name) + + self.file = values["file"] + self.refresh_ms = int(values["refresh_ms"]) + self.loop = values["loop"].lower() == "on" + self.autostart = values["autostart"] + self.parsemap(values["map"]) + self.script_start = values["script_start"] + self.script_pause = values["script_pause"] + self.script_stop = values["script_stop"] logger.info("ns-2 scripted mobility configured for WLAN %d using file: %s", object_id, self.file) self.readscriptfile() self.copywaypoints() self.setendtime() - @classmethod - def configure_mob(cls, session, config_data): - """ - Handle configuration messages for setting up a model. - Pass the MobilityManager object as the manager object. - - :param core.session.Session session: current session calling function - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - """ - return cls.configure(session.mobility, config_data) - def readscriptfile(self): """ Read in mobility script from a file. This adds waypoints to a @@ -1234,4 +1233,4 @@ class Ns2ScriptedMobility(WayPointMobility): return filename = self.findfile(filename) args = ["/bin/sh", filename, typestr] - utils.check_cmd(args, cwd=self.session.sessiondir, env=self.session.get_environment()) + utils.check_cmd(args, cwd=self.session.session_dir, env=self.session.get_environment()) diff --git a/daemon/core/netns/nodes.py b/daemon/core/netns/nodes.py index a225eb6f..9b32d9fc 100644 --- a/daemon/core/netns/nodes.py +++ b/daemon/core/netns/nodes.py @@ -398,12 +398,12 @@ class WlanNode(LxBrNet): elif model.config_type == RegisterTlvs.MOBILITY.value: self.mobility = model(session=self.session, object_id=self.objid, values=config) - def updatemodel(self, model_name, values): + def updatemodel(self, model_name, config): """ Allow for model updates during runtime (similar to setmodel().) - :param model_name: model name to update - :param values: values to update model with + :param str model_name: model name to update + :param dict config: values to update model with :return: nothing """ logger.info("updating model %s" % model_name) @@ -412,14 +412,14 @@ class WlanNode(LxBrNet): model = self.model if model.config_type == RegisterTlvs.WIRELESS.value: - if not model.updateconfig(values): + if not model.updateconfig(config): return if self.model.position_callback: for netif in self.netifs(): netif.poshook = self.model.position_callback if netif.node is not None: - (x, y, z) = netif.node.position.get() + x, y, z = netif.node.position.get() netif.poshook(netif, x, y, z) self.model.setlinkparams() diff --git a/daemon/core/sdt.py b/daemon/core/sdt.py index 054b7a58..5f5da01e 100644 --- a/daemon/core/sdt.py +++ b/daemon/core/sdt.py @@ -118,12 +118,7 @@ class Sdt(object): :return: True if enabled, False otherwise :rtype: bool """ - if not hasattr(self.session.options, "enablesdt"): - return False - enabled = self.session.options.enablesdt - if enabled in ("1", "true", 1, True): - return True - return False + return self.session.options.get_config("enablesdt") == "1" def seturl(self): """ @@ -132,11 +127,8 @@ class Sdt(object): :return: nothing """ - url = None - if hasattr(self.session.options, "sdturl"): - if self.session.options.sdturl != "": - url = self.session.options.sdturl - if url is None or url == "": + url = self.session.options.get_config("stdurl") + if not url: url = self.DEFAULT_SDT_URL self.url = urlparse(url) self.address = (self.url.hostname, self.url.port) diff --git a/daemon/core/service.py b/daemon/core/service.py index a62cf84b..78cd9b6d 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -7,17 +7,11 @@ a list of available services to the GUI and for configuring individual services. """ import time -from itertools import repeat from core import CoreCommandError from core import logger -from core.conf import Configurable -from core.conf import ConfigurableManager -from core.data import ConfigData from core.data import EventData from core.data import FileData -from core.enumerations import ConfigDataTypes -from core.enumerations import ConfigFlags from core.enumerations import EventTypes from core.enumerations import MessageFlags from core.enumerations import RegisterTlvs @@ -78,7 +72,7 @@ class ServiceManager(object): cls.add(service) -class CoreServices(ConfigurableManager): +class CoreServices(object): """ Class for interacting with a list of available startup services for nodes. Mostly used to convert a CoreService into a Config API @@ -88,7 +82,6 @@ class CoreServices(ConfigurableManager): """ name = "services" config_type = RegisterTlvs.UTILITY.value - _invalid_custom_names = ("core", "api", "emane", "misc", "netns", "phys", "services") def __init__(self, session): @@ -98,7 +91,7 @@ class CoreServices(ConfigurableManager): :param core.session.Session session: session this manager is tied to :return: nothing """ - ConfigurableManager.__init__(self) + # ConfigurableManager.__init__(self) self.session = session # dict of default services tuples, key is node type self.defaultservices = {} @@ -154,34 +147,31 @@ class CoreServices(ConfigurableManager): return s return service - def setcustomservice(self, object_id, service, values): + def setcustomservice(self, object_id, service, config): """ Store service customizations in an instantiated service object using a list of values that came from a config message. :param int object_id: object id to set custom service for :param class service: service to set - :param list values: values to + :param dict config: values to :return: """ + logger.debug("setting custom service(%s) for node(%s): %s", object_id, service, config) if service._custom: s = service else: # instantiate the class, for storing config customization s = service() - # values are new key=value format; not all keys need to be present - # a missing key means go with the default - if Configurable.haskeyvalues(values): - for v in values: - key, value = v.split('=', 1) - s.setvalue(key, value) - # old-style config, list of values - else: - s.fromvaluelist(values) + + # set custom service configuration + for name, value in config.iteritems(): + s.setvalue(name, value) # assume custom service already in dict if service._custom: return + # add the custom service to dict if object_id in self.customservices: self.customservices[object_id] += (s,) @@ -436,130 +426,6 @@ class CoreServices(ConfigurableManager): status = "-1" return status - def configure_request(self, config_data): - """ - Receive configuration message for configuring services. - With a request flag set, a list of services has been requested. - When the opaque field is present, a specific service is being - configured or requested. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: response messages - :rtype: ConfigData - """ - node_id = config_data.node - session_id = config_data.session - opaque = config_data.opaque - - logger.debug("configuration request: node(%s) session(%s) opaque(%s)", node_id, session_id, opaque) - - # send back a list of available services - if opaque is None: - type_flag = ConfigFlags.NONE.value - data_types = tuple(repeat(ConfigDataTypes.BOOL.value, len(ServiceManager.services))) - values = "|".join(repeat('0', len(ServiceManager.services))) - names = map(lambda x: x._name, ServiceManager.services) - captions = "|".join(names) - possible_values = "" - for s in ServiceManager.services: - if s._custom_needed: - possible_values += '1' - possible_values += '|' - groups = self.buildgroups(ServiceManager.services) - # send back the properties for this service - else: - if node_id is None: - return None - node = self.session.get_object(node_id) - if node is None: - logger.warn("Request to configure service for unknown node %s", node_id) - return None - servicesstring = opaque.split(':') - services, unknown = self.servicesfromopaque(opaque, node.objid) - for u in unknown: - logger.warn("Request for unknown service '%s'" % u) - - if len(services) < 1: - return None - - if len(servicesstring) == 3: - # a file request: e.g. "service:zebra:quagga.conf" - file_data = self.getservicefile(services, node, servicesstring[2]) - self.session.broadcast_file(file_data) - - # short circuit this request early to avoid returning response below - return None - - # the first service in the list is the one being configured - svc = services[0] - # send back: - # dirs, configs, startindex, startup, shutdown, metadata, config - type_flag = ConfigFlags.UPDATE.value - data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(svc.keys))) - values = svc.tovaluelist(node, services) - captions = None - possible_values = None - groups = None - - return ConfigData( - message_type=0, - node=node_id, - object=self.name, - type=type_flag, - data_types=data_types, - data_values=values, - captions=captions, - possible_values=possible_values, - groups=groups, - session=session_id, - opaque=opaque - ) - - def configure_values(self, config_data): - """ - Receive configuration message for configuring services. - With a request flag set, a list of services has been requested. - When the opaque field is present, a specific service is being - configured or requested. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: None - """ - data_types = config_data.data_types - values = config_data.data_values - node_id = config_data.node - opaque = config_data.opaque - - error_message = "services config message that I don't know how to handle" - if values is None: - logger.error(error_message) - return None - else: - values = values.split('|') - - if opaque is None: - # store default services for a node type in self.defaultservices[] - if data_types is None or data_types[0] != ConfigDataTypes.STRING.value: - logger.info(error_message) - return None - key = values.pop(0) - self.defaultservices[key] = values - logger.debug("default services for type %s set to %s", key, values) - else: - # store service customized config in self.customservices[] - if node_id is None: - return None - services, unknown = self.servicesfromopaque(opaque, node_id) - for u in unknown: - logger.warn("Request for unknown service '%s'" % u) - - if len(services) < 1: - return None - svc = services[0] - self.setcustomservice(node_id, svc, values) - - return None - def servicesfromopaque(self, opaque, object_id): """ Build a list of services from an opaque data string. diff --git a/daemon/core/session.py b/daemon/core/session.py index 55bd6c38..27d01e0d 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -11,6 +11,7 @@ import subprocess import tempfile import threading import time +from itertools import repeat import pwd @@ -18,8 +19,10 @@ from core import constants from core import logger from core.api import coreapi from core.broker import CoreBroker -from core.conf import Configurable -from core.conf import ConfigurableManager +from core.conf import ConfigShim +from core.conf import ConfigurableOptions +from core.conf import Configuration +from core.conf import NewConfigurableManager from core.data import ConfigData from core.data import EventData from core.data import ExceptionData @@ -37,11 +40,10 @@ from core.misc import nodeutils from core.misc import utils from core.misc.event import EventLoop from core.misc.ipaddress import MacAddress -from core.mobility import BasicRangeModel from core.mobility import MobilityManager -from core.mobility import Ns2ScriptedMobility from core.netns import nodes from core.sdt import Sdt +from core.service import CoreService from core.service import CoreServices from core.xml.xmlsession import save_session_xml @@ -81,10 +83,6 @@ class Session(object): self.objects = {} self._objects_lock = threading.Lock() - # dict of configurable objects - self.config_objects = {} - self._config_objects_lock = threading.Lock() - # TODO: should the default state be definition? self.state = EventTypes.NONE.value self._state_time = time.time() @@ -106,60 +104,30 @@ class Session(object): self.config_handlers = [] self.shutdown_handlers = [] - # setup broker + # initialize feature helpers self.broker = CoreBroker(session=self) - self.add_config_object(CoreBroker.name, CoreBroker.config_type, self.broker.configure) - - # setup location self.location = CoreLocation() - self.add_config_object(CoreLocation.name, CoreLocation.config_type, self.location.configure) - - # setup mobiliy self.mobility = MobilityManager(session=self) - self.add_config_object(MobilityManager.name, MobilityManager.config_type, self.mobility.configure) - self.add_config_object(BasicRangeModel.name, BasicRangeModel.config_type, BasicRangeModel.configure_mob) - self.add_config_object(Ns2ScriptedMobility.name, Ns2ScriptedMobility.config_type, - Ns2ScriptedMobility.configure_mob) - - # setup services self.services = CoreServices(session=self) - self.add_config_object(CoreServices.name, CoreServices.config_type, self.services.configure) - - # setup emane self.emane = EmaneManager(session=self) - self.add_config_object(EmaneManager.name, EmaneManager.config_type, self.emane.configure) - - # setup sdt - self.sdt = Sdt(session=self) - - # future parameters set by the GUI may go here - self.options = SessionConfig(session=self) - self.add_config_object(SessionConfig.name, SessionConfig.config_type, self.options.configure) + self.options = SessionConfig() self.metadata = SessionMetaData() - self.add_config_object(SessionMetaData.name, SessionMetaData.config_type, self.metadata.configure) + self.sdt = Sdt(session=self) def shutdown(self): """ Shutdown all emulation objects and remove the session directory. """ - - # shutdown emane + # shutdown/cleanup feature helpers self.emane.shutdown() - - # shutdown broker self.broker.shutdown() - - # shutdown NRL's SDT3D self.sdt.shutdown() # delete all current objects self.delete_objects() - preserve = False - if hasattr(self.options, "preservedir") and self.options.preservedir == "1": - preserve = True - # remove this sessions working directory + preserve = self.options.get_config("preservedir") == "1" if not preserve: shutil.rmtree(self.session_dir, ignore_errors=True) @@ -379,12 +347,7 @@ class Session(object): except: message = "exception occured when running %s state hook: %s" % (coreapi.state_name(state), hook) logger.exception(message) - self.exception( - ExceptionLevels.ERROR, - "Session.run_state_hooks", - None, - message - ) + self.exception(ExceptionLevels.ERROR, "Session.run_state_hooks", None, message) def add_state_hook(self, state, hook): """ @@ -422,7 +385,7 @@ class Session(object): if state == EventTypes.RUNTIME_STATE.value: self.emane.poststartup() xml_file_version = self.get_config_item("xmlfilever") - if xml_file_version in ('1.0',): + if xml_file_version in ("1.0",): xml_file_name = os.path.join(self.session_dir, "session-deployed.xml") save_session_xml(self, xml_file_name, xml_file_version) @@ -597,64 +560,6 @@ class Session(object): except IOError: logger.exception("error writing nodes file") - def add_config_object(self, name, object_type, callback): - """ - Objects can register configuration objects that are included in - the Register Message and may be configured via the Configure - Message. The callback is invoked when receiving a Configure Message. - - :param str name: name of configuration object to add - :param int object_type: register tlv type - :param func callback: callback function for object - :return: nothing - """ - register_tlv = RegisterTlvs(object_type) - logger.debug("adding config object callback: %s - %s", name, register_tlv) - with self._config_objects_lock: - self.config_objects[name] = (object_type, callback) - - def config_object(self, config_data): - """ - Invoke the callback for an object upon receipt of configuration data for that object. - A no-op if the object doesn't exist. - - :param core.data.ConfigData config_data: configuration data to execute against - :return: responses to the configuration data - :rtype: list - """ - name = config_data.object - logger.info("session(%s) setting config(%s)", self.session_id, name) - for key, value in config_data.__dict__.iteritems(): - logger.debug("%s = %s", key, value) - - replies = [] - - if name == "all": - with self._config_objects_lock: - for name in self.config_objects: - config_type, callback = self.config_objects[name] - reply = callback(self, config_data) - - if reply: - replies.append(reply) - - return replies - - if name in self.config_objects: - with self._config_objects_lock: - config_type, callback = self.config_objects[name] - - reply = callback(self, config_data) - - if reply: - replies.append(reply) - - return replies - else: - logger.info("session object doesn't own model '%s', ignoring", name) - - return replies - def dump_session(self): """ Log information about the session in its current state. @@ -742,10 +647,8 @@ class Session(object): if self.emane.startup() == self.emane.NOT_READY: return - # startup broker + # start feature helpers self.broker.startup() - - # startup mobility self.mobility.startup() # boot the services on each node @@ -901,11 +804,25 @@ class Session(object): :return: control net prefix list :rtype: list """ - p = getattr(self.options, "controlnet", self.config.get("controlnet")) - p0 = getattr(self.options, "controlnet0", self.config.get("controlnet0")) - p1 = getattr(self.options, "controlnet1", self.config.get("controlnet1")) - p2 = getattr(self.options, "controlnet2", self.config.get("controlnet2")) - p3 = getattr(self.options, "controlnet3", self.config.get("controlnet3")) + p = self.options.get_config("controlnet") + if not p: + p = self.config.get("controlnet") + + p0 = self.options.get_config("controlnet0") + if not p0: + p0 = self.config.get("controlnet0") + + p1 = self.options.get_config("controlnet1") + if not p1: + p1 = self.config.get("controlnet1") + + p2 = self.options.get_config("controlnet2") + if not p2: + p2 = self.config.get("controlnet2") + + p3 = self.options.get_config("controlnet3") + if not p3: + p3 = self.config.get("controlnet3") if not p0 and p: p0 = p @@ -1000,7 +917,7 @@ class Session(object): logger.warning("controlnet updown script not configured") # check if session option set, overwrite if so - options_updown_script = getattr(self.options, "controlnet_updown_script", None) + options_updown_script = self.options.get_config("controlnet_updown_script") if options_updown_script: updown_script = options_updown_script @@ -1194,6 +1111,7 @@ class Session(object): node = self.get_object(node_id) node.cmd(data, wait=False) + # TODO: move to core handlers def send_objects(self): """ Return API messages that describe the current session. @@ -1222,36 +1140,48 @@ class Session(object): logger.info(pprint.pformat(dict(link_data._asdict()))) self.broadcast_link(link_data) - # send model info - configs = self.mobility.getallconfigs() - configs += self.emane.getallconfigs() - logger.info("sending model configs:") - for node_number, cls, values in configs: - logger.info("config: node(%s) class(%s) values(%s)", node_number, cls, values) - config_data = cls.config_data( - flags=0, - node_id=node_number, - type_flags=ConfigFlags.UPDATE.value, - values=values - ) - logger.info(pprint.pformat(dict(config_data._asdict()))) - self.broadcast_config(config_data) + # send mobility model info + for node_id in self.mobility.nodes(): + node = self.get_object(node_id) + for model_class, config in self.mobility.getmodels(node): + logger.info("mobility config: node(%s) class(%s) values(%s)", node_id, model_class, config) + config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) + self.broadcast_config(config_data) + + # send emane model info + for node_id in self.emane.nodes(): + if node_id not in self.objects: + continue + + node = self.get_object(node_id) + for model_class, config in self.emane.getmodels(node): + logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) + config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) + self.broadcast_config(config_data) # service customizations service_configs = self.services.getallconfigs() - for node_number, service in service_configs: + for node_id, service in service_configs: opaque = "service:%s" % service._name + data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(CoreService.keys))) + node = self.get_object(node_id) + values = CoreService.tovaluelist(node, node.services) config_data = ConfigData( - node=node_number, + message_type=0, + node=node_id, + object=self.services.name, + type=ConfigFlags.UPDATE.value, + data_types=data_types, + data_values=values, + session=self.session_id, opaque=opaque ) - config_response = self.services.configure_request(config_data) - self.broadcast_config(config_response) + self.broadcast_config(config_data) for file_name, config_data in self.services.getallfiles(service): file_data = FileData( message_type=MessageFlags.ADD.value, - node=node_number, + node=node_id, name=str(file_name), type=opaque, data=str(config_data) @@ -1271,91 +1201,62 @@ class Session(object): ) self.broadcast_file(file_data) - config_data = ConfigData() + # send session configuration + session_config = self.options.get_configs() + config_data = ConfigShim.config_data(0, None, ConfigFlags.UPDATE.value, self.options, session_config) + self.broadcast_config(config_data) - # retrieve session configuration data - options_config = self.options.configure_request(config_data, type_flags=ConfigFlags.UPDATE.value) - self.broadcast_config(options_config) - - # retrieve session metadata - metadata_config = self.metadata.configure_request(config_data, type_flags=ConfigFlags.UPDATE.value) - self.broadcast_config(metadata_config) + # send session metadata + data_values = "|".join(["%s=%s" % item for item in self.metadata.get_configs().iteritems()]) + data_types = tuple(ConfigDataTypes.STRING.value for _ in self.metadata.get_configs()) + config_data = ConfigData( + message_type=0, + object=self.metadata.name, + type=ConfigFlags.NONE.value, + data_types=data_types, + data_values=data_values + ) + self.broadcast_config(config_data) logger.info("informed GUI about %d nodes and %d links", len(nodes_data), len(links_data)) -class SessionConfig(ConfigurableManager, Configurable): +class SessionConfig(NewConfigurableManager, ConfigurableOptions): """ Session configuration object. """ name = "session" config_type = RegisterTlvs.UTILITY.value - config_matrix = [ - ("controlnet", ConfigDataTypes.STRING.value, "", "", "Control network"), - ("controlnet_updown_script", ConfigDataTypes.STRING.value, "", "", "Control network script"), - ("enablerj45", ConfigDataTypes.BOOL.value, "1", "On,Off", "Enable RJ45s"), - ("preservedir", ConfigDataTypes.BOOL.value, "0", "On,Off", "Preserve session dir"), - ("enablesdt", ConfigDataTypes.BOOL.value, "0", "On,Off", "Enable SDT3D output"), - ("sdturl", ConfigDataTypes.STRING.value, Sdt.DEFAULT_SDT_URL, "", "SDT3D URL"), - ] - config_groups = "Options:1-%d" % len(config_matrix) - def __init__(self, session): - """ - Creates a SessionConfig instance. + @classmethod + def configurations(cls): + return [ + Configuration(_id="controlnet", _type=ConfigDataTypes.STRING, label="Control Network"), + Configuration(_id="controlnet0", _type=ConfigDataTypes.STRING, label="Control Network 0"), + Configuration(_id="controlnet1", _type=ConfigDataTypes.STRING, label="Control Network 1"), + Configuration(_id="controlnet2", _type=ConfigDataTypes.STRING, label="Control Network 2"), + Configuration(_id="controlnet3", _type=ConfigDataTypes.STRING, label="Control Network 3"), + Configuration(_id="controlnet_updown_script", _type=ConfigDataTypes.STRING, label="Control Network Script"), + Configuration(_id="enablerj45", _type=ConfigDataTypes.BOOL, default="1", options=["On", "Off"], + label="Enable RJ45s"), + Configuration(_id="preservedir", _type=ConfigDataTypes.BOOL, default="0", options=["On", "Off"], + label="Preserve session dir"), + Configuration(_id="enablesdt", _type=ConfigDataTypes.BOOL, default="0", options=["On", "Off"], + label="Enable SDT3D output"), + Configuration(_id="sdturl", _type=ConfigDataTypes.STRING, default=Sdt.DEFAULT_SDT_URL, label="SDT3D URL") + ] - :param core.session.Session session: session this manager is tied to - :return: nothing - """ - ConfigurableManager.__init__(self) - self.session = session - self.reset() + @classmethod + def config_groups(cls): + return "Options:1-%d" % len(cls.configurations()) - def reset(self): - """ - Reset the session configuration. - - :return: nothing - """ - defaults = self.getdefaultvalues() - for key in self.getnames(): - # value may come from config file - value = self.session.get_config_item(key) - if value is None: - value = self.valueof(key, defaults) - value = self.offontobool(value) - setattr(self, key, value) - - def configure_values(self, config_data): - """ - Handle configuration values. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: None - """ - return self.configure_values_keyvalues(config_data, self, self.getnames()) - - def configure_request(self, config_data, type_flags=ConfigFlags.NONE.value): - """ - Handle a configuration request. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :param type_flags: - :return: - """ - node_id = config_data.node - values = [] - - for key in self.getnames(): - value = getattr(self, key) - if value is None: - value = "" - values.append("%s" % value) - - return self.config_data(0, node_id, type_flags, values) + def __init__(self): + super(SessionConfig, self).__init__() + config = self.default_values() + self.set_configs(config) -class SessionMetaData(ConfigurableManager): +class SessionMetaData(NewConfigurableManager): """ Metadata is simply stored in a configs[] dict. Key=value pairs are passed in from configure messages destined to the "metadata" object. @@ -1363,92 +1264,3 @@ class SessionMetaData(ConfigurableManager): """ name = "metadata" config_type = RegisterTlvs.UTILITY.value - - def configure_values(self, config_data): - """ - Handle configuration values. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :return: None - """ - values = config_data.data_values - if values is None: - return None - - key_values = values.split('|') - for key_value in key_values: - try: - key, value = key_value.split('=', 1) - except ValueError: - raise ValueError("invalid key in metdata: %s", key_value) - - self.add_item(key, value) - - return None - - def configure_request(self, config_data, type_flags=ConfigFlags.NONE.value): - """ - Handle a configuration request. - - :param core.conf.ConfigData config_data: configuration data for carrying out a configuration - :param int type_flags: configuration request flag value - :return: configuration data - :rtype: ConfigData - """ - node_number = config_data.node - values_str = "|".join(map(lambda item: "%s=%s" % item, self.items())) - return self.config_data(0, node_number, type_flags, values_str) - - def config_data(self, flags, node_id, type_flags, values_str): - """ - Retrieve configuration data object, leveraging provided data. - - :param flags: configuration data flags - :param int node_id: node id - :param type_flags: type flags - :param values_str: values string - :return: configuration data - :rtype: ConfigData - """ - data_types = tuple(map(lambda (k, v): ConfigDataTypes.STRING.value, self.items())) - - return ConfigData( - message_type=flags, - node=node_id, - object=self.name, - type=type_flags, - data_types=data_types, - data_values=values_str - ) - - def add_item(self, key, value): - """ - Add configuration key/value pair. - - :param key: configuration key - :param value: configuration value - :return: nothing - """ - self.configs[key] = value - - def get_item(self, key): - """ - Retrieve configuration value. - - :param key: key for configuration value to retrieve - :return: configuration value - """ - try: - return self.configs[key] - except KeyError: - logger.exception("error retrieving item from configs: %s", key) - - return None - - def items(self): - """ - Retrieve configuration items. - - :return: configuration items iterator - """ - return self.configs.iteritems() diff --git a/daemon/core/xml/xmlparser0.py b/daemon/core/xml/xmlparser0.py index e13e61c5..c93f4102 100644 --- a/daemon/core/xml/xmlparser0.py +++ b/daemon/core/xml/xmlparser0.py @@ -1,6 +1,7 @@ from xml.dom.minidom import parse from core import logger +from core.conf import ConfigShim from core.enumerations import NodeTypes from core.misc import nodeutils from core.service import ServiceManager @@ -369,6 +370,7 @@ class CoreDocumentParser0(object): values.append("files=%s" % files) if not bool(service.getAttribute("custom")): return True + values = ConfigShim.str_to_dict(values) self.session.services.setcustomservice(n.objid, svc, values) return True diff --git a/daemon/core/xml/xmlparser1.py b/daemon/core/xml/xmlparser1.py index 0ee96cd6..adc3d06a 100644 --- a/daemon/core/xml/xmlparser1.py +++ b/daemon/core/xml/xmlparser1.py @@ -4,6 +4,7 @@ from xml.dom.minidom import parse from core import constants from core import logger +from core.conf import ConfigShim from core.enumerations import NodeTypes from core.misc import nodeutils from core.misc.ipaddress import MacAddress @@ -615,6 +616,7 @@ class CoreDocumentParser1(object): values.append('cmddown=%s' % shutdown) if validate: values.append('cmdval=%s' % validate) + filenames = [] files = [] for f in xmlutils.iter_children_with_name(service, 'file'): @@ -629,11 +631,15 @@ class CoreDocumentParser1(object): data = None typestr = 'service:%s:%s' % (name, filename) files.append((typestr, filename, data)) + if filenames: values.append('files=%s' % filenames) + custom = service.getAttribute('custom') if custom and custom.lower() == 'true': + values = ConfigShim.str_to_dict(values) self.session.services.setcustomservice(node.objid, session_service, values) + # NOTE: if a custom service is used, setservicefile() must be # called after the custom service exists for typestr, filename, data in files: @@ -812,7 +818,7 @@ class CoreDocumentParser1(object): params = self.parse_parameter_children(options) for name, value in params.iteritems(): if name and value: - setattr(self.session.options, str(name), str(value)) + self.session.options.set_config(str(name), str(value)) def parse_session_hooks(self, session_config): """ diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index 235b4d56..d2459bdf 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -217,13 +217,7 @@ class ScenarioPlan(XmlElement): self.last_network_id = 0 self.addNetworks() self.addDevices() - - # XXX Do we need these? - # self.session.emane.setup() # not during runtime? - # self.addorigin() - self.addDefaultServices() - self.addSessionConfiguration() def addNetworks(self): @@ -318,10 +312,12 @@ class ScenarioPlan(XmlElement): # options options = self.createElement("options") - defaults = self.coreSession.options.getdefaultvalues() - for i, (k, v) in enumerate(self.coreSession.options.getkeyvaluelist()): - if str(v) != str(defaults[i]): - XmlElement.add_parameter(self.document, options, k, v) + options_config = self.coreSession.options.get_configs() + for _id, default_value in self.coreSession.options.default_values().iteritems(): + value = options_config[_id] + if value != default_value: + XmlElement.add_parameter(self.document, options, _id, value) + if options.hasChildNodes(): config.appendChild(options) @@ -340,7 +336,7 @@ class ScenarioPlan(XmlElement): # metadata meta = self.createElement("metadata") - for k, v in self.coreSession.metadata.items(): + for k, v in self.coreSession.metadata.get_configs().iteritems(): XmlElement.add_parameter(self.document, meta, k, v) if meta.hasChildNodes(): config.appendChild(meta) @@ -482,6 +478,7 @@ class NetworkElement(NamedXmlElement): modelconfigs = network_object.session.mobility.getmodels(network_object) modelconfigs += network_object.session.emane.getmodels(network_object) chan = None + for model, conf in modelconfigs: # Handle mobility parameters below if model.config_type == RegisterTlvs.MOBILITY.value: @@ -496,10 +493,9 @@ class NetworkElement(NamedXmlElement): channel_domain="CORE") # Add wireless model parameters - for i, key in enumerate(model.getnames()): - value = conf[i] + for key, value in conf.iteritems(): if value is not None: - chan.addParameter(key, model.valueof(key, conf)) + chan.addParameter(key, value) for model, conf in modelconfigs: if model.config_type == RegisterTlvs.MOBILITY.value: @@ -509,8 +505,8 @@ class NetworkElement(NamedXmlElement): type_element = self.createElement("type") type_element.appendChild(self.createTextNode(model.name)) mobility.appendChild(type_element) - for i, key in enumerate(model.getnames()): - value = conf[i] + + for key, value in conf.iteritems(): if value is not None: mobility.addParameter(key, value) From e4aaebfefb8a6a11fb97a7726e9e0b3268fed2c8 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 7 Jun 2018 09:09:56 -0700 Subject: [PATCH 28/83] fixed xml 0.0 failing --- daemon/core/xml/xmlwriter0.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/daemon/core/xml/xmlwriter0.py b/daemon/core/xml/xmlwriter0.py index e7155ce8..d81005fe 100644 --- a/daemon/core/xml/xmlwriter0.py +++ b/daemon/core/xml/xmlwriter0.py @@ -370,18 +370,20 @@ class CoreDocumentWriter0(Document): """ # options options = self.createElement("SessionOptions") - defaults = self.session.options.getdefaultvalues() - for i, (k, v) in enumerate(self.session.options.getkeyvaluelist()): - if str(v) != str(defaults[i]): - xmlutils.add_text_param_to_parent(self, options, k, v) - # addparamtoparent(self, options, k, v) + defaults = self.session.options.default_values() + for name, current_value in self.session.options.get_configs().iteritems(): + default_value = defaults[name] + if current_value != default_value: + xmlutils.add_text_param_to_parent(self, options, name, current_value) + if options.hasChildNodes(): self.meta.appendChild(options) + # hook scripts self.addhooks() + # meta meta = self.createElement("MetaData") self.meta.appendChild(meta) - for k, v in self.session.metadata.items(): - xmlutils.add_text_param_to_parent(self, meta, k, v) - # addparamtoparent(self, meta, k, v) + for name, current_value in self.session.metadata.get_configs().iteritems(): + xmlutils.add_text_param_to_parent(self, meta, name, current_value) From 7abf4bca166dd0692cbf711f57926468462629df Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 7 Jun 2018 12:57:32 -0700 Subject: [PATCH 29/83] updates to get all tests working with config changes --- daemon/core/conf.py | 11 ++++++----- daemon/core/emane/tdma.py | 2 +- daemon/core/emulator/coreemu.py | 8 ++++---- daemon/core/mobility.py | 9 ++++----- daemon/core/netns/nodes.py | 2 +- daemon/data/logging.conf | 2 +- daemon/tests/test_core.py | 22 ++++++++++++---------- 7 files changed, 29 insertions(+), 27 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index fae75d6c..5e461f86 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -140,6 +140,7 @@ class NewConfigurableManager(object): return [node_id for node_id in self._configuration_maps.iterkeys() if node_id != self._default_node] def config_reset(self, node_id=None): + logger.debug("resetting all configurations: %s", self.__class__.__name__) if not node_id: self._configuration_maps.clear() elif node_id in self._configuration_maps: @@ -152,9 +153,8 @@ class NewConfigurableManager(object): def set_configs(self, config, node_id=_default_node, config_type=_default_type): logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) - node_type_map = self.get_configs(node_id, config_type) - node_type_map.clear() - node_type_map.update(config) + node_configs = self.get_config_types(node_id) + node_configs[config_type] = config def get_config(self, _id, node_id=_default_node, config_type=_default_type): logger.debug("getting config for node(%s) type(%s): %s", node_id, config_type, _id) @@ -163,8 +163,9 @@ class NewConfigurableManager(object): def get_configs(self, node_id=_default_node, config_type=_default_type): logger.debug("getting configs for node(%s) type(%s)", node_id, config_type) - node_map = self._configuration_maps.setdefault(node_id, {}) + node_map = self.get_config_types(node_id) return node_map.setdefault(config_type, {}) def get_config_types(self, node_id=_default_node): - return self._configuration_maps.get(node_id, {}) + logger.debug("getting all configs for node(%s)", node_id) + return self._configuration_maps.setdefault(node_id, {}) diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index 5d43a692..d72936a2 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -47,7 +47,7 @@ class EmaneTdmaModel(emanemodel.EmaneModel): :return: nothing """ # get configured schedule - config = emane_manager.get_configs() + config = emane_manager.get_configs(self.object_id, self.name) if not config: return schedule = config[self.schedule_name] diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index ff814e4d..949a2a19 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -812,8 +812,8 @@ class EmuSession(Session): :param emane_model: emane model to set :return: nothing """ - values = list(emane_model.getdefaultvalues()) - self.emane.setconfig(emane_node.objid, emane_model.name, values) + config = emane_model.default_values() + self.emane.set_configs(config, emane_node.objid, emane_model.name) def set_wireless_model(self, node, model): """ @@ -823,8 +823,8 @@ class EmuSession(Session): :param core.mobility.WirelessModel model: wireless model to set node to :return: nothing """ - values = list(model.getdefaultvalues()) - node.setmodel(model, values) + config = model.default_values() + node.setmodel(model, config) def wireless_link_all(self, network, nodes): """ diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 442eaeef..07c84ecc 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -104,11 +104,11 @@ class MobilityManager(NewConfigurableManager): for model_name in node_configs.iterkeys(): try: clazz = self._modelclsmap[model_name] - model_config = self.get_config(node_id, model_name) + model_config = self.get_configs(node_id, model_name) logger.info("setting mobility model(%s) to node: %s", model_name, model_config) node.setmodel(clazz, model_config) except KeyError: - logger.warn("skipping mobility configuration for unknown model: %s", model_name) + logger.error("skipping mobility configuration for unknown model: %s", model_name) continue if self.session.master: @@ -145,7 +145,6 @@ class MobilityManager(NewConfigurableManager): if self.session.state == EventTypes.RUNTIME_STATE.value: try: node = self.session.get_object(node_id) - # TODO: this need to be updated node.updatemodel(config_type, config) except KeyError: logger.exception("skipping mobility configuration for unknown node %s", node_id) @@ -409,10 +408,10 @@ class BasicRangeModel(WirelessModel): self.wlan = session.get_object(object_id) self._netifs = {} self._netifslock = threading.Lock() - if not values: - values = self.default_values() # TODO: can this be handled in a better spot + if not values: + values = self.default_values() self.session.mobility.set_configs(values, node_id=object_id, config_type=self.name) self.range = None diff --git a/daemon/core/netns/nodes.py b/daemon/core/netns/nodes.py index 9b32d9fc..00c54bae 100644 --- a/daemon/core/netns/nodes.py +++ b/daemon/core/netns/nodes.py @@ -382,7 +382,7 @@ class WlanNode(LxBrNet): Sets the mobility and wireless model. :param core.mobility.WirelessModel.cls model: wireless model to set to - :param config: model configuration + :param dict config: model configuration :return: nothing """ logger.info("adding model: %s", model.name) diff --git a/daemon/data/logging.conf b/daemon/data/logging.conf index 7f3d496f..46de6e92 100644 --- a/daemon/data/logging.conf +++ b/daemon/data/logging.conf @@ -14,7 +14,7 @@ } }, "root": { - "level": "INFO", + "level": "DEBUG", "handlers": ["console"] } } diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 544b05f9..29caba4c 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -10,10 +10,10 @@ from xml.etree import ElementTree import pytest from mock import MagicMock -from core.data import ConfigData from core.emulator.emudata import NodeOptions from core.enumerations import MessageFlags, NodeTypes from core.mobility import BasicRangeModel +from core.mobility import Ns2ScriptedMobility from core.netns.vnodeclient import VnodeClient from core.service import ServiceManager @@ -306,15 +306,17 @@ class TestCore: session.wireless_link_all(wlan_node, [node_one, node_two]) # configure mobility script for session - config = ConfigData( - node=wlan_node.objid, - object="ns2script", - type=0, - data_types=(10, 3, 11, 10, 10, 10, 10, 10, 0), - data_values="file=%s|refresh_ms=50|loop=1|autostart=0.0|" - "map=|script_start=|script_pause=|script_stop=" % _MOBILITY_FILE - ) - session.config_object(config) + config = { + "file": _MOBILITY_FILE, + "refresh_ms": "50", + "loop": "1", + "autostart": "0.0", + "map": "", + "script_start": "", + "script_pause": "", + "script_stop": "", + } + session.mobility.set_configs(config, wlan_node.objid, Ns2ScriptedMobility.name) # add handler for receiving node updates event = threading.Event() From 4b9cf996d1b8ad9ff0a02fcac5621bd3c117ae33 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 7 Jun 2018 15:32:16 -0700 Subject: [PATCH 30/83] updates to account for config messages in core handlers that dont contain all configuration options --- daemon/core/api/dataconversion.py | 34 +++++++++++++--- daemon/core/corehandlers.py | 65 +++++++++---------------------- daemon/core/emane/emanemanager.py | 10 +++-- daemon/core/emane/nodes.py | 2 +- daemon/core/mobility.py | 54 ++++++++++++------------- daemon/core/netns/nodes.py | 4 +- daemon/core/netns/openvswitch.py | 4 +- ns3/corens3/obj.py | 2 +- 8 files changed, 85 insertions(+), 90 deletions(-) diff --git a/daemon/core/api/dataconversion.py b/daemon/core/api/dataconversion.py index 9080eee8..5bc33fa9 100644 --- a/daemon/core/api/dataconversion.py +++ b/daemon/core/api/dataconversion.py @@ -2,21 +2,19 @@ Converts CORE data objects into legacy API messages. """ -from core import logger from core.api import coreapi +from core.enumerations import ConfigTlvs from core.enumerations import NodeTlvs from core.misc import structutils def convert_node(node_data): """ - Callback to handle an node broadcast out from a session. + Convenience method for converting NodeData to a packed TLV message. - :param core.data.NodeData node_data: node data to handle + :param core.data.NodeData node_data: node data to convert :return: packed node message """ - logger.debug("converting node data to message: %s", node_data) - tlv_data = structutils.pack_values(coreapi.CoreNodeTlv, [ (NodeTlvs.NUMBER, node_data.id), (NodeTlvs.TYPE, node_data.node_type), @@ -39,5 +37,29 @@ def convert_node(node_data): (NodeTlvs.ICON, node_data.icon), (NodeTlvs.OPAQUE, node_data.opaque) ]) - return coreapi.CoreNodeMessage.pack(node_data.message_type, tlv_data) + + +def convert_config(config_data): + """ + Convenience method for converting ConfigData to a packed TLV message. + + :param core.data.ConfigData config_data: config data to convert + :return: packed message + """ + tlv_data = structutils.pack_values(coreapi.CoreConfigTlv, [ + (ConfigTlvs.NODE, config_data.node), + (ConfigTlvs.OBJECT, config_data.object), + (ConfigTlvs.TYPE, config_data.type), + (ConfigTlvs.DATA_TYPES, config_data.data_types), + (ConfigTlvs.VALUES, config_data.data_values), + (ConfigTlvs.CAPTIONS, config_data.captions), + (ConfigTlvs.BITMAP, config_data.bitmap), + (ConfigTlvs.POSSIBLE_VALUES, config_data.possible_values), + (ConfigTlvs.GROUPS, config_data.groups), + (ConfigTlvs.SESSION, config_data.session), + (ConfigTlvs.INTERFACE_NUMBER, config_data.interface_number), + (ConfigTlvs.NETWORK_ID, config_data.network_id), + (ConfigTlvs.OPAQUE, config_data.opaque), + ]) + return coreapi.CoreConfMessage.pack(config_data.message_type, tlv_data) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 141ec3e2..2ef7d780 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -14,13 +14,16 @@ from itertools import repeat from core import logger from core.api import coreapi +from core.api import dataconversion from core.conf import ConfigShim from core.data import ConfigData from core.data import EventData from core.emulator.emudata import InterfaceData from core.emulator.emudata import LinkOptions from core.emulator.emudata import NodeOptions -from core.enumerations import ConfigTlvs, ConfigFlags, ConfigDataTypes +from core.enumerations import ConfigDataTypes +from core.enumerations import ConfigFlags +from core.enumerations import ConfigTlvs from core.enumerations import EventTlvs from core.enumerations import EventTypes from core.enumerations import ExceptionTlvs @@ -37,7 +40,8 @@ from core.enumerations import SessionTlvs from core.misc import nodeutils from core.misc import structutils from core.misc import utils -from core.mobility import BasicRangeModel, Ns2ScriptedMobility +from core.mobility import BasicRangeModel +from core.mobility import Ns2ScriptedMobility from core.service import ServiceManager @@ -265,24 +269,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): :return: nothing """ logger.debug("handling broadcast config: %s", config_data) - - tlv_data = structutils.pack_values(coreapi.CoreConfigTlv, [ - (ConfigTlvs.NODE, config_data.node), - (ConfigTlvs.OBJECT, config_data.object), - (ConfigTlvs.TYPE, config_data.type), - (ConfigTlvs.DATA_TYPES, config_data.data_types), - (ConfigTlvs.VALUES, config_data.data_values), - (ConfigTlvs.CAPTIONS, config_data.captions), - (ConfigTlvs.BITMAP, config_data.bitmap), - (ConfigTlvs.POSSIBLE_VALUES, config_data.possible_values), - (ConfigTlvs.GROUPS, config_data.groups), - (ConfigTlvs.SESSION, config_data.session), - (ConfigTlvs.INTERFACE_NUMBER, config_data.interface_number), - (ConfigTlvs.NETWORK_ID, config_data.network_id), - (ConfigTlvs.OPAQUE, config_data.opaque), - ]) - message = coreapi.CoreConfMessage.pack(config_data.message_type, tlv_data) - + message = dataconversion.convert_config(config_data) try: self.sendall(message) except IOError: @@ -319,31 +306,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): :return: nothing """ logger.debug("handling broadcast node: %s", node_data) - - tlv_data = structutils.pack_values(coreapi.CoreNodeTlv, [ - (NodeTlvs.NUMBER, node_data.id), - (NodeTlvs.TYPE, node_data.node_type), - (NodeTlvs.NAME, node_data.name), - (NodeTlvs.IP_ADDRESS, node_data.ip_address), - (NodeTlvs.MAC_ADDRESS, node_data.mac_address), - (NodeTlvs.IP6_ADDRESS, node_data.ip6_address), - (NodeTlvs.MODEL, node_data.model), - (NodeTlvs.EMULATION_ID, node_data.emulation_id), - (NodeTlvs.EMULATION_SERVER, node_data.emulation_server), - (NodeTlvs.SESSION, node_data.session), - (NodeTlvs.X_POSITION, node_data.x_position), - (NodeTlvs.Y_POSITION, node_data.y_position), - (NodeTlvs.CANVAS, node_data.canvas), - (NodeTlvs.NETWORK_ID, node_data.network_id), - (NodeTlvs.SERVICES, node_data.services), - (NodeTlvs.LATITUDE, node_data.latitude), - (NodeTlvs.LONGITUDE, node_data.longitude), - (NodeTlvs.ALTITUDE, node_data.altitude), - (NodeTlvs.ICON, node_data.icon), - (NodeTlvs.OPAQUE, node_data.opaque) - ]) - message = coreapi.CoreNodeMessage.pack(node_data.message_type, tlv_data) - + message = dataconversion.convert_node(node_data) try: self.sendall(message) except IOError: @@ -1247,7 +1210,11 @@ class CoreHandler(SocketServer.BaseRequestHandler): if values_str: config = ConfigShim.str_to_dict(values_str) else: - config = model_class.default_values() + config = self.session.mobility.get_configs(node_id, object_name) + + for name, value in model_class.default_values().iteritems(): + if name not in config: + config[name] = value self.session.mobility.set_configs(config, node_id, object_name) @@ -1339,7 +1306,11 @@ class CoreHandler(SocketServer.BaseRequestHandler): if values_str: config = ConfigShim.str_to_dict(values_str) else: - config = model_class.default_values() + config = self.session.emane.get_configs(node_id, object_name) + + for name, value in model_class.default_values().iteritems(): + if name not in config: + config[name] = value self.session.emane.set_configs(config, node_id, object_name) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 567d84d6..c8e4101f 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -10,6 +10,7 @@ from core import CoreCommandError from core import constants from core import logger from core.api import coreapi +from core.api import dataconversion from core.conf import ConfigShim from core.conf import Configuration from core.conf import NewConfigurableManager @@ -20,11 +21,12 @@ from core.emane.emanemodel import EmaneModel from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel -from core.enumerations import ConfigDataTypes, NodeTypes +from core.enumerations import ConfigDataTypes from core.enumerations import ConfigFlags from core.enumerations import ConfigTlvs from core.enumerations import MessageFlags from core.enumerations import MessageTypes +from core.enumerations import NodeTypes from core.enumerations import RegisterTlvs from core.misc import nodeutils from core.misc import utils @@ -459,9 +461,9 @@ class EmaneManager(NewConfigurableManager): typeflags = ConfigFlags.UPDATE.value self.set_config("platform_id_start", str(platformid)) self.set_config("nem_id_start", str(nemid)) - msg = ConfigShim.config_data(0, None, typeflags, self.emane_config, self.get_configs()) - # TODO: this needs to be converted into a sendable TLV message - server.sock.send(msg) + config_data = ConfigShim.config_data(0, None, typeflags, self.emane_config, self.get_configs()) + message = dataconversion.convert_config(config_data) + server.sock.send(message) # increment nemid for next server by number of interfaces with self._ifccountslock: if server in self._ifccounts: diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 0dbf0721..9f7119e1 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -77,7 +77,7 @@ class EmaneNode(EmaneNet): # when buildnemxml() is called, not during init() self.model = model(session=self.session, object_id=self.objid) elif model.config_type == RegisterTlvs.MOBILITY.value: - self.mobility = model(session=self.session, object_id=self.objid, values=config) + self.mobility = model(session=self.session, object_id=self.objid, config=config) def setnemid(self, netif, nemid): """ diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 07c84ecc..bd378116 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -108,7 +108,7 @@ class MobilityManager(NewConfigurableManager): logger.info("setting mobility model(%s) to node: %s", model_name, model_config) node.setmodel(clazz, model_config) except KeyError: - logger.error("skipping mobility configuration for unknown model: %s", model_name) + logger.exception("skipping mobility configuration for unknown model: %s", model_name) continue if self.session.master: @@ -328,13 +328,13 @@ class WirelessModel(ConfigurableOptions): bitmap = None position_callback = None - def __init__(self, session, object_id, values=None): + def __init__(self, session, object_id, config=None): """ Create a WirelessModel instance. :param core.session.Session session: core session we are tied to :param int object_id: object id - :param values: values + :param dict config: values """ self.session = session self.object_id = object_id @@ -395,13 +395,13 @@ class BasicRangeModel(WirelessModel): def config_groups(cls): return "Basic Range Parameters:1-%d" % len(cls.configurations()) - def __init__(self, session, object_id, values=None): + def __init__(self, session, object_id, config=None): """ Create a BasicRangeModel instance. :param core.session.Session session: related core session :param int object_id: object id - :param dict values: values + :param dict config: values """ super(BasicRangeModel, self).__init__(session=session, object_id=object_id) self.session = session @@ -410,9 +410,9 @@ class BasicRangeModel(WirelessModel): self._netifslock = threading.Lock() # TODO: can this be handled in a better spot - if not values: - values = self.default_values() - self.session.mobility.set_configs(values, node_id=object_id, config_type=self.name) + if not config: + config = self.default_values() + self.session.mobility.set_configs(config, node_id=object_id, config_type=self.name) self.range = None self.bw = None @@ -420,7 +420,7 @@ class BasicRangeModel(WirelessModel): self.loss = None self.jitter = None - self.values_from_config(values) + self.values_from_config(config) def values_from_config(self, config): """ @@ -430,7 +430,7 @@ class BasicRangeModel(WirelessModel): :return: nothing """ self.range = float(config["range"]) - logger.info("Basic range model configured for WLAN %d using range %d", self.wlan.objid, self.range) + logger.info("basic range model configured for WLAN %d using range %d", self.wlan.objid, self.range) self.bw = int(config["bandwidth"]) if self.bw == 0.0: self.bw = None @@ -680,16 +680,16 @@ class WayPointMobility(WirelessModel): STATE_RUNNING = 1 STATE_PAUSED = 2 - def __init__(self, session, object_id, values=None): + def __init__(self, session, object_id, config=None): """ Create a WayPointMobility instance. :param core.session.Session session: CORE session instance :param int object_id: object id - :param values: values for this model + :param config: values for this model :return: """ - super(WayPointMobility, self).__init__(session=session, object_id=object_id, values=values) + super(WayPointMobility, self).__init__(session=session, object_id=object_id, config=config) self.state = self.STATE_STOPPED self.queue = [] @@ -1002,30 +1002,30 @@ class Ns2ScriptedMobility(WayPointMobility): def config_groups(cls): return "ns-2 Mobility Script Parameters:1-%d" % len(cls.configurations()) - def __init__(self, session, object_id, values=None): + def __init__(self, session, object_id, config=None): """ Creates a Ns2ScriptedMobility instance. :param core.session.Session session: CORE session instance :param int object_id: object id - :param values: values + :param config: values """ - super(Ns2ScriptedMobility, self).__init__(session=session, object_id=object_id, values=values) + super(Ns2ScriptedMobility, self).__init__(session=session, object_id=object_id, config=config) self._netifs = {} self._netifslock = threading.Lock() - if not values: - values = self.default_values() - self.session.mobility.set_configs(values, node_id=object_id, config_type=self.name) + if not config: + config = self.default_values() + self.session.mobility.set_configs(config, node_id=object_id, config_type=self.name) - self.file = values["file"] - self.refresh_ms = int(values["refresh_ms"]) - self.loop = values["loop"].lower() == "on" - self.autostart = values["autostart"] - self.parsemap(values["map"]) - self.script_start = values["script_start"] - self.script_pause = values["script_pause"] - self.script_stop = values["script_stop"] + self.file = config["file"] + self.refresh_ms = int(config["refresh_ms"]) + self.loop = config["loop"].lower() == "on" + self.autostart = config["autostart"] + self.parsemap(config["map"]) + self.script_start = config["script_start"] + self.script_pause = config["script_pause"] + self.script_stop = config["script_stop"] logger.info("ns-2 scripted mobility configured for WLAN %d using file: %s", object_id, self.file) self.readscriptfile() self.copywaypoints() diff --git a/daemon/core/netns/nodes.py b/daemon/core/netns/nodes.py index 00c54bae..b1b393ee 100644 --- a/daemon/core/netns/nodes.py +++ b/daemon/core/netns/nodes.py @@ -387,7 +387,7 @@ class WlanNode(LxBrNet): """ logger.info("adding model: %s", model.name) if model.config_type == RegisterTlvs.WIRELESS.value: - self.model = model(session=self.session, object_id=self.objid, values=config) + self.model = model(session=self.session, object_id=self.objid, config=config) if self.model.position_callback: for netif in self.netifs(): netif.poshook = self.model.position_callback @@ -396,7 +396,7 @@ class WlanNode(LxBrNet): netif.poshook(netif, x, y, z) self.model.setlinkparams() elif model.config_type == RegisterTlvs.MOBILITY.value: - self.mobility = model(session=self.session, object_id=self.objid, values=config) + self.mobility = model(session=self.session, object_id=self.objid, config=config) def updatemodel(self, model_name, config): """ diff --git a/daemon/core/netns/openvswitch.py b/daemon/core/netns/openvswitch.py index 46ba2f2c..01a3a142 100644 --- a/daemon/core/netns/openvswitch.py +++ b/daemon/core/netns/openvswitch.py @@ -600,7 +600,7 @@ class OvsWlanNode(OvsNet): logger.info("adding model %s", model.name) if model.type == RegisterTlvs.WIRELESS.value: - self.model = model(session=self.session, object_id=self.objid, values=config) + self.model = model(session=self.session, object_id=self.objid, config=config) if self.model.position_callback: for interface in self.netifs(): interface.poshook = self.model.position_callback @@ -609,7 +609,7 @@ class OvsWlanNode(OvsNet): interface.poshook(interface, x, y, z) self.model.setlinkparams() elif model.type == RegisterTlvs.MOBILITY.value: - self.mobility = model(session=self.session, object_id=self.objid, values=config) + self.mobility = model(session=self.session, object_id=self.objid, config=config) def updatemodel(self, model_name, values): """ diff --git a/ns3/corens3/obj.py b/ns3/corens3/obj.py index 229288d2..a3b0e921 100644 --- a/ns3/corens3/obj.py +++ b/ns3/corens3/obj.py @@ -488,7 +488,7 @@ class Ns3Session(Session): Start a tracing thread using the ASCII output from the ns3 mobility helper. """ - net.mobility = WayPointMobility(session=self, object_id=net.objid, values=None) + net.mobility = WayPointMobility(session=self, object_id=net.objid, config=None) net.mobility.setendtime() net.mobility.refresh_ms = 300 net.mobility.empty_queue_stop = False From b696cf16e9df9d5de4f14a878bb20badfcf01d28 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 7 Jun 2018 16:55:44 -0700 Subject: [PATCH 31/83] updates to make how core_handlers handles model config messages a bit simpler --- daemon/core/corehandlers.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 2ef7d780..0fdb5996 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1207,14 +1207,13 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] + config = model_class.default_values() if values_str: - config = ConfigShim.str_to_dict(values_str) - else: - config = self.session.mobility.get_configs(node_id, object_name) - - for name, value in model_class.default_values().iteritems(): - if name not in config: + parsed_config = ConfigShim.str_to_dict(values_str) + for name, value in parsed_config.iteritems(): config[name] = value + else: + config = self.session.mobility.get_configs(node_id, object_name) or config self.session.mobility.set_configs(config, node_id, object_name) @@ -1303,14 +1302,13 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] + config = model_class.default_values() if values_str: - config = ConfigShim.str_to_dict(values_str) - else: - config = self.session.emane.get_configs(node_id, object_name) - - for name, value in model_class.default_values().iteritems(): - if name not in config: + parsed_config = ConfigShim.str_to_dict(values_str) + for name, value in parsed_config.iteritems(): config[name] = value + else: + config = self.session.emane.get_configs(node_id, object_name) or config self.session.emane.set_configs(config, node_id, object_name) From f6656f024546a0f96505a5bce7eaa93010afc330 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 8 Jun 2018 10:25:56 -0700 Subject: [PATCH 32/83] added some initial tests for conf objects --- daemon/core/emulator/coreemu.py | 2 + daemon/tests/test_conf.py | 142 ++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 daemon/tests/test_conf.py diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 949a2a19..f7732c1b 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -813,6 +813,7 @@ class EmuSession(Session): :return: nothing """ config = emane_model.default_values() + emane_node.setmodel(emane_model, config) self.emane.set_configs(config, emane_node.objid, emane_model.name) def set_wireless_model(self, node, model): @@ -825,6 +826,7 @@ class EmuSession(Session): """ config = model.default_values() node.setmodel(model, config) + self.mobility.set_configs(config, node.objid, model.name) def wireless_link_all(self, network, nodes): """ diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py new file mode 100644 index 00000000..e56ef86c --- /dev/null +++ b/daemon/tests/test_conf.py @@ -0,0 +1,142 @@ +from core.conf import NewConfigurableManager, ConfigurableOptions, Configuration +from core.enumerations import ConfigDataTypes + + +class TestConfigurableOptions(ConfigurableOptions): + name_one = "value1" + name_two = "value2" + + @classmethod + def configurations(cls): + return [ + Configuration(_id=TestConfigurableOptions.name_one, _type=ConfigDataTypes.STRING, label="value1"), + Configuration(_id=TestConfigurableOptions.name_two, _type=ConfigDataTypes.STRING, label="value2") + ] + + +class TestConf: + def test_configurable_options_default(self): + # given + configurable_options = TestConfigurableOptions() + + # when + default_values = TestConfigurableOptions.default_values() + instance_default_values = configurable_options.default_values() + + # then + assert len(default_values) == 2 + assert TestConfigurableOptions.name_one in default_values + assert TestConfigurableOptions.name_two in default_values + assert len(instance_default_values) == 2 + assert TestConfigurableOptions.name_one in instance_default_values + assert TestConfigurableOptions.name_two in instance_default_values + + def test_nodes(self): + # given + config_manager = NewConfigurableManager() + test_config = {1: 2} + node_id = 1 + config_manager.set_configs(test_config) + config_manager.set_configs(test_config, node_id=node_id) + + # when + nodes = config_manager.nodes() + + # then + assert len(nodes) == 1 + assert nodes[0] == node_id + + def test_config_reset_all(self): + # given + config_manager = NewConfigurableManager() + test_config = {1: 2} + node_id = 1 + config_manager.set_configs(test_config) + config_manager.set_configs(test_config, node_id=node_id) + + # when + config_manager.config_reset() + + # then + assert not config_manager._configuration_maps + + def test_config_reset_node(self): + # given + config_manager = NewConfigurableManager() + test_config = {1: 2} + node_id = 1 + config_manager.set_configs(test_config) + config_manager.set_configs(test_config, node_id=node_id) + + # when + config_manager.config_reset(node_id) + + # then + assert not config_manager.get_configs(node_id) + + def test_configs_setget(self): + # given + config_manager = NewConfigurableManager() + test_config = {1: 2} + node_id = 1 + config_manager.set_configs(test_config) + config_manager.set_configs(test_config, node_id=node_id) + + # when + default_config = config_manager.get_configs() + node_config = config_manager.get_configs(node_id) + + # then + assert default_config + assert node_config + + def test_config_setget(self): + # given + config_manager = NewConfigurableManager() + name = "test" + value = "1" + node_id = 1 + config_manager.set_config(name, value) + config_manager.set_config(name, value, node_id=node_id) + + # when + defaults_value = config_manager.get_config(name) + node_value = config_manager.get_config(name, node_id=node_id) + + # then + assert defaults_value == value + assert node_value == value + + def test_all_configs(self): + # given + config_manager = NewConfigurableManager() + name = "test" + value_one = "1" + value_two = "2" + node_id = 1 + config_one = "config1" + config_two = "config2" + config_manager.set_config(name, value_one, config_type=config_one) + config_manager.set_config(name, value_two, config_type=config_two) + config_manager.set_config(name, value_one, node_id=node_id, config_type=config_one) + config_manager.set_config(name, value_two, node_id=node_id, config_type=config_two) + + # when + defaults_value_one = config_manager.get_config(name, config_type=config_one) + defaults_value_two = config_manager.get_config(name, config_type=config_two) + node_value_one = config_manager.get_config(name, node_id=node_id, config_type=config_one) + node_value_two = config_manager.get_config(name, node_id=node_id, config_type=config_two) + default_all_configs = config_manager.get_config_types() + node_all_configs = config_manager.get_config_types(node_id=node_id) + + # then + assert defaults_value_one == value_one + assert defaults_value_two == value_two + assert node_value_one == value_one + assert node_value_two == value_two + assert len(default_all_configs) == 2 + assert config_one in default_all_configs + assert config_two in default_all_configs + assert len(node_all_configs) == 2 + assert config_one in node_all_configs + assert config_two in node_all_configs From 52bfd1edf48955016202d1b89f6ea718504ae376 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 8 Jun 2018 10:30:41 -0700 Subject: [PATCH 33/83] refactored get_config_types to get_all_configs and NewConfigurableManager back to ConfigurableManager --- daemon/core/conf.py | 8 ++++---- daemon/core/emane/emanemanager.py | 8 ++++---- daemon/core/mobility.py | 8 ++++---- daemon/core/session.py | 6 +++--- daemon/tests/test_conf.py | 30 +++++++++++++++++++----------- 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 5e461f86..bba983c7 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -129,7 +129,7 @@ class ConfigurableOptions(object): return OrderedDict([(config.id, config.default) for config in cls.configurations()]) -class NewConfigurableManager(object): +class ConfigurableManager(object): _default_node = -1 _default_type = "default" @@ -153,7 +153,7 @@ class NewConfigurableManager(object): def set_configs(self, config, node_id=_default_node, config_type=_default_type): logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) - node_configs = self.get_config_types(node_id) + node_configs = self.get_all_configs(node_id) node_configs[config_type] = config def get_config(self, _id, node_id=_default_node, config_type=_default_type): @@ -163,9 +163,9 @@ class NewConfigurableManager(object): def get_configs(self, node_id=_default_node, config_type=_default_type): logger.debug("getting configs for node(%s) type(%s)", node_id, config_type) - node_map = self.get_config_types(node_id) + node_map = self.get_all_configs(node_id) return node_map.setdefault(config_type, {}) - def get_config_types(self, node_id=_default_node): + def get_all_configs(self, node_id=_default_node): logger.debug("getting all configs for node(%s)", node_id) return self._configuration_maps.setdefault(node_id, {}) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index c8e4101f..d6feabb2 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -13,7 +13,7 @@ from core.api import coreapi from core.api import dataconversion from core.conf import ConfigShim from core.conf import Configuration -from core.conf import NewConfigurableManager +from core.conf import ConfigurableManager from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel @@ -54,7 +54,7 @@ EMANE_MODELS = [ ] -class EmaneManager(NewConfigurableManager): +class EmaneManager(ConfigurableManager): """ EMANE controller object. Lives in a Session instance and is used for building EMANE config files from all of the EmaneNode objects in this @@ -208,7 +208,7 @@ class EmaneManager(NewConfigurableManager): """ Used with XML export. """ - configs = self.get_config_types(node.objid) + configs = self.get_all_configs(node.objid) models = [] for model_name, config in configs.iteritems(): model_class = self._modelclsmap[model_name] @@ -572,7 +572,7 @@ class EmaneManager(NewConfigurableManager): def setnodemodel(self, node_id): logger.debug("setting emane models for node: %s", node_id) - node_config_types = self.get_config_types(node_id) + node_config_types = self.get_all_configs(node_id) if not node_config_types: logger.debug("no emane node model configuration, leaving: %s", node_id) return False diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index bd378116..eaf6b610 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -11,7 +11,7 @@ import time from core import logger from core.conf import ConfigurableOptions from core.conf import Configuration -from core.conf import NewConfigurableManager +from core.conf import ConfigurableManager from core.coreobj import PyCoreNode from core.data import EventData from core.data import LinkData @@ -26,7 +26,7 @@ from core.misc import utils from core.misc.ipaddress import IpAddress -class MobilityManager(NewConfigurableManager): +class MobilityManager(ConfigurableManager): """ Member of session class for handling configuration data for mobility and range models. @@ -69,7 +69,7 @@ class MobilityManager(NewConfigurableManager): :return: list of model and values tuples for the network node :rtype: list """ - configs = self.get_config_types(node.objid) + configs = self.get_all_configs(node.objid) models = [] for model_name, config in configs.iteritems(): model_class = self._modelclsmap[model_name] @@ -96,7 +96,7 @@ class MobilityManager(NewConfigurableManager): logger.warn("skipping mobility configuration for unknown node: %s", node_id) continue - node_configs = self.get_config_types(node_id) + node_configs = self.get_all_configs(node_id) if not node_configs: logger.warn("missing mobility configuration for node: %s", node_id) continue diff --git a/daemon/core/session.py b/daemon/core/session.py index 27d01e0d..9d2c241f 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -22,7 +22,7 @@ from core.broker import CoreBroker from core.conf import ConfigShim from core.conf import ConfigurableOptions from core.conf import Configuration -from core.conf import NewConfigurableManager +from core.conf import ConfigurableManager from core.data import ConfigData from core.data import EventData from core.data import ExceptionData @@ -1221,7 +1221,7 @@ class Session(object): logger.info("informed GUI about %d nodes and %d links", len(nodes_data), len(links_data)) -class SessionConfig(NewConfigurableManager, ConfigurableOptions): +class SessionConfig(ConfigurableManager, ConfigurableOptions): """ Session configuration object. """ @@ -1256,7 +1256,7 @@ class SessionConfig(NewConfigurableManager, ConfigurableOptions): self.set_configs(config) -class SessionMetaData(NewConfigurableManager): +class SessionMetaData(ConfigurableManager): """ Metadata is simply stored in a configs[] dict. Key=value pairs are passed in from configure messages destined to the "metadata" object. diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index e56ef86c..0d56ce3f 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -1,4 +1,4 @@ -from core.conf import NewConfigurableManager, ConfigurableOptions, Configuration +from core.conf import ConfigurableManager, ConfigurableOptions, Configuration from core.enumerations import ConfigDataTypes @@ -9,8 +9,16 @@ class TestConfigurableOptions(ConfigurableOptions): @classmethod def configurations(cls): return [ - Configuration(_id=TestConfigurableOptions.name_one, _type=ConfigDataTypes.STRING, label="value1"), - Configuration(_id=TestConfigurableOptions.name_two, _type=ConfigDataTypes.STRING, label="value2") + Configuration( + _id=TestConfigurableOptions.name_one, + _type=ConfigDataTypes.STRING, + label=TestConfigurableOptions.name_one + ), + Configuration( + _id=TestConfigurableOptions.name_two, + _type=ConfigDataTypes.STRING, + label=TestConfigurableOptions.name_two + ) ] @@ -33,7 +41,7 @@ class TestConf: def test_nodes(self): # given - config_manager = NewConfigurableManager() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -48,7 +56,7 @@ class TestConf: def test_config_reset_all(self): # given - config_manager = NewConfigurableManager() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -62,7 +70,7 @@ class TestConf: def test_config_reset_node(self): # given - config_manager = NewConfigurableManager() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -76,7 +84,7 @@ class TestConf: def test_configs_setget(self): # given - config_manager = NewConfigurableManager() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -92,7 +100,7 @@ class TestConf: def test_config_setget(self): # given - config_manager = NewConfigurableManager() + config_manager = ConfigurableManager() name = "test" value = "1" node_id = 1 @@ -109,7 +117,7 @@ class TestConf: def test_all_configs(self): # given - config_manager = NewConfigurableManager() + config_manager = ConfigurableManager() name = "test" value_one = "1" value_two = "2" @@ -126,8 +134,8 @@ class TestConf: defaults_value_two = config_manager.get_config(name, config_type=config_two) node_value_one = config_manager.get_config(name, node_id=node_id, config_type=config_one) node_value_two = config_manager.get_config(name, node_id=node_id, config_type=config_two) - default_all_configs = config_manager.get_config_types() - node_all_configs = config_manager.get_config_types(node_id=node_id) + default_all_configs = config_manager.get_all_configs() + node_all_configs = config_manager.get_all_configs(node_id=node_id) # then assert defaults_value_one == value_one From 044e7de5e3c013406520471178720843e617e63b Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 8 Jun 2018 13:53:16 -0700 Subject: [PATCH 34/83] update to support consistently retrieving the last set configuration --- daemon/core/conf.py | 4 +++- daemon/tests/test_conf.py | 27 ++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index bba983c7..988225e1 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -154,6 +154,8 @@ class ConfigurableManager(object): def set_configs(self, config, node_id=_default_node, config_type=_default_type): logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) node_configs = self.get_all_configs(node_id) + if config_type in node_configs: + node_configs.pop(config_type) node_configs[config_type] = config def get_config(self, _id, node_id=_default_node, config_type=_default_type): @@ -168,4 +170,4 @@ class ConfigurableManager(object): def get_all_configs(self, node_id=_default_node): logger.debug("getting all configs for node(%s)", node_id) - return self._configuration_maps.setdefault(node_id, {}) + return self._configuration_maps.setdefault(node_id, OrderedDict()) diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index 0d56ce3f..2849f6ec 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -1,4 +1,10 @@ -from core.conf import ConfigurableManager, ConfigurableOptions, Configuration +from random import shuffle + +import pytest + +from core.conf import ConfigurableManager +from core.conf import ConfigurableOptions +from core.conf import Configuration from core.enumerations import ConfigDataTypes @@ -148,3 +154,22 @@ class TestConf: assert len(node_all_configs) == 2 assert config_one in node_all_configs assert config_two in node_all_configs + + @pytest.mark.parametrize("_", xrange(10)) + def test_config_last_key(self, _): + # given + config_manager = ConfigurableManager() + config = {1: 2} + node_id = 1 + config_types = [1, 2, 3] + shuffle(config_types) + config_manager.set_configs(config, node_id=node_id, config_type=config_types[0]) + config_manager.set_configs(config, node_id=node_id, config_type=config_types[1]) + config_manager.set_configs(config, node_id=node_id, config_type=config_types[2]) + + # when + keys = config_manager.get_all_configs(node_id=node_id).keys() + + # then + assert keys + assert keys[-1] == config_types[2] From 1b843e2868c348b0af0ef25c10bbb976146a7639 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 8 Jun 2018 14:21:41 -0700 Subject: [PATCH 35/83] updated logic so emane will always use the last model configured for a node, fixes issue when using gui and configuring multiple models --- daemon/core/emane/emanemanager.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index d6feabb2..2c0bb69e 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -12,8 +12,8 @@ from core import logger from core.api import coreapi from core.api import dataconversion from core.conf import ConfigShim -from core.conf import Configuration from core.conf import ConfigurableManager +from core.conf import Configuration from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel @@ -80,7 +80,7 @@ class EmaneManager(ConfigurableManager): self._emane_node_lock = threading.Lock() self._ifccounts = {} self._ifccountslock = threading.Lock() - # Port numbers are allocated from these counters + # port numbers are allocated from these counters self.platformport = self.session.get_config_item_int("emane_platform_port", 8100) self.transformport = self.session.get_config_item_int("emane_transform_port", 8200) self.doeventloop = False @@ -98,6 +98,10 @@ class EmaneManager(ConfigurableManager): self.service = None self.emane_check() + def config_reset(self, node_id=None): + super(EmaneManager, self).config_reset(node_id) + self.set_configs(self.emane_config.default_values()) + def emane_models(self): return self._modelclsmap.keys() @@ -577,8 +581,12 @@ class EmaneManager(ConfigurableManager): logger.debug("no emane node model configuration, leaving: %s", node_id) return False + # retrieve and use the last set model configured for this node + # this supports behavior from the legacy gui due to there not being a formal set model message emane_node = self._emane_nodes[node_id] - for model_class, config in self.getmodels(emane_node): + models = self.getmodels(emane_node) + if models: + model_class, config = models[-1] logger.debug("setting emane model(%s) config(%s)", model_class, config) emane_node.setmodel(model_class, config) return True From 52230bc02688580d8a0a5247e6e759b3810321e8 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Mon, 11 Jun 2018 12:26:12 -0700 Subject: [PATCH 36/83] refactored configuration managers and options into a single class, now the models that can be configured will deal with and handle configurations for nodes themselves --- daemon/core/conf.py | 91 +++++++----------- daemon/core/corehandlers.py | 34 +++---- daemon/core/emane/bypass.py | 1 + daemon/core/emane/commeffect.py | 4 +- daemon/core/emane/emanemanager.py | 151 ++++++++++++------------------ daemon/core/emane/emanemodel.py | 48 ++++++++-- daemon/core/emane/ieee80211abg.py | 1 + daemon/core/emane/nodes.py | 13 ++- daemon/core/emane/rfpipe.py | 1 + daemon/core/emane/tdma.py | 3 +- daemon/core/emulator/coreemu.py | 32 +------ daemon/core/mobility.py | 97 +++++++------------ daemon/core/netns/nodes.py | 42 ++++----- daemon/core/netns/openvswitch.py | 39 +++----- daemon/core/session.py | 35 ++++--- daemon/core/xml/xmlparser1.py | 9 +- daemon/core/xml/xmlwriter0.py | 3 +- daemon/core/xml/xmlwriter1.py | 3 +- daemon/examples/api/wlan.py | 2 +- daemon/tests/conftest.py | 6 ++ daemon/tests/test_conf.py | 76 ++------------- daemon/tests/test_core.py | 15 +-- 22 files changed, 284 insertions(+), 422 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 988225e1..7e5a4115 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -94,80 +94,55 @@ class ConfigurableOptions(object): # unique name to receive configuration changes name = None bitmap = None + configuration_maps = None + _default_node = -1 @classmethod def configurations(cls): - """ - Returns configuration options supported by this class. - - :return: list of configuration options - :rtype: list[Configuration] - """ return [] @classmethod def config_groups(cls): - """ - String formatted to specify configuration groupings, using list index positions. - - Example: - "Group1:start-stop|Group2:start-stop" - - :return: config groups - :rtype: str - """ return None @classmethod def default_values(cls): - """ - Retrieves default values for configurations. - - :return: mapping of configuration options that can also be iterated in order of definition - :rtype: OrderedDict - """ return OrderedDict([(config.id, config.default) for config in cls.configurations()]) + @classmethod + def nodes(cls): + return {node_id for node_id in cls.configuration_maps.iterkeys() if node_id != cls._default_node} -class ConfigurableManager(object): - _default_node = -1 - _default_type = "default" - - def __init__(self): - self._configuration_maps = {} - - def nodes(self): - return [node_id for node_id in self._configuration_maps.iterkeys() if node_id != self._default_node] - - def config_reset(self, node_id=None): - logger.debug("resetting all configurations: %s", self.__class__.__name__) + @classmethod + def config_reset(cls, node_id=None): if not node_id: - self._configuration_maps.clear() - elif node_id in self._configuration_maps: - self._configuration_maps.pop(node_id) + logger.debug("resetting all configurations: %s", cls.__name__) + cls.configuration_maps.clear() + elif node_id in cls.configuration_maps: + logger.debug("resetting node(%s) configurations: %s", node_id, cls.__name__) + cls.configuration_maps.pop(node_id) - def set_config(self, _id, value, node_id=_default_node, config_type=_default_type): - logger.debug("setting config for node(%s) type(%s): %s=%s", node_id, config_type, _id, value) - node_type_map = self.get_configs(node_id, config_type) - node_type_map[_id] = value + @classmethod + def set_config(cls, _id, value, node_id=_default_node): + logger.debug("setting config for node(%s) type(%s): %s=%s", node_id, _id, value) + node_configs = cls.get_configs(node_id) + node_configs[_id] = value - def set_configs(self, config, node_id=_default_node, config_type=_default_type): - logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) - node_configs = self.get_all_configs(node_id) - if config_type in node_configs: - node_configs.pop(config_type) - node_configs[config_type] = config + @classmethod + def get_config(cls, _id, node_id=_default_node): + logger.debug("getting config for node(%s): %s", node_id, _id) + node_configs = cls.get_configs(node_id) + return node_configs.get(_id) - def get_config(self, _id, node_id=_default_node, config_type=_default_type): - logger.debug("getting config for node(%s) type(%s): %s", node_id, config_type, _id) - node_type_map = self.get_configs(node_id, config_type) - return node_type_map.get(_id) + @classmethod + def set_configs(cls, config=None, node_id=_default_node): + logger.debug("setting config for node(%s): %s", node_id, config) + node_config = cls.get_configs(node_id) + if config: + for key, value in config.iteritems(): + node_config[key] = value - def get_configs(self, node_id=_default_node, config_type=_default_type): - logger.debug("getting configs for node(%s) type(%s)", node_id, config_type) - node_map = self.get_all_configs(node_id) - return node_map.setdefault(config_type, {}) - - def get_all_configs(self, node_id=_default_node): - logger.debug("getting all configs for node(%s)", node_id) - return self._configuration_maps.setdefault(node_id, OrderedDict()) + @classmethod + def get_configs(cls, node_id=_default_node): + logger.debug("getting configs for node(%s)", node_id) + return cls.configuration_maps.setdefault(node_id, cls.default_values()) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 0fdb5996..f50bafe6 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -955,7 +955,6 @@ class CoreHandler(SocketServer.BaseRequestHandler): node_id = config_data.node self.session.location.reset() self.session.services.reset() - self.session.mobility.reset() self.session.mobility.config_reset(node_id) self.session.emane.config_reset(node_id) else: @@ -1139,7 +1138,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.error(error_message) else: if opaque is None: - values = values.split('|') + values = values.split("|") # store default services for a node type in self.defaultservices[] if data_types is None or data_types[0] != ConfigDataTypes.STRING.value: logger.info(error_message) @@ -1151,7 +1150,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): # store service customized config in self.customservices[] services, unknown = self.session.services.servicesfromopaque(opaque, node_id) for u in unknown: - logger.warn("Request for unknown service '%s'" % u) + logger.warn("request for unknown service: %s", u) if services: svc = services[0] @@ -1187,10 +1186,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] - config = self.session.mobility.get_configs(node_id, object_name) - if not config: - config = model_class.default_values() - + config = model_class.get_configs(node_id=node_id) config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) replies.append(config_response) elif message_type == ConfigFlags.RESET: @@ -1207,15 +1203,12 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] - config = model_class.default_values() if values_str: parsed_config = ConfigShim.str_to_dict(values_str) - for name, value in parsed_config.iteritems(): - config[name] = value - else: - config = self.session.mobility.get_configs(node_id, object_name) or config + model_class.set_configs(parsed_config, node_id=node_id) - self.session.mobility.set_configs(config, node_id, object_name) + config = model_class.get_configs(node_id) + model_class.set_configs(config, node_id=node_id) return replies @@ -1282,10 +1275,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] - config = self.session.emane.get_configs(node_id, object_name) - if not config: - config = model_class.default_values() - + config = model_class.get_configs(node_id=node_id) config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) replies.append(config_response) elif message_type == ConfigFlags.RESET: @@ -1302,15 +1292,13 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] - config = model_class.default_values() if values_str: parsed_config = ConfigShim.str_to_dict(values_str) - for name, value in parsed_config.iteritems(): - config[name] = value - else: - config = self.session.emane.get_configs(node_id, object_name) or config + model_class.set_configs(parsed_config, node_id=node_id) - self.session.emane.set_configs(config, node_id, object_name) + config = model_class.get_configs(node_id) + model_class.set_configs(config, node_id=node_id) + self.session.emane.set_node_model(node_id, object_name) return replies diff --git a/daemon/core/emane/bypass.py b/daemon/core/emane/bypass.py index 8f60f238..44e7c078 100644 --- a/daemon/core/emane/bypass.py +++ b/daemon/core/emane/bypass.py @@ -8,6 +8,7 @@ from core.enumerations import ConfigDataTypes class EmaneBypassModel(emanemodel.EmaneModel): name = "emane_bypass" + configuration_maps = {} # values to ignore, when writing xml files config_ignore = {"none"} diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index 4b737832..03adbbd0 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -29,6 +29,7 @@ def convert_none(x): class EmaneCommEffectModel(emanemodel.EmaneModel): name = "emane_commeffect" + configuration_maps = {} shim_library = "commeffectshim" shim_xml = "/usr/share/emane/manifest/commeffectshim.xml" @@ -54,8 +55,7 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): :param interface: interface for the emane node :return: nothing """ - default_values = self.default_values() - config = emane_manager.getifcconfig(self.object_id, self.name, default_values, interface) + config = self.getifcconfig(self.object_id, interface) if not config: return diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 2c0bb69e..2dce8b5f 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -12,7 +12,6 @@ from core import logger from core.api import coreapi from core.api import dataconversion from core.conf import ConfigShim -from core.conf import ConfigurableManager from core.conf import Configuration from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel @@ -54,7 +53,7 @@ EMANE_MODELS = [ ] -class EmaneManager(ConfigurableManager): +class EmaneManager(object): """ EMANE controller object. Lives in a Session instance and is used for building EMANE config files from all of the EmaneNode objects in this @@ -74,7 +73,6 @@ class EmaneManager(ConfigurableManager): :param core.session.Session session: session this manager is tied to :return: nothing """ - super(EmaneManager, self).__init__() self.session = session self._emane_nodes = {} self._emane_node_lock = threading.Lock() @@ -87,8 +85,10 @@ class EmaneManager(ConfigurableManager): self.eventmonthread = None # model for global EMANE configuration options - self.emane_config = EmaneGlobalModel(session, None) - self.set_configs(self.emane_config.default_values()) + self.emane_config = EmaneGlobalModel(session) + + # store the last configured model for a node, used during startup + self.node_models = {} session.broker.handlers.add(self.handledistributed) self.service = None @@ -98,9 +98,19 @@ class EmaneManager(ConfigurableManager): self.service = None self.emane_check() + def set_node_model(self, node_id, model_name): + if model_name not in self._modelclsmap: + raise ValueError("unknown emane model: %s", model_name) + self.node_models[node_id] = model_name + def config_reset(self, node_id=None): - super(EmaneManager, self).config_reset(node_id) - self.set_configs(self.emane_config.default_values()) + # clear and reset current emane configuration + self.emane_config.config_reset() + self.emane_config.set_configs() + + # reset model configurations + for model_class in self._modelclsmap.itervalues(): + model_class.config_reset(node_id=node_id) def emane_models(self): return self._modelclsmap.keys() @@ -152,8 +162,8 @@ class EmaneManager(ConfigurableManager): return # Get the control network to be used for events - group, port = self.get_config("eventservicegroup").split(":") - self.event_device = self.get_config("eventservicedevice") + group, port = self.emane_config.get_config("eventservicegroup").split(":") + self.event_device = self.emane_config.get_config("eventservicedevice") eventnetidx = self.session.get_control_net_index(self.event_device) if eventnetidx < 0: logger.error("invalid emane event service device provided: %s", self.event_device) @@ -212,54 +222,14 @@ class EmaneManager(ConfigurableManager): """ Used with XML export. """ - configs = self.get_all_configs(node.objid) models = [] - for model_name, config in configs.iteritems(): - model_class = self._modelclsmap[model_name] - models.append((model_class, config)) + for model_class in self._modelclsmap.itervalues(): + if node.objid in model_class.configuration_maps: + config = model_class.get_configs(node_id=node.objid) + models.append((model_class, config)) logger.debug("emane models: %s", models) return models - def getifcconfig(self, node_id, config_type, default_values, ifc): - """ - Retrieve interface configuration or node configuration if not provided. - - :param int node_id: node id - :param str config_type: configuration type - :param dict default_values: default configuration values - :param ifc: node interface - :return: - """ - # use the network-wide config values or interface(NEM)-specific values? - if ifc is None: - return self.get_configs(node_id, config_type) or default_values - else: - # don"t use default values when interface config is the same as net - # note here that using ifc.node.objid as key allows for only one type - # of each model per node; - # TODO: use both node and interface as key - - # Adamson change: first check for iface config keyed by "node:ifc.name" - # (so that nodes w/ multiple interfaces of same conftype can have - # different configs for each separate interface) - key = 1000 * ifc.node.objid - if ifc.netindex is not None: - key += ifc.netindex - - # try retrieve interface specific configuration - config = self.get_configs(key, config_type) - - # otherwise retrieve the interfaces node configuration - if not config: - config = self.get_configs(ifc.node.objid, config_type) - - if not config and ifc.transport_type == "raw": - # with EMANE 0.9.2+, we need an extra NEM XML from - # model.buildnemxmlfiles(), so defaults are returned here - config = self.get_configs(node_id, config_type) or default_values - - return config - def setup(self): """ Populate self._objs with EmaneNodes; perform distributed setup; @@ -284,7 +254,7 @@ class EmaneManager(ConfigurableManager): # - needs to be configured before checkdistributed() for distributed # - needs to exist when eventservice binds to it (initeventservice) if self.session.master: - otadev = self.get_config("otamanagerdevice") + otadev = self.emane_config.get_config("otamanagerdevice") netidx = self.session.get_control_net_index(otadev) logger.debug("emane ota manager device: index(%s) otadev(%s)", netidx, otadev) if netidx < 0: @@ -293,7 +263,7 @@ class EmaneManager(ConfigurableManager): ctrlnet = self.session.add_remove_control_net(net_index=netidx, remove=False, conf_required=False) self.distributedctrlnet(ctrlnet) - eventdev = self.get_config("eventservicedevice") + eventdev = self.emane_config.get_config("eventservicedevice") logger.debug("emane event service device: eventdev(%s)", eventdev) if eventdev != otadev: netidx = self.session.get_control_net_index(eventdev) @@ -309,11 +279,11 @@ class EmaneManager(ConfigurableManager): # we are slave, but haven't received a platformid yet platform_id_start = "platform_id_start" default_values = self.emane_config.default_values() - value = self.get_config(platform_id_start) + value = self.emane_config.get_config(platform_id_start) if value == default_values[platform_id_start]: return EmaneManager.NOT_READY - self.setnodemodels() + self.check_node_models() return EmaneManager.SUCCESS def startup(self): @@ -441,10 +411,10 @@ class EmaneManager(ConfigurableManager): emane_node = self._emane_nodes[key] nemcount += emane_node.numnetif() - nemid = int(self.get_config("nem_id_start")) + nemid = int(self.emane_config.get_config("nem_id_start")) nemid += nemcount - platformid = int(self.get_config("platform_id_start")) + platformid = int(self.emane_config.get_config("platform_id_start")) # build an ordered list of servers so platform ID is deterministic servers = [] @@ -463,8 +433,8 @@ class EmaneManager(ConfigurableManager): platformid += 1 typeflags = ConfigFlags.UPDATE.value - self.set_config("platform_id_start", str(platformid)) - self.set_config("nem_id_start", str(nemid)) + self.emane_config.set_config("platform_id_start", str(platformid)) + self.emane_config.set_config("nem_id_start", str(nemid)) config_data = ConfigShim.config_data(0, None, typeflags, self.emane_config, self.get_configs()) message = dataconversion.convert_config(config_data) server.sock.send(message) @@ -489,6 +459,7 @@ class EmaneManager(ConfigurableManager): self.buildnemxml() self.buildeventservicexml() + # TODO: remove need for tlv messaging def distributedctrlnet(self, ctrlnet): """ Distributed EMANE requires multiple control network prefixes to @@ -566,33 +537,30 @@ class EmaneManager(ConfigurableManager): with open(pathname, "w") as xml_file: doc.writexml(writer=xml_file, indent="", addindent=" ", newl="\n", encoding="UTF-8") - def setnodemodels(self): + def check_node_models(self): """ Associate EmaneModel classes with EmaneNode nodes. The model configurations are stored in self.configs. """ - for key in self._emane_nodes: - self.setnodemodel(key) + for node_id in self._emane_nodes: + emane_node = self._emane_nodes[node_id] + logger.debug("checking emane model for node: %s", node_id) - def setnodemodel(self, node_id): - logger.debug("setting emane models for node: %s", node_id) - node_config_types = self.get_all_configs(node_id) - if not node_config_types: - logger.debug("no emane node model configuration, leaving: %s", node_id) - return False + # skip nodes that already have a model set + if emane_node.model: + logger.debug("node(%s) already has model(%s)", emane_node.objid, emane_node.model.name) + continue - # retrieve and use the last set model configured for this node - # this supports behavior from the legacy gui due to there not being a formal set model message - emane_node = self._emane_nodes[node_id] - models = self.getmodels(emane_node) - if models: - model_class, config = models[-1] + # set model configured for node, due to legacy messaging configuration before nodes exist + model_name = self.node_models.get(node_id) + if not model_name: + logger.error("emane node(%s) has no node model", node_id) + raise ValueError("emane node has no model set") + + model_class = self._modelclsmap[model_name] + config = model_class.get_configs(node_id=node_id) logger.debug("setting emane model(%s) config(%s)", model_class, config) emane_node.setmodel(model_class, config) - return True - - # no model has been configured for this EmaneNode - return False def nemlookup(self, nemid): """ @@ -631,10 +599,10 @@ class EmaneManager(ConfigurableManager): plat = doc.getElementsByTagName("platform").pop() if otadev: - self.set_config("otamanagerdevice", otadev) + self.emane_config.set_config("otamanagerdevice", otadev) if eventdev: - self.set_config("eventservicedevice", eventdev) + self.emane_config.set_config("eventservicedevice", eventdev) # append all platform options (except starting id) to doc for configuration in self.emane_config.emulator_config: @@ -642,7 +610,7 @@ class EmaneManager(ConfigurableManager): if name == "platform_id_start": continue - value = self.get_config(name) + value = self.emane_config.get_config(name) param = self.xmlparam(doc, name, value) plat.appendChild(param) @@ -652,7 +620,7 @@ class EmaneManager(ConfigurableManager): """ Build a platform.xml file now that all nodes are configured. """ - nemid = int(self.get_config("nem_id_start")) + nemid = int(self.emane_config.get_config("nem_id_start")) platformxmls = {} # assume self._objslock is already held here @@ -729,7 +697,7 @@ class EmaneManager(ConfigurableManager): default_values = self.emane_config.default_values() for name in ["eventservicegroup", "eventservicedevice"]: a = default_values[name] - b = self.get_config(name) + b = self.emane_config.get_config(name) if a != b: need_xml = True @@ -739,12 +707,12 @@ class EmaneManager(ConfigurableManager): return try: - group, port = self.get_config("eventservicegroup").split(":") + group, port = self.emane_config.get_config("eventservicegroup").split(":") except ValueError: logger.exception("invalid eventservicegroup in EMANE config") return - dev = self.get_config("eventservicedevice") + dev = self.emane_config.get_config("eventservicedevice") doc = self.xmldoc("emaneeventmsgsvc") es = doc.getElementsByTagName("emaneeventmsgsvc").pop() kvs = (("group", group), ("port", port), ("device", dev), ("mcloop", "1"), ("ttl", "32")) @@ -771,12 +739,12 @@ class EmaneManager(ConfigurableManager): if realtime: emanecmd += "-r", - otagroup, otaport = self.get_config("otamanagergroup").split(":") - otadev = self.get_config("otamanagerdevice") + otagroup, otaport = self.emane_config.get_config("otamanagergroup").split(":") + otadev = self.emane_config.get_config("otamanagerdevice") otanetidx = self.session.get_control_net_index(otadev) - eventgroup, eventport = self.get_config("eventservicegroup").split(":") - eventdev = self.get_config("eventservicedevice") + eventgroup, eventport = self.emane_config.get_config("eventservicegroup").split(":") + eventdev = self.emane_config.get_config("eventservicedevice") eventservicenetidx = self.session.get_control_net_index(eventdev) run_emane_on_host = False @@ -1013,6 +981,7 @@ class EmaneGlobalModel(EmaneModel): _DEFAULT_DEV = "ctrl0" name = "emane" + configuration_maps = {} emulator_xml = "/usr/share/emane/manifest/nemmanager.xml" emulator_defaults = { diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index b02468fd..349044d2 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -70,9 +70,6 @@ class EmaneModel(WirelessModel): config_len = len(cls.configurations()) return "MAC Parameters:1-%d|PHY Parameters:%d-%d" % (mac_len, mac_len + 1, config_len) - def __init__(self, session, object_id=None): - super(EmaneModel, self).__init__(session, object_id) - def build_xml_files(self, emane_manager, interface): """ Builds xml files for emane. Includes a nem.xml file that points to both mac.xml and phy.xml definitions. @@ -82,7 +79,7 @@ class EmaneModel(WirelessModel): :return: nothing """ # retrieve configuration values - config = emane_manager.getifcconfig(self.object_id, self.name, self.default_values(), interface) + config = self.getifcconfig(self.object_id, interface) if not config: return @@ -270,11 +267,10 @@ class EmaneModel(WirelessModel): :rtype: str """ name = "n%s" % self.object_id - emane_manager = self.session.emane if interface: node_id = interface.node.objid - if emane_manager.getifcconfig(node_id, self.name, {}, interface): + if self.getifcconfig(node_id, interface): name = interface.localname.replace(".", "_") return "%s%s" % (name, self.name) @@ -353,3 +349,43 @@ class EmaneModel(WirelessModel): :return: nothing """ logger.warn("emane model(%s) does not support link configuration", self.name) + + def getifcconfig(self, node_id, ifc): + """ + Retrieve interface configuration or node configuration if not provided. + + :param int node_id: node id + :param ifc: node interface + :return: + """ + # use the network-wide config values or interface(NEM)-specific values? + if ifc is None: + return self.get_configs(node_id) + else: + # don"t use default values when interface config is the same as net + # note here that using ifc.node.objid as key allows for only one type + # of each model per node; + # TODO: use both node and interface as key + + # Adamson change: first check for iface config keyed by "node:ifc.name" + # (so that nodes w/ multiple interfaces of same conftype can have + # different configs for each separate interface) + key = 1000 * ifc.node.objid + if ifc.netindex is not None: + key += ifc.netindex + + # try retrieve interface specific configuration, avoid getting defaults + config = {} + if key in self.configuration_maps: + config = self.get_configs(key) + + # otherwise retrieve the interfaces node configuration, avoid using defaults + if not config and ifc.node.objid in self.configuration_maps: + config = self.get_configs(ifc.node.objid) + + if not config and ifc.transport_type == "raw": + # with EMANE 0.9.2+, we need an extra NEM XML from + # model.buildnemxmlfiles(), so defaults are returned here + config = self.get_configs(node_id) + + return config diff --git a/daemon/core/emane/ieee80211abg.py b/daemon/core/emane/ieee80211abg.py index 9b4e6ea6..d14ae4cd 100644 --- a/daemon/core/emane/ieee80211abg.py +++ b/daemon/core/emane/ieee80211abg.py @@ -9,6 +9,7 @@ from core.emane import emanemodel class EmaneIeee80211abgModel(emanemodel.EmaneModel): # model name name = "emane_ieee80211abg" + configuration_maps = {} # mac configuration mac_library = "ieee80211abgmaclayer" diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 9f7119e1..4159e33f 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -67,7 +67,13 @@ class EmaneNode(EmaneNet): def unlink(self, netif1, netif2): pass - def setmodel(self, model, config): + def updatemodel(self, config): + if not self.model: + raise ValueError("no model set to update for node(%s)", self.objid) + logger.info("node(%s) updating model(%s): %s", self.objid, self.model.name, config) + self.model.set_configs(config, node_id=self.objid) + + def setmodel(self, model, config=None): """ set the EmaneModel associated with this node """ @@ -75,7 +81,7 @@ class EmaneNode(EmaneNet): if model.config_type == RegisterTlvs.WIRELESS.value: # EmaneModel really uses values from ConfigurableManager # when buildnemxml() is called, not during init() - self.model = model(session=self.session, object_id=self.objid) + self.model = model(session=self.session, object_id=self.objid, config=config) elif model.config_type == RegisterTlvs.MOBILITY.value: self.mobility = model(session=self.session, object_id=self.objid, config=config) @@ -174,7 +180,8 @@ class EmaneNode(EmaneNet): trans.setAttribute("library", "trans%s" % transport_type.lower()) trans.appendChild(emane.xmlparam(transdoc, "bitrate", "0")) - config = emane.get_configs(self.objid, self.model.name) + model_class = emane.get_model_class(self.model.name) + config = model_class.get_configs(self.objid) logger.debug("transport xml config: %s", config) flowcontrol = config.get("flowcontrolenable", "0") == "1" diff --git a/daemon/core/emane/rfpipe.py b/daemon/core/emane/rfpipe.py index 3c600ddd..cc24d31a 100644 --- a/daemon/core/emane/rfpipe.py +++ b/daemon/core/emane/rfpipe.py @@ -9,6 +9,7 @@ from core.emane import emanemodel class EmaneRfPipeModel(emanemodel.EmaneModel): # model name name = "emane_rfpipe" + configuration_maps = {} # mac configuration mac_library = "rfpipemaclayer" diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index d72936a2..47fbff43 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -16,6 +16,7 @@ from core.misc import utils class EmaneTdmaModel(emanemodel.EmaneModel): # model name name = "emane_tdma" + configuration_maps = {} # mac configuration mac_library = "tdmaeventschedulerradiomodel" @@ -47,7 +48,7 @@ class EmaneTdmaModel(emanemodel.EmaneModel): :return: nothing """ # get configured schedule - config = emane_manager.get_configs(self.object_id, self.name) + config = self.get_configs(self.object_id) if not config: return schedule = config[self.schedule_name] diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index f7732c1b..fa21953f 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -325,9 +325,6 @@ class EmuSession(Session): :param core.enumerations.LinkTypes link_type: link type to delete :return: nothing """ - # interface data - # interface_one_data, interface_two_data = get_interfaces(link_data) - # get node objects identified by link data node_one, node_two, net_one, net_two, tunnel = self._link_nodes(node_one_id, node_two_id) @@ -390,9 +387,6 @@ class EmuSession(Session): :param core.emulator.emudata.LinkOptions link_options: data to update link with :return: nothing """ - # interface data - # interface_one_data, interface_two_data = get_interfaces(link_data) - # get node objects identified by link data node_one, node_two, net_one, net_two, tunnel = self._link_nodes(node_one_id, node_two_id) @@ -801,33 +795,9 @@ class EmuSession(Session): # create and return network emane_network = self.add_node(_type=NodeTypes.EMANE, node_options=node_options) - self.set_emane_model(emane_network, model) + emane_network.setmodel(model) return emane_network - def set_emane_model(self, emane_node, emane_model): - """ - Set emane model for a given emane node. - - :param emane_node: emane node to set model for - :param emane_model: emane model to set - :return: nothing - """ - config = emane_model.default_values() - emane_node.setmodel(emane_model, config) - self.emane.set_configs(config, emane_node.objid, emane_model.name) - - def set_wireless_model(self, node, model): - """ - Convenience method for setting a wireless model. - - :param node: node to set wireless model for - :param core.mobility.WirelessModel model: wireless model to set node to - :return: nothing - """ - config = model.default_values() - node.setmodel(model, config) - self.mobility.set_configs(config, node.objid, model.name) - def wireless_link_all(self, network, nodes): """ Link all nodes to the provided wireless network. diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index eaf6b610..08d2153d 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -11,7 +11,6 @@ import time from core import logger from core.conf import ConfigurableOptions from core.conf import Configuration -from core.conf import ConfigurableManager from core.coreobj import PyCoreNode from core.data import EventData from core.data import LinkData @@ -26,7 +25,7 @@ from core.misc import utils from core.misc.ipaddress import IpAddress -class MobilityManager(ConfigurableManager): +class MobilityManager(object): """ Member of session class for handling configuration data for mobility and range models. @@ -40,7 +39,6 @@ class MobilityManager(ConfigurableManager): :param core.session.Session session: session this manager is tied to """ - super(MobilityManager, self).__init__() self.session = session # configurations for basic range, indexed by WLAN node number, are stored in configurations # mapping from model names to their classes @@ -69,13 +67,19 @@ class MobilityManager(ConfigurableManager): :return: list of model and values tuples for the network node :rtype: list """ - configs = self.get_all_configs(node.objid) models = [] - for model_name, config in configs.iteritems(): - model_class = self._modelclsmap[model_name] - models.append((model_class, config)) + for model_class in self._modelclsmap.itervalues(): + if node.objid in model_class.configuration_maps: + config = model_class.get_configs(node_id=node.objid) + models.append((model_class, config)) return models + def nodes(self): + node_ids = set() + for model_class in self._modelclsmap.itervalues(): + node_ids |= model_class.nodes() + return node_ids + def startup(self, node_ids=None): """ Session is transitioning from instantiation to runtime state. @@ -96,20 +100,13 @@ class MobilityManager(ConfigurableManager): logger.warn("skipping mobility configuration for unknown node: %s", node_id) continue - node_configs = self.get_all_configs(node_id) - if not node_configs: - logger.warn("missing mobility configuration for node: %s", node_id) - continue - - for model_name in node_configs.iterkeys(): - try: - clazz = self._modelclsmap[model_name] - model_config = self.get_configs(node_id, model_name) - logger.info("setting mobility model(%s) to node: %s", model_name, model_config) - node.setmodel(clazz, model_config) - except KeyError: - logger.exception("skipping mobility configuration for unknown model: %s", model_name) + for model_class in self._modelclsmap.itervalues(): + logger.debug("model(%s) configurations: %s", model_class, model_class.configuration_maps) + if node_id not in model_class.configuration_maps: continue + config = model_class.get_configs(node_id=node_id) + logger.info("setting mobility model(%s) to node: %s", model_class.name, config) + node.setmodel(model_class, config) if self.session.master: self.installphysnodes(node) @@ -117,37 +114,15 @@ class MobilityManager(ConfigurableManager): if node.mobility: self.session.event_loop.add_event(0.0, node.mobility.startup) - def reset(self): + def config_reset(self, node_id=None): """ Reset all configs. + :param int node_id: node configuration to reset or None for all configurations :return: nothing """ - self.config_reset() - - def set_configs(self, config, node_id=None, config_type=None): - """ - Adds a check for run-time updates for WLANs after providing normal set configs. - - :param dict config: configuration value - :param int node_id: node id - :param str config_type: configuration type - :return: nothing - """ - if not node_id or not config_type: - raise ValueError("mobility manager invalid node id or config type: %s - %s", node_id, config_type) - - super(MobilityManager, self).set_configs(config, node_id, config_type) - - if self.session is None: - return - - if self.session.state == EventTypes.RUNTIME_STATE.value: - try: - node = self.session.get_object(node_id) - node.updatemodel(config_type, config) - except KeyError: - logger.exception("skipping mobility configuration for unknown node %s", node_id) + for model in self._modelclsmap.itervalues(): + model.config_reset(node_id=node_id) def handleevent(self, event_data): """ @@ -338,6 +313,8 @@ class WirelessModel(ConfigurableOptions): """ self.session = session self.object_id = object_id + if config: + self.set_configs(config, node_id=self.object_id) def all_link_data(self, flags): """ @@ -360,12 +337,11 @@ class WirelessModel(ConfigurableOptions): """ raise NotImplementedError - def updateconfig(self, config): + def updateconfig(self): """ For run-time updates of model config. Returns True when position callback and set link parameters should be invoked. - :param dict config: value to update :return: False :rtype: bool """ @@ -379,6 +355,7 @@ class BasicRangeModel(WirelessModel): the GUI. """ name = "basic_range" + configuration_maps = {} @classmethod def configurations(cls): @@ -409,10 +386,8 @@ class BasicRangeModel(WirelessModel): self._netifs = {} self._netifslock = threading.Lock() - # TODO: can this be handled in a better spot - if not config: - config = self.default_values() - self.session.mobility.set_configs(config, node_id=object_id, config_type=self.name) + # retrieve current configuration + config = self.get_configs(node_id=self.object_id) self.range = None self.bw = None @@ -571,17 +546,17 @@ class BasicRangeModel(WirelessModel): c = p1[2] - p2[2] return math.hypot(math.hypot(a, b), c) - def updateconfig(self, config): + def updateconfig(self): """ Configuration has changed during runtime. - MobilityManager.setconfig() -> WlanNode.updatemodel() -> - WirelessModel.updateconfig() :param dict config: values to update configuration :return: was update successful :rtype: bool """ + config = self.get_configs(node_id=self.object_id) self.values_from_config(config) + self.setlinkparams() return True def create_link_data(self, interface1, interface2, message_type): @@ -840,7 +815,6 @@ class WayPointMobility(WirelessModel): self.setnodeposition(node, x, y, z) moved.append(node) moved_netifs.append(netif) - # self.wlan.model.update(moved) self.session.mobility.updatewlans(moved, moved_netifs) def addwaypoint(self, time, nodenum, x, y, z, speed): @@ -914,7 +888,6 @@ class WayPointMobility(WirelessModel): :return: nothing """ # this would cause PyCoreNetIf.poshook() callback (range calculation) - # node.setposition(x, y, z) node.position.set(x, y, z) node_data = node.data(message_type=0) self.session.broadcast_node(node_data) @@ -984,6 +957,7 @@ class Ns2ScriptedMobility(WayPointMobility): BonnMotion. """ name = "ns2script" + configuration_maps = {} @classmethod def configurations(cls): @@ -1014,9 +988,8 @@ class Ns2ScriptedMobility(WayPointMobility): self._netifs = {} self._netifslock = threading.Lock() - if not config: - config = self.default_values() - self.session.mobility.set_configs(config, node_id=object_id, config_type=self.name) + # retrieve current configuration + config = self.get_configs(self.object_id) self.file = config["file"] self.refresh_ms = int(config["refresh_ms"]) @@ -1041,9 +1014,9 @@ class Ns2ScriptedMobility(WayPointMobility): """ filename = self.findfile(self.file) try: - f = open(filename, 'r') + f = open(filename, "r") except IOError: - logger.exception("ns-2 scripted mobility failed to load file '%s'", self.file) + logger.exception("ns-2 scripted mobility failed to load file: %s", self.file) return logger.info("reading ns-2 script file: %s" % filename) ln = 0 diff --git a/daemon/core/netns/nodes.py b/daemon/core/netns/nodes.py index b1b393ee..3cf6a224 100644 --- a/daemon/core/netns/nodes.py +++ b/daemon/core/netns/nodes.py @@ -377,7 +377,7 @@ class WlanNode(LxBrNet): # invokes any netif.poshook netif.setposition(x, y, z) - def setmodel(self, model, config): + def setmodel(self, model, config=None): """ Sets the mobility and wireless model. @@ -398,31 +398,23 @@ class WlanNode(LxBrNet): elif model.config_type == RegisterTlvs.MOBILITY.value: self.mobility = model(session=self.session, object_id=self.objid, config=config) - def updatemodel(self, model_name, config): - """ - Allow for model updates during runtime (similar to setmodel().) + def update_mobility(self, config): + if not self.mobility: + raise ValueError("no mobility set to update for node(%s)", self.objid) + self.mobility.set_configs(config, node_id=self.objid) - :param str model_name: model name to update - :param dict config: values to update model with - :return: nothing - """ - logger.info("updating model %s" % model_name) - if self.model is None or self.model.name != model_name: - return - - model = self.model - if model.config_type == RegisterTlvs.WIRELESS.value: - if not model.updateconfig(config): - return - - if self.model.position_callback: - for netif in self.netifs(): - netif.poshook = self.model.position_callback - if netif.node is not None: - x, y, z = netif.node.position.get() - netif.poshook(netif, x, y, z) - - self.model.setlinkparams() + def updatemodel(self, config): + if not self.model: + raise ValueError("no model set to update for node(%s)", self.objid) + logger.info("node(%s) updating model(%s): %s", self.objid, self.model.name, config) + self.model.set_configs(config, node_id=self.objid) + if self.model.position_callback: + for netif in self.netifs(): + netif.poshook = self.model.position_callback + if netif.node is not None: + x, y, z = netif.node.position.get() + netif.poshook(netif, x, y, z) + self.model.updateconfig() def all_link_data(self, flags): """ diff --git a/daemon/core/netns/openvswitch.py b/daemon/core/netns/openvswitch.py index 01a3a142..ba47131f 100644 --- a/daemon/core/netns/openvswitch.py +++ b/daemon/core/netns/openvswitch.py @@ -305,7 +305,7 @@ class OvsNet(PyCoreNet): utils.check_cmd([constants.OVS_BIN, "add-port", network.bridge_name, interface.name]) utils.check_cmd([constants.IP_BIN, "link", "set", interface.name, "up"]) - # TODO: is there a native method for this? see if this causes issues + # TODO: is there a native method for this? see if this causes issues # i = network.newifindex() # network._netif[i] = interface # with network._linked_lock: @@ -593,7 +593,7 @@ class OvsWlanNode(OvsNet): interface.setposition(x, y, z) # self.model.setlinkparams() - def setmodel(self, model, config): + def setmodel(self, model, config=None): """ Mobility and wireless model. """ @@ -611,29 +611,18 @@ class OvsWlanNode(OvsNet): elif model.type == RegisterTlvs.MOBILITY.value: self.mobility = model(session=self.session, object_id=self.objid, config=config) - def updatemodel(self, model_name, values): - """ - Allow for model updates during runtime (similar to setmodel().) - """ - logger.info("updating model %s", model_name) - if self.model is None or self.model.name != model_name: - logger.info( - "failure to update model, model doesn't exist or invalid name: model(%s) - name(%s)", - self.model, model_name - ) - return - - model = self.model - if model.type == RegisterTlvs.WIRELESS.value: - if not model.updateconfig(values): - return - if self.model.position_callback: - for interface in self.netifs(): - interface.poshook = self.model.position_callback - if interface.node is not None: - x, y, z = interface.node.position.get() - interface.poshook(interface, x, y, z) - self.model.setlinkparams() + def updatemodel(self, config): + if not self.model: + raise ValueError("no model set to update for node(%s)", self.objid) + logger.info("node(%s) updating model(%s): %s", self.objid, self.model.name, config) + self.model.set_configs(config, node_id=self.objid) + if self.model.position_callback: + for netif in self.netifs(): + netif.poshook = self.model.position_callback + if netif.node is not None: + x, y, z = netif.node.position.get() + netif.poshook(netif, x, y, z) + self.model.updateconfig() def all_link_data(self, flags): all_links = OvsNet.all_link_data(self, flags) diff --git a/daemon/core/session.py b/daemon/core/session.py index 9d2c241f..dcfdf79b 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -19,10 +19,8 @@ from core import constants from core import logger from core.api import coreapi from core.broker import CoreBroker -from core.conf import ConfigShim -from core.conf import ConfigurableOptions +from core.conf import ConfigShim, ConfigurableOptions from core.conf import Configuration -from core.conf import ConfigurableManager from core.data import ConfigData from core.data import EventData from core.data import ExceptionData @@ -1149,16 +1147,24 @@ class Session(object): self.broadcast_config(config_data) # send emane model info - for node_id in self.emane.nodes(): - if node_id not in self.objects: - continue - - node = self.get_object(node_id) - for model_class, config in self.emane.getmodels(node): + for model_name in self.emane.emane_models(): + model_class = self.emane.get_model_class(model_name) + for node_id in model_class.nodes(): + config = model_class.get_configs(node_id) logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) self.broadcast_config(config_data) + # for node_id in self.emane.nodes(): + # if node_id not in self.objects: + # continue + # + # node = self.get_object(node_id) + # for model_class, config in self.emane.getmodels(node): + # logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) + # config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) + # self.broadcast_config(config_data) + # service customizations service_configs = self.services.getallconfigs() for node_id, service in service_configs: @@ -1221,11 +1227,12 @@ class Session(object): logger.info("informed GUI about %d nodes and %d links", len(nodes_data), len(links_data)) -class SessionConfig(ConfigurableManager, ConfigurableOptions): +class SessionConfig(ConfigurableOptions): """ Session configuration object. """ name = "session" + configuration_maps = {} config_type = RegisterTlvs.UTILITY.value @classmethod @@ -1251,16 +1258,16 @@ class SessionConfig(ConfigurableManager, ConfigurableOptions): return "Options:1-%d" % len(cls.configurations()) def __init__(self): - super(SessionConfig, self).__init__() - config = self.default_values() - self.set_configs(config) + self.set_configs() -class SessionMetaData(ConfigurableManager): +class SessionMetaData(ConfigurableOptions): """ Metadata is simply stored in a configs[] dict. Key=value pairs are passed in from configure messages destined to the "metadata" object. The data is not otherwise interpreted or processed. """ name = "metadata" + configuration_maps = {} + config_type = RegisterTlvs.UTILITY.value diff --git a/daemon/core/xml/xmlparser1.py b/daemon/core/xml/xmlparser1.py index adc3d06a..9f86255a 100644 --- a/daemon/core/xml/xmlparser1.py +++ b/daemon/core/xml/xmlparser1.py @@ -210,9 +210,12 @@ class CoreDocumentParser1(object): raise NotImplementedError logger.info("setting wireless link params: node(%s) model(%s) mobility_model(%s)", nodenum, model_name, mobility_model_name) - mgr.setconfig_keyvalues(nodenum, model_name, link_params.items()) + model_class = mgr.get_model_class(model_name) + model_class.set_configs(link_params, node_id=nodenum) + if mobility_model_name and mobility_params: - mgr.setconfig_keyvalues(nodenum, mobility_model_name, mobility_params.items()) + model_class = mgr.get_model_class(mobility_model_name) + model_class.set_configs(mobility_params, node_id=nodenum) def link_layer2_devices(self, device1, ifname1, device2, ifname2): """ @@ -843,7 +846,7 @@ class CoreDocumentParser1(object): params = self.parse_parameter_children(metadata) for name, value in params.iteritems(): if name and value: - self.session.metadata.add_item(str(name), str(value)) + self.session.metadata.set_config(str(name), str(value)) def parse_session_config(self): session_config = xmlutils.get_first_child_by_tag_name(self.scenario, 'CORE:sessionconfig') diff --git a/daemon/core/xml/xmlwriter0.py b/daemon/core/xml/xmlwriter0.py index d81005fe..a54278b0 100644 --- a/daemon/core/xml/xmlwriter0.py +++ b/daemon/core/xml/xmlwriter0.py @@ -194,8 +194,7 @@ class CoreDocumentWriter0(Document): self.addaddresses(i, ifc) # per-interface models if netmodel and netmodel._name[:6] == "emane_": - cfg = self.session.emane.getifcconfig(node.objid, netmodel._name, - None, ifc) + cfg = netmodel.getifcconfig(node.objid, ifc) if cfg: self.addmodels(i, ((netmodel, cfg),)) diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index d2459bdf..59988346 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -654,8 +654,7 @@ class DeviceElement(NamedXmlElement): # per-interface models # XXX Remove??? if netmodel and netmodel.name[:6] == "emane_": - cfg = self.coreSession.emane.getifcconfig(device_object.objid, netmodel.name, - None, interface_object) + cfg = netmodel.getifcconfig(device_object.objid, interface_object) if cfg: interface_element.addModels(((netmodel, cfg),)) diff --git a/daemon/examples/api/wlan.py b/daemon/examples/api/wlan.py index c3b29431..0aab4673 100644 --- a/daemon/examples/api/wlan.py +++ b/daemon/examples/api/wlan.py @@ -27,7 +27,7 @@ def example(options): # create wlan network node wlan = session.add_node(_type=NodeTypes.WIRELESS_LAN) - session.set_wireless_model(wlan, BasicRangeModel) + wlan.setmodel(BasicRangeModel) # create nodes wireless_nodes = [] diff --git a/daemon/tests/conftest.py b/daemon/tests/conftest.py index e1f3fb34..6923807a 100644 --- a/daemon/tests/conftest.py +++ b/daemon/tests/conftest.py @@ -214,6 +214,12 @@ def session(): # return created session yield session_fixture + # clear session configurations + session_fixture.location.reset() + session_fixture.services.reset() + session_fixture.mobility.config_reset() + session_fixture.emane.config_reset() + # shutdown coreemu coreemu.shutdown() diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index 2849f6ec..b25f4feb 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -1,8 +1,3 @@ -from random import shuffle - -import pytest - -from core.conf import ConfigurableManager from core.conf import ConfigurableOptions from core.conf import Configuration from core.enumerations import ConfigDataTypes @@ -11,6 +6,7 @@ from core.enumerations import ConfigDataTypes class TestConfigurableOptions(ConfigurableOptions): name_one = "value1" name_two = "value2" + configuration_maps = {} @classmethod def configurations(cls): @@ -47,7 +43,7 @@ class TestConf: def test_nodes(self): # given - config_manager = ConfigurableManager() + config_manager = TestConfigurableOptions() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -58,11 +54,11 @@ class TestConf: # then assert len(nodes) == 1 - assert nodes[0] == node_id + assert node_id in nodes def test_config_reset_all(self): # given - config_manager = ConfigurableManager() + config_manager = TestConfigurableOptions() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -72,11 +68,11 @@ class TestConf: config_manager.config_reset() # then - assert not config_manager._configuration_maps + assert not config_manager.configuration_maps def test_config_reset_node(self): # given - config_manager = ConfigurableManager() + config_manager = TestConfigurableOptions() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -86,11 +82,12 @@ class TestConf: config_manager.config_reset(node_id) # then - assert not config_manager.get_configs(node_id) + assert node_id not in config_manager.configuration_maps + assert config_manager.get_configs() def test_configs_setget(self): # given - config_manager = ConfigurableManager() + config_manager = TestConfigurableOptions() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -106,7 +103,7 @@ class TestConf: def test_config_setget(self): # given - config_manager = ConfigurableManager() + config_manager = TestConfigurableOptions() name = "test" value = "1" node_id = 1 @@ -120,56 +117,3 @@ class TestConf: # then assert defaults_value == value assert node_value == value - - def test_all_configs(self): - # given - config_manager = ConfigurableManager() - name = "test" - value_one = "1" - value_two = "2" - node_id = 1 - config_one = "config1" - config_two = "config2" - config_manager.set_config(name, value_one, config_type=config_one) - config_manager.set_config(name, value_two, config_type=config_two) - config_manager.set_config(name, value_one, node_id=node_id, config_type=config_one) - config_manager.set_config(name, value_two, node_id=node_id, config_type=config_two) - - # when - defaults_value_one = config_manager.get_config(name, config_type=config_one) - defaults_value_two = config_manager.get_config(name, config_type=config_two) - node_value_one = config_manager.get_config(name, node_id=node_id, config_type=config_one) - node_value_two = config_manager.get_config(name, node_id=node_id, config_type=config_two) - default_all_configs = config_manager.get_all_configs() - node_all_configs = config_manager.get_all_configs(node_id=node_id) - - # then - assert defaults_value_one == value_one - assert defaults_value_two == value_two - assert node_value_one == value_one - assert node_value_two == value_two - assert len(default_all_configs) == 2 - assert config_one in default_all_configs - assert config_two in default_all_configs - assert len(node_all_configs) == 2 - assert config_one in node_all_configs - assert config_two in node_all_configs - - @pytest.mark.parametrize("_", xrange(10)) - def test_config_last_key(self, _): - # given - config_manager = ConfigurableManager() - config = {1: 2} - node_id = 1 - config_types = [1, 2, 3] - shuffle(config_types) - config_manager.set_configs(config, node_id=node_id, config_type=config_types[0]) - config_manager.set_configs(config, node_id=node_id, config_type=config_types[1]) - config_manager.set_configs(config, node_id=node_id, config_type=config_types[2]) - - # when - keys = config_manager.get_all_configs(node_id=node_id).keys() - - # then - assert keys - assert keys[-1] == config_types[2] diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 29caba4c..cfe5f0ef 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -12,15 +12,17 @@ from mock import MagicMock from core.emulator.emudata import NodeOptions from core.enumerations import MessageFlags, NodeTypes -from core.mobility import BasicRangeModel -from core.mobility import Ns2ScriptedMobility +from core.mobility import BasicRangeModel, Ns2ScriptedMobility from core.netns.vnodeclient import VnodeClient from core.service import ServiceManager _PATH = os.path.abspath(os.path.dirname(__file__)) _SERVICES_PATH = os.path.join(_PATH, "myservices") _MOBILITY_FILE = os.path.join(_PATH, "mobility.scen") -_XML_VERSIONS = ["0.0", "1.0"] +_XML_VERSIONS = [ + "0.0", + "1.0" +] _WIRED = [ NodeTypes.PEER_TO_PEER, NodeTypes.HUB, @@ -101,7 +103,6 @@ class TestCore: :param str version: xml version to write and parse :param ip_prefixes: generates ip addresses for nodes """ - # create ptp ptp_node = session.add_node(_type=NodeTypes.PEER_TO_PEER) @@ -256,7 +257,7 @@ class TestCore: # create wlan wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) - session.set_wireless_model(wlan_node, BasicRangeModel) + wlan_node.setmodel(BasicRangeModel) # create nodes node_options = NodeOptions() @@ -289,7 +290,7 @@ class TestCore: # create wlan wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) - session.set_wireless_model(wlan_node, BasicRangeModel) + wlan_node.setmodel(BasicRangeModel) # create nodes node_options = NodeOptions() @@ -316,7 +317,7 @@ class TestCore: "script_pause": "", "script_stop": "", } - session.mobility.set_configs(config, wlan_node.objid, Ns2ScriptedMobility.name) + wlan_node.setmodel(Ns2ScriptedMobility, config) # add handler for receiving node updates event = threading.Event() From 981e48ed30b22ed325a02253390bad56e3d0cdb2 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Mon, 11 Jun 2018 12:26:51 -0700 Subject: [PATCH 37/83] removed the emane manager from post_startup on emane models, it was not needed, since we can access it from our local session object, if needed --- daemon/core/emane/emanemanager.py | 2 +- daemon/core/emane/emanemodel.py | 3 +-- daemon/core/emane/tdma.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 2dce8b5f..94a7cc58 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -335,7 +335,7 @@ class EmaneManager(object): for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] logger.debug("post startup for emane node: %s - %s", emane_node.objid, emane_node.name) - emane_node.model.post_startup(self) + emane_node.model.post_startup() for netif in emane_node.netifs(): x, y, z = netif.node.position.get() emane_node.setnemposition(netif, x, y, z) diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 349044d2..e9db964a 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -191,11 +191,10 @@ class EmaneModel(WirelessModel): return phy_document - def post_startup(self, emane_manager): + def post_startup(self): """ Logic to execute after the emane manager is finished with startup. - :param core.emane.emanemanager.EmaneManager emane_manager: emane manager for the session :return: nothing """ logger.info("emane model(%s) has no post setup tasks", self.name) diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index 47fbff43..99820cf8 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -40,11 +40,10 @@ class EmaneTdmaModel(emanemodel.EmaneModel): ) config_ignore = {schedule_name} - def post_startup(self, emane_manager): + def post_startup(self): """ Logic to execute after the emane manager is finished with startup. - :param core.emane.emanemanager.EmaneManager emane_manager: emane manager for the session :return: nothing """ # get configured schedule @@ -53,7 +52,8 @@ class EmaneTdmaModel(emanemodel.EmaneModel): return schedule = config[self.schedule_name] - event_device = emane_manager.event_device + # get the set event device + event_device = self.session.emane.event_device # initiate tdma schedule logger.info("setting up tdma schedule: schedule(%s) device(%s)", schedule, event_device) From 719904480944994af31160e96c2e2a5e6e99f995 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Mon, 11 Jun 2018 12:34:19 -0700 Subject: [PATCH 38/83] removed all checks from within specific config handlers --- daemon/core/corehandlers.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index f50bafe6..eae0fa6e 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1176,10 +1176,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.debug("received configure message for %s nodenum: %s", object_name, node_id) if message_type == ConfigFlags.REQUEST: logger.info("replying to configure request for model: %s", object_name) - if object_name == "all": - typeflags = ConfigFlags.UPDATE.value - else: - typeflags = ConfigFlags.NONE.value + typeflags = ConfigFlags.NONE.value model_class = self.session.mobility.get_model_class(object_name) if not model_class: @@ -1189,10 +1186,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): config = model_class.get_configs(node_id=node_id) config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) replies.append(config_response) - elif message_type == ConfigFlags.RESET: - if object_name == "all": - self.session.mobility.config_reset(node_id) - else: + elif message_type != ConfigFlags.RESET: # store the configuration values for later use, when the node if not object_name: logger.warn("no configuration object for node: %s", node_id) @@ -1226,17 +1220,11 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.debug("received configure message for %s nodenum: %s", object_name, node_id) if message_type == ConfigFlags.REQUEST: logger.info("replying to configure request for %s model", object_name) - if object_name == "all": - typeflags = ConfigFlags.UPDATE.value - else: - typeflags = ConfigFlags.NONE.value + typeflags = ConfigFlags.NONE.value config = self.session.emane.get_configs() config_response = ConfigShim.config_data(0, node_id, typeflags, self.session.emane.emane_config, config) replies.append(config_response) - elif config_type == ConfigFlags.RESET.value: - if object_name == "all": - self.session.emane.config_reset(node_id) - else: + elif message_type != ConfigFlags.RESET: if not object_name: logger.info("no configuration object for node %s", node_id) return [] @@ -1265,10 +1253,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.debug("received configure message for %s nodenum: %s", object_name, node_id) if message_type == ConfigFlags.REQUEST: logger.info("replying to configure request for model: %s", object_name) - if object_name == "all": - typeflags = ConfigFlags.UPDATE.value - else: - typeflags = ConfigFlags.NONE.value + typeflags = ConfigFlags.NONE.value model_class = self.session.emane.get_model_class(object_name) if not model_class: @@ -1278,10 +1263,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): config = model_class.get_configs(node_id=node_id) config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) replies.append(config_response) - elif message_type == ConfigFlags.RESET: - if object_name == "all": - self.session.emane.config_reset(node_id) - else: + elif message_type != ConfigFlags.RESET: # store the configuration values for later use, when the node if not object_name: logger.warn("no configuration object for node: %s", node_id) From bb8c3fe27544fdf424e755d8b8bf3dfeb21a8a07 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Mon, 11 Jun 2018 12:58:42 -0700 Subject: [PATCH 39/83] moved send_objects from session to corehandlers, since it was used for tlv messaging specifically --- daemon/core/corehandlers.py | 113 +++++++++++++++++++++++++++++++- daemon/core/session.py | 126 +----------------------------------- 2 files changed, 112 insertions(+), 127 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index eae0fa6e..3308122e 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -5,6 +5,7 @@ socket server request handlers leveraged by core servers. import Queue import SocketServer import os +import pprint import shlex import shutil import sys @@ -18,6 +19,7 @@ from core.api import dataconversion from core.conf import ConfigShim from core.data import ConfigData from core.data import EventData +from core.data import FileData from core.emulator.emudata import InterfaceData from core.emulator.emudata import LinkOptions from core.emulator.emudata import NodeOptions @@ -42,6 +44,7 @@ from core.misc import structutils from core.misc import utils from core.mobility import BasicRangeModel from core.mobility import Ns2ScriptedMobility +from core.service import CoreService from core.service import ServiceManager @@ -1428,7 +1431,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): elif event_type == EventTypes.FILE_OPEN: filename = event_data.name self.session.open_xml(filename, start=False) - self.session.send_objects() + self.send_objects() return () elif event_type == EventTypes.FILE_SAVE: filename = event_data.name @@ -1530,7 +1533,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): self.session.set_user(user) if message.flags & MessageFlags.STRING.value: - self.session.send_objects() + self.send_objects() elif message.flags & MessageFlags.DELETE.value: # shut down the specified session(s) logger.info("request to terminate session %s" % session_id) @@ -1559,3 +1562,109 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.exception("error sending node emulation id message: %s", node_id) del self.node_status_request[node_id] + + def send_objects(self): + """ + Return API messages that describe the current session. + """ + # find all nodes and links + nodes_data = [] + links_data = [] + with self.session._objects_lock: + for obj in self.session.objects.itervalues(): + node_data = obj.data(message_type=MessageFlags.ADD.value) + if node_data: + nodes_data.append(node_data) + + node_links = obj.all_link_data(flags=MessageFlags.ADD.value) + for link_data in node_links: + links_data.append(link_data) + + # send all nodes first, so that they will exist for any links + logger.info("sending nodes:") + for node_data in nodes_data: + logger.info(pprint.pformat(dict(node_data._asdict()))) + self.session.broadcast_node(node_data) + + logger.info("sending links:") + for link_data in links_data: + logger.info(pprint.pformat(dict(link_data._asdict()))) + self.session.broadcast_link(link_data) + + # send mobility model info + for node_id in self.session.mobility.nodes(): + node = self.session.get_object(node_id) + for model_class, config in self.session.mobility.getmodels(node): + logger.info("mobility config: node(%s) class(%s) values(%s)", node_id, model_class, config) + config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) + self.session.broadcast_config(config_data) + + # send emane model info + for model_name in self.session.emane.emane_models(): + model_class = self.session.emane.get_model_class(model_name) + for node_id in model_class.nodes(): + config = model_class.get_configs(node_id) + logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) + config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) + self.session.broadcast_config(config_data) + + # service customizations + service_configs = self.session.services.getallconfigs() + for node_id, service in service_configs: + opaque = "service:%s" % service._name + data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(CoreService.keys))) + node = self.session.get_object(node_id) + values = CoreService.tovaluelist(node, node.services) + config_data = ConfigData( + message_type=0, + node=node_id, + object=self.session.services.name, + type=ConfigFlags.UPDATE.value, + data_types=data_types, + data_values=values, + session=self.session.session_id, + opaque=opaque + ) + self.session.broadcast_config(config_data) + + for file_name, config_data in self.session.services.getallfiles(service): + file_data = FileData( + message_type=MessageFlags.ADD.value, + node=node_id, + name=str(file_name), + type=opaque, + data=str(config_data) + ) + self.session.broadcast_file(file_data) + + # TODO: send location info + + # send hook scripts + for state in sorted(self.session._hooks.keys()): + for file_name, config_data in self.session._hooks[state]: + file_data = FileData( + message_type=MessageFlags.ADD.value, + name=str(file_name), + type="hook:%s" % state, + data=str(config_data) + ) + self.session.broadcast_file(file_data) + + # send session configuration + session_config = self.session.options.get_configs() + config_data = ConfigShim.config_data(0, None, ConfigFlags.UPDATE.value, self.session.options, session_config) + self.session.broadcast_config(config_data) + + # send session metadata + data_values = "|".join(["%s=%s" % item for item in self.session.metadata.get_configs().iteritems()]) + data_types = tuple(ConfigDataTypes.STRING.value for _ in self.session.metadata.get_configs()) + config_data = ConfigData( + message_type=0, + object=self.session.metadata.name, + type=ConfigFlags.NONE.value, + data_types=data_types, + data_values=data_values + ) + self.session.broadcast_config(config_data) + + logger.info("informed GUI about %d nodes and %d links", len(nodes_data), len(links_data)) diff --git a/daemon/core/session.py b/daemon/core/session.py index dcfdf79b..fda1266d 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -4,14 +4,12 @@ that manages a CORE session. """ import os -import pprint import random import shutil import subprocess import tempfile import threading import time -from itertools import repeat import pwd @@ -19,18 +17,14 @@ from core import constants from core import logger from core.api import coreapi from core.broker import CoreBroker -from core.conf import ConfigShim, ConfigurableOptions +from core.conf import ConfigurableOptions from core.conf import Configuration -from core.data import ConfigData from core.data import EventData from core.data import ExceptionData -from core.data import FileData from core.emane.emanemanager import EmaneManager from core.enumerations import ConfigDataTypes -from core.enumerations import ConfigFlags from core.enumerations import EventTypes from core.enumerations import ExceptionLevels -from core.enumerations import MessageFlags from core.enumerations import NodeTypes from core.enumerations import RegisterTlvs from core.location import CoreLocation @@ -41,7 +35,6 @@ from core.misc.ipaddress import MacAddress from core.mobility import MobilityManager from core.netns import nodes from core.sdt import Sdt -from core.service import CoreService from core.service import CoreServices from core.xml.xmlsession import save_session_xml @@ -1109,123 +1102,6 @@ class Session(object): node = self.get_object(node_id) node.cmd(data, wait=False) - # TODO: move to core handlers - def send_objects(self): - """ - Return API messages that describe the current session. - """ - # find all nodes and links - nodes_data = [] - links_data = [] - with self._objects_lock: - for obj in self.objects.itervalues(): - node_data = obj.data(message_type=MessageFlags.ADD.value) - if node_data: - nodes_data.append(node_data) - - node_links = obj.all_link_data(flags=MessageFlags.ADD.value) - for link_data in node_links: - links_data.append(link_data) - - # send all nodes first, so that they will exist for any links - logger.info("sending nodes:") - for node_data in nodes_data: - logger.info(pprint.pformat(dict(node_data._asdict()))) - self.broadcast_node(node_data) - - logger.info("sending links:") - for link_data in links_data: - logger.info(pprint.pformat(dict(link_data._asdict()))) - self.broadcast_link(link_data) - - # send mobility model info - for node_id in self.mobility.nodes(): - node = self.get_object(node_id) - for model_class, config in self.mobility.getmodels(node): - logger.info("mobility config: node(%s) class(%s) values(%s)", node_id, model_class, config) - config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) - self.broadcast_config(config_data) - - # send emane model info - for model_name in self.emane.emane_models(): - model_class = self.emane.get_model_class(model_name) - for node_id in model_class.nodes(): - config = model_class.get_configs(node_id) - logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) - config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) - self.broadcast_config(config_data) - - # for node_id in self.emane.nodes(): - # if node_id not in self.objects: - # continue - # - # node = self.get_object(node_id) - # for model_class, config in self.emane.getmodels(node): - # logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) - # config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) - # self.broadcast_config(config_data) - - # service customizations - service_configs = self.services.getallconfigs() - for node_id, service in service_configs: - opaque = "service:%s" % service._name - data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(CoreService.keys))) - node = self.get_object(node_id) - values = CoreService.tovaluelist(node, node.services) - config_data = ConfigData( - message_type=0, - node=node_id, - object=self.services.name, - type=ConfigFlags.UPDATE.value, - data_types=data_types, - data_values=values, - session=self.session_id, - opaque=opaque - ) - self.broadcast_config(config_data) - - for file_name, config_data in self.services.getallfiles(service): - file_data = FileData( - message_type=MessageFlags.ADD.value, - node=node_id, - name=str(file_name), - type=opaque, - data=str(config_data) - ) - self.broadcast_file(file_data) - - # TODO: send location info - - # send hook scripts - for state in sorted(self._hooks.keys()): - for file_name, config_data in self._hooks[state]: - file_data = FileData( - message_type=MessageFlags.ADD.value, - name=str(file_name), - type="hook:%s" % state, - data=str(config_data) - ) - self.broadcast_file(file_data) - - # send session configuration - session_config = self.options.get_configs() - config_data = ConfigShim.config_data(0, None, ConfigFlags.UPDATE.value, self.options, session_config) - self.broadcast_config(config_data) - - # send session metadata - data_values = "|".join(["%s=%s" % item for item in self.metadata.get_configs().iteritems()]) - data_types = tuple(ConfigDataTypes.STRING.value for _ in self.metadata.get_configs()) - config_data = ConfigData( - message_type=0, - object=self.metadata.name, - type=ConfigFlags.NONE.value, - data_types=data_types, - data_values=data_values - ) - self.broadcast_config(config_data) - - logger.info("informed GUI about %d nodes and %d links", len(nodes_data), len(links_data)) - class SessionConfig(ConfigurableOptions): """ From eb415aa4d483c206d56f7230dd90f9161b232e59 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Tue, 12 Jun 2018 08:37:39 -0700 Subject: [PATCH 40/83] merged session config/options together --- daemon/core/conf.py | 7 +- daemon/core/emane/emanemanager.py | 20 ++--- daemon/core/services/quagga.py | 10 +-- daemon/core/session.py | 108 ++++++++---------------- daemon/examples/netns/wlanemanetests.py | 1 - 5 files changed, 50 insertions(+), 96 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 7e5a4115..0678a2a9 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -129,10 +129,11 @@ class ConfigurableOptions(object): node_configs[_id] = value @classmethod - def get_config(cls, _id, node_id=_default_node): - logger.debug("getting config for node(%s): %s", node_id, _id) + def get_config(cls, _id, node_id=_default_node, default=None): node_configs = cls.get_configs(node_id) - return node_configs.get(_id) + value = node_configs.get(_id, default) + logger.debug("getting config for node(%s): %s = %s", node_id, _id, value) + return value @classmethod def set_configs(cls, config=None, node_id=_default_node): diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 94a7cc58..38a26cef 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -79,8 +79,8 @@ class EmaneManager(object): self._ifccounts = {} self._ifccountslock = threading.Lock() # port numbers are allocated from these counters - self.platformport = self.session.get_config_item_int("emane_platform_port", 8100) - self.transformport = self.session.get_config_item_int("emane_transform_port", 8200) + self.platformport = self.session.options.get_config_int("emane_platform_port", 8100) + self.transformport = self.session.options.get_config_int("emane_transform_port", 8200) self.doeventloop = False self.eventmonthread = None @@ -133,7 +133,7 @@ class EmaneManager(object): self.load_models(EMANE_MODELS) # load custom models - custom_models_path = self.session.config.get("emane_models_dir") + custom_models_path = self.session.options.get_config("emane_models_dir") if custom_models_path: emane_models = utils.load_classes(custom_models_path, EmaneModel) self.load_models(emane_models) @@ -349,8 +349,8 @@ class EmaneManager(object): self._emane_nodes.clear() # don't clear self._ifccounts here; NEM counts are needed for buildxml - self.platformport = self.session.get_config_item_int("emane_platform_port", 8100) - self.transformport = self.session.get_config_item_int("emane_transform_port", 8200) + self.platformport = self.session.options.get_config_int("emane_platform_port", 8100) + self.transformport = self.session.options.get_config_int("emane_transform_port", 8200) def shutdown(self): """ @@ -477,8 +477,6 @@ class EmaneManager(object): return prefix = session.options.get_config("controlnet") - if not prefix: - prefix = session.config.get("controlnet") prefixes = prefix.split() # normal Config messaging will distribute controlnets if len(prefixes) >= len(servers): @@ -729,8 +727,8 @@ class EmaneManager(object): """ logger.info("starting emane daemons...") loglevel = str(EmaneManager.DEFAULT_LOG_LEVEL) - cfgloglevel = self.session.get_config_item_int("emane_log_level") - realtime = self.session.get_config_item_bool("emane_realtime", True) + cfgloglevel = self.session.options.get_config_int("emane_log_level") + realtime = self.session.options.get_config_bool("emane_realtime", default=True) if cfgloglevel: logger.info("setting user-defined EMANE log level: %d", cfgloglevel) loglevel = str(cfgloglevel) @@ -837,7 +835,7 @@ class EmaneManager(object): """ # this support must be explicitly turned on; by default, CORE will # generate the EMANE events when nodes are moved - return self.session.get_config_item_bool("emane_event_monitor", False) + return self.session.options.get_config_bool("emane_event_monitor") def genlocationevents(self): """ @@ -845,7 +843,7 @@ class EmaneManager(object): """ # By default, CORE generates EMANE location events when nodes # are moved; this can be explicitly disabled in core.conf - tmp = self.session.get_config_item_bool("emane_event_generate") + tmp = self.session.options.get_config_bool("emane_event_generate") if tmp is None: tmp = not self.doeventmonitor() return tmp diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index feb4eac7..73f797a5 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -115,12 +115,10 @@ class Zebra(CoreService): """ Generate a shell script used to boot the Quagga daemons. """ - try: - quagga_bin_search = node.session.config['quagga_bin_search'] - quagga_sbin_search = node.session.config['quagga_sbin_search'] - except KeyError: - quagga_bin_search = '"/usr/local/bin /usr/bin /usr/lib/quagga"' - quagga_sbin_search = '"/usr/local/sbin /usr/sbin /usr/lib/quagga"' + quagga_bin_search = node.session.options.get_config("quagga_bin_search", + default='"/usr/local/bin /usr/bin /usr/lib/quagga"') + quagga_sbin_search = node.session.options.get_config('quagga_sbin_search', + default='"/usr/local/sbin /usr/sbin /usr/lib/quagga"') return """\ #!/bin/sh # auto-generated by zebra service (quagga.py) diff --git a/daemon/core/session.py b/daemon/core/session.py index fda1266d..8a158168 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -54,11 +54,6 @@ class Session(object): """ self.session_id = session_id - # dict of configuration items from /etc/core/core.conf config file - if not config: - config = {} - self.config = config - # define and create session directory when desired self.session_dir = os.path.join(tempfile.gettempdir(), "pycore.%s" % self.session_id) if mkdir: @@ -95,14 +90,19 @@ class Session(object): self.config_handlers = [] self.shutdown_handlers = [] - # initialize feature helpers + # session options/metadata + self.options = SessionConfig() + if not config: + config = {} + self.options.set_configs(config) + self.metadata = SessionMetaData() + + # initialize session feature helpers self.broker = CoreBroker(session=self) self.location = CoreLocation() self.mobility = MobilityManager(session=self) self.services = CoreServices(session=self) self.emane = EmaneManager(session=self) - self.options = SessionConfig() - self.metadata = SessionMetaData() self.sdt = Sdt(session=self) def shutdown(self): @@ -375,7 +375,7 @@ class Session(object): """ if state == EventTypes.RUNTIME_STATE.value: self.emane.poststartup() - xml_file_version = self.get_config_item("xmlfilever") + xml_file_version = self.options.get_config("xmlfilever") if xml_file_version in ("1.0",): xml_file_name = os.path.join(self.session_dir, "session-deployed.xml") save_session_xml(self, xml_file_name, xml_file_version) @@ -581,46 +581,6 @@ class Session(object): self.broadcast_exception(exception_data) - def get_config_item(self, name): - """ - Return an entry from the configuration dictionary that comes from - command-line arguments and/or the core.conf config file. - - :param str name: name of configuration to retrieve - :return: config value - """ - return self.config.get(name) - - def get_config_item_bool(self, name, default=None): - """ - Return a boolean entry from the configuration dictionary, may - return None if undefined. - - :param str name: configuration item name - :param default: default value to return if not found - :return: boolean value of the configuration item - :rtype: bool - """ - item = self.get_config_item(name) - if item is None: - return default - return bool(item.lower() == "true") - - def get_config_item_int(self, name, default=None): - """ - Return an integer entry from the configuration dictionary, may - return None if undefined. - - :param str name: configuration item name - :param default: default value to return if not found - :return: integer value of the configuration item - :rtype: int - """ - item = self.get_config_item(name) - if item is None: - return default - return int(item) - def instantiate(self): """ We have entered the instantiation state, invoke startup methods @@ -796,24 +756,10 @@ class Session(object): :rtype: list """ p = self.options.get_config("controlnet") - if not p: - p = self.config.get("controlnet") - p0 = self.options.get_config("controlnet0") - if not p0: - p0 = self.config.get("controlnet0") - p1 = self.options.get_config("controlnet1") - if not p1: - p1 = self.config.get("controlnet1") - p2 = self.options.get_config("controlnet2") - if not p2: - p2 = self.config.get("controlnet2") - p3 = self.options.get_config("controlnet3") - if not p3: - p3 = self.config.get("controlnet3") if not p0 and p: p0 = p @@ -827,12 +773,12 @@ class Session(object): :return: list of control net server interfaces :rtype: list """ - d0 = self.config.get("controlnetif0") + d0 = self.options.get_config("controlnetif0") if d0: logger.error("controlnet0 cannot be assigned with a host interface") - d1 = self.config.get("controlnetif1") - d2 = self.config.get("controlnetif2") - d3 = self.config.get("controlnetif3") + d1 = self.options.get_config("controlnetif1") + d2 = self.options.get_config("controlnetif2") + d3 = self.options.get_config("controlnetif3") return [None, d1, d2, d3] def get_control_net_index(self, dev): @@ -903,15 +849,10 @@ class Session(object): updown_script = None if net_index == 0: - updown_script = self.config.get("controlnet_updown_script") + updown_script = self.options.get_config("controlnet_updown_script") if not updown_script: logger.warning("controlnet updown script not configured") - # check if session option set, overwrite if so - options_updown_script = self.options.get_config("controlnet_updown_script") - if options_updown_script: - updown_script = options_updown_script - prefixes = prefix_spec.split() if len(prefixes) > 1: # a list of per-host prefixes is provided @@ -1020,7 +961,7 @@ class Session(object): :param bool remove: flag to check if it should be removed :return: nothing """ - if not self.get_config_item_bool("update_etc_hosts", False): + if not self.options.get_config_bool("update_etc_hosts", default=False): return try: @@ -1136,6 +1077,24 @@ class SessionConfig(ConfigurableOptions): def __init__(self): self.set_configs() + def get_config(cls, _id, node_id=ConfigurableOptions._default_node, default=None): + value = super(SessionConfig, cls).get_config(_id, node_id, default) + if value == "": + value = default + return value + + def get_config_bool(self, name, default=None): + value = self.get_config(name) + if value is None: + return default + return value.lower() == "true" + + def get_config_int(self, name, default=None): + value = self.get_config(name, default=default) + if value is not None: + value = int(value) + return value + class SessionMetaData(ConfigurableOptions): """ @@ -1145,5 +1104,4 @@ class SessionMetaData(ConfigurableOptions): """ name = "metadata" configuration_maps = {} - config_type = RegisterTlvs.UTILITY.value diff --git a/daemon/examples/netns/wlanemanetests.py b/daemon/examples/netns/wlanemanetests.py index 172732e3..ba6f2aa2 100755 --- a/daemon/examples/netns/wlanemanetests.py +++ b/daemon/examples/netns/wlanemanetests.py @@ -440,7 +440,6 @@ class Experiment(object): self.session.master = True self.session.location.setrefgeo(47.57917, -122.13232, 2.00000) self.session.location.refscale = 150.0 - self.session.config["emane_models"] = "RfPipe, Ieee80211abg, Bypass" self.session.emane.loadmodels() self.net = self.session.add_object(cls=EmaneNode, objid=numnodes + 1, name="wlan1") self.net.verbose = verbose From 3a39432fc7b1c500838412be26a8b297abb1504d Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 13 Jun 2018 11:59:50 -0700 Subject: [PATCH 41/83] refactored configs back to be able to provide instance conifgurations for sessions --- daemon/core/conf.py | 87 +++++++------ daemon/core/corehandlers.py | 36 ++---- daemon/core/emane/bypass.py | 1 - daemon/core/emane/commeffect.py | 3 +- daemon/core/emane/emanemanager.py | 171 +++++++++++++++++++------ daemon/core/emane/emanemodel.py | 48 +------ daemon/core/emane/ieee80211abg.py | 1 - daemon/core/emane/nodes.py | 11 +- daemon/core/emane/rfpipe.py | 1 - daemon/core/emane/tdma.py | 3 +- daemon/core/emulator/coreemu.py | 6 +- daemon/core/mobility.py | 152 +++++++++++++--------- daemon/core/netns/nodes.py | 10 +- daemon/core/session.py | 16 ++- daemon/core/xml/xmlparser0.py | 15 ++- daemon/core/xml/xmlparser1.py | 7 +- daemon/core/xml/xmlwriter0.py | 17 +-- daemon/core/xml/xmlwriter1.py | 2 +- daemon/examples/api/wlan.py | 2 +- daemon/tests/test_conf.py | 16 +-- daemon/tests/test_core.py | 71 ++--------- daemon/tests/test_xml.py | 203 ++++++++++++++++++++++++++++++ 22 files changed, 560 insertions(+), 319 deletions(-) create mode 100644 daemon/tests/test_xml.py diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 0678a2a9..863fc868 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -90,11 +90,57 @@ class Configuration(object): self.__class__.__name__, self.id, self.type, self.default, self.options) +class ConfigurableManager(object): + _default_node = -1 + _default_type = _default_node + + def __init__(self): + self._configuration_maps = {} + + def nodes(self): + return [node_id for node_id in self._configuration_maps.iterkeys() if node_id != self._default_node] + + def has_configs(self, node_id): + return node_id in self._configuration_maps + + def config_reset(self, node_id=None): + logger.debug("resetting all configurations: %s", self.__class__.__name__) + if not node_id: + self._configuration_maps.clear() + elif node_id in self._configuration_maps: + self._configuration_maps.pop(node_id) + + def set_config(self, _id, value, node_id=_default_node, config_type=_default_type): + logger.debug("setting config for node(%s) type(%s): %s=%s", node_id, config_type, _id, value) + node_type_map = self.get_configs(node_id, config_type) + node_type_map[_id] = value + + def set_configs(self, config, node_id=_default_node, config_type=_default_type): + logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) + node_configs = self.get_all_configs(node_id) + if config_type in node_configs: + node_configs.pop(config_type) + node_configs[config_type] = config + + def get_config(self, _id, node_id=_default_node, config_type=_default_type, default=None): + logger.debug("getting config for node(%s) type(%s): %s", node_id, config_type, _id) + node_type_map = self.get_configs(node_id, config_type) + return node_type_map.get(_id, default) + + def get_configs(self, node_id=_default_node, config_type=_default_type): + logger.debug("getting configs for node(%s) type(%s)", node_id, config_type) + node_map = self.get_all_configs(node_id) + return node_map.setdefault(config_type, {}) + + def get_all_configs(self, node_id=_default_node): + logger.debug("getting all configs for node(%s)", node_id) + return self._configuration_maps.setdefault(node_id, OrderedDict()) + + class ConfigurableOptions(object): # unique name to receive configuration changes name = None bitmap = None - configuration_maps = None _default_node = -1 @classmethod @@ -108,42 +154,3 @@ class ConfigurableOptions(object): @classmethod def default_values(cls): return OrderedDict([(config.id, config.default) for config in cls.configurations()]) - - @classmethod - def nodes(cls): - return {node_id for node_id in cls.configuration_maps.iterkeys() if node_id != cls._default_node} - - @classmethod - def config_reset(cls, node_id=None): - if not node_id: - logger.debug("resetting all configurations: %s", cls.__name__) - cls.configuration_maps.clear() - elif node_id in cls.configuration_maps: - logger.debug("resetting node(%s) configurations: %s", node_id, cls.__name__) - cls.configuration_maps.pop(node_id) - - @classmethod - def set_config(cls, _id, value, node_id=_default_node): - logger.debug("setting config for node(%s) type(%s): %s=%s", node_id, _id, value) - node_configs = cls.get_configs(node_id) - node_configs[_id] = value - - @classmethod - def get_config(cls, _id, node_id=_default_node, default=None): - node_configs = cls.get_configs(node_id) - value = node_configs.get(_id, default) - logger.debug("getting config for node(%s): %s = %s", node_id, _id, value) - return value - - @classmethod - def set_configs(cls, config=None, node_id=_default_node): - logger.debug("setting config for node(%s): %s", node_id, config) - node_config = cls.get_configs(node_id) - if config: - for key, value in config.iteritems(): - node_config[key] = value - - @classmethod - def get_configs(cls, node_id=_default_node): - logger.debug("getting configs for node(%s)", node_id) - return cls.configuration_maps.setdefault(node_id, cls.default_values()) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 3308122e..23eb1a94 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1186,7 +1186,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] - config = model_class.get_configs(node_id=node_id) + config = self.session.mobility.get_model_config(node_id, object_name) config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) replies.append(config_response) elif message_type != ConfigFlags.RESET: @@ -1195,17 +1195,11 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("no configuration object for node: %s", node_id) return [] - model_class = self.session.mobility.get_model_class(object_name) - if not model_class: - logger.warn("model class does not exist: %s", object_name) - return [] - + parsed_config = {} if values_str: parsed_config = ConfigShim.str_to_dict(values_str) - model_class.set_configs(parsed_config, node_id=node_id) - config = model_class.get_configs(node_id) - model_class.set_configs(config, node_id=node_id) + self.session.mobility.set_model_config(node_id, object_name, parsed_config) return replies @@ -1213,7 +1207,6 @@ class CoreHandler(SocketServer.BaseRequestHandler): replies = [] node_id = config_data.node object_name = config_data.object - config_type = config_data.type interface_id = config_data.interface_number values_str = config_data.data_values @@ -1263,7 +1256,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("model class does not exist: %s", object_name) return [] - config = model_class.get_configs(node_id=node_id) + config = self.session.emane.get_model_config(node_id, object_name) config_response = ConfigShim.config_data(0, node_id, typeflags, model_class, config) replies.append(config_response) elif message_type != ConfigFlags.RESET: @@ -1272,18 +1265,11 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.warn("no configuration object for node: %s", node_id) return [] - model_class = self.session.emane.get_model_class(object_name) - if not model_class: - logger.warn("model class does not exist: %s", object_name) - return [] - + parsed_config = {} if values_str: parsed_config = ConfigShim.str_to_dict(values_str) - model_class.set_configs(parsed_config, node_id=node_id) - config = model_class.get_configs(node_id) - model_class.set_configs(config, node_id=node_id) - self.session.emane.set_node_model(node_id, object_name) + self.session.emane.set_model_config(node_id, object_name, parsed_config) return replies @@ -1435,7 +1421,8 @@ class CoreHandler(SocketServer.BaseRequestHandler): return () elif event_type == EventTypes.FILE_SAVE: filename = event_data.name - self.session.save_xml(filename, self.session.config["xmlfilever"]) + xml_version = self.session.options.get_config("xmlfilever") + self.session.save_xml(filename, xml_version) elif event_type == EventTypes.SCHEDULED: etime = event_data.time node = event_data.node @@ -1600,10 +1587,9 @@ class CoreHandler(SocketServer.BaseRequestHandler): self.session.broadcast_config(config_data) # send emane model info - for model_name in self.session.emane.emane_models(): - model_class = self.session.emane.get_model_class(model_name) - for node_id in model_class.nodes(): - config = model_class.get_configs(node_id) + for node_id in self.session.emane.nodes(): + node = self.session.get_object(node_id) + for model_class, config in self.session.emane.getmodels(node): logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) self.session.broadcast_config(config_data) diff --git a/daemon/core/emane/bypass.py b/daemon/core/emane/bypass.py index 44e7c078..8f60f238 100644 --- a/daemon/core/emane/bypass.py +++ b/daemon/core/emane/bypass.py @@ -8,7 +8,6 @@ from core.enumerations import ConfigDataTypes class EmaneBypassModel(emanemodel.EmaneModel): name = "emane_bypass" - configuration_maps = {} # values to ignore, when writing xml files config_ignore = {"none"} diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index 03adbbd0..fbce2046 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -29,7 +29,6 @@ def convert_none(x): class EmaneCommEffectModel(emanemodel.EmaneModel): name = "emane_commeffect" - configuration_maps = {} shim_library = "commeffectshim" shim_xml = "/usr/share/emane/manifest/commeffectshim.xml" @@ -55,7 +54,7 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): :param interface: interface for the emane node :return: nothing """ - config = self.getifcconfig(self.object_id, interface) + config = emane_manager.getifcconfig(self.object_id, interface, self.name) if not config: return diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 38a26cef..78b770b4 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -11,7 +11,7 @@ from core import constants from core import logger from core.api import coreapi from core.api import dataconversion -from core.conf import ConfigShim +from core.conf import ConfigShim, ConfigurableManager from core.conf import Configuration from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel @@ -53,7 +53,7 @@ EMANE_MODELS = [ ] -class EmaneManager(object): +class EmaneManager(ConfigurableManager): """ EMANE controller object. Lives in a Session instance and is used for building EMANE config files from all of the EmaneNode objects in this @@ -73,6 +73,7 @@ class EmaneManager(object): :param core.session.Session session: session this manager is tied to :return: nothing """ + super(EmaneManager, self).__init__() self.session = session self._emane_nodes = {} self._emane_node_lock = threading.Lock() @@ -86,6 +87,7 @@ class EmaneManager(object): # model for global EMANE configuration options self.emane_config = EmaneGlobalModel(session) + self.set_configs(self.emane_config.default_values()) # store the last configured model for a node, used during startup self.node_models = {} @@ -98,19 +100,106 @@ class EmaneManager(object): self.service = None self.emane_check() - def set_node_model(self, node_id, model_name): - if model_name not in self._modelclsmap: - raise ValueError("unknown emane model: %s", model_name) + def set_model_config(self, node_id, model_name, config): + """ + Set configuration data for a model. + + :param int node_id: node id to set model configuration for + :param str model_name: model to set configuration for + :param dict config: configuration data to set for model + :return: nothing + """ + # get model class to configure + model_class = self._modelclsmap.get(model_name) + if not model_class: + raise ValueError("%s is an invalid model" % model_name) + + # retrieve default values + node_config = self.get_model_config(node_id, model_name) + for key, value in config.iteritems(): + node_config[key] = value + + # set as node model for startup self.node_models[node_id] = model_name - def config_reset(self, node_id=None): - # clear and reset current emane configuration - self.emane_config.config_reset() - self.emane_config.set_configs() + # set configuration + self.set_configs(node_config, node_id=node_id, config_type=model_name) - # reset model configurations - for model_class in self._modelclsmap.itervalues(): - model_class.config_reset(node_id=node_id) + def get_model_config(self, node_id, model_name): + """ + Set configuration data for a model. + + :param int node_id: node id to set model configuration for + :param str model_name: model to set configuration for + :return: current model configuration for node + :rtype: dict + """ + # get model class to configure + model_class = self._modelclsmap.get(model_name) + if not model_class: + raise ValueError("%s is an invalid model" % model_name) + + config = self.get_configs(node_id=node_id, config_type=model_name) + if not config: + # set default values, when not already set + config = model_class.default_values() + self.set_configs(config, node_id=node_id, config_type=model_name) + + return config + + def getifcconfig(self, node_id, interface, model_name): + """ + Retrieve interface configuration or node configuration if not provided. + + :param int node_id: node id + :param interface: node interface + :param str model_name: model to get configuration for + :return: node/interface model configuration + :rtype: dict + """ + # use the network-wide config values or interface(NEM)-specific values? + if interface is None: + return self.get_configs(node_id=node_id, config_type=model_name) + else: + # don"t use default values when interface config is the same as net + # note here that using ifc.node.objid as key allows for only one type + # of each model per node; + # TODO: use both node and interface as key + + # Adamson change: first check for iface config keyed by "node:ifc.name" + # (so that nodes w/ multiple interfaces of same conftype can have + # different configs for each separate interface) + key = 1000 * interface.node.objid + if interface.netindex is not None: + key += interface.netindex + + # try retrieve interface specific configuration, avoid getting defaults + config = {} + if self.has_configs(key): + config = self.get_configs(key) + + # otherwise retrieve the interfaces node configuration, avoid using defaults + if not config and self.has_configs(interface.node.objid): + config = self.get_configs(node_id=interface.node.objid, config_type=model_name) + + if not config and interface.transport_type == "raw": + # with EMANE 0.9.2+, we need an extra NEM XML from + # model.buildnemxmlfiles(), so defaults are returned here + config = self.get_configs(node_id=node_id, config_type=model_name) + + return config + + def set_model(self, node, model_class, config=None): + logger.info("setting emane model(%s) for node(%s): %s", model_class.name, node.objid, config) + if not config: + config = {} + self.set_model_config(node.objid, model_class.name, config) + config = self.get_model_config(node.objid, model_class.name) + node.setmodel(model_class, config) + + def config_reset(self, node_id=None): + super(EmaneManager, self).config_reset(node_id) + self.set_configs(self.emane_config.default_values()) def emane_models(self): return self._modelclsmap.keys() @@ -162,8 +251,8 @@ class EmaneManager(object): return # Get the control network to be used for events - group, port = self.emane_config.get_config("eventservicegroup").split(":") - self.event_device = self.emane_config.get_config("eventservicedevice") + group, port = self.get_config("eventservicegroup").split(":") + self.event_device = self.get_config("eventservicedevice") eventnetidx = self.session.get_control_net_index(self.event_device) if eventnetidx < 0: logger.error("invalid emane event service device provided: %s", self.event_device) @@ -223,11 +312,15 @@ class EmaneManager(object): Used with XML export. """ models = [] - for model_class in self._modelclsmap.itervalues(): - if node.objid in model_class.configuration_maps: - config = model_class.get_configs(node_id=node.objid) - models.append((model_class, config)) - logger.debug("emane models: %s", models) + all_configs = {} + if self.has_configs(node_id=node.objid): + all_configs = self.get_all_configs(node_id=node.objid) + + for model_name in all_configs.iterkeys(): + model_class = self._modelclsmap[model_name] + config = self.get_configs(node_id=node.objid, config_type=model_name) + models.append((model_class, config)) + logger.debug("emane models for node(%s): %s", node.objid, models) return models def setup(self): @@ -254,7 +347,7 @@ class EmaneManager(object): # - needs to be configured before checkdistributed() for distributed # - needs to exist when eventservice binds to it (initeventservice) if self.session.master: - otadev = self.emane_config.get_config("otamanagerdevice") + otadev = self.get_config("otamanagerdevice") netidx = self.session.get_control_net_index(otadev) logger.debug("emane ota manager device: index(%s) otadev(%s)", netidx, otadev) if netidx < 0: @@ -263,7 +356,7 @@ class EmaneManager(object): ctrlnet = self.session.add_remove_control_net(net_index=netidx, remove=False, conf_required=False) self.distributedctrlnet(ctrlnet) - eventdev = self.emane_config.get_config("eventservicedevice") + eventdev = self.get_config("eventservicedevice") logger.debug("emane event service device: eventdev(%s)", eventdev) if eventdev != otadev: netidx = self.session.get_control_net_index(eventdev) @@ -279,7 +372,7 @@ class EmaneManager(object): # we are slave, but haven't received a platformid yet platform_id_start = "platform_id_start" default_values = self.emane_config.default_values() - value = self.emane_config.get_config(platform_id_start) + value = self.get_config(platform_id_start) if value == default_values[platform_id_start]: return EmaneManager.NOT_READY @@ -411,10 +504,10 @@ class EmaneManager(object): emane_node = self._emane_nodes[key] nemcount += emane_node.numnetif() - nemid = int(self.emane_config.get_config("nem_id_start")) + nemid = int(self.get_config("nem_id_start")) nemid += nemcount - platformid = int(self.emane_config.get_config("platform_id_start")) + platformid = int(self.get_config("platform_id_start")) # build an ordered list of servers so platform ID is deterministic servers = [] @@ -433,8 +526,8 @@ class EmaneManager(object): platformid += 1 typeflags = ConfigFlags.UPDATE.value - self.emane_config.set_config("platform_id_start", str(platformid)) - self.emane_config.set_config("nem_id_start", str(nemid)) + self.set_config("platform_id_start", str(platformid)) + self.set_config("nem_id_start", str(nemid)) config_data = ConfigShim.config_data(0, None, typeflags, self.emane_config, self.get_configs()) message = dataconversion.convert_config(config_data) server.sock.send(message) @@ -555,9 +648,9 @@ class EmaneManager(object): logger.error("emane node(%s) has no node model", node_id) raise ValueError("emane node has no model set") + config = self.get_model_config(node_id=node_id, model_name=model_name) + logger.debug("setting emane model(%s) config(%s)", model_name, config) model_class = self._modelclsmap[model_name] - config = model_class.get_configs(node_id=node_id) - logger.debug("setting emane model(%s) config(%s)", model_class, config) emane_node.setmodel(model_class, config) def nemlookup(self, nemid): @@ -597,10 +690,10 @@ class EmaneManager(object): plat = doc.getElementsByTagName("platform").pop() if otadev: - self.emane_config.set_config("otamanagerdevice", otadev) + self.set_config("otamanagerdevice", otadev) if eventdev: - self.emane_config.set_config("eventservicedevice", eventdev) + self.set_config("eventservicedevice", eventdev) # append all platform options (except starting id) to doc for configuration in self.emane_config.emulator_config: @@ -608,7 +701,7 @@ class EmaneManager(object): if name == "platform_id_start": continue - value = self.emane_config.get_config(name) + value = self.get_config(name) param = self.xmlparam(doc, name, value) plat.appendChild(param) @@ -618,7 +711,7 @@ class EmaneManager(object): """ Build a platform.xml file now that all nodes are configured. """ - nemid = int(self.emane_config.get_config("nem_id_start")) + nemid = int(self.get_config("nem_id_start")) platformxmls = {} # assume self._objslock is already held here @@ -695,7 +788,7 @@ class EmaneManager(object): default_values = self.emane_config.default_values() for name in ["eventservicegroup", "eventservicedevice"]: a = default_values[name] - b = self.emane_config.get_config(name) + b = self.get_config(name) if a != b: need_xml = True @@ -705,12 +798,12 @@ class EmaneManager(object): return try: - group, port = self.emane_config.get_config("eventservicegroup").split(":") + group, port = self.get_config("eventservicegroup").split(":") except ValueError: logger.exception("invalid eventservicegroup in EMANE config") return - dev = self.emane_config.get_config("eventservicedevice") + dev = self.get_config("eventservicedevice") doc = self.xmldoc("emaneeventmsgsvc") es = doc.getElementsByTagName("emaneeventmsgsvc").pop() kvs = (("group", group), ("port", port), ("device", dev), ("mcloop", "1"), ("ttl", "32")) @@ -737,12 +830,12 @@ class EmaneManager(object): if realtime: emanecmd += "-r", - otagroup, otaport = self.emane_config.get_config("otamanagergroup").split(":") - otadev = self.emane_config.get_config("otamanagerdevice") + otagroup, otaport = self.get_config("otamanagergroup").split(":") + otadev = self.get_config("otamanagerdevice") otanetidx = self.session.get_control_net_index(otadev) - eventgroup, eventport = self.emane_config.get_config("eventservicegroup").split(":") - eventdev = self.emane_config.get_config("eventservicedevice") + eventgroup, eventport = self.get_config("eventservicegroup").split(":") + eventdev = self.get_config("eventservicedevice") eventservicenetidx = self.session.get_control_net_index(eventdev) run_emane_on_host = False diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index e9db964a..02726fe8 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -79,7 +79,7 @@ class EmaneModel(WirelessModel): :return: nothing """ # retrieve configuration values - config = self.getifcconfig(self.object_id, interface) + config = emane_manager.getifcconfig(self.object_id, interface, self.name) if not config: return @@ -149,7 +149,7 @@ class EmaneModel(WirelessModel): continue # check if value is a multi param - value = config[name] + value = str(config[name]) param = value_to_params(mac_document, name, value) if not param: param = emane_manager.xmlparam(mac_document, name, value) @@ -182,7 +182,7 @@ class EmaneModel(WirelessModel): continue # check if value is a multi param - value = config[name] + value = str(config[name]) param = value_to_params(phy_document, name, value) if not param: param = emane_manager.xmlparam(phy_document, name, value) @@ -269,7 +269,7 @@ class EmaneModel(WirelessModel): if interface: node_id = interface.node.objid - if self.getifcconfig(node_id, interface): + if self.session.emane.getifcconfig(node_id, interface, self.name): name = interface.localname.replace(".", "_") return "%s%s" % (name, self.name) @@ -348,43 +348,3 @@ class EmaneModel(WirelessModel): :return: nothing """ logger.warn("emane model(%s) does not support link configuration", self.name) - - def getifcconfig(self, node_id, ifc): - """ - Retrieve interface configuration or node configuration if not provided. - - :param int node_id: node id - :param ifc: node interface - :return: - """ - # use the network-wide config values or interface(NEM)-specific values? - if ifc is None: - return self.get_configs(node_id) - else: - # don"t use default values when interface config is the same as net - # note here that using ifc.node.objid as key allows for only one type - # of each model per node; - # TODO: use both node and interface as key - - # Adamson change: first check for iface config keyed by "node:ifc.name" - # (so that nodes w/ multiple interfaces of same conftype can have - # different configs for each separate interface) - key = 1000 * ifc.node.objid - if ifc.netindex is not None: - key += ifc.netindex - - # try retrieve interface specific configuration, avoid getting defaults - config = {} - if key in self.configuration_maps: - config = self.get_configs(key) - - # otherwise retrieve the interfaces node configuration, avoid using defaults - if not config and ifc.node.objid in self.configuration_maps: - config = self.get_configs(ifc.node.objid) - - if not config and ifc.transport_type == "raw": - # with EMANE 0.9.2+, we need an extra NEM XML from - # model.buildnemxmlfiles(), so defaults are returned here - config = self.get_configs(node_id) - - return config diff --git a/daemon/core/emane/ieee80211abg.py b/daemon/core/emane/ieee80211abg.py index d14ae4cd..9b4e6ea6 100644 --- a/daemon/core/emane/ieee80211abg.py +++ b/daemon/core/emane/ieee80211abg.py @@ -9,7 +9,6 @@ from core.emane import emanemodel class EmaneIeee80211abgModel(emanemodel.EmaneModel): # model name name = "emane_ieee80211abg" - configuration_maps = {} # mac configuration mac_library = "ieee80211abgmaclayer" diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 4159e33f..be99f375 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -73,7 +73,7 @@ class EmaneNode(EmaneNet): logger.info("node(%s) updating model(%s): %s", self.objid, self.model.name, config) self.model.set_configs(config, node_id=self.objid) - def setmodel(self, model, config=None): + def setmodel(self, model, config): """ set the EmaneModel associated with this node """ @@ -81,9 +81,11 @@ class EmaneNode(EmaneNet): if model.config_type == RegisterTlvs.WIRELESS.value: # EmaneModel really uses values from ConfigurableManager # when buildnemxml() is called, not during init() - self.model = model(session=self.session, object_id=self.objid, config=config) + self.model = model(session=self.session, object_id=self.objid) + self.model.update_config(config) elif model.config_type == RegisterTlvs.MOBILITY.value: - self.mobility = model(session=self.session, object_id=self.objid, config=config) + self.mobility = model(session=self.session, object_id=self.objid) + self.mobility.update_config(config) def setnemid(self, netif, nemid): """ @@ -180,8 +182,7 @@ class EmaneNode(EmaneNet): trans.setAttribute("library", "trans%s" % transport_type.lower()) trans.appendChild(emane.xmlparam(transdoc, "bitrate", "0")) - model_class = emane.get_model_class(self.model.name) - config = model_class.get_configs(self.objid) + config = emane.get_configs(node_id=self.objid, config_type=self.model.name) logger.debug("transport xml config: %s", config) flowcontrol = config.get("flowcontrolenable", "0") == "1" diff --git a/daemon/core/emane/rfpipe.py b/daemon/core/emane/rfpipe.py index cc24d31a..3c600ddd 100644 --- a/daemon/core/emane/rfpipe.py +++ b/daemon/core/emane/rfpipe.py @@ -9,7 +9,6 @@ from core.emane import emanemodel class EmaneRfPipeModel(emanemodel.EmaneModel): # model name name = "emane_rfpipe" - configuration_maps = {} # mac configuration mac_library = "rfpipemaclayer" diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index 99820cf8..1239182b 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -16,7 +16,6 @@ from core.misc import utils class EmaneTdmaModel(emanemodel.EmaneModel): # model name name = "emane_tdma" - configuration_maps = {} # mac configuration mac_library = "tdmaeventschedulerradiomodel" @@ -47,7 +46,7 @@ class EmaneTdmaModel(emanemodel.EmaneModel): :return: nothing """ # get configured schedule - config = self.get_configs(self.object_id) + config = self.session.emane.get_configs(node_id=self.object_id, config_type=self.name) if not config: return schedule = config[self.schedule_name] diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index fa21953f..182527ad 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -739,6 +739,7 @@ class EmuSession(Session): self.delete_objects() self.del_hooks() self.broker.reset() + self.emane.reset() def start_events(self): """ @@ -778,7 +779,7 @@ class EmuSession(Session): node_options.model = "mdr" return self.add_node(_type=NodeTypes.DEFAULT, _id=_id, node_options=node_options) - def create_emane_network(self, model, geo_reference, geo_scale=None, node_options=NodeOptions()): + def create_emane_network(self, model, geo_reference, geo_scale=None, node_options=NodeOptions(), config=None): """ Convenience method for creating an emane network. @@ -786,6 +787,7 @@ class EmuSession(Session): :param geo_reference: geo reference point to use for emane node locations :param geo_scale: geo scale to use for emane node locations, defaults to 1.0 :param core.emulator.emudata.NodeOptions node_options: options for emane node being created + :param dict config: emane model configuration :return: create emane network """ # required to be set for emane to function properly @@ -795,7 +797,7 @@ class EmuSession(Session): # create and return network emane_network = self.add_node(_type=NodeTypes.EMANE, node_options=node_options) - emane_network.setmodel(model) + self.emane.set_model(emane_network, model, config) return emane_network def wireless_link_all(self, network, nodes): diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 08d2153d..8c0e69b4 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -9,7 +9,7 @@ import threading import time from core import logger -from core.conf import ConfigurableOptions +from core.conf import ConfigurableOptions, ConfigurableManager from core.conf import Configuration from core.coreobj import PyCoreNode from core.data import EventData @@ -25,7 +25,7 @@ from core.misc import utils from core.misc.ipaddress import IpAddress -class MobilityManager(object): +class MobilityManager(ConfigurableManager): """ Member of session class for handling configuration data for mobility and range models. @@ -39,6 +39,7 @@ class MobilityManager(object): :param core.session.Session session: session this manager is tied to """ + super(MobilityManager, self).__init__() self.session = session # configurations for basic range, indexed by WLAN node number, are stored in configurations # mapping from model names to their classes @@ -52,6 +53,50 @@ class MobilityManager(object): self.physnets = {} self.session.broker.handlers.add(self.physnodehandlelink) + def set_model_config(self, node_id, model_name, config): + """ + Set configuration data for a model. + + :param int node_id: node id to set model configuration for + :param str model_name: model to set configuration for + :param dict config: configuration data to set for model + :return: nothing + """ + # get model class to configure + model_class = self._modelclsmap.get(model_name) + if not model_class: + raise ValueError("%s is an invalid model" % model_name) + + # retrieve default values + node_config = self.get_model_config(node_id, model_name) + for key, value in config.iteritems(): + node_config[key] = value + + # set configuration + self.set_configs(node_config, node_id=node_id, config_type=model_name) + + def get_model_config(self, node_id, model_name): + """ + Set configuration data for a model. + + :param int node_id: node id to set model configuration for + :param str model_name: model to set configuration for + :return: current model configuration for node + :rtype: dict + """ + # get model class to configure + model_class = self._modelclsmap.get(model_name) + if not model_class: + raise ValueError("%s is an invalid model" % model_name) + + config = self.get_configs(node_id=node_id, config_type=model_name) + if not config: + # set default values, when not already set + config = model_class.default_values() + self.set_configs(config, node_id=node_id, config_type=model_name) + + return config + def mobility_models(self): return self._modelclsmap.keys() @@ -68,17 +113,17 @@ class MobilityManager(object): :rtype: list """ models = [] - for model_class in self._modelclsmap.itervalues(): - if node.objid in model_class.configuration_maps: - config = model_class.get_configs(node_id=node.objid) - models.append((model_class, config)) - return models + all_configs = {} + if self.has_configs(node_id=node.objid): + all_configs = self.get_all_configs(node_id=node.objid) - def nodes(self): - node_ids = set() - for model_class in self._modelclsmap.itervalues(): - node_ids |= model_class.nodes() - return node_ids + for model_name in all_configs.iterkeys(): + model_class = self._modelclsmap[model_name] + config = self.get_configs(node_id=node.objid, config_type=model_name) + models.append((model_class, config)) + + logger.debug("mobility models for node(%s): %s", node.objid, models) + return models def startup(self, node_ids=None): """ @@ -93,6 +138,7 @@ class MobilityManager(object): for node_id in node_ids: logger.info("checking mobility startup for node: %s", node_id) + logger.info("node mobility configurations: %s", self.get_all_configs(node_id)) try: node = self.session.get_object(node_id) @@ -100,13 +146,12 @@ class MobilityManager(object): logger.warn("skipping mobility configuration for unknown node: %s", node_id) continue - for model_class in self._modelclsmap.itervalues(): - logger.debug("model(%s) configurations: %s", model_class, model_class.configuration_maps) - if node_id not in model_class.configuration_maps: + for model_name in self._modelclsmap.iterkeys(): + config = self.get_configs(node_id=node_id, config_type=model_name) + if not config: continue - config = model_class.get_configs(node_id=node_id) - logger.info("setting mobility model(%s) to node: %s", model_class.name, config) - node.setmodel(model_class, config) + model_class = self._modelclsmap[model_name] + self.set_model(node, model_class, config) if self.session.master: self.installphysnodes(node) @@ -114,15 +159,13 @@ class MobilityManager(object): if node.mobility: self.session.event_loop.add_event(0.0, node.mobility.startup) - def config_reset(self, node_id=None): - """ - Reset all configs. - - :param int node_id: node configuration to reset or None for all configurations - :return: nothing - """ - for model in self._modelclsmap.itervalues(): - model.config_reset(node_id=node_id) + def set_model(self, node, model_class, config=None): + logger.info("setting mobility model(%s) for node(%s): %s", model_class.name, node.objid, config) + if not config: + config = {} + self.set_model_config(node.objid, model_class.name, config) + config = self.get_model_config(node.objid, model_class.name) + node.setmodel(model_class, config) def handleevent(self, event_data): """ @@ -233,7 +276,8 @@ class MobilityManager(object): else: self.physnets[netnum].append(node_id) - # TODO: remove need for handling old style message + # TODO: remove need for handling old style message + def physnodehandlelink(self, message): """ Broker handler. Snoop Link add messages to get @@ -254,7 +298,8 @@ class MobilityManager(object): dummy = PyCoreNode(session=self.session, objid=nn[1], name="n%d" % nn[1], start=False) self.addphys(nn[0], dummy) - # TODO: remove need to handling old style messages + # TODO: remove need to handling old style messages + def physnodeupdateposition(self, message): """ Snoop node messages belonging to physical nodes. The dummy object @@ -303,7 +348,7 @@ class WirelessModel(ConfigurableOptions): bitmap = None position_callback = None - def __init__(self, session, object_id, config=None): + def __init__(self, session, object_id): """ Create a WirelessModel instance. @@ -313,8 +358,6 @@ class WirelessModel(ConfigurableOptions): """ self.session = session self.object_id = object_id - if config: - self.set_configs(config, node_id=self.object_id) def all_link_data(self, flags): """ @@ -337,15 +380,15 @@ class WirelessModel(ConfigurableOptions): """ raise NotImplementedError - def updateconfig(self): + def update_config(self, config): """ For run-time updates of model config. Returns True when position callback and set link parameters should be invoked. - :return: False - :rtype: bool + :param dict config: configuration values to update + :return: nothing """ - return False + pass class BasicRangeModel(WirelessModel): @@ -355,7 +398,6 @@ class BasicRangeModel(WirelessModel): the GUI. """ name = "basic_range" - configuration_maps = {} @classmethod def configurations(cls): @@ -372,7 +414,7 @@ class BasicRangeModel(WirelessModel): def config_groups(cls): return "Basic Range Parameters:1-%d" % len(cls.configurations()) - def __init__(self, session, object_id, config=None): + def __init__(self, session, object_id): """ Create a BasicRangeModel instance. @@ -386,17 +428,12 @@ class BasicRangeModel(WirelessModel): self._netifs = {} self._netifslock = threading.Lock() - # retrieve current configuration - config = self.get_configs(node_id=self.object_id) - self.range = None self.bw = None self.delay = None self.loss = None self.jitter = None - self.values_from_config(config) - def values_from_config(self, config): """ Values to convert to link parameters. @@ -546,15 +583,13 @@ class BasicRangeModel(WirelessModel): c = p1[2] - p2[2] return math.hypot(math.hypot(a, b), c) - def updateconfig(self): + def update_config(self, config): """ Configuration has changed during runtime. :param dict config: values to update configuration - :return: was update successful - :rtype: bool + :return: nothing """ - config = self.get_configs(node_id=self.object_id) self.values_from_config(config) self.setlinkparams() return True @@ -655,16 +690,15 @@ class WayPointMobility(WirelessModel): STATE_RUNNING = 1 STATE_PAUSED = 2 - def __init__(self, session, object_id, config=None): + def __init__(self, session, object_id): """ Create a WayPointMobility instance. :param core.session.Session session: CORE session instance :param int object_id: object id - :param config: values for this model :return: """ - super(WayPointMobility, self).__init__(session=session, object_id=object_id, config=config) + super(WayPointMobility, self).__init__(session=session, object_id=object_id) self.state = self.STATE_STOPPED self.queue = [] @@ -957,7 +991,6 @@ class Ns2ScriptedMobility(WayPointMobility): BonnMotion. """ name = "ns2script" - configuration_maps = {} @classmethod def configurations(cls): @@ -976,7 +1009,7 @@ class Ns2ScriptedMobility(WayPointMobility): def config_groups(cls): return "ns-2 Mobility Script Parameters:1-%d" % len(cls.configurations()) - def __init__(self, session, object_id, config=None): + def __init__(self, session, object_id): """ Creates a Ns2ScriptedMobility instance. @@ -984,14 +1017,22 @@ class Ns2ScriptedMobility(WayPointMobility): :param int object_id: object id :param config: values """ - super(Ns2ScriptedMobility, self).__init__(session=session, object_id=object_id, config=config) + super(Ns2ScriptedMobility, self).__init__(session=session, object_id=object_id) self._netifs = {} self._netifslock = threading.Lock() - # retrieve current configuration - config = self.get_configs(self.object_id) + self.file = None + self.refresh_ms = None + self.loop = None + self.autostart = None + self.nodemap = {} + self.script_start = None + self.script_pause = None + self.script_stop = None + def update_config(self, config): self.file = config["file"] + logger.info("ns-2 scripted mobility configured for WLAN %d using file: %s", self.object_id, self.file) self.refresh_ms = int(config["refresh_ms"]) self.loop = config["loop"].lower() == "on" self.autostart = config["autostart"] @@ -999,7 +1040,6 @@ class Ns2ScriptedMobility(WayPointMobility): self.script_start = config["script_start"] self.script_pause = config["script_pause"] self.script_stop = config["script_stop"] - logger.info("ns-2 scripted mobility configured for WLAN %d using file: %s", object_id, self.file) self.readscriptfile() self.copywaypoints() self.setendtime() diff --git a/daemon/core/netns/nodes.py b/daemon/core/netns/nodes.py index 3cf6a224..6890bb57 100644 --- a/daemon/core/netns/nodes.py +++ b/daemon/core/netns/nodes.py @@ -377,17 +377,18 @@ class WlanNode(LxBrNet): # invokes any netif.poshook netif.setposition(x, y, z) - def setmodel(self, model, config=None): + def setmodel(self, model, config): """ Sets the mobility and wireless model. :param core.mobility.WirelessModel.cls model: wireless model to set to - :param dict config: model configuration + :param dict config: configuration for model being set :return: nothing """ logger.info("adding model: %s", model.name) if model.config_type == RegisterTlvs.WIRELESS.value: - self.model = model(session=self.session, object_id=self.objid, config=config) + self.model = model(session=self.session, object_id=self.objid) + self.model.update_config(config) if self.model.position_callback: for netif in self.netifs(): netif.poshook = self.model.position_callback @@ -396,7 +397,8 @@ class WlanNode(LxBrNet): netif.poshook(netif, x, y, z) self.model.setlinkparams() elif model.config_type == RegisterTlvs.MOBILITY.value: - self.mobility = model(session=self.session, object_id=self.objid, config=config) + self.mobility = model(session=self.session, object_id=self.objid) + self.mobility.update_config(config) def update_mobility(self, config): if not self.mobility: diff --git a/daemon/core/session.py b/daemon/core/session.py index 8a158168..b37f05d5 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -17,6 +17,7 @@ from core import constants from core import logger from core.api import coreapi from core.broker import CoreBroker +from core.conf import ConfigurableManager from core.conf import ConfigurableOptions from core.conf import Configuration from core.data import EventData @@ -94,7 +95,8 @@ class Session(object): self.options = SessionConfig() if not config: config = {} - self.options.set_configs(config) + for key, value in config.iteritems(): + self.options.set_config(key, value) self.metadata = SessionMetaData() # initialize session feature helpers @@ -1044,7 +1046,7 @@ class Session(object): node.cmd(data, wait=False) -class SessionConfig(ConfigurableOptions): +class SessionConfig(ConfigurableManager, ConfigurableOptions): """ Session configuration object. """ @@ -1075,10 +1077,12 @@ class SessionConfig(ConfigurableOptions): return "Options:1-%d" % len(cls.configurations()) def __init__(self): - self.set_configs() + super(SessionConfig, self).__init__() + self.set_configs(self.default_values()) - def get_config(cls, _id, node_id=ConfigurableOptions._default_node, default=None): - value = super(SessionConfig, cls).get_config(_id, node_id, default) + def get_config(self, _id, node_id=ConfigurableManager._default_node, + config_type=ConfigurableManager._default_type, default=None): + value = super(SessionConfig, self).get_config(_id, node_id, config_type, default) if value == "": value = default return value @@ -1096,7 +1100,7 @@ class SessionConfig(ConfigurableOptions): return value -class SessionMetaData(ConfigurableOptions): +class SessionMetaData(ConfigurableManager): """ Metadata is simply stored in a configs[] dict. Key=value pairs are passed in from configure messages destined to the "metadata" object. diff --git a/daemon/core/xml/xmlparser0.py b/daemon/core/xml/xmlparser0.py index c93f4102..6ab25912 100644 --- a/daemon/core/xml/xmlparser0.py +++ b/daemon/core/xml/xmlparser0.py @@ -72,7 +72,12 @@ class CoreDocumentParser0(object): """ Helper to return tuple of attributes common to nodes and nets. """ - node_id = int(obj.getAttribute("id")) + node_id = obj.getAttribute("id") + try: + node_id = int(node_id) + except: + logger.debug("parsing node without integer id: %s", node_id) + name = str(obj.getAttribute("name")) node_type = str(obj.getAttribute("type")) return node_id, name, node_type @@ -205,7 +210,8 @@ class CoreDocumentParser0(object): # TODO: assign other config managers here if mgr: - mgr.setconfig_keyvalues(nodenum, name, kvs) + for k, v in kvs: + mgr.set_config(k, v, node_id=nodenum, config_type=name) def parsenetem(self, model, obj, kvs): """ @@ -218,7 +224,6 @@ class CoreDocumentParser0(object): # nodes and interfaces do not exist yet, at this point of the parsing, # save (key, value) pairs for later try: - # kvs = map(lambda(k, v): (int(v)), kvs) kvs = map(self.numericvalue, kvs) except ValueError: logger.warn("error parsing link parameters for '%s' on '%s'", ifname, peer) @@ -394,7 +399,7 @@ class CoreDocumentParser0(object): v = str(param.getAttribute("value")) if v == '': v = xmlutils.get_text_child(param) # allow attribute/text for newlines - setattr(self.session.options, k, v) + self.session.options.set_config(k, v) hooks = xmlutils.get_one_element(self.meta, "Hooks") if hooks: self.parsehooks(hooks) @@ -405,4 +410,4 @@ class CoreDocumentParser0(object): v = str(param.getAttribute("value")) if v == '': v = xmlutils.get_text_child(param) - self.session.metadata.add_item(k, v) + self.session.metadata.set_config(k, v) diff --git a/daemon/core/xml/xmlparser1.py b/daemon/core/xml/xmlparser1.py index 9f86255a..46f6bcf9 100644 --- a/daemon/core/xml/xmlparser1.py +++ b/daemon/core/xml/xmlparser1.py @@ -210,12 +210,11 @@ class CoreDocumentParser1(object): raise NotImplementedError logger.info("setting wireless link params: node(%s) model(%s) mobility_model(%s)", nodenum, model_name, mobility_model_name) - model_class = mgr.get_model_class(model_name) - model_class.set_configs(link_params, node_id=nodenum) + mgr.set_model_config(node_id=nodenum, model_name=model_name, config=link_params) if mobility_model_name and mobility_params: - model_class = mgr.get_model_class(mobility_model_name) - model_class.set_configs(mobility_params, node_id=nodenum) + self.session.mobility.set_model_config(node_id=nodenum, model_name=mobility_model_name, + config=mobility_params) def link_layer2_devices(self, device1, ifname1, device2, ifname2): """ diff --git a/daemon/core/xml/xmlwriter0.py b/daemon/core/xml/xmlwriter0.py index a54278b0..acbfb537 100644 --- a/daemon/core/xml/xmlwriter0.py +++ b/daemon/core/xml/xmlwriter0.py @@ -6,7 +6,7 @@ import pwd from core import logger from core.coreobj import PyCoreNet from core.coreobj import PyCoreNode -from core.enumerations import RegisterTlvs +from core.enumerations import RegisterTlvs, EventTypes from core.xml import xmlutils @@ -38,7 +38,8 @@ class CoreDocumentWriter0(Document): self.populatefromsession() def populatefromsession(self): - self.session.emane.setup() # not during runtime? + if self.session.state != EventTypes.RUNTIME_STATE.value: + self.session.emane.setup() # not during runtime? self.addorigin() self.adddefaultservices() self.addnets() @@ -136,14 +137,14 @@ class CoreDocumentWriter0(Document): for m, conf in configs: model = self.createElement("model") n.appendChild(model) - model.setAttribute("name", m._name) + model.setAttribute("name", m.name) type = "wireless" - if m._type == RegisterTlvs.MOBILITY.value: + if m.config_type == RegisterTlvs.MOBILITY.value: type = "mobility" model.setAttribute("type", type) - for i, k in enumerate(m.getnames()): + + for k, value in conf.iteritems(): key = self.createElement(k) - value = conf[i] if value is None: value = "" key.appendChild(self.createTextNode("%s" % value)) @@ -193,8 +194,8 @@ class CoreDocumentWriter0(Document): # could use ifc.params, transport_type self.addaddresses(i, ifc) # per-interface models - if netmodel and netmodel._name[:6] == "emane_": - cfg = netmodel.getifcconfig(node.objid, ifc) + if netmodel and netmodel.name[:6] == "emane_": + cfg = self.session.emane.getifcconfig(node.objid, ifc, netmodel.name) if cfg: self.addmodels(i, ((netmodel, cfg),)) diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index 59988346..98d7de89 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -654,7 +654,7 @@ class DeviceElement(NamedXmlElement): # per-interface models # XXX Remove??? if netmodel and netmodel.name[:6] == "emane_": - cfg = netmodel.getifcconfig(device_object.objid, interface_object) + cfg = self.coreSession.emane.getifcconfig(device_object.objid, interface_object, netmodel.name) if cfg: interface_element.addModels(((netmodel, cfg),)) diff --git a/daemon/examples/api/wlan.py b/daemon/examples/api/wlan.py index 0aab4673..ebec9d34 100644 --- a/daemon/examples/api/wlan.py +++ b/daemon/examples/api/wlan.py @@ -27,7 +27,7 @@ def example(options): # create wlan network node wlan = session.add_node(_type=NodeTypes.WIRELESS_LAN) - wlan.setmodel(BasicRangeModel) + session.mobility.set_model(wlan, BasicRangeModel) # create nodes wireless_nodes = [] diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index b25f4feb..058b7964 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -1,4 +1,4 @@ -from core.conf import ConfigurableOptions +from core.conf import ConfigurableOptions, ConfigurableManager from core.conf import Configuration from core.enumerations import ConfigDataTypes @@ -43,7 +43,7 @@ class TestConf: def test_nodes(self): # given - config_manager = TestConfigurableOptions() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -58,7 +58,7 @@ class TestConf: def test_config_reset_all(self): # given - config_manager = TestConfigurableOptions() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -68,11 +68,11 @@ class TestConf: config_manager.config_reset() # then - assert not config_manager.configuration_maps + assert not config_manager._configuration_maps def test_config_reset_node(self): # given - config_manager = TestConfigurableOptions() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -82,12 +82,12 @@ class TestConf: config_manager.config_reset(node_id) # then - assert node_id not in config_manager.configuration_maps + assert node_id not in config_manager._configuration_maps assert config_manager.get_configs() def test_configs_setget(self): # given - config_manager = TestConfigurableOptions() + config_manager = ConfigurableManager() test_config = {1: 2} node_id = 1 config_manager.set_configs(test_config) @@ -103,7 +103,7 @@ class TestConf: def test_config_setget(self): # given - config_manager = TestConfigurableOptions() + config_manager = ConfigurableManager() name = "test" value = "1" node_id = 1 diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index cfe5f0ef..6b657304 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -5,24 +5,21 @@ Unit tests for testing basic CORE networks. import os import stat import threading -from xml.etree import ElementTree import pytest from mock import MagicMock from core.emulator.emudata import NodeOptions -from core.enumerations import MessageFlags, NodeTypes -from core.mobility import BasicRangeModel, Ns2ScriptedMobility +from core.enumerations import MessageFlags +from core.enumerations import NodeTypes +from core.mobility import BasicRangeModel +from core.mobility import Ns2ScriptedMobility from core.netns.vnodeclient import VnodeClient from core.service import ServiceManager _PATH = os.path.abspath(os.path.dirname(__file__)) _SERVICES_PATH = os.path.join(_PATH, "myservices") _MOBILITY_FILE = os.path.join(_PATH, "mobility.scen") -_XML_VERSIONS = [ - "0.0", - "1.0" -] _WIRED = [ NodeTypes.PEER_TO_PEER, NodeTypes.HUB, @@ -93,60 +90,6 @@ class TestCore: status = ping(node_one, node_two, ip_prefixes) assert not status - @pytest.mark.parametrize("version", _XML_VERSIONS) - def test_xml(self, session, tmpdir, version, ip_prefixes): - """ - Test xml client methods. - - :param session: session for test - :param tmpdir: tmpdir to create data in - :param str version: xml version to write and parse - :param ip_prefixes: generates ip addresses for nodes - """ - # create ptp - ptp_node = session.add_node(_type=NodeTypes.PEER_TO_PEER) - - # create nodes - node_one = session.add_node() - node_two = session.add_node() - - # link nodes to ptp net - for node in [node_one, node_two]: - interface = ip_prefixes.create_interface(node) - session.add_link(node.objid, ptp_node.objid, interface_one=interface) - - # instantiate session - session.instantiate() - - # get ids for nodes - n1_id = node_one.objid - n2_id = node_two.objid - - # save xml - xml_file = tmpdir.join("session.xml") - file_path = xml_file.strpath - session.save_xml(file_path, version) - - # verify xml file was created and can be parsed - assert xml_file.isfile() - assert ElementTree.parse(file_path) - - # stop current session, clearing data - session.shutdown() - - # verify nodes have been removed from session - with pytest.raises(KeyError): - assert not session.get_object(n1_id) - with pytest.raises(KeyError): - assert not session.get_object(n2_id) - - # load saved xml - session.open_xml(file_path, start=True) - - # verify nodes have been recreated - assert session.get_object(n1_id) - assert session.get_object(n2_id) - def test_vnode_client(self, session, ip_prefixes): """ Test vnode client methods. @@ -257,7 +200,7 @@ class TestCore: # create wlan wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) - wlan_node.setmodel(BasicRangeModel) + session.mobility.set_model(wlan_node, BasicRangeModel) # create nodes node_options = NodeOptions() @@ -290,7 +233,7 @@ class TestCore: # create wlan wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) - wlan_node.setmodel(BasicRangeModel) + session.mobility.set_model(wlan_node, BasicRangeModel) # create nodes node_options = NodeOptions() @@ -317,7 +260,7 @@ class TestCore: "script_pause": "", "script_stop": "", } - wlan_node.setmodel(Ns2ScriptedMobility, config) + session.mobility.set_model(wlan_node, Ns2ScriptedMobility, config) # add handler for receiving node updates event = threading.Event() diff --git a/daemon/tests/test_xml.py b/daemon/tests/test_xml.py new file mode 100644 index 00000000..4d0cfa7f --- /dev/null +++ b/daemon/tests/test_xml.py @@ -0,0 +1,203 @@ +from xml.etree import ElementTree + +import pytest + +from core.emane.ieee80211abg import EmaneIeee80211abgModel +from core.emulator.emudata import NodeOptions +from core.enumerations import NodeTypes +from core.mobility import BasicRangeModel + +_XML_VERSIONS = [ + "0.0", + "1.0" +] + + +class TestXml: + @pytest.mark.parametrize("version", _XML_VERSIONS) + def test_xml_ptp(self, session, tmpdir, version, ip_prefixes): + """ + Test xml client methods for a ptp neetwork. + + :param session: session for test + :param tmpdir: tmpdir to create data in + :param str version: xml version to write and parse + :param ip_prefixes: generates ip addresses for nodes + """ + # create ptp + ptp_node = session.add_node(_type=NodeTypes.PEER_TO_PEER) + + # create nodes + node_one = session.add_node() + node_two = session.add_node() + + # link nodes to ptp net + for node in [node_one, node_two]: + interface = ip_prefixes.create_interface(node) + session.add_link(node.objid, ptp_node.objid, interface_one=interface) + + # instantiate session + session.instantiate() + + # get ids for nodes + n1_id = node_one.objid + n2_id = node_two.objid + + # save xml + xml_file = tmpdir.join("session.xml") + file_path = xml_file.strpath + session.save_xml(file_path, version) + + # verify xml file was created and can be parsed + assert xml_file.isfile() + assert ElementTree.parse(file_path) + + # stop current session, clearing data + session.shutdown() + + # verify nodes have been removed from session + with pytest.raises(KeyError): + assert not session.get_object(n1_id) + with pytest.raises(KeyError): + assert not session.get_object(n2_id) + + # load saved xml + session.open_xml(file_path, start=True) + + # verify nodes have been recreated + assert session.get_object(n1_id) + assert session.get_object(n2_id) + + @pytest.mark.parametrize("version", _XML_VERSIONS) + def test_xml_mobility(self, session, tmpdir, version, ip_prefixes): + """ + Test xml client methods for mobility. + + :param session: session for test + :param tmpdir: tmpdir to create data in + :param str version: xml version to write and parse + :param ip_prefixes: generates ip addresses for nodes + """ + # create wlan + wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) + session.mobility.set_model(wlan_node, BasicRangeModel, {"test": "1"}) + + # create nodes + node_options = NodeOptions() + node_options.set_position(0, 0) + node_one = session.create_wireless_node(node_options=node_options) + node_two = session.create_wireless_node(node_options=node_options) + + # link nodes + for node in [node_one, node_two]: + interface = ip_prefixes.create_interface(node) + session.add_link(node.objid, wlan_node.objid, interface_one=interface) + + # link nodes in wlan + session.wireless_link_all(wlan_node, [node_one, node_two]) + + # instantiate session + session.instantiate() + + # get ids for nodes + wlan_id = wlan_node.objid + n1_id = node_one.objid + n2_id = node_two.objid + + # save xml + xml_file = tmpdir.join("session.xml") + file_path = xml_file.strpath + session.save_xml(file_path, version) + + # verify xml file was created and can be parsed + assert xml_file.isfile() + assert ElementTree.parse(file_path) + + # stop current session, clearing data + session.shutdown() + + # verify nodes have been removed from session + with pytest.raises(KeyError): + assert not session.get_object(n1_id) + with pytest.raises(KeyError): + assert not session.get_object(n2_id) + + # load saved xml + session.open_xml(file_path, start=True) + + # retrieve configuration we set originally + value = str(session.mobility.get_config("test", wlan_id, BasicRangeModel.name)) + + # verify nodes and configuration were restored + assert session.get_object(n1_id) + assert session.get_object(n2_id) + assert session.get_object(wlan_id) + assert value == "1" + + @pytest.mark.parametrize("version", ["1.0"]) + def test_xml_emane(self, session, tmpdir, version, ip_prefixes): + """ + Test xml client methods for emane. + + :param session: session for test + :param tmpdir: tmpdir to create data in + :param str version: xml version to write and parse + :param ip_prefixes: generates ip addresses for nodes + """ + # create emane node for networking the core nodes + emane_network = session.create_emane_network( + EmaneIeee80211abgModel, + geo_reference=(47.57917, -122.13232, 2.00000), + config={"test": "1"} + ) + emane_network.setposition(x=80, y=50) + + # create nodes + node_options = NodeOptions() + node_options.set_position(150, 150) + node_one = session.create_wireless_node(node_options=node_options) + node_options.set_position(300, 150) + node_two = session.create_wireless_node(node_options=node_options) + + for i, node in enumerate([node_one, node_two]): + node.setposition(x=150 * (i + 1), y=150) + interface = ip_prefixes.create_interface(node) + session.add_link(node.objid, emane_network.objid, interface_one=interface) + + # instantiate session + session.instantiate() + + # get ids for nodes + emane_id = emane_network.objid + n1_id = node_one.objid + n2_id = node_two.objid + + # save xml + xml_file = tmpdir.join("session.xml") + file_path = xml_file.strpath + session.save_xml(file_path, version) + + # verify xml file was created and can be parsed + assert xml_file.isfile() + assert ElementTree.parse(file_path) + + # stop current session, clearing data + session.shutdown() + + # verify nodes have been removed from session + with pytest.raises(KeyError): + assert not session.get_object(n1_id) + with pytest.raises(KeyError): + assert not session.get_object(n2_id) + + # load saved xml + session.open_xml(file_path, start=True) + + # retrieve configuration we set originally + value = str(session.emane.get_config("test", emane_id, EmaneIeee80211abgModel.name)) + + # verify nodes and configuration were restored + assert session.get_object(n1_id) + assert session.get_object(n2_id) + assert session.get_object(emane_id) + assert value == "1" From 7dbc2c40f8469afe45f7e07b239a6b7ab6d2b806 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 13 Jun 2018 12:32:01 -0700 Subject: [PATCH 42/83] added documentation to configuration classes --- daemon/core/conf.py | 117 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 3 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 863fc868..e6329092 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -9,8 +9,19 @@ from core.data import ConfigData class ConfigShim(object): + """ + Provides helper methods for converting newer configuration values into TLV compatible formats. + """ + @classmethod def str_to_dict(cls, key_values): + """ + Converts a TLV key/value string into an ordered mapping. + + :param str key_values: + :return: ordered mapping of key/value pairs + :rtype: OrderedDict + """ key_values = key_values.split("|") values = OrderedDict() for key_value in key_values: @@ -25,9 +36,9 @@ class ConfigShim(object): by the class, but node number, conf type flags, and values must be passed in. - :param flags: message flags + :param int flags: message flags :param int node_id: node id - :param type_flags: type flags + :param int type_flags: type flags :param ConfigurableOptions configurable_options: options to create config data for :param dict config: configuration values for options :return: configuration data object @@ -74,7 +85,20 @@ class ConfigShim(object): class Configuration(object): + """ + Represents a configuration options. + """ + def __init__(self, _id, _type, label, default="", options=None): + """ + Creates a Configuration object. + + :param str _id: unique name for configuration + :param core.enumerations.ConfigDataTypes _type: configuration data type + :param str label: configuration label for display + :param str default: default value for configuration + :param list options: list options if this is a configuration with a combobox + """ self.id = _id self.type = _type self.default = default @@ -91,19 +115,44 @@ class Configuration(object): class ConfigurableManager(object): + """ + Provides convenience methods for storing and retrieving configuration options for nodes. + """ _default_node = -1 _default_type = _default_node def __init__(self): + """ + Creates a ConfigurableManager object. + """ self._configuration_maps = {} def nodes(self): + """ + Retrieves the ids of all node configurations known by this manager. + + :return: list of node ids + :rtype: list + """ return [node_id for node_id in self._configuration_maps.iterkeys() if node_id != self._default_node] def has_configs(self, node_id): + """ + Checks if this manager contains a configuration for the node id. + + :param int node_id: node id to check for a configuration + :return: True if a node configuration exists, False otherwise + :rtype: bool + """ return node_id in self._configuration_maps def config_reset(self, node_id=None): + """ + Clears all configurations or configuration for a specific node. + + :param int node_id: node id to clear configurations for, default is None and clears all configurations + :return: nothing + """ logger.debug("resetting all configurations: %s", self.__class__.__name__) if not node_id: self._configuration_maps.clear() @@ -111,11 +160,28 @@ class ConfigurableManager(object): self._configuration_maps.pop(node_id) def set_config(self, _id, value, node_id=_default_node, config_type=_default_type): + """ + Set a specific configuration value for a node and configuration type. + + :param str _id: configuration key + :param str value: configuration value + :param int node_id: node id to store configuration for + :param str config_type: configuration type to store configuration for + :return: nothing + """ logger.debug("setting config for node(%s) type(%s): %s=%s", node_id, config_type, _id, value) node_type_map = self.get_configs(node_id, config_type) node_type_map[_id] = value def set_configs(self, config, node_id=_default_node, config_type=_default_type): + """ + Set configurations for a node and configuration type. + + :param dict config: configurations to set + :param int node_id: node id to store configuration for + :param str config_type: configuration type to store configuration for + :return: nothing + """ logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) node_configs = self.get_all_configs(node_id) if config_type in node_configs: @@ -123,34 +189,79 @@ class ConfigurableManager(object): node_configs[config_type] = config def get_config(self, _id, node_id=_default_node, config_type=_default_type, default=None): + """ + Retrieves a specific configuration for a node and configuration type. + + :param str _id: specific configuration to retrieve + :param int node_id: node id to store configuration for + :param str config_type: configuration type to store configuration for + :param default: + :return: configuration value + :rtype str + """ logger.debug("getting config for node(%s) type(%s): %s", node_id, config_type, _id) node_type_map = self.get_configs(node_id, config_type) return node_type_map.get(_id, default) def get_configs(self, node_id=_default_node, config_type=_default_type): + """ + Retrieve configurations for a node and configuration type. + + :param int node_id: node id to store configuration for + :param str config_type: configuration type to store configuration for + :return: configurations + :rtype: dict + """ logger.debug("getting configs for node(%s) type(%s)", node_id, config_type) node_map = self.get_all_configs(node_id) return node_map.setdefault(config_type, {}) def get_all_configs(self, node_id=_default_node): + """ + Retrieve all current configuration types for a node. + + :param int node_id: node id to retrieve configurations for + :return: all configuration types for a node + :rtype: dict + """ logger.debug("getting all configs for node(%s)", node_id) return self._configuration_maps.setdefault(node_id, OrderedDict()) class ConfigurableOptions(object): - # unique name to receive configuration changes + """ + Provides a base for defining configuration options within CORE. + """ name = None bitmap = None _default_node = -1 @classmethod def configurations(cls): + """ + Provides the configurations for this class. + + :return: configurations + :rtype: list[Configuration] + """ return [] @classmethod def config_groups(cls): + """ + Defines how configurations are grouped. + + :return: configuration group definition + :rtype: str + """ return None @classmethod def default_values(cls): + """ + Provides an ordered mapping of configuration keys to default values. + + :return: ordered configuration mapping default values + :rtype: OrderedDict + """ return OrderedDict([(config.id, config.default) for config in cls.configurations()]) From a52e454111340bc1ee5cd0bd373370b851c2d884 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 13 Jun 2018 12:39:50 -0700 Subject: [PATCH 43/83] renamed dict variable in ConfigurationManager --- daemon/core/conf.py | 14 +++++++------- daemon/tests/test_conf.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index e6329092..631d2e5d 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -125,7 +125,7 @@ class ConfigurableManager(object): """ Creates a ConfigurableManager object. """ - self._configuration_maps = {} + self.node_configurations = {} def nodes(self): """ @@ -134,7 +134,7 @@ class ConfigurableManager(object): :return: list of node ids :rtype: list """ - return [node_id for node_id in self._configuration_maps.iterkeys() if node_id != self._default_node] + return [node_id for node_id in self.node_configurations.iterkeys() if node_id != self._default_node] def has_configs(self, node_id): """ @@ -144,7 +144,7 @@ class ConfigurableManager(object): :return: True if a node configuration exists, False otherwise :rtype: bool """ - return node_id in self._configuration_maps + return node_id in self.node_configurations def config_reset(self, node_id=None): """ @@ -155,9 +155,9 @@ class ConfigurableManager(object): """ logger.debug("resetting all configurations: %s", self.__class__.__name__) if not node_id: - self._configuration_maps.clear() - elif node_id in self._configuration_maps: - self._configuration_maps.pop(node_id) + self.node_configurations.clear() + elif node_id in self.node_configurations: + self.node_configurations.pop(node_id) def set_config(self, _id, value, node_id=_default_node, config_type=_default_type): """ @@ -225,7 +225,7 @@ class ConfigurableManager(object): :rtype: dict """ logger.debug("getting all configs for node(%s)", node_id) - return self._configuration_maps.setdefault(node_id, OrderedDict()) + return self.node_configurations.setdefault(node_id, OrderedDict()) class ConfigurableOptions(object): diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index 058b7964..8c215e03 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -68,7 +68,7 @@ class TestConf: config_manager.config_reset() # then - assert not config_manager._configuration_maps + assert not config_manager.node_configurations def test_config_reset_node(self): # given @@ -82,7 +82,7 @@ class TestConf: config_manager.config_reset(node_id) # then - assert node_id not in config_manager._configuration_maps + assert not config_manager.has_configs(node_id) assert config_manager.get_configs() def test_configs_setget(self): From 25cfb21586755dc675c61a4766e68999bce8ec4b Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 13 Jun 2018 16:17:47 -0700 Subject: [PATCH 44/83] added modelmanager for both mobility and emane to leverage and reduce duplicate logic --- daemon/core/conf.py | 84 ++++++++++++++++++++++++ daemon/core/corehandlers.py | 16 ++--- daemon/core/emane/emanemanager.py | 92 ++------------------------- daemon/core/mobility.py | 102 +++--------------------------- daemon/tests/test_conf.py | 69 +++++++++++++++++++- 5 files changed, 171 insertions(+), 192 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 631d2e5d..99c9a738 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -265,3 +265,87 @@ class ConfigurableOptions(object): :rtype: OrderedDict """ return OrderedDict([(config.id, config.default) for config in cls.configurations()]) + + +class ModelManager(ConfigurableManager): + def __init__(self): + super(ModelManager, self).__init__() + self.models = {} + self.node_models = {} + + def set_model_config(self, node_id, model_name, config=None): + """ + Set configuration data for a model. + + :param int node_id: node id to set model configuration for + :param str model_name: model to set configuration for + :param dict config: configuration data to set for model + :return: nothing + """ + # get model class to configure + model_class = self.models.get(model_name) + if not model_class: + raise ValueError("%s is an invalid model" % model_name) + + # retrieve default values + node_config = self.get_model_config(node_id, model_name) + if not config: + config = {} + for key, value in config.iteritems(): + node_config[key] = value + + # set as node model for startup + self.node_models[node_id] = model_name + + # set configuration + self.set_configs(node_config, node_id=node_id, config_type=model_name) + + def get_model_config(self, node_id, model_name): + """ + Set configuration data for a model. + + :param int node_id: node id to set model configuration for + :param str model_name: model to set configuration for + :return: current model configuration for node + :rtype: dict + """ + # get model class to configure + model_class = self.models.get(model_name) + if not model_class: + raise ValueError("%s is an invalid model" % model_name) + + config = self.get_configs(node_id=node_id, config_type=model_name) + if not config: + # set default values, when not already set + config = model_class.default_values() + self.set_configs(config, node_id=node_id, config_type=model_name) + + return config + + def set_model(self, node, model_class, config=None): + logger.info("setting mobility model(%s) for node(%s): %s", model_class.name, node.objid, config) + self.set_model_config(node.objid, model_class.name, config) + config = self.get_model_config(node.objid, model_class.name) + node.setmodel(model_class, config) + + def getmodels(self, node): + """ + Return a list of model classes and values for a net if one has been + configured. This is invoked when exporting a session to XML. + + :param node: network node to get models for + :return: list of model and values tuples for the network node + :rtype: list + """ + models = [] + all_configs = {} + if self.has_configs(node_id=node.objid): + all_configs = self.get_all_configs(node_id=node.objid) + + for model_name in all_configs.iterkeys(): + model_class = self.models[model_name] + config = self.get_configs(node_id=node.objid, config_type=model_name) + models.append((model_class, config)) + + logger.debug("mobility models for node(%s): %s", node.objid, models) + return models diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 23eb1a94..bb7967fb 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -42,8 +42,6 @@ from core.enumerations import SessionTlvs from core.misc import nodeutils from core.misc import structutils from core.misc import utils -from core.mobility import BasicRangeModel -from core.mobility import Ns2ScriptedMobility from core.service import CoreService from core.service import ServiceManager @@ -380,13 +378,11 @@ class CoreHandler(SocketServer.BaseRequestHandler): tlv_data += coreapi.CoreRegisterTlv.pack(self.session.broker.config_type, self.session.broker.name) tlv_data += coreapi.CoreRegisterTlv.pack(self.session.location.config_type, self.session.location.name) tlv_data += coreapi.CoreRegisterTlv.pack(self.session.mobility.config_type, self.session.mobility.name) - for model_name in self.session.mobility.mobility_models(): - model_class = self.session.mobility.get_model_class(model_name) + for model_class in self.session.mobility.models.itervalues(): tlv_data += coreapi.CoreRegisterTlv.pack(model_class.config_type, model_class.name) tlv_data += coreapi.CoreRegisterTlv.pack(self.session.services.config_type, self.session.services.name) tlv_data += coreapi.CoreRegisterTlv.pack(self.session.emane.config_type, self.session.emane.name) - for model_name in self.session.emane.emane_models(): - model_class = self.session.emane.get_model_class(model_name) + for model_class in self.session.emane.models.itervalues(): tlv_data += coreapi.CoreRegisterTlv.pack(model_class.config_type, model_class.name) tlv_data += coreapi.CoreRegisterTlv.pack(self.session.options.config_type, self.session.options.name) tlv_data += coreapi.CoreRegisterTlv.pack(self.session.metadata.config_type, self.session.metadata.name) @@ -937,11 +933,11 @@ class CoreHandler(SocketServer.BaseRequestHandler): replies = self.handle_config_services(message_type, config_data) elif config_data.object == self.session.mobility.name: self.handle_config_mobility(message_type, config_data) - elif config_data.object in [BasicRangeModel.name, Ns2ScriptedMobility.name]: + elif config_data.object in self.session.mobility.models: replies = self.handle_config_mobility_models(message_type, config_data) elif config_data.object == self.session.emane.name: replies = self.handle_config_emane(message_type, config_data) - elif config_data.object in self.session.emane.emane_models(): + elif config_data.object in self.session.emane.models: replies = self.handle_config_emane_models(message_type, config_data) else: raise Exception("no handler for configuration: %s", config_data.object) @@ -1181,7 +1177,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.info("replying to configure request for model: %s", object_name) typeflags = ConfigFlags.NONE.value - model_class = self.session.mobility.get_model_class(object_name) + model_class = self.session.mobility.models.get(object_name) if not model_class: logger.warn("model class does not exist: %s", object_name) return [] @@ -1251,7 +1247,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.info("replying to configure request for model: %s", object_name) typeflags = ConfigFlags.NONE.value - model_class = self.session.emane.get_model_class(object_name) + model_class = self.session.emane.models.get(object_name) if not model_class: logger.warn("model class does not exist: %s", object_name) return [] diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 78b770b4..ae58d1b5 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -11,8 +11,9 @@ from core import constants from core import logger from core.api import coreapi from core.api import dataconversion -from core.conf import ConfigShim, ConfigurableManager +from core.conf import ConfigShim from core.conf import Configuration +from core.conf import ModelManager from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel @@ -53,7 +54,7 @@ EMANE_MODELS = [ ] -class EmaneManager(ConfigurableManager): +class EmaneManager(ModelManager): """ EMANE controller object. Lives in a Session instance and is used for building EMANE config files from all of the EmaneNode objects in this @@ -89,64 +90,11 @@ class EmaneManager(ConfigurableManager): self.emane_config = EmaneGlobalModel(session) self.set_configs(self.emane_config.default_values()) - # store the last configured model for a node, used during startup - self.node_models = {} - session.broker.handlers.add(self.handledistributed) self.service = None self.event_device = None - self._modelclsmap = {} - - self.service = None self.emane_check() - def set_model_config(self, node_id, model_name, config): - """ - Set configuration data for a model. - - :param int node_id: node id to set model configuration for - :param str model_name: model to set configuration for - :param dict config: configuration data to set for model - :return: nothing - """ - # get model class to configure - model_class = self._modelclsmap.get(model_name) - if not model_class: - raise ValueError("%s is an invalid model" % model_name) - - # retrieve default values - node_config = self.get_model_config(node_id, model_name) - for key, value in config.iteritems(): - node_config[key] = value - - # set as node model for startup - self.node_models[node_id] = model_name - - # set configuration - self.set_configs(node_config, node_id=node_id, config_type=model_name) - - def get_model_config(self, node_id, model_name): - """ - Set configuration data for a model. - - :param int node_id: node id to set model configuration for - :param str model_name: model to set configuration for - :return: current model configuration for node - :rtype: dict - """ - # get model class to configure - model_class = self._modelclsmap.get(model_name) - if not model_class: - raise ValueError("%s is an invalid model" % model_name) - - config = self.get_configs(node_id=node_id, config_type=model_name) - if not config: - # set default values, when not already set - config = model_class.default_values() - self.set_configs(config, node_id=node_id, config_type=model_name) - - return config - def getifcconfig(self, node_id, interface, model_name): """ Retrieve interface configuration or node configuration if not provided. @@ -189,24 +137,10 @@ class EmaneManager(ConfigurableManager): return config - def set_model(self, node, model_class, config=None): - logger.info("setting emane model(%s) for node(%s): %s", model_class.name, node.objid, config) - if not config: - config = {} - self.set_model_config(node.objid, model_class.name, config) - config = self.get_model_config(node.objid, model_class.name) - node.setmodel(model_class, config) - def config_reset(self, node_id=None): super(EmaneManager, self).config_reset(node_id) self.set_configs(self.emane_config.default_values()) - def emane_models(self): - return self._modelclsmap.keys() - - def get_model_class(self, model_name): - return self._modelclsmap[model_name] - def emane_check(self): """ Check if emane is installed and load models. @@ -281,7 +215,7 @@ class EmaneManager(ConfigurableManager): """ for emane_model in emane_models: logger.info("loading emane model: %s", emane_model.__name__) - self._modelclsmap[emane_model.name] = emane_model + self.models[emane_model.name] = emane_model def add_node(self, emane_node): """ @@ -307,22 +241,6 @@ class EmaneManager(ConfigurableManager): nodes.add(netif.node) return nodes - def getmodels(self, node): - """ - Used with XML export. - """ - models = [] - all_configs = {} - if self.has_configs(node_id=node.objid): - all_configs = self.get_all_configs(node_id=node.objid) - - for model_name in all_configs.iterkeys(): - model_class = self._modelclsmap[model_name] - config = self.get_configs(node_id=node.objid, config_type=model_name) - models.append((model_class, config)) - logger.debug("emane models for node(%s): %s", node.objid, models) - return models - def setup(self): """ Populate self._objs with EmaneNodes; perform distributed setup; @@ -650,7 +568,7 @@ class EmaneManager(ConfigurableManager): config = self.get_model_config(node_id=node_id, model_name=model_name) logger.debug("setting emane model(%s) config(%s)", model_name, config) - model_class = self._modelclsmap[model_name] + model_class = self.models[model_name] emane_node.setmodel(model_class, config) def nemlookup(self, nemid): diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 8c0e69b4..70efb539 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -9,8 +9,9 @@ import threading import time from core import logger -from core.conf import ConfigurableOptions, ConfigurableManager +from core.conf import ConfigurableOptions from core.conf import Configuration +from core.conf import ModelManager from core.coreobj import PyCoreNode from core.data import EventData from core.data import LinkData @@ -25,7 +26,7 @@ from core.misc import utils from core.misc.ipaddress import IpAddress -class MobilityManager(ConfigurableManager): +class MobilityManager(ModelManager): """ Member of session class for handling configuration data for mobility and range models. @@ -41,90 +42,14 @@ class MobilityManager(ConfigurableManager): """ super(MobilityManager, self).__init__() self.session = session - # configurations for basic range, indexed by WLAN node number, are stored in configurations - # mapping from model names to their classes - self._modelclsmap = { - BasicRangeModel.name: BasicRangeModel, - Ns2ScriptedMobility.name: Ns2ScriptedMobility - } + self.models[BasicRangeModel.name] = BasicRangeModel + self.models[Ns2ScriptedMobility.name] = Ns2ScriptedMobility # dummy node objects for tracking position of nodes on other servers self.phys = {} self.physnets = {} self.session.broker.handlers.add(self.physnodehandlelink) - def set_model_config(self, node_id, model_name, config): - """ - Set configuration data for a model. - - :param int node_id: node id to set model configuration for - :param str model_name: model to set configuration for - :param dict config: configuration data to set for model - :return: nothing - """ - # get model class to configure - model_class = self._modelclsmap.get(model_name) - if not model_class: - raise ValueError("%s is an invalid model" % model_name) - - # retrieve default values - node_config = self.get_model_config(node_id, model_name) - for key, value in config.iteritems(): - node_config[key] = value - - # set configuration - self.set_configs(node_config, node_id=node_id, config_type=model_name) - - def get_model_config(self, node_id, model_name): - """ - Set configuration data for a model. - - :param int node_id: node id to set model configuration for - :param str model_name: model to set configuration for - :return: current model configuration for node - :rtype: dict - """ - # get model class to configure - model_class = self._modelclsmap.get(model_name) - if not model_class: - raise ValueError("%s is an invalid model" % model_name) - - config = self.get_configs(node_id=node_id, config_type=model_name) - if not config: - # set default values, when not already set - config = model_class.default_values() - self.set_configs(config, node_id=node_id, config_type=model_name) - - return config - - def mobility_models(self): - return self._modelclsmap.keys() - - def get_model_class(self, model_name): - return self._modelclsmap[model_name] - - def getmodels(self, node): - """ - Return a list of model classes and values for a net if one has been - configured. This is invoked when exporting a session to XML. - - :param node: network node to get models for - :return: list of model and values tuples for the network node - :rtype: list - """ - models = [] - all_configs = {} - if self.has_configs(node_id=node.objid): - all_configs = self.get_all_configs(node_id=node.objid) - - for model_name in all_configs.iterkeys(): - model_class = self._modelclsmap[model_name] - config = self.get_configs(node_id=node.objid, config_type=model_name) - models.append((model_class, config)) - - logger.debug("mobility models for node(%s): %s", node.objid, models) - return models - def startup(self, node_ids=None): """ Session is transitioning from instantiation to runtime state. @@ -146,11 +71,11 @@ class MobilityManager(ConfigurableManager): logger.warn("skipping mobility configuration for unknown node: %s", node_id) continue - for model_name in self._modelclsmap.iterkeys(): + for model_name in self.models.iterkeys(): config = self.get_configs(node_id=node_id, config_type=model_name) if not config: continue - model_class = self._modelclsmap[model_name] + model_class = self.models[model_name] self.set_model(node, model_class, config) if self.session.master: @@ -159,14 +84,6 @@ class MobilityManager(ConfigurableManager): if node.mobility: self.session.event_loop.add_event(0.0, node.mobility.startup) - def set_model(self, node, model_class, config=None): - logger.info("setting mobility model(%s) for node(%s): %s", model_class.name, node.objid, config) - if not config: - config = {} - self.set_model_config(node.objid, model_class.name, config) - config = self.get_model_config(node.objid, model_class.name) - node.setmodel(model_class, config) - def handleevent(self, event_data): """ Handle an Event Message used to start, stop, or pause @@ -189,7 +106,7 @@ class MobilityManager(ConfigurableManager): models = name[9:].split(',') for model in models: try: - cls = self._modelclsmap[model] + cls = self.models[model] except KeyError: logger.warn("Ignoring event for unknown model '%s'", model) continue @@ -298,8 +215,7 @@ class MobilityManager(ConfigurableManager): dummy = PyCoreNode(session=self.session, objid=nn[1], name="n%d" % nn[1], start=False) self.addphys(nn[0], dummy) - # TODO: remove need to handling old style messages - + # TODO: remove need to handling old style messages def physnodeupdateposition(self, message): """ Snoop node messages belonging to physical nodes. The dummy object diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index 8c215e03..f19fc168 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -1,6 +1,10 @@ -from core.conf import ConfigurableOptions, ConfigurableManager +import pytest + +from core.conf import ConfigurableOptions, ConfigurableManager, ModelManager from core.conf import Configuration -from core.enumerations import ConfigDataTypes +from core.emane.ieee80211abg import EmaneIeee80211abgModel +from core.enumerations import ConfigDataTypes, NodeTypes +from core.mobility import BasicRangeModel class TestConfigurableOptions(ConfigurableOptions): @@ -117,3 +121,64 @@ class TestConf: # then assert defaults_value == value assert node_value == value + + def test_model_setget_config(self): + # given + manager = ModelManager() + manager.models[BasicRangeModel.name] = BasicRangeModel + + # when + manager.set_model_config(1, BasicRangeModel.name) + + # then + assert manager.get_model_config(1, BasicRangeModel.name) + + def test_model_set_config_error(self): + # given + manager = ModelManager() + manager.models[BasicRangeModel.name] = BasicRangeModel + bad_name = "bad-model" + + # when/then + with pytest.raises(ValueError): + manager.set_model_config(1, bad_name) + + def test_model_get_config_error(self): + # given + manager = ModelManager() + manager.models[BasicRangeModel.name] = BasicRangeModel + bad_name = "bad-model" + + # when/then + with pytest.raises(ValueError): + manager.get_model_config(1, bad_name) + + def test_model_set(self, session): + # given + wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) + + # when + session.mobility.set_model(wlan_node, BasicRangeModel) + + # then + assert session.mobility.get_model_config(wlan_node.objid, BasicRangeModel.name) + + def test_model_set_error(self, session): + # given + wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) + + # when / then + with pytest.raises(ValueError): + session.mobility.set_model(wlan_node, EmaneIeee80211abgModel) + + def test_get_models(self, session): + # given + wlan_node = session.add_node(_type=NodeTypes.WIRELESS_LAN) + session.mobility.set_model(wlan_node, BasicRangeModel) + + # when + models = session.mobility.getmodels(wlan_node) + + # then + assert models + assert len(models) == 1 From 8e3cd0e0130739c35a209a9a20e7f3e2ff094b5d Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 13 Jun 2018 16:23:51 -0700 Subject: [PATCH 45/83] refactored modelmanager getmodels to get_models --- daemon/core/conf.py | 2 +- daemon/core/corehandlers.py | 4 ++-- daemon/core/xml/xmlwriter0.py | 4 ++-- daemon/core/xml/xmlwriter1.py | 4 ++-- daemon/tests/test_conf.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 99c9a738..2a9e30eb 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -328,7 +328,7 @@ class ModelManager(ConfigurableManager): config = self.get_model_config(node.objid, model_class.name) node.setmodel(model_class, config) - def getmodels(self, node): + def get_models(self, node): """ Return a list of model classes and values for a net if one has been configured. This is invoked when exporting a session to XML. diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index bb7967fb..182ffaaf 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1577,7 +1577,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): # send mobility model info for node_id in self.session.mobility.nodes(): node = self.session.get_object(node_id) - for model_class, config in self.session.mobility.getmodels(node): + for model_class, config in self.session.mobility.get_models(node): logger.info("mobility config: node(%s) class(%s) values(%s)", node_id, model_class, config) config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) self.session.broadcast_config(config_data) @@ -1585,7 +1585,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): # send emane model info for node_id in self.session.emane.nodes(): node = self.session.get_object(node_id) - for model_class, config in self.session.emane.getmodels(node): + for model_class, config in self.session.emane.get_models(node): logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) self.session.broadcast_config(config_data) diff --git a/daemon/core/xml/xmlwriter0.py b/daemon/core/xml/xmlwriter0.py index acbfb537..a11e099a 100644 --- a/daemon/core/xml/xmlwriter0.py +++ b/daemon/core/xml/xmlwriter0.py @@ -84,8 +84,8 @@ class CoreDocumentWriter0(Document): for netif in net.netifs(sort=True): self.addnetem(n, netif) # wireless/mobility models - modelconfigs = net.session.mobility.getmodels(net) - modelconfigs += net.session.emane.getmodels(net) + modelconfigs = net.session.mobility.get_models(net) + modelconfigs += net.session.emane.get_models(net) self.addmodels(n, modelconfigs) self.addposition(net) diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index 98d7de89..6f5deb50 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -475,8 +475,8 @@ class NetworkElement(NamedXmlElement): """ if nodeutils.is_node(network_object, (NodeTypes.WIRELESS_LAN, NodeTypes.EMANE)): - modelconfigs = network_object.session.mobility.getmodels(network_object) - modelconfigs += network_object.session.emane.getmodels(network_object) + modelconfigs = network_object.session.mobility.get_models(network_object) + modelconfigs += network_object.session.emane.get_models(network_object) chan = None for model, conf in modelconfigs: diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index f19fc168..9f21da78 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -177,7 +177,7 @@ class TestConf: session.mobility.set_model(wlan_node, BasicRangeModel) # when - models = session.mobility.getmodels(wlan_node) + models = session.mobility.get_models(wlan_node) # then assert models From 82c3d57dd31f2f6c22e63d86ba09edf512a6853f Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 14 Jun 2018 08:41:48 -0700 Subject: [PATCH 46/83] setup a simple default way for dealing with configurable options and added conifg group opbjects as a better way to access the same information formatted within a string --- daemon/core/conf.py | 45 ++++++++++++++++++++++++--- daemon/core/emane/commeffect.py | 5 ++- daemon/core/emane/emanemanager.py | 1 - daemon/core/emane/emanemodel.py | 6 +++- daemon/core/mobility.py | 51 +++++++++++++++---------------- daemon/core/session.py | 39 +++++++++-------------- daemon/tests/test_conf.py | 35 +++++++++++---------- 7 files changed, 107 insertions(+), 75 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 2a9e30eb..1e579804 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -29,6 +29,21 @@ class ConfigShim(object): values[key] = value return values + @classmethod + def groups_to_str(cls, config_groups): + """ + Converts configuration groups to a TLV formatted string. + + :param list[ConfigGroup] config_groups: configuration groups to format + :return: TLV configuration group string + :rtype: str + """ + group_strings = [] + for config_group in config_groups: + group_string = "%s:%s-%s" % (config_group.name, config_group.start, config_group.stop) + group_strings.append(group_string) + return "|".join(group_strings) + @classmethod def config_data(cls, flags, node_id, type_flags, configurable_options, config): """ @@ -70,6 +85,7 @@ class ConfigShim(object): else: key_values += "|%s" % key_value + groups_str = cls.groups_to_str(configurable_options.config_groups()) return ConfigData( message_type=flags, node=node_id, @@ -80,7 +96,7 @@ class ConfigShim(object): captions=captions, possible_values="|".join(possible_values), bitmap=configurable_options.bitmap, - groups=configurable_options.config_groups() + groups=groups_str ) @@ -228,12 +244,31 @@ class ConfigurableManager(object): return self.node_configurations.setdefault(node_id, OrderedDict()) +class ConfigGroup(object): + """ + Defines configuration group tabs used for display by ConfigurationOptions. + """ + + def __init__(self, name, start, stop): + """ + Creates a ConfigGroup object. + + :param str name: configuration group display name + :param int start: configurations start index for this group + :param int stop: configurations stop index for this group + """ + self.name = name + self.start = start + self.stop = stop + + class ConfigurableOptions(object): """ Provides a base for defining configuration options within CORE. """ name = None bitmap = None + options = [] _default_node = -1 @classmethod @@ -244,7 +279,7 @@ class ConfigurableOptions(object): :return: configurations :rtype: list[Configuration] """ - return [] + return cls.options @classmethod def config_groups(cls): @@ -252,9 +287,11 @@ class ConfigurableOptions(object): Defines how configurations are grouped. :return: configuration group definition - :rtype: str + :rtype: list[ConfigGroup] """ - return None + return [ + ConfigGroup("Options", 1, len(cls.configurations())) + ] @classmethod def default_values(cls): diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index fbce2046..bc699099 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -3,6 +3,7 @@ commeffect.py: EMANE CommEffect model for CORE """ from core import logger +from core.conf import ConfigGroup from core.emane import emanemanifest from core.emane import emanemodel @@ -41,7 +42,9 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): @classmethod def config_groups(cls): - return "CommEffect SHIM Parameters:1-%d" % len(cls.configurations()) + return [ + ConfigGroup("CommEffect SHIM Parameters", 1, len(cls.configurations())) + ] def build_xml_files(self, emane_manager, interface): """ diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index ae58d1b5..29de1be2 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -990,7 +990,6 @@ class EmaneGlobalModel(EmaneModel): _DEFAULT_DEV = "ctrl0" name = "emane" - configuration_maps = {} emulator_xml = "/usr/share/emane/manifest/nemmanager.xml" emulator_defaults = { diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 02726fe8..77aafcec 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -3,6 +3,7 @@ Defines Emane Models used within CORE. """ from core import logger +from core.conf import ConfigGroup from core.emane import emanemanifest from core.misc import utils from core.mobility import WirelessModel @@ -68,7 +69,10 @@ class EmaneModel(WirelessModel): def config_groups(cls): mac_len = len(cls.mac_config) config_len = len(cls.configurations()) - return "MAC Parameters:1-%d|PHY Parameters:%d-%d" % (mac_len, mac_len + 1, config_len) + return [ + ConfigGroup("MAC Parameters", 1, mac_len), + ConfigGroup("PHY Parameters", mac_len + 1, config_len) + ] def build_xml_files(self, emane_manager, interface): """ diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 70efb539..46f1f966 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -9,6 +9,7 @@ import threading import time from core import logger +from core.conf import ConfigGroup from core.conf import ConfigurableOptions from core.conf import Configuration from core.conf import ModelManager @@ -314,21 +315,20 @@ class BasicRangeModel(WirelessModel): the GUI. """ name = "basic_range" - - @classmethod - def configurations(cls): - return [ - Configuration(_id="range", _type=ConfigDataTypes.UINT32, default="275", label="wireless range (pixels)"), - Configuration(_id="bandwidth", _type=ConfigDataTypes.UINT32, default="54000", label="bandwidth (bps)"), - Configuration(_id="jitter", _type=ConfigDataTypes.FLOAT, default="0.0", label="transmission jitter (usec)"), - Configuration(_id="delay", _type=ConfigDataTypes.FLOAT, default="5000.0", - label="transmission delay (usec)"), - Configuration(_id="error", _type=ConfigDataTypes.FLOAT, default="0.0", label="error rate (%)") - ] + options = [ + Configuration(_id="range", _type=ConfigDataTypes.UINT32, default="275", label="wireless range (pixels)"), + Configuration(_id="bandwidth", _type=ConfigDataTypes.UINT32, default="54000", label="bandwidth (bps)"), + Configuration(_id="jitter", _type=ConfigDataTypes.FLOAT, default="0.0", label="transmission jitter (usec)"), + Configuration(_id="delay", _type=ConfigDataTypes.FLOAT, default="5000.0", + label="transmission delay (usec)"), + Configuration(_id="error", _type=ConfigDataTypes.FLOAT, default="0.0", label="error rate (%)") + ] @classmethod def config_groups(cls): - return "Basic Range Parameters:1-%d" % len(cls.configurations()) + return [ + ConfigGroup("Basic Range Parameters", 1, len(cls.configurations())) + ] def __init__(self, session, object_id): """ @@ -907,23 +907,22 @@ class Ns2ScriptedMobility(WayPointMobility): BonnMotion. """ name = "ns2script" - - @classmethod - def configurations(cls): - return [ - Configuration(_id="file", _type=ConfigDataTypes.STRING, label="mobility script file"), - Configuration(_id="refresh_ms", _type=ConfigDataTypes.UINT32, default="50", label="mobility script file"), - Configuration(_id="loop", _type=ConfigDataTypes.BOOL, default="1", options=["On", "Off"], label="loop"), - Configuration(_id="autostart", _type=ConfigDataTypes.STRING, label="auto-start seconds (0.0 for runtime)"), - Configuration(_id="map", _type=ConfigDataTypes.STRING, label="node mapping (optional, e.g. 0:1,1:2,2:3)"), - Configuration(_id="script_start", _type=ConfigDataTypes.STRING, label="script file to run upon start"), - Configuration(_id="script_pause", _type=ConfigDataTypes.STRING, label="script file to run upon pause"), - Configuration(_id="script_stop", _type=ConfigDataTypes.STRING, label="script file to run upon stop") - ] + options = [ + Configuration(_id="file", _type=ConfigDataTypes.STRING, label="mobility script file"), + Configuration(_id="refresh_ms", _type=ConfigDataTypes.UINT32, default="50", label="mobility script file"), + Configuration(_id="loop", _type=ConfigDataTypes.BOOL, default="1", options=["On", "Off"], label="loop"), + Configuration(_id="autostart", _type=ConfigDataTypes.STRING, label="auto-start seconds (0.0 for runtime)"), + Configuration(_id="map", _type=ConfigDataTypes.STRING, label="node mapping (optional, e.g. 0:1,1:2,2:3)"), + Configuration(_id="script_start", _type=ConfigDataTypes.STRING, label="script file to run upon start"), + Configuration(_id="script_pause", _type=ConfigDataTypes.STRING, label="script file to run upon pause"), + Configuration(_id="script_stop", _type=ConfigDataTypes.STRING, label="script file to run upon stop") + ] @classmethod def config_groups(cls): - return "ns-2 Mobility Script Parameters:1-%d" % len(cls.configurations()) + return [ + ConfigGroup("ns-2 Mobility Script Parameters", 1, len(cls.configurations())) + ] def __init__(self, session, object_id): """ diff --git a/daemon/core/session.py b/daemon/core/session.py index b37f05d5..57a007cd 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -1051,31 +1051,23 @@ class SessionConfig(ConfigurableManager, ConfigurableOptions): Session configuration object. """ name = "session" - configuration_maps = {} + options = [ + Configuration(_id="controlnet", _type=ConfigDataTypes.STRING, label="Control Network"), + Configuration(_id="controlnet0", _type=ConfigDataTypes.STRING, label="Control Network 0"), + Configuration(_id="controlnet1", _type=ConfigDataTypes.STRING, label="Control Network 1"), + Configuration(_id="controlnet2", _type=ConfigDataTypes.STRING, label="Control Network 2"), + Configuration(_id="controlnet3", _type=ConfigDataTypes.STRING, label="Control Network 3"), + Configuration(_id="controlnet_updown_script", _type=ConfigDataTypes.STRING, label="Control Network Script"), + Configuration(_id="enablerj45", _type=ConfigDataTypes.BOOL, default="1", options=["On", "Off"], + label="Enable RJ45s"), + Configuration(_id="preservedir", _type=ConfigDataTypes.BOOL, default="0", options=["On", "Off"], + label="Preserve session dir"), + Configuration(_id="enablesdt", _type=ConfigDataTypes.BOOL, default="0", options=["On", "Off"], + label="Enable SDT3D output"), + Configuration(_id="sdturl", _type=ConfigDataTypes.STRING, default=Sdt.DEFAULT_SDT_URL, label="SDT3D URL") + ] config_type = RegisterTlvs.UTILITY.value - @classmethod - def configurations(cls): - return [ - Configuration(_id="controlnet", _type=ConfigDataTypes.STRING, label="Control Network"), - Configuration(_id="controlnet0", _type=ConfigDataTypes.STRING, label="Control Network 0"), - Configuration(_id="controlnet1", _type=ConfigDataTypes.STRING, label="Control Network 1"), - Configuration(_id="controlnet2", _type=ConfigDataTypes.STRING, label="Control Network 2"), - Configuration(_id="controlnet3", _type=ConfigDataTypes.STRING, label="Control Network 3"), - Configuration(_id="controlnet_updown_script", _type=ConfigDataTypes.STRING, label="Control Network Script"), - Configuration(_id="enablerj45", _type=ConfigDataTypes.BOOL, default="1", options=["On", "Off"], - label="Enable RJ45s"), - Configuration(_id="preservedir", _type=ConfigDataTypes.BOOL, default="0", options=["On", "Off"], - label="Preserve session dir"), - Configuration(_id="enablesdt", _type=ConfigDataTypes.BOOL, default="0", options=["On", "Off"], - label="Enable SDT3D output"), - Configuration(_id="sdturl", _type=ConfigDataTypes.STRING, default=Sdt.DEFAULT_SDT_URL, label="SDT3D URL") - ] - - @classmethod - def config_groups(cls): - return "Options:1-%d" % len(cls.configurations()) - def __init__(self): super(SessionConfig, self).__init__() self.set_configs(self.default_values()) @@ -1107,5 +1099,4 @@ class SessionMetaData(ConfigurableManager): The data is not otherwise interpreted or processed. """ name = "metadata" - configuration_maps = {} config_type = RegisterTlvs.UTILITY.value diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index 9f21da78..96e3f6d9 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -1,31 +1,30 @@ import pytest -from core.conf import ConfigurableOptions, ConfigurableManager, ModelManager +from core.conf import ConfigurableManager +from core.conf import ConfigurableOptions from core.conf import Configuration +from core.conf import ModelManager from core.emane.ieee80211abg import EmaneIeee80211abgModel -from core.enumerations import ConfigDataTypes, NodeTypes +from core.enumerations import ConfigDataTypes +from core.enumerations import NodeTypes from core.mobility import BasicRangeModel class TestConfigurableOptions(ConfigurableOptions): name_one = "value1" name_two = "value2" - configuration_maps = {} - - @classmethod - def configurations(cls): - return [ - Configuration( - _id=TestConfigurableOptions.name_one, - _type=ConfigDataTypes.STRING, - label=TestConfigurableOptions.name_one - ), - Configuration( - _id=TestConfigurableOptions.name_two, - _type=ConfigDataTypes.STRING, - label=TestConfigurableOptions.name_two - ) - ] + options = [ + Configuration( + _id=name_one, + _type=ConfigDataTypes.STRING, + label=name_one + ), + Configuration( + _id=name_two, + _type=ConfigDataTypes.STRING, + label=name_two + ) + ] class TestConf: From 0bf9c99910573d7ea31443d729a83d5649dd8226 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 14 Jun 2018 12:50:48 -0700 Subject: [PATCH 47/83] updated service manager to use a dict and throw and error on duplicate service names --- daemon/core/corehandlers.py | 45 +++++++++++++++++++----- daemon/core/service.py | 69 +++++++++++-------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 182ffaaf..74534af5 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1068,15 +1068,42 @@ class CoreHandler(SocketServer.BaseRequestHandler): if opaque is None: type_flag = ConfigFlags.NONE.value data_types = tuple(repeat(ConfigDataTypes.BOOL.value, len(ServiceManager.services))) - values = "|".join(repeat('0', len(ServiceManager.services))) - names = map(lambda x: x._name, ServiceManager.services) - captions = "|".join(names) - possible_values = "" - for s in ServiceManager.services: - if s._custom_needed: - possible_values += '1' - possible_values += '|' - groups = self.session.services.buildgroups(ServiceManager.services) + + # sort groups by name and map services to groups + groups = set() + group_map = {} + for service in ServiceManager.services.itervalues(): + group = service._group + groups.add(group) + group_map.setdefault(group, []).append(service) + groups = sorted(groups, key=lambda x: x.lower()) + + # define tlv values in proper order + captions = [] + possible_values = [] + values = [] + group_strings = [] + start_index = 1 + logger.info("sorted groups: %s", groups) + for group in groups: + services = sorted(group_map[group], key=lambda x: x._name.lower()) + logger.info("sorted services for group(%s): %s", group, services) + end_index = start_index + len(services) - 1 + group_strings.append("%s:%s-%s" % (group, start_index, end_index)) + start_index = start_index + len(services) + for service in services: + captions.append(service._name) + values.append("0") + if service._custom_needed: + possible_values.append("1") + else: + possible_values.append("") + + # format for tlv + captions = "|".join(captions) + possible_values = "|".join(possible_values) + values = "|".join(values) + groups = "|".join(group_strings) # send back the properties for this service else: if not node_id: diff --git a/daemon/core/service.py b/daemon/core/service.py index 78cd9b6d..ffdaf7cc 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -22,7 +22,7 @@ class ServiceManager(object): """ Manages services available for CORE nodes to use. """ - services = [] + services = {} @classmethod def add(cls, service): @@ -32,14 +32,11 @@ class ServiceManager(object): :param CoreService service: service to add :return: nothing """ - insert = 0 - for index, known_service in enumerate(cls.services): - if known_service._group == service._group: - insert = index + 1 - break - logger.info("loading service: %s", service.__name__) - cls.services.insert(insert, service) + name = service._name + if name in cls.services: + raise ValueError("duplicate service being added: %s" % name) + cls.services[name] = service @classmethod def get(cls, name): @@ -50,10 +47,7 @@ class ServiceManager(object): :return: service if it exists, None otherwise :rtype: CoreService """ - for service in cls.services: - if service._name == name: - return service - return None + return cls.services.get(name) @classmethod def add_services(cls, path): @@ -89,7 +83,6 @@ class CoreServices(object): Creates a CoreServices instance. :param core.session.Session session: session this manager is tied to - :return: nothing """ # ConfigurableManager.__init__(self) self.session = session @@ -450,39 +443,6 @@ class CoreServices(object): services.append(s) return services, unknown - def buildgroups(self, servicelist): - """ - Build a string of groups for use in a configuration message given - a list of services. The group list string has the format - "title1:1-5|title2:6-9|10-12", where title is an optional group title - and i-j is a numeric range of value indices; groups are - separated by commas. - - :param list servicelist: service list to build group string from - :return: groups string - :rtype: str - """ - i = 0 - r = "" - lastgroup = "" - for service in servicelist: - i += 1 - group = service._group - if group != lastgroup: - lastgroup = group - # finish previous group - if i > 1: - r += "-%d|" % (i - 1) - # optionally include group title - if group == "": - r += "%d" % i - else: - r += "%s:%d" % (group, i) - # finish the last group list - if i > 0: - r += "-%d" % i - return r - def getservicefile(self, services, node, filename): """ Send a File Message when the GUI has requested a service file. @@ -662,27 +622,40 @@ class CoreService(object): """ # service name should not include spaces _name = "" + # group string allows grouping services together _group = "" + # list name(s) of services that this service depends upon _depends = () keys = ["dirs", "files", "startidx", "cmdup", "cmddown", "cmdval", "meta", "starttime"] + # private, per-node directories required by this service _dirs = () + # config files written by this service _configs = () + # configs = [] + # index used to determine start order with other services _startindex = 0 + # time in seconds after runtime to run startup commands _starttime = "" + # list of startup commands _startup = () + # startup = [] + # list of shutdown commands _shutdown = () + # list of validate commands _validate = () + # metadata associated with this service _meta = "" + # custom configuration text _configtxt = () _custom = False @@ -701,13 +674,13 @@ class CoreService(object): pass @classmethod - def getconfigfilenames(cls, nodenum, services): + def getconfigfilenames(cls, node_id, services): """ Return the tuple of configuration file filenames. This default method returns the cls._configs tuple, but this method may be overriden to provide node-specific filenames that may be based on other services. - :param int nodenum: node id to get config file names for + :param int node_id: node id to get config file names for :param list services: node services :return: class configuration files :rtype: tuple From e80736061fdd391bf5d921bd226bcfe6b4ed7084 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 15 Jun 2018 14:03:27 -0700 Subject: [PATCH 48/83] refactored service interaction use names with a _, and cleanup up some of the CoreServices methods --- daemon/core/corehandlers.py | 133 +++-- daemon/core/coreobj.py | 2 +- daemon/core/emulator/coreemu.py | 27 +- daemon/core/service.py | 684 +++++++++++------------- daemon/core/services/bird.py | 65 +-- daemon/core/services/dockersvc.py | 22 +- daemon/core/services/nrl.py | 129 +++-- daemon/core/services/quagga.py | 138 ++--- daemon/core/services/sdn.py | 52 +- daemon/core/services/security.py | 58 +- daemon/core/services/startup.py | 24 +- daemon/core/services/ucarp.py | 26 +- daemon/core/services/utility.py | 168 +++--- daemon/core/services/xorp.py | 58 +- daemon/core/xml/xmlparser0.py | 23 +- daemon/core/xml/xmlparser1.py | 21 +- daemon/core/xml/xmlwriter0.py | 24 +- daemon/core/xml/xmlwriter1.py | 29 +- daemon/examples/netns/howmanynodes.py | 2 +- daemon/examples/netns/wlanemanetests.py | 3 +- daemon/tests/conftest.py | 14 +- daemon/tests/myservices/sample.py | 32 +- daemon/tests/test_core.py | 12 - daemon/tests/test_gui.py | 3 +- daemon/tests/test_services.py | 18 + daemon/tests/test_xml.py | 70 +++ ns3/examples/ns3wifirandomwalk.py | 6 +- 27 files changed, 958 insertions(+), 885 deletions(-) create mode 100644 daemon/tests/test_services.py diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 74534af5..9a220fe0 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -42,8 +42,8 @@ from core.enumerations import SessionTlvs from core.misc import nodeutils from core.misc import structutils from core.misc import utils -from core.service import CoreService from core.service import ServiceManager +from core.service import ServiceShim class CoreHandler(SocketServer.BaseRequestHandler): @@ -1072,10 +1072,10 @@ class CoreHandler(SocketServer.BaseRequestHandler): # sort groups by name and map services to groups groups = set() group_map = {} - for service in ServiceManager.services.itervalues(): - group = service._group + for service_name in ServiceManager.services.itervalues(): + group = service_name.group groups.add(group) - group_map.setdefault(group, []).append(service) + group_map.setdefault(group, []).append(service_name) groups = sorted(groups, key=lambda x: x.lower()) # define tlv values in proper order @@ -1086,15 +1086,15 @@ class CoreHandler(SocketServer.BaseRequestHandler): start_index = 1 logger.info("sorted groups: %s", groups) for group in groups: - services = sorted(group_map[group], key=lambda x: x._name.lower()) + services = sorted(group_map[group], key=lambda x: x.name.lower()) logger.info("sorted services for group(%s): %s", group, services) end_index = start_index + len(services) - 1 group_strings.append("%s:%s-%s" % (group, start_index, end_index)) - start_index = start_index + len(services) - for service in services: - captions.append(service._name) + start_index += len(services) + for service_name in services: + captions.append(service_name.name) values.append("0") - if service._custom_needed: + if service_name.custom_needed: possible_values.append("1") else: possible_values.append("") @@ -1111,30 +1111,31 @@ class CoreHandler(SocketServer.BaseRequestHandler): node = self.session.get_object(node_id) if node is None: - logger.warn("Request to configure service for unknown node %s", node_id) + logger.warn("request to configure service for unknown node %s", node_id) return replies - servicesstring = opaque.split(':') - services, unknown = self.session.services.servicesfromopaque(opaque, node.objid) - for u in unknown: - logger.warn("Request for unknown service '%s'" % u) + services = ServiceShim.servicesfromopaque(opaque) if not services: return replies + servicesstring = opaque.split(":") if len(servicesstring) == 3: # a file request: e.g. "service:zebra:quagga.conf" - file_data = self.session.services.getservicefile(services, node, servicesstring[2]) + file_name = servicesstring[2] + service_name = services[0] + file_data = self.session.services.getservicefile(service_name, node, file_name, services) self.session.broadcast_file(file_data) # short circuit this request early to avoid returning response below return replies # the first service in the list is the one being configured - svc = services[0] + service_name = services[0] # send back: # dirs, configs, startindex, startup, shutdown, metadata, config type_flag = ConfigFlags.UPDATE.value - data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(svc.keys))) - values = svc.tovaluelist(node, services) + data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(ServiceShim.keys))) + service = self.session.services.getcustomservice(node_id, service_name, default_service=True) + values = ServiceShim.tovaluelist(service, node, services) captions = None possible_values = None groups = None @@ -1173,15 +1174,21 @@ class CoreHandler(SocketServer.BaseRequestHandler): self.session.services.defaultservices[key] = values logger.debug("default services for type %s set to %s", key, values) elif node_id: - # store service customized config in self.customservices[] - services, unknown = self.session.services.servicesfromopaque(opaque, node_id) - for u in unknown: - logger.warn("request for unknown service: %s", u) - + services = ServiceShim.servicesfromopaque(opaque) if services: - svc = services[0] + service_name = services[0] + + # set custom service for node + self.session.services.setcustomservice(node_id, service_name) + + # set custom values for custom service + service = self.session.services.getcustomservice(node_id, service_name) + if not service: + raise ValueError("custom service(%s) for node(%s) does not exist", service_name, node_id) + values = ConfigShim.str_to_dict(values) - self.session.services.setcustomservice(node_id, svc, values) + for name, value in values.iteritems(): + ServiceShim.setvalue(service, name, value) return replies @@ -1324,7 +1331,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): if file_type is not None: if file_type.startswith("service:"): _, service_name = file_type.split(':')[:2] - self.session.add_node_service_file(node_num, service_name, file_name, source_name, data) + self.session.services.setservicefile(node_num, service_name, file_name, data) return () elif file_type.startswith("hook:"): _, state = file_type.split(':')[:2] @@ -1430,7 +1437,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): # TODO: register system for event message handlers, # like confobjs if name.startswith("service:"): - self.session.services_event(event_data) + self.handle_service_event(event_data) handled = True elif name.startswith("mobility:"): self.session.mobility_event(event_data) @@ -1463,6 +1470,72 @@ class CoreHandler(SocketServer.BaseRequestHandler): return () + def handle_service_event(self, event_data): + """ + Handle an Event Message used to start, stop, restart, or validate + a service on a given node. + + :param EventData event_data: event data to handle + :return: nothing + """ + event_type = event_data.event_type + node_id = event_data.node + name = event_data.name + + try: + node = self.session.get_object(node_id) + except KeyError: + logger.warn("ignoring event for service '%s', unknown node '%s'", name, node_id) + return + + fail = "" + unknown = [] + services = ServiceShim.servicesfromopaque(name) + for service_name in services: + service = self.session.services.getcustomservice(node_id, service_name, default_service=True) + if not service: + unknown.append(service_name) + continue + + if event_type == EventTypes.STOP.value or event_type == EventTypes.RESTART.value: + status = self.session.services.stopnodeservice(node, service) + if status != "0": + fail += "Stop %s," % service.name + if event_type == EventTypes.START.value or event_type == EventTypes.RESTART.value: + status = self.session.services.node_service_startup(node, service, services) + if status != "0": + fail += "Start %s(%s)," % service.name + if event_type == EventTypes.PAUSE.value: + status = self.session.services.validatenodeservice(node, service, services) + if status != 0: + fail += "%s," % service.name + if event_type == EventTypes.RECONFIGURE.value: + self.session.services.node_service_reconfigure(node, service, services) + + fail_data = "" + if len(fail) > 0: + fail_data += "Fail:" + fail + unknown_data = "" + num = len(unknown) + if num > 0: + for u in unknown: + unknown_data += u + if num > 1: + unknown_data += ", " + num -= 1 + logger.warn("Event requested for unknown service(s): %s", unknown_data) + unknown_data = "Unknown:" + unknown_data + + event_data = EventData( + node=node_id, + event_type=event_type, + name=name, + data=fail_data + ";" + unknown_data, + time="%s" % time.time() + ) + + self.session.broadcast_event(event_data) + def handle_session_message(self, message): """ Session Message handler @@ -1620,10 +1693,10 @@ class CoreHandler(SocketServer.BaseRequestHandler): # service customizations service_configs = self.session.services.getallconfigs() for node_id, service in service_configs: - opaque = "service:%s" % service._name - data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(CoreService.keys))) + opaque = "service:%s" % service.name + data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(ServiceShim.keys))) node = self.session.get_object(node_id) - values = CoreService.tovaluelist(node, node.services) + values = ServiceShim.tovaluelist(service, node, node.services) config_data = ConfigData( message_type=0, node=node_id, diff --git a/daemon/core/coreobj.py b/daemon/core/coreobj.py index badd22ce..c5031260 100644 --- a/daemon/core/coreobj.py +++ b/daemon/core/coreobj.py @@ -218,7 +218,7 @@ class PyCoreObj(object): if hasattr(self, "services") and len(self.services) != 0: nodeservices = [] for s in self.services: - nodeservices.append(s._name) + nodeservices.append(s.name) services = "|".join(nodeservices) node_data = NodeData( diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 182527ad..7d948552 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -507,8 +507,7 @@ class EmuSession(Session): if _type in [NodeTypes.DEFAULT, NodeTypes.PHYSICAL]: node.type = node_options.model logger.debug("set node type: %s", node.type) - services = "|".join(node_options.services) or None - self.services.addservicestonode(node, node.type, services) + self.services.addservicestonode(node, node.type, node_options.services) # boot nodes if created after runtime, LcxNodes, Physical, and RJ45 are all PyCoreNodes is_boot_node = isinstance(node, PyCoreNode) and not nodeutils.is_node(node, NodeTypes.RJ45) @@ -697,21 +696,6 @@ class EmuSession(Session): state = ":%s" % state self.set_hook(state, file_name, source_name, data) - def add_node_service_file(self, node_id, service_name, file_name, source_name, data): - """ - Add a service file for a node. - - :param int node_id: node to add service file to - :param str service_name: service file to add - :param str file_name: file name to use - :param str source_name: source file - :param str data: file data to save - :return: nothing - """ - # hack to conform with old logic until updated - service_name = ":%s" % service_name - self.services.setservicefile(node_id, service_name, file_name, source_name, data) - def add_node_file(self, node_id, source_name, file_name, data): """ Add a file to a node. @@ -749,15 +733,6 @@ class EmuSession(Session): """ self.event_loop.run() - def services_event(self, event_data): - """ - Handle a service event. - - :param core.data.EventData event_data: event data to handle - :return: - """ - self.services.handleevent(event_data) - def mobility_event(self, event_data): """ Handle a mobility event. diff --git a/daemon/core/service.py b/daemon/core/service.py index ffdaf7cc..21f94a24 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -6,18 +6,110 @@ The CoreServices class handles configuration messages for sending a list of available services to the GUI and for configuring individual services. """ -import time from core import CoreCommandError from core import logger -from core.data import EventData from core.data import FileData -from core.enumerations import EventTypes from core.enumerations import MessageFlags from core.enumerations import RegisterTlvs from core.misc import utils +class ServiceShim(object): + keys = ["dirs", "files", "startidx", "cmdup", "cmddown", "cmdval", "meta", "starttime"] + + @classmethod + def tovaluelist(cls, service, node, services): + """ + Convert service properties into a string list of key=value pairs, + separated by "|". + + :param CoreService service: service to get value list for + :param core.netns.nodes.CoreNode node: node to get value list for + :param list[CoreService] services: services for node + :return: value list string + :rtype: str + """ + valmap = [service.dirs, service.configs, service.startindex, service.startup, + service.shutdown, service.validate, service.meta, service.starttime] + if not service.custom: + # this is always reached due to classmethod + valmap[valmap.index(service.configs)] = service.getconfigfilenames(node.objid, services) + valmap[valmap.index(service.startup)] = service.getstartup(node, services) + vals = map(lambda a, b: "%s=%s" % (a, str(b)), cls.keys, valmap) + return "|".join(vals) + + @classmethod + def fromvaluelist(cls, service, values): + """ + Convert list of values into properties for this instantiated + (customized) service. + + :param CoreService service: service to get value list for + :param dict values: value list to set properties from + :return: nothing + """ + # TODO: support empty value? e.g. override default meta with '' + for key in cls.keys: + try: + cls.setvalue(service, key, values[cls.keys.index(key)]) + except IndexError: + # old config does not need to have new keys + logger.exception("error indexing into key") + + @classmethod + def setvalue(cls, service, key, value): + """ + Set values for this service. + + :param CoreService service: service to get value list for + :param str key: key to set value for + :param value: value of key to set + :return: nothing + """ + if key not in cls.keys: + raise ValueError('key `%s` not in `%s`' % (key, cls.keys)) + # this handles data conversion to int, string, and tuples + if value: + if key == "startidx": + value = int(value) + elif key == "meta": + value = str(value) + else: + value = utils.make_tuple_fromstr(value, str) + + if key == "dirs": + service.dirs = value + elif key == "files": + service.configs = value + elif key == "startidx": + service.startindex = value + elif key == "cmdup": + service.startup = value + elif key == "cmddown": + service.shutdown = value + elif key == "cmdval": + service.validate = value + elif key == "meta": + service.meta = value + elif key == "starttime": + service.starttime = value + + @classmethod + def servicesfromopaque(self, opaque): + """ + Build a list of services from an opaque data string. + + :param str opaque: opaque data string + :return: services + :rtype: list + """ + servicesstring = opaque.split(':') + if servicesstring[0] != "service": + return [] + return servicesstring[1].split(',') + + class ServiceManager(object): """ Manages services available for CORE nodes to use. @@ -33,7 +125,7 @@ class ServiceManager(object): :return: nothing """ logger.info("loading service: %s", service.__name__) - name = service._name + name = service.name if name in cls.services: raise ValueError("duplicate service being added: %s" % name) cls.services[name] = service @@ -45,7 +137,7 @@ class ServiceManager(object): :param str name: name of the service to retrieve :return: service if it exists, None otherwise - :rtype: CoreService + :rtype: CoreService.class """ return cls.services.get(name) @@ -60,7 +152,7 @@ class ServiceManager(object): """ services = utils.load_classes(path, CoreService) for service in services: - if not service._name: + if not service.name: continue service.on_load() cls.add(service) @@ -76,7 +168,6 @@ class CoreServices(object): """ name = "services" config_type = RegisterTlvs.UTILITY.value - _invalid_custom_names = ("core", "api", "emane", "misc", "netns", "phys", "services") def __init__(self, session): """ @@ -84,7 +175,6 @@ class CoreServices(object): :param core.session.Session session: session this manager is tied to """ - # ConfigurableManager.__init__(self) self.session = session # dict of default services tuples, key is node type self.defaultservices = {} @@ -109,99 +199,79 @@ class CoreServices(object): :param service_type: service type to get default services for :return: default services - :rtype: list + :rtype: list[CoreService] """ logger.debug("getting default services for type: %s", service_type) results = [] - if service_type in self.defaultservices: - defaults = self.defaultservices[service_type] - for name in defaults: - logger.debug("checking for service with service manager: %s", name) - service = ServiceManager.get(name) - if not service: - logger.warn("default service %s is unknown", name) - else: - results.append(service) + defaults = self.defaultservices.get(service_type, []) + for name in defaults: + logger.debug("checking for service with service manager: %s", name) + service = ServiceManager.get(name) + if not service: + logger.warn("default service %s is unknown", name) + else: + results.append(service) return results - def getcustomservice(self, object_id, service): + def getcustomservice(self, node_id, service_name, default_service=False): """ Get any custom service configured for the given node that matches the specified service name. If no custom service is found, return the specified service. - :param int object_id: object id to get service from - :param CoreService service: custom service to retrieve + :param int node_id: object id to get service from + :param str service_name: name of service to retrieve + :param bool default_service: True to return default service when custom does not exist, False returns None :return: custom service from the node :rtype: CoreService """ - if object_id in self.customservices: - for s in self.customservices[object_id]: - if s._name == service._name: - return s - return service + node_services = self.customservices.setdefault(node_id, {}) + default = None + if default_service: + default = ServiceManager.get(service_name) + return node_services.get(service_name, default) - def setcustomservice(self, object_id, service, config): + def setcustomservice(self, node_id, service_name): """ Store service customizations in an instantiated service object using a list of values that came from a config message. - :param int object_id: object id to set custom service for - :param class service: service to set - :param dict config: values to - :return: - """ - logger.debug("setting custom service(%s) for node(%s): %s", object_id, service, config) - if service._custom: - s = service - else: - # instantiate the class, for storing config customization - s = service() - - # set custom service configuration - for name, value in config.iteritems(): - s.setvalue(name, value) - - # assume custom service already in dict - if service._custom: - return - - # add the custom service to dict - if object_id in self.customservices: - self.customservices[object_id] += (s,) - else: - self.customservices[object_id] = (s,) - - def addservicestonode(self, node, nodetype, services_str): - """ - Populate the node.service list using (1) the list of services - requested from the services TLV, (2) using any custom service - configuration, or (3) using the default services for this node type. - - :param core.coreobj.PyCoreNode node: node to add services to - :param str nodetype: node type to add services to - :param str services_str: string formatted service list + :param int node_id: object id to set custom service for + :param str service_name: name of service to set :return: nothing """ - if services_str is not None: - logger.info("setting custom services for node(%s)", node.name) - services = services_str.split("|") - for name in services: - s = ServiceManager.get(name) - if s is None: - logger.warn("configured service %s for node %s is unknown", name, node.name) - continue - logger.info("adding service to node(%s): %s", node.name, s._name) - s = self.getcustomservice(node.objid, s) - node.addservice(s) - else: - logger.info("setting default services for node(%s) type (%s)", node.name, nodetype) - services = self.getdefaultservices(nodetype) - for s in services: - logger.info("adding service to node(%s): %s", node.name, s._name) - s = self.getcustomservice(node.objid, s) - node.addservice(s) + logger.debug("setting custom service(%s) for node: %s", node_id, service_name) + service = self.getcustomservice(node_id, service_name) + if not service: + service_class = ServiceManager.get(service_name) + service = service_class() - def getallconfigs(self, use_clsmap=True): + # add the custom service to dict + node_services = self.customservices.setdefault(node_id, {}) + node_services[service.name] = service + + def addservicestonode(self, node, node_type, services=None): + """ + Add services to a node. + + :param core.coreobj.PyCoreNode node: node to add services to + :param str node_type: node type to add services to + :param list[str] services: services to add to node + :return: nothing + """ + if not services: + logger.info("using default services for node(%s) type(%s)", node.name, node_type) + services = self.defaultservices.get(node_type, []) + + logger.info("setting services for node(%s): %s", node.name, services) + for service_name in services: + service = self.getcustomservice(node.objid, service_name, default_service=True) + if not service: + logger.warn("unknown service(%s) for node(%s)", service_name, node.name) + continue + logger.info("adding service to node(%s): %s", node.name, service_name) + node.addservice(service) + + def getallconfigs(self): """ Return (nodenum, service) tuples for all stored configs. Used when reconnecting to a session or opening XML. @@ -211,9 +281,9 @@ class CoreServices(object): :rtype: list """ configs = [] - for nodenum in self.customservices: - for service in self.customservices[nodenum]: - configs.append((nodenum, service)) + for node_id in self.customservices.iterkeys(): + for service in self.customservices[node_id].itervalues(): + configs.append((node_id, service)) return configs def getallfiles(self, service): @@ -226,11 +296,11 @@ class CoreServices(object): """ files = [] - if not service._custom: + if not service.custom: return files - for filename in service._configs: - data = self.getservicefiledata(service, filename) + for filename in service.configs: + data = service.configtxt.get(filename) if data is None: continue files.append((filename, data)) @@ -244,19 +314,19 @@ class CoreServices(object): :param core.netns.vnode.LxcNode node: node to start services on :return: nothing """ - services = sorted(node.services, key=lambda service: service._startindex) + services = sorted(node.services, key=lambda x: x.startindex) use_startup_service = any(map(self.is_startup_service, services)) - for s in services: - if len(str(s._starttime)) > 0: + for service in services: + if len(str(service.starttime)) > 0: try: - t = float(s._starttime) - if t > 0.0: + starttime = float(service.starttime) + if starttime > 0.0: fn = self.bootnodeservice - self.session.event_loop.add_event(t, fn, node, s, services, False) + self.session.event_loop.add_event(starttime, fn, node, service, services, False) continue except ValueError: logger.exception("error converting start time to float") - self.bootnodeservice(node, s, services, use_startup_service) + self.bootnodeservice(node, service, services, use_startup_service) def bootnodeservice(self, node, service, services, use_startup_service): """ @@ -269,12 +339,12 @@ class CoreServices(object): :param bool use_startup_service: flag to use startup services or not :return: nothing """ - if service._custom: + if service.custom: self.bootnodecustomservice(node, service, services, use_startup_service) return - logger.info("starting node(%s) service: %s (%s)", node.name, service._name, service._startindex) - for directory in service._dirs: + logger.info("starting node(%s) service: %s (%s)", node.name, service.name, service.startindex) + for directory in service.dirs: node.privatedir(directory) for filename in service.getconfigfilenames(node.objid, services): @@ -299,18 +369,17 @@ class CoreServices(object): :param bool use_startup_service: flag to use startup services or not :return: nothing """ - logger.info("starting service(%s) %s (%s)(custom)", service, service._name, service._startindex) - for directory in service._dirs: + logger.info("starting node(%s) service(custom): %s (%s)", node.name, service.name, service.startindex) + for directory in service.dirs: node.privatedir(directory) - logger.info("service configurations: %s", service._configs) - for i, filename in enumerate(service._configs): + logger.info("service configurations: %s", service.configs) + for filename in service.configs: logger.info("generating service config: %s", filename) - if len(filename) == 0: - continue - cfg = self.getservicefiledata(service, filename) + cfg = service.configtxt.get(filename) if cfg is None: cfg = service.generateconfig(node, filename, services) + # cfg may have a file:/// url for copying from a file try: if self.copyservicefile(node, filename, cfg): @@ -323,7 +392,7 @@ class CoreServices(object): if use_startup_service and not self.is_startup_service(service): return - for args in service._startup: + for args in service.startup: # TODO: this wait=False can be problematic! node.cmd(args, wait=False) @@ -355,9 +424,9 @@ class CoreServices(object): :param core.netns.vnode.LxcNode node: node to validate services for :return: nothing """ - services = sorted(node.services, key=lambda service: service._startindex) - for s in services: - self.validatenodeservice(node, s, services) + services = sorted(node.services, key=lambda service: service.startindex) + for service in services: + self.validatenodeservice(node, service, services) def validatenodeservice(self, node, service, services): """ @@ -369,22 +438,20 @@ class CoreServices(object): :return: service validation status :rtype: int """ - logger.info("validating service for node (%s): %s (%s)", node.name, service._name, service._startindex) - if service._custom: - validate_cmds = service._validate + logger.info("validating service for node (%s): %s (%s)", node.name, service.name, service.startindex) + if service.custom: + validate_cmds = service.validate else: validate_cmds = service.getvalidate(node, services) status = 0 - # has validate commands - if validate_cmds: - for args in validate_cmds: - logger.info("validating service %s using: %s", service._name, args) - try: - node.check_cmd(args) - except CoreCommandError: - logger.exception("validate command failed") - status = -1 + for args in validate_cmds: + logger.info("validating service %s using: %s", service.name, args) + try: + node.check_cmd(args) + except CoreCommandError: + logger.exception("validate command failed") + status = -1 return status @@ -395,9 +462,9 @@ class CoreServices(object): :param core.netns.nodes.CoreNode node: node to stop services on :return: nothing """ - services = sorted(node.services, key=lambda service: service._startindex) - for s in services: - self.stopnodeservice(node, s) + services = sorted(node.services, key=lambda x: x.startindex) + for service in services: + self.stopnodeservice(node, service) def stopnodeservice(self, node, service): """ @@ -409,68 +476,57 @@ class CoreServices(object): :rtype: str """ status = "0" - if service._shutdown: - for args in service._shutdown: - try: - node.check_cmd(args) - except CoreCommandError: - logger.exception("error running stop command %s", args) - # TODO: determine if its ok to just return the bad exit status - status = "-1" + for args in service.shutdown: + try: + node.check_cmd(args) + except CoreCommandError: + logger.exception("error running stop command %s", args) + # TODO: determine if its ok to just return the bad exit status + status = "-1" return status - def servicesfromopaque(self, opaque, object_id): - """ - Build a list of services from an opaque data string. - - :param str opaque: opaque data string - :param int object_id: object id - :return: services and unknown services lists tuple - :rtype: tuple - """ - services = [] - unknown = [] - servicesstring = opaque.split(':') - if servicesstring[0] != "service": - return [] - servicenames = servicesstring[1].split(',') - for name in servicenames: - s = ServiceManager.get(name) - s = self.getcustomservice(object_id, s) - if s is None: - unknown.append(name) - else: - services.append(s) - return services, unknown - - def getservicefile(self, services, node, filename): + def getservicefile(self, service_name, node, filename, services): """ Send a File Message when the GUI has requested a service file. The file data is either auto-generated or comes from an existing config. - :param list services: service list + :param str service_name: service to get file from :param core.netns.vnode.LxcNode node: node to get service file from :param str filename: file name to retrieve + :param list[str] services: list of services associated with node :return: file message for node """ - svc = services[0] - # get the filename and determine the config file index - if svc._custom: - cfgfiles = svc._configs + # get service to get file from + service = self.getcustomservice(node.objid, service_name, default_service=True) + if not service: + raise ValueError("invalid service: %s", service_name) + + # get service for node + node_services = [] + for service_name in services: + node_service = self.getcustomservice(node.objid, service_name, default_service=True) + if not node_service: + logger.warn("unknown service: %s", service) + continue + node_services.append(node_service) + + # retrieve config files for default/custom service + if service.custom: + config_files = service.configs else: - cfgfiles = svc.getconfigfilenames(node.objid, services) - if filename not in cfgfiles: - logger.warn("Request for unknown file '%s' for service '%s'" % (filename, services[0])) - return None + config_files = service.getconfigfilenames(node.objid, node_services) + + if filename not in config_files: + raise ValueError("unknown service(%s) config file: %s", service_name, filename) # get the file data - data = self.getservicefiledata(svc, filename) + data = service.configtxt.get(filename) if data is None: - data = "%s" % svc.generateconfig(node, filename, services) + data = "%s" % service.generateconfig(node, filename, services) else: data = "%s" % data - filetypestr = "service:%s" % svc._name + filetypestr = "service:%s" % service.name return FileData( message_type=MessageFlags.ADD.value, node=node.objid, @@ -479,141 +535,85 @@ class CoreServices(object): data=data ) - def getservicefiledata(self, service, filename): - """ - Get the customized file data associated with a service. Return None - for invalid filenames or missing file data. - - :param CoreService service: service to get file data from - :param str filename: file name to get data from - :return: file data - """ - try: - i = service._configs.index(filename) - except ValueError: - return None - if i >= len(service._configtxt) or service._configtxt[i] is None: - return None - return service._configtxt[i] - - def setservicefile(self, nodenum, type, filename, srcname, data): + def setservicefile(self, node_id, service_name, filename, data): """ Receive a File Message from the GUI and store the customized file in the service config. The filename must match one from the list of config files in the service. - :param int nodenum: node id to set service file - :param str type: file type to set + :param int node_id: node id to set service file + :param str service_name: service name to set file for :param str filename: file name to set - :param str srcname: source name of file to set :param data: data for file to set :return: nothing """ - if len(type.split(':')) < 2: - logger.warn("Received file type did not contain service info.") - return - if srcname is not None: - raise NotImplementedError - svcid, svcname = type.split(':')[:2] - svc = ServiceManager.get(svcname) - svc = self.getcustomservice(nodenum, svc) + # attempt to set custom service, if needed + self.setcustomservice(node_id, service_name) + + # retrieve custom service + svc = self.getcustomservice(node_id, service_name) if svc is None: - logger.warn("Received filename for unknown service '%s'" % svcname) + logger.warn("received filename for unknown service: %s", service_name) return - cfgfiles = svc._configs + + # validate file being set is valid + cfgfiles = svc.configs if filename not in cfgfiles: - logger.warn("Received unknown file '%s' for service '%s'" % (filename, svcname)) + logger.warn("received unknown file '%s' for service '%s'", filename, service_name) return - i = cfgfiles.index(filename) - configtxtlist = list(svc._configtxt) - numitems = len(configtxtlist) - if numitems < i + 1: - # add empty elements to list to support index assignment - for j in range(1, (i + 2) - numitems): - configtxtlist += None, - configtxtlist[i] = data - svc._configtxt = configtxtlist - def handleevent(self, event_data): + # set custom service file data + svc.configtxt[filename] = data + + def node_service_startup(self, node, service, services): """ - Handle an Event Message used to start, stop, restart, or validate - a service on a given node. + Startup a node service. - :param EventData event_data: event data to handle + :param PyCoreNode node: node to reconfigure service for + :param CoreService service: service to reconfigure + :param list[CoreService] services: node services + :return: status of startup + :rtype: str + """ + + if service.custom: + cmds = service.startup + else: + cmds = service.getstartup(node, services) + + status = "0" + for args in cmds: + try: + node.check_cmd(args) + except CoreCommandError: + logger.exception("error starting command") + status = "-1" + return status + + def node_service_reconfigure(self, node, service, services): + """ + Reconfigure a node service. + + :param PyCoreNode node: node to reconfigure service for + :param CoreService service: service to reconfigure + :param list[CoreService] services: node services :return: nothing """ - event_type = event_data.event_type - node_id = event_data.node - name = event_data.name + if service.custom: + cfgfiles = service.configs + else: + cfgfiles = service.getconfigfilenames(node.objid, services) - try: - node = self.session.get_object(node_id) - except KeyError: - logger.warn("Ignoring event for service '%s', unknown node '%s'", name, node_id) - return + for filename in cfgfiles: + if filename[:7] == "file:///": + # TODO: implement this + raise NotImplementedError - fail = "" - services, unknown = self.servicesfromopaque(name, node_id) - for s in services: - if event_type == EventTypes.STOP.value or event_type == EventTypes.RESTART.value: - status = self.stopnodeservice(node, s) - if status != "0": - fail += "Stop %s," % s._name - if event_type == EventTypes.START.value or event_type == EventTypes.RESTART.value: - if s._custom: - cmds = s._startup - else: - cmds = s.getstartup(node, services) - if len(cmds) > 0: - for args in cmds: - try: - node.check_cmd(args) - except CoreCommandError: - logger.exception("error starting command") - fail += "Start %s(%s)," % (s._name, args) - if event_type == EventTypes.PAUSE.value: - status = self.validatenodeservice(node, s, services) - if status != 0: - fail += "%s," % s._name - if event_type == EventTypes.RECONFIGURE.value: - if s._custom: - cfgfiles = s._configs - else: - cfgfiles = s.getconfigfilenames(node.objid, services) - if len(cfgfiles) > 0: - for filename in cfgfiles: - if filename[:7] == "file:///": - # TODO: implement this - raise NotImplementedError - cfg = self.getservicefiledata(s, filename) - if cfg is None: - cfg = s.generateconfig(node, filename, services) + cfg = service.configtxt.get(filename) + if cfg is None: + cfg = service.generateconfig(node, filename, services) - node.nodefile(filename, cfg) - - fail_data = "" - if len(fail) > 0: - fail_data += "Fail:" + fail - unknown_data = "" - num = len(unknown) - if num > 0: - for u in unknown: - unknown_data += u - if num > 1: - unknown_data += ", " - num -= 1 - logger.warn("Event requested for unknown service(s): %s", unknown_data) - unknown_data = "Unknown:" + unknown_data - - event_data = EventData( - node=node_id, - event_type=event_type, - name=name, - data=fail_data + ";" + unknown_data, - time="%s" % time.time() - ) - - self.session.broadcast_event(event_data) + node.nodefile(filename, cfg) class CoreService(object): @@ -621,45 +621,42 @@ class CoreService(object): Parent class used for defining services. """ # service name should not include spaces - _name = "" + name = None # group string allows grouping services together - _group = "" + group = None # list name(s) of services that this service depends upon - _depends = () - keys = ["dirs", "files", "startidx", "cmdup", "cmddown", "cmdval", "meta", "starttime"] + depends = () # private, per-node directories required by this service - _dirs = () + dirs = () # config files written by this service - _configs = () - # configs = [] + configs = () # index used to determine start order with other services - _startindex = 0 + startindex = 0 # time in seconds after runtime to run startup commands - _starttime = "" + starttime = 0 # list of startup commands - _startup = () - # startup = [] + startup = () # list of shutdown commands - _shutdown = () + shutdown = () # list of validate commands - _validate = () + validate = () # metadata associated with this service - _meta = "" + meta = None # custom configuration text - _configtxt = () - _custom = False - _custom_needed = False + configtxt = {} + custom = False + custom_needed = False def __init__(self): """ @@ -667,7 +664,16 @@ class CoreService(object): against their config. Services are instantiated when a custom configuration is used to override their default parameters. """ - self._custom = True + self.custom = True + self.dirs = self.__class__.dirs + self.configs = self.__class__.configs + self.startindex = self.__class__.startindex + self.startup = self.__class__.startup + self.shutdown = self.__class__.shutdown + self.validate = self.__class__.validate + self.meta = self.__class__.meta + self.starttime = self.__class__.starttime + self.configtxt = self.__class__.configtxt @classmethod def on_load(cls): @@ -685,7 +691,7 @@ class CoreService(object): :return: class configuration files :rtype: tuple """ - return cls._configs + return cls.configs @classmethod def generateconfig(cls, node, filename, services): @@ -716,7 +722,7 @@ class CoreService(object): :return: startup commands :rtype: tuple """ - return cls._startup + return cls.startup @classmethod def getvalidate(cls, node, services): @@ -731,76 +737,4 @@ class CoreService(object): :return: validation commands :rtype: tuple """ - return cls._validate - - @classmethod - def tovaluelist(cls, node, services): - """ - Convert service properties into a string list of key=value pairs, - separated by "|". - - :param core.netns.nodes.CoreNode node: node to get value list for - :param list services: services for node - :return: value list string - :rtype: str - """ - valmap = [cls._dirs, cls._configs, cls._startindex, cls._startup, - cls._shutdown, cls._validate, cls._meta, cls._starttime] - if not cls._custom: - # this is always reached due to classmethod - valmap[valmap.index(cls._configs)] = cls.getconfigfilenames(node.objid, services) - valmap[valmap.index(cls._startup)] = cls.getstartup(node, services) - vals = map(lambda a, b: "%s=%s" % (a, str(b)), cls.keys, valmap) - return "|".join(vals) - - def fromvaluelist(self, values): - """ - Convert list of values into properties for this instantiated - (customized) service. - - :param list values: value list to set properties from - :return: nothing - """ - # TODO: support empty value? e.g. override default meta with '' - for key in self.keys: - try: - self.setvalue(key, values[self.keys.index(key)]) - except IndexError: - # old config does not need to have new keys - logger.exception("error indexing into key") - - def setvalue(self, key, value): - """ - Set values for this service. - - :param str key: key to set value for - :param value: value of key to set - :return: nothing - """ - if key not in self.keys: - raise ValueError('key `%s` not in `%s`' % (key, self.keys)) - # this handles data conversion to int, string, and tuples - if value: - if key == "startidx": - value = int(value) - elif key == "meta": - value = str(value) - else: - value = utils.make_tuple_fromstr(value, str) - - if key == "dirs": - self._dirs = value - elif key == "files": - self._configs = value - elif key == "startidx": - self._startindex = value - elif key == "cmdup": - self._startup = value - elif key == "cmddown": - self._shutdown = value - elif key == "cmdval": - self._validate = value - elif key == "meta": - self._meta = value - elif key == "starttime": - self._starttime = value + return cls.validate diff --git a/daemon/core/services/bird.py b/daemon/core/services/bird.py index 097053c4..d56bc14c 100644 --- a/daemon/core/services/bird.py +++ b/daemon/core/services/bird.py @@ -9,22 +9,22 @@ class Bird(CoreService): """ Bird router support """ - _name = "bird" - _group = "BIRD" - _depends = () - _dirs = ("/etc/bird",) - _configs = ("/etc/bird/bird.conf",) - _startindex = 35 - _startup = ("bird -c %s" % (_configs[0]),) - _shutdown = ("killall bird",) - _validate = ("pidof bird",) + name = "bird" + group = "BIRD" + depends = () + dirs = ("/etc/bird",) + configs = ("/etc/bird/bird.conf",) + startindex = 35 + startup = ("bird -c %s" % (configs[0]),) + shutdown = ("killall bird",) + validate = ("pidof bird",) @classmethod def generateconfig(cls, node, filename, services): """ Return the bird.conf file contents. """ - if filename == cls._configs[0]: + if filename == cls.configs[0]: return cls.generateBirdConf(node, services) else: raise ValueError @@ -35,7 +35,7 @@ class Bird(CoreService): Helper to return the first IPv4 address of a node as its router ID. """ for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue for a in ifc.addrlist: if a.find(".") >= 0: @@ -73,11 +73,11 @@ protocol device { scan time 10; # Scan interfaces every 10 seconds } -""" % (cls._name, cls.routerid(node)) +""" % (cls.name, cls.routerid(node)) # Generate protocol specific configurations for s in services: - if cls._name not in s._depends: + if cls.name not in s.depends: continue cfg += s.generatebirdconfig(node) @@ -90,15 +90,15 @@ class BirdService(CoreService): common to Bird's routing daemons. """ - _name = None - _group = "BIRD" - _depends = ("bird",) - _dirs = () - _configs = () - _startindex = 40 - _startup = () - _shutdown = () - _meta = "The config file for this service can be found in the bird service." + name = None + group = "BIRD" + depends = ("bird",) + dirs = () + configs = () + startindex = 40 + startup = () + shutdown = () + meta = "The config file for this service can be found in the bird service." @classmethod def generatebirdconfig(cls, node): @@ -106,14 +106,15 @@ class BirdService(CoreService): @classmethod def generatebirdifcconfig(cls, node): - ''' Use only bare interfaces descriptions in generated protocol + """ + Use only bare interfaces descriptions in generated protocol configurations. This has the slight advantage of being the same everywhere. - ''' + """ cfg = "" for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue cfg += ' interface "%s";\n' % ifc.name @@ -125,8 +126,8 @@ class BirdBgp(BirdService): BGP BIRD Service (configuration generation) """ - _name = "BIRD_BGP" - _custom_needed = True + name = "BIRD_BGP" + custom_needed = True @classmethod def generatebirdconfig(cls, node): @@ -156,7 +157,7 @@ class BirdOspf(BirdService): OSPF BIRD Service (configuration generation) """ - _name = "BIRD_OSPFv2" + name = "BIRD_OSPFv2" @classmethod def generatebirdconfig(cls, node): @@ -181,7 +182,7 @@ class BirdRadv(BirdService): RADV BIRD Service (configuration generation) """ - _name = "BIRD_RADV" + name = "BIRD_RADV" @classmethod def generatebirdconfig(cls, node): @@ -209,7 +210,7 @@ class BirdRip(BirdService): RIP BIRD Service (configuration generation) """ - _name = "BIRD_RIP" + name = "BIRD_RIP" @classmethod def generatebirdconfig(cls, node): @@ -231,8 +232,8 @@ class BirdStatic(BirdService): Static Bird Service (configuration generation) """ - _name = "BIRD_static" - _custom_needed = True + name = "BIRD_static" + custom_needed = True @classmethod def generatebirdconfig(cls, node): diff --git a/daemon/core/services/dockersvc.py b/daemon/core/services/dockersvc.py index 1733f04b..9699465c 100644 --- a/daemon/core/services/dockersvc.py +++ b/daemon/core/services/dockersvc.py @@ -112,16 +112,16 @@ class DockerService(CoreService): This is a service which will allow running docker containers in a CORE node. """ - _name = "Docker" - _group = "Docker" - _depends = () - _dirs = ('/var/lib/docker/containers/', '/run/shm', '/run/resolvconf',) - _configs = ('docker.sh',) - _startindex = 50 - _startup = ('sh docker.sh',) - _shutdown = ('service docker stop',) + name = "Docker" + group = "Docker" + depends = () + dirs = ('/var/lib/docker/containers/', '/run/shm', '/run/resolvconf',) + configs = ('docker.sh',) + startindex = 50 + startup = ('sh docker.sh',) + shutdown = ('service docker stop',) # Container image to start - _image = "" + image = "" @classmethod def generateconfig(cls, node, filename, services): @@ -139,7 +139,7 @@ class DockerService(CoreService): # distros may just be docker cfg += 'service docker start\n' cfg += "# you could add a command to start a image here eg:\n" - if not cls._image: + if not cls.image: cfg += "# docker run -d --net host --name coreDock \n" else: cfg += """\ @@ -150,7 +150,7 @@ until [ $result -eq 0 ]; do # this is to alleviate contention to docker's SQLite database sleep 0.3 done -""" % (cls._image,) +""" % (cls.image,) return cfg @classmethod diff --git a/daemon/core/services/nrl.py b/daemon/core/services/nrl.py index 5fe74ecd..07152404 100644 --- a/daemon/core/services/nrl.py +++ b/daemon/core/services/nrl.py @@ -13,14 +13,14 @@ class NrlService(CoreService): Parent class for NRL services. Defines properties and methods common to NRL's routing daemons. """"" - _name = None - _group = "ProtoSvc" - _depends = () - _dirs = () - _configs = () - _startindex = 45 - _startup = () - _shutdown = () + name = None + group = "ProtoSvc" + depends = () + dirs = () + configs = () + startindex = 45 + startup = () + shutdown = () @classmethod def generateconfig(cls, node, filename, services): @@ -34,7 +34,7 @@ class NrlService(CoreService): interface's prefix length, so e.g. '/32' can turn into '/24'. """ for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue for a in ifc.addrlist: if a.find(".") >= 0: @@ -46,12 +46,12 @@ class NrlService(CoreService): class MgenSinkService(NrlService): - _name = "MGEN_Sink" - _configs = ("sink.mgen",) - _startindex = 5 - _startup = ("mgen input sink.mgen",) - _validate = ("pidof mgen",) - _shutdown = ("killall mgen",) + name = "MGEN_Sink" + configs = ("sink.mgen",) + startindex = 5 + startup = ("mgen input sink.mgen",) + validate = ("pidof mgen",) + shutdown = ("killall mgen",) @classmethod def generateconfig(cls, node, filename, services): @@ -63,7 +63,7 @@ class MgenSinkService(NrlService): @classmethod def getstartup(cls, node, services): - cmd = cls._startup[0] + cmd = cls.startup[0] cmd += " output /tmp/mgen_%s.log" % node.name return cmd, @@ -72,27 +72,26 @@ class NrlNhdp(NrlService): """ NeighborHood Discovery Protocol for MANET networks. """ - _name = "NHDP" - _startup = ("nrlnhdp",) - _shutdown = ("killall nrlnhdp",) - _validate = ("pidof nrlnhdp",) + name = "NHDP" + startup = ("nrlnhdp",) + shutdown = ("killall nrlnhdp",) + validate = ("pidof nrlnhdp",) @classmethod def getstartup(cls, node, services): """ Generate the appropriate command-line based on node interfaces. """ - cmd = cls._startup[0] + cmd = cls.startup[0] cmd += " -l /var/log/nrlnhdp.log" cmd += " -rpipe %s_nhdp" % node.name - servicenames = map(lambda x: x._name, services) + servicenames = map(lambda x: x.name, services) if "SMF" in servicenames: cmd += " -flooding ecds" cmd += " -smfClient %s_smf" % node.name - netifs = filter(lambda x: not getattr(x, 'control', False), \ - node.netifs()) + netifs = filter(lambda x: not getattr(x, 'control', False), node.netifs()) if len(netifs) > 0: interfacenames = map(lambda x: x.name, netifs) cmd += " -i " @@ -105,11 +104,11 @@ class NrlSmf(NrlService): """ Simplified Multicast Forwarding for MANET networks. """ - _name = "SMF" - _startup = ("sh startsmf.sh",) - _shutdown = ("killall nrlsmf",) - _validate = ("pidof nrlsmf",) - _configs = ("startsmf.sh",) + name = "SMF" + startup = ("sh startsmf.sh",) + shutdown = ("killall nrlsmf",) + validate = ("pidof nrlsmf",) + configs = ("startsmf.sh",) @classmethod def generateconfig(cls, node, filename, services): @@ -123,7 +122,7 @@ class NrlSmf(NrlService): comments = "" cmd = "nrlsmf instance %s_smf" % node.name - servicenames = map(lambda x: x._name, services) + servicenames = map(lambda x: x.name, services) netifs = filter(lambda x: not getattr(x, 'control', False), node.netifs()) if len(netifs) == 0: return "" @@ -156,17 +155,17 @@ class NrlOlsr(NrlService): """ Optimized Link State Routing protocol for MANET networks. """ - _name = "OLSR" - _startup = ("nrlolsrd",) - _shutdown = ("killall nrlolsrd",) - _validate = ("pidof nrlolsrd",) + name = "OLSR" + startup = ("nrlolsrd",) + shutdown = ("killall nrlolsrd",) + validate = ("pidof nrlolsrd",) @classmethod def getstartup(cls, node, services): """ Generate the appropriate command-line based on node interfaces. """ - cmd = cls._startup[0] + cmd = cls.startup[0] # are multiple interfaces supported? No. netifs = list(node.netifs()) if len(netifs) > 0: @@ -175,7 +174,7 @@ class NrlOlsr(NrlService): cmd += " -l /var/log/nrlolsrd.log" cmd += " -rpipe %s_olsr" % node.name - servicenames = map(lambda x: x._name, services) + servicenames = map(lambda x: x.name, services) if "SMF" in servicenames and not "NHDP" in servicenames: cmd += " -flooding s-mpr" cmd += " -smfClient %s_smf" % node.name @@ -189,21 +188,21 @@ class NrlOlsrv2(NrlService): """ Optimized Link State Routing protocol version 2 for MANET networks. """ - _name = "OLSRv2" - _startup = ("nrlolsrv2",) - _shutdown = ("killall nrlolsrv2",) - _validate = ("pidof nrlolsrv2",) + name = "OLSRv2" + startup = ("nrlolsrv2",) + shutdown = ("killall nrlolsrv2",) + validate = ("pidof nrlolsrv2",) @classmethod def getstartup(cls, node, services): """ Generate the appropriate command-line based on node interfaces. """ - cmd = cls._startup[0] + cmd = cls.startup[0] cmd += " -l /var/log/nrlolsrv2.log" cmd += " -rpipe %s_olsrv2" % node.name - servicenames = map(lambda x: x._name, services) + servicenames = map(lambda x: x.name, services) if "SMF" in servicenames: cmd += " -flooding ecds" cmd += " -smfClient %s_smf" % node.name @@ -223,19 +222,19 @@ class OlsrOrg(NrlService): """ Optimized Link State Routing protocol from olsr.org for MANET networks. """ - _name = "OLSRORG" - _configs = ("/etc/olsrd/olsrd.conf",) - _dirs = ("/etc/olsrd",) - _startup = ("olsrd",) - _shutdown = ("killall olsrd",) - _validate = ("pidof olsrd",) + name = "OLSRORG" + configs = ("/etc/olsrd/olsrd.conf",) + dirs = ("/etc/olsrd",) + startup = ("olsrd",) + shutdown = ("killall olsrd",) + validate = ("pidof olsrd",) @classmethod def getstartup(cls, node, services): """ Generate the appropriate command-line based on node interfaces. """ - cmd = cls._startup[0] + cmd = cls.startup[0] netifs = filter(lambda x: not getattr(x, 'control', False), node.netifs()) if len(netifs) > 0: interfacenames = map(lambda x: x.name, netifs) @@ -572,24 +571,24 @@ class MgenActor(NrlService): """ # a unique name is required, without spaces - _name = "MgenActor" + name = "MgenActor" # you can create your own group here - _group = "ProtoSvc" + group = "ProtoSvc" # list of other services this service depends on - _depends = () + depends = () # per-node directories - _dirs = () + dirs = () # generated files (without a full path this file goes in the node's dir, # e.g. /tmp/pycore.12345/n1.conf/) - _configs = ('start_mgen_actor.sh',) + configs = ('start_mgen_actor.sh',) # this controls the starting order vs other enabled services - _startindex = 50 + startindex = 50 # list of startup commands, also may be generated during startup - _startup = ("sh start_mgen_actor.sh",) + startup = ("sh start_mgen_actor.sh",) # list of validation commands - _validate = ("pidof mgen",) + validate = ("pidof mgen",) # list of shutdown commands - _shutdown = ("killall mgen",) + shutdown = ("killall mgen",) @classmethod def generateconfig(cls, node, filename, services): @@ -603,7 +602,7 @@ class MgenActor(NrlService): comments = "" cmd = "mgenBasicActor.py -n %s -a 0.0.0.0" % node.name - servicenames = map(lambda x: x._name, services) + servicenames = map(lambda x: x.name, services) netifs = filter(lambda x: not getattr(x, 'control', False), node.netifs()) if len(netifs) == 0: return "" @@ -616,12 +615,12 @@ class Arouted(NrlService): """ Adaptive Routing """ - _name = "arouted" - _configs = ("startarouted.sh",) - _startindex = NrlService._startindex + 10 - _startup = ("sh startarouted.sh",) - _shutdown = ("pkill arouted",) - _validate = ("pidof arouted",) + name = "arouted" + configs = ("startarouted.sh",) + startindex = NrlService.startindex + 10 + startup = ("sh startarouted.sh",) + shutdown = ("pkill arouted",) + validate = ("pidof arouted",) @classmethod def generateconfig(cls, node, filename, services): diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index 73f797a5..791d892d 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -10,33 +10,33 @@ from core.service import CoreService class Zebra(CoreService): - _name = "zebra" - _group = "Quagga" - _dirs = ("/usr/local/etc/quagga", "/var/run/quagga") - _configs = ( + name = "zebra" + group = "Quagga" + dirs = ("/usr/local/etc/quagga", "/var/run/quagga") + configs = ( "/usr/local/etc/quagga/Quagga.conf", "quaggaboot.sh", "/usr/local/etc/quagga/vtysh.conf" ) - _startindex = 35 - _startup = ("sh quaggaboot.sh zebra",) - _shutdown = ("killall zebra",) - _validate = ("pidof zebra",) + startindex = 35 + startup = ("sh quaggaboot.sh zebra",) + shutdown = ("killall zebra",) + validate = ("pidof zebra",) @classmethod def generateconfig(cls, node, filename, services): """ Return the Quagga.conf or quaggaboot.sh file contents. """ - if filename == cls._configs[0]: + if filename == cls.configs[0]: return cls.generateQuaggaConf(node, services) - elif filename == cls._configs[1]: + elif filename == cls.configs[1]: return cls.generateQuaggaBoot(node, services) - elif filename == cls._configs[2]: + elif filename == cls.configs[2]: return cls.generateVtyshConf(node, services) else: raise ValueError("file name (%s) is not a known configuration: %s", - filename, cls._configs) + filename, cls.configs) @classmethod def generateVtyshConf(cls, node, services): @@ -67,12 +67,12 @@ class Zebra(CoreService): want_ipv4 = False want_ipv6 = False for s in services: - if cls._name not in s._depends: + if cls.name not in s.depends: continue ifccfg = s.generatequaggaifcconfig(node, ifc) - if s._ipv4_routing: + if s.ipv4_routing: want_ipv4 = True - if s._ipv6_routing: + if s.ipv6_routing: want_ipv6 = True cfgv6 += ifccfg else: @@ -93,7 +93,7 @@ class Zebra(CoreService): cfg += "!\n" for s in services: - if cls._name not in s._depends: + if cls.name not in s.depends: continue cfg += s.generatequaggaconfig(node) return cfg @@ -212,7 +212,7 @@ if [ "$1" != "zebra" ]; then fi confcheck bootquagga -""" % (cls._configs[0], quagga_sbin_search, quagga_bin_search, constants.QUAGGA_STATE_DIR) +""" % (cls.configs[0], quagga_sbin_search, quagga_bin_search, constants.QUAGGA_STATE_DIR) class QuaggaService(CoreService): @@ -220,18 +220,18 @@ class QuaggaService(CoreService): Parent class for Quagga services. Defines properties and methods common to Quagga's routing daemons. """ - _name = None - _group = "Quagga" - _depends = ("zebra",) - _dirs = () - _configs = () - _startindex = 40 - _startup = () - _shutdown = () - _meta = "The config file for this service can be found in the Zebra service." + name = None + group = "Quagga" + depends = ("zebra",) + dirs = () + configs = () + startindex = 40 + startup = () + shutdown = () + meta = "The config file for this service can be found in the Zebra service." - _ipv4_routing = False - _ipv6_routing = False + ipv4_routing = False + ipv6_routing = False @staticmethod def routerid(node): @@ -239,7 +239,7 @@ class QuaggaService(CoreService): Helper to return the first IPv4 address of a node as its router ID. """ for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue for a in ifc.addrlist: if a.find(".") >= 0: @@ -280,11 +280,11 @@ class Ospfv2(QuaggaService): not build its own configuration file but has hooks for adding to the unified Quagga.conf file. """ - _name = "OSPFv2" - _startup = () - _shutdown = ("killall ospfd",) - _validate = ("pidof ospfd",) - _ipv4_routing = True + name = "OSPFv2" + startup = () + shutdown = ("killall ospfd",) + validate = ("pidof ospfd",) + ipv4_routing = True @staticmethod def mtucheck(ifc): @@ -355,12 +355,12 @@ class Ospfv3(QuaggaService): not build its own configuration file but has hooks for adding to the unified Quagga.conf file. """ - _name = "OSPFv3" - _startup = () - _shutdown = ("killall ospf6d",) - _validate = ("pidof ospf6d",) - _ipv4_routing = True - _ipv6_routing = True + name = "OSPFv3" + startup = () + shutdown = ("killall ospf6d",) + validate = ("pidof ospf6d",) + ipv4_routing = True + ipv6_routing = True @staticmethod def minmtu(ifc): @@ -436,8 +436,8 @@ class Ospfv3mdr(Ospfv3): configuration file but has hooks for adding to the unified Quagga.conf file. """ - _name = "OSPFv3MDR" - _ipv4_routing = True + name = "OSPFv3MDR" + ipv4_routing = True @classmethod def generatequaggaifcconfig(cls, node, ifc): @@ -464,13 +464,13 @@ class Bgp(QuaggaService): Peers must be manually configured, with a full mesh for those having the same AS number. """ - _name = "BGP" - _startup = () - _shutdown = ("killall bgpd",) - _validate = ("pidof bgpd",) - _custom_needed = True - _ipv4_routing = True - _ipv6_routing = True + name = "BGP" + startup = () + shutdown = ("killall bgpd",) + validate = ("pidof bgpd",) + custom_needed = True + ipv4_routing = True + ipv6_routing = True @classmethod def generatequaggaconfig(cls, node): @@ -489,11 +489,11 @@ class Rip(QuaggaService): """ The RIP service provides IPv4 routing for wired networks. """ - _name = "RIP" - _startup = () - _shutdown = ("killall ripd",) - _validate = ("pidof ripd",) - _ipv4_routing = True + name = "RIP" + startup = () + shutdown = ("killall ripd",) + validate = ("pidof ripd",) + ipv4_routing = True @classmethod def generatequaggaconfig(cls, node): @@ -512,11 +512,11 @@ class Ripng(QuaggaService): """ The RIP NG service provides IPv6 routing for wired networks. """ - _name = "RIPNG" - _startup = () - _shutdown = ("killall ripngd",) - _validate = ("pidof ripngd",) - _ipv6_routing = True + name = "RIPNG" + startup = () + shutdown = ("killall ripngd",) + validate = ("pidof ripngd",) + ipv6_routing = True @classmethod def generatequaggaconfig(cls, node): @@ -536,11 +536,11 @@ class Babel(QuaggaService): The Babel service provides a loop-avoiding distance-vector routing protocol for IPv6 and IPv4 with fast convergence properties. """ - _name = "Babel" - _startup = () - _shutdown = ("killall babeld",) - _validate = ("pidof babeld",) - _ipv6_routing = True + name = "Babel" + startup = () + shutdown = ("killall babeld",) + validate = ("pidof babeld",) + ipv6_routing = True @classmethod def generatequaggaconfig(cls, node): @@ -565,11 +565,11 @@ class Xpimd(QuaggaService): """ PIM multicast routing based on XORP. """ - _name = 'Xpimd' - _startup = () - _shutdown = ('killall xpimd',) - _validate = ('pidof xpimd',) - _ipv4_routing = True + name = 'Xpimd' + startup = () + shutdown = ('killall xpimd',) + validate = ('pidof xpimd',) + ipv4_routing = True @classmethod def generatequaggaconfig(cls, node): diff --git a/daemon/core/services/sdn.py b/daemon/core/services/sdn.py index 9d8a26ba..33f50109 100644 --- a/daemon/core/services/sdn.py +++ b/daemon/core/services/sdn.py @@ -11,14 +11,14 @@ class SdnService(CoreService): """ Parent class for SDN services. """ - _name = None - _group = "SDN" - _depends = () - _dirs = () - _configs = () - _startindex = 50 - _startup = () - _shutdown = () + name = None + group = "SDN" + depends = () + dirs = () + configs = () + startindex = 50 + startup = () + shutdown = () @classmethod def generateconfig(cls, node, filename, services): @@ -26,27 +26,27 @@ class SdnService(CoreService): class OvsService(SdnService): - _name = "OvsService" - _group = "SDN" - _depends = () - _dirs = ("/etc/openvswitch", "/var/run/openvswitch", "/var/log/openvswitch") - _configs = ('OvsService.sh',) - _startindex = 50 - _startup = ('sh OvsService.sh',) - _shutdown = ('killall ovs-vswitchd', 'killall ovsdb-server') + name = "OvsService" + group = "SDN" + depends = () + dirs = ("/etc/openvswitch", "/var/run/openvswitch", "/var/log/openvswitch") + configs = ('OvsService.sh',) + startindex = 50 + startup = ('sh OvsService.sh',) + shutdown = ('killall ovs-vswitchd', 'killall ovsdb-server') @classmethod def generateconfig(cls, node, filename, services): # Check whether the node is running zebra has_zebra = 0 for s in services: - if s._name == "zebra": + if s.name == "zebra": has_zebra = 1 # Check whether the node is running an SDN controller has_sdn_ctrlr = 0 for s in services: - if s._name == "ryuService": + if s.name == "ryuService": has_sdn_ctrlr = 1 cfg = "#!/bin/sh\n" @@ -100,14 +100,14 @@ class OvsService(SdnService): class RyuService(SdnService): - _name = "ryuService" - _group = "SDN" - _depends = () - _dirs = () - _configs = ('ryuService.sh',) - _startindex = 50 - _startup = ('sh ryuService.sh',) - _shutdown = ('killall ryu-manager',) + name = "ryuService" + group = "SDN" + depends = () + dirs = () + configs = ('ryuService.sh',) + startindex = 50 + startup = ('sh ryuService.sh',) + shutdown = ('killall ryu-manager',) @classmethod def generateconfig(cls, node, filename, services): diff --git a/daemon/core/services/security.py b/daemon/core/services/security.py index 34a37624..a8318e42 100644 --- a/daemon/core/services/security.py +++ b/daemon/core/services/security.py @@ -9,14 +9,14 @@ from core.service import CoreService class VPNClient(CoreService): - _name = "VPNClient" - _group = "Security" - _configs = ('vpnclient.sh',) - _startindex = 60 - _startup = ('sh vpnclient.sh',) - _shutdown = ("killall openvpn",) - _validate = ("pidof openvpn",) - _custom_needed = True + name = "VPNClient" + group = "Security" + configs = ('vpnclient.sh',) + startindex = 60 + startup = ('sh vpnclient.sh',) + shutdown = ("killall openvpn",) + validate = ("pidof openvpn",) + custom_needed = True @classmethod def generateconfig(cls, node, filename, services): @@ -36,14 +36,14 @@ class VPNClient(CoreService): class VPNServer(CoreService): - _name = "VPNServer" - _group = "Security" - _configs = ('vpnserver.sh',) - _startindex = 50 - _startup = ('sh vpnserver.sh',) - _shutdown = ("killall openvpn",) - _validate = ("pidof openvpn",) - _custom_needed = True + name = "VPNServer" + group = "Security" + configs = ('vpnserver.sh',) + startindex = 50 + startup = ('sh vpnserver.sh',) + shutdown = ("killall openvpn",) + validate = ("pidof openvpn",) + custom_needed = True @classmethod def generateconfig(cls, node, filename, services): @@ -64,13 +64,13 @@ class VPNServer(CoreService): class IPsec(CoreService): - _name = "IPsec" - _group = "Security" - _configs = ('ipsec.sh',) - _startindex = 60 - _startup = ('sh ipsec.sh',) - _shutdown = ("killall racoon",) - _custom_needed = True + name = "IPsec" + group = "Security" + configs = ('ipsec.sh',) + startindex = 60 + startup = ('sh ipsec.sh',) + shutdown = ("killall racoon",) + custom_needed = True @classmethod def generateconfig(cls, node, filename, services): @@ -92,12 +92,12 @@ class IPsec(CoreService): class Firewall(CoreService): - _name = "Firewall" - _group = "Security" - _configs = ('firewall.sh',) - _startindex = 20 - _startup = ('sh firewall.sh',) - _custom_needed = True + name = "Firewall" + group = "Security" + configs = ('firewall.sh',) + startindex = 20 + startup = ('sh firewall.sh',) + custom_needed = True @classmethod def generateconfig(cls, node, filename, services): diff --git a/daemon/core/services/startup.py b/daemon/core/services/startup.py index a26cd6f4..6346c541 100644 --- a/daemon/core/services/startup.py +++ b/daemon/core/services/startup.py @@ -8,15 +8,15 @@ class Startup(CoreService): """ A CORE service to start other services in order, serially """ - _name = 'startup' - _group = 'Utility' - _depends = () - _dirs = () - _configs = ('startup.sh',) - _startindex = maxint - _startup = ('sh startup.sh',) - _shutdown = () - _validate = () + name = 'startup' + group = 'Utility' + depends = () + dirs = () + configs = ('startup.sh',) + startindex = maxint + startup = ('sh startup.sh',) + shutdown = () + validate = () @staticmethod def is_startup_service(s): @@ -24,13 +24,13 @@ class Startup(CoreService): @classmethod def generateconfig(cls, node, filename, services): - if filename != cls._configs[0]: + if filename != cls.configs[0]: return '' script = '#!/bin/sh\n' \ '# auto-generated by Startup (startup.py)\n\n' \ 'exec > startup.log 2>&1\n\n' - for s in sorted(services, key=lambda x: x._startindex): - if cls.is_startup_service(s) or len(str(s._starttime)) > 0: + for s in sorted(services, key=lambda x: x.startindex): + if cls.is_startup_service(s) or len(str(s.starttime)) > 0: continue start = '\n'.join(s.getstartup(node, services)) if start: diff --git a/daemon/core/services/ucarp.py b/daemon/core/services/ucarp.py index ee73e7ca..5009ea6d 100644 --- a/daemon/core/services/ucarp.py +++ b/daemon/core/services/ucarp.py @@ -8,29 +8,29 @@ UCARP_ETC = "/usr/local/etc/ucarp" class Ucarp(CoreService): - _name = "ucarp" - _group = "Utility" - _depends = ( ) - _dirs = (UCARP_ETC,) - _configs = ( + name = "ucarp" + group = "Utility" + depends = ( ) + dirs = (UCARP_ETC,) + configs = ( UCARP_ETC + "/default.sh", UCARP_ETC + "/default-up.sh", UCARP_ETC + "/default-down.sh", "ucarpboot.sh",) - _startindex = 65 - _startup = ("sh ucarpboot.sh",) - _shutdown = ("killall ucarp",) - _validate = ("pidof ucarp",) + startindex = 65 + startup = ("sh ucarpboot.sh",) + shutdown = ("killall ucarp",) + validate = ("pidof ucarp",) @classmethod def generateconfig(cls, node, filename, services): """ Return the default file contents """ - if filename == cls._configs[0]: + if filename == cls.configs[0]: return cls.generateUcarpConf(node, services) - elif filename == cls._configs[1]: + elif filename == cls.configs[1]: return cls.generateVipUp(node, services) - elif filename == cls._configs[2]: + elif filename == cls.configs[2]: return cls.generateVipDown(node, services) - elif filename == cls._configs[3]: + elif filename == cls.configs[3]: return cls.generateUcarpBoot(node, services) else: raise ValueError diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index e1495ca9..2bc11893 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -16,14 +16,14 @@ class UtilService(CoreService): """ Parent class for utility services. """ - _name = None - _group = "Utility" - _depends = () - _dirs = () - _configs = () - _startindex = 80 - _startup = () - _shutdown = () + name = None + group = "Utility" + depends = () + dirs = () + configs = () + startindex = 80 + startup = () + shutdown = () @classmethod def generateconfig(cls, node, filename, services): @@ -31,10 +31,10 @@ class UtilService(CoreService): class IPForwardService(UtilService): - _name = "IPForward" - _configs = ("ipforward.sh",) - _startindex = 5 - _startup = ("sh ipforward.sh",) + name = "IPForward" + configs = ("ipforward.sh",) + startindex = 5 + startup = ("sh ipforward.sh",) @classmethod def generateconfig(cls, node, filename, services): @@ -67,9 +67,9 @@ class IPForwardService(UtilService): class DefaultRouteService(UtilService): - _name = "DefaultRoute" - _configs = ("defaultroute.sh",) - _startup = ("sh defaultroute.sh",) + name = "DefaultRoute" + configs = ("defaultroute.sh",) + startup = ("sh defaultroute.sh",) @classmethod def generateconfig(cls, node, filename, services): @@ -101,9 +101,9 @@ class DefaultRouteService(UtilService): class DefaultMulticastRouteService(UtilService): - _name = "DefaultMulticastRoute" - _configs = ("defaultmroute.sh",) - _startup = ("sh defaultmroute.sh",) + name = "DefaultMulticastRoute" + configs = ("defaultmroute.sh",) + startup = ("sh defaultmroute.sh",) @classmethod def generateconfig(cls, node, filename, services): @@ -113,7 +113,7 @@ class DefaultMulticastRouteService(UtilService): cfg += "as needed\n" for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue if os.uname()[0] == "Linux": rtcmd = "ip route add 224.0.0.0/4 dev" @@ -126,10 +126,10 @@ class DefaultMulticastRouteService(UtilService): class StaticRouteService(UtilService): - _name = "StaticRoute" - _configs = ("staticroute.sh",) - _startup = ("sh staticroute.sh",) - _custom_needed = True + name = "StaticRoute" + configs = ("staticroute.sh",) + startup = ("sh staticroute.sh",) + custom_needed = True @classmethod def generateconfig(cls, node, filename, services): @@ -165,12 +165,12 @@ class StaticRouteService(UtilService): class SshService(UtilService): - _name = "SSH" - _configs = ("startsshd.sh", "/etc/ssh/sshd_config",) - _dirs = ("/etc/ssh", "/var/run/sshd",) - _startup = ("sh startsshd.sh",) - _shutdown = ("killall sshd",) - _validate = () + name = "SSH" + configs = ("startsshd.sh", "/etc/ssh/sshd_config",) + dirs = ("/etc/ssh", "/var/run/sshd",) + startup = ("sh startsshd.sh",) + shutdown = ("killall sshd",) + validate = () @classmethod def generateconfig(cls, node, filename, services): @@ -178,8 +178,8 @@ class SshService(UtilService): Use a startup script for launching sshd in order to wait for host key generation. """ - sshcfgdir = cls._dirs[0] - sshstatedir = cls._dirs[1] + sshcfgdir = cls.dirs[0] + sshstatedir = cls.dirs[1] sshlibdir = "/usr/lib/openssh" if filename == "startsshd.sh": return """\ @@ -233,12 +233,12 @@ UseDNS no class DhcpService(UtilService): - _name = "DHCP" - _configs = ("/etc/dhcp/dhcpd.conf",) - _dirs = ("/etc/dhcp",) - _startup = ("dhcpd",) - _shutdown = ("killall dhcpd",) - _validate = ("pidof dhcpd",) + name = "DHCP" + configs = ("/etc/dhcp/dhcpd.conf",) + dirs = ("/etc/dhcp",) + startup = ("dhcpd",) + shutdown = ("killall dhcpd",) + validate = ("pidof dhcpd",) @classmethod def generateconfig(cls, node, filename, services): @@ -296,11 +296,11 @@ class DhcpClientService(UtilService): """ Use a DHCP client for all interfaces for addressing. """ - _name = "DHCPClient" - _configs = ("startdhcpclient.sh",) - _startup = ("sh startdhcpclient.sh",) - _shutdown = ("killall dhclient",) - _validate = ("pidof dhclient",) + name = "DHCPClient" + configs = ("startdhcpclient.sh",) + startup = ("sh startdhcpclient.sh",) + shutdown = ("killall dhclient",) + validate = ("pidof dhclient",) @classmethod def generateconfig(cls, node, filename, services): @@ -327,12 +327,12 @@ class FtpService(UtilService): """ Start a vsftpd server. """ - _name = "FTP" - _configs = ("vsftpd.conf",) - _dirs = ("/var/run/vsftpd/empty", "/var/ftp",) - _startup = ("vsftpd ./vsftpd.conf",) - _shutdown = ("killall vsftpd",) - _validate = ("pidof vsftpd",) + name = "FTP" + configs = ("vsftpd.conf",) + dirs = ("/var/run/vsftpd/empty", "/var/ftp",) + startup = ("vsftpd ./vsftpd.conf",) + shutdown = ("killall vsftpd",) + validate = ("pidof vsftpd",) @classmethod def generateconfig(cls, node, filename, services): @@ -359,14 +359,14 @@ class HttpService(UtilService): """ Start an apache server. """ - _name = "HTTP" - _configs = ("/etc/apache2/apache2.conf", "/etc/apache2/envvars", - "/var/www/index.html",) - _dirs = ("/etc/apache2", "/var/run/apache2", "/var/log/apache2", - "/run/lock", "/var/lock/apache2", "/var/www",) - _startup = ("chown www-data /var/lock/apache2", "apache2ctl start",) - _shutdown = ("apache2ctl stop",) - _validate = ("pidof apache2",) + name = "HTTP" + configs = ("/etc/apache2/apache2.conf", "/etc/apache2/envvars", + "/var/www/index.html",) + dirs = ("/etc/apache2", "/var/run/apache2", "/var/log/apache2", + "/run/lock", "/var/lock/apache2", "/var/www",) + startup = ("chown www-data /var/lock/apache2", "apache2ctl start",) + shutdown = ("apache2ctl stop",) + validate = ("pidof apache2",) APACHEVER22, APACHEVER24 = (22, 24) @@ -375,11 +375,11 @@ class HttpService(UtilService): """ Generate an apache2.conf configuration file. """ - if filename == cls._configs[0]: + if filename == cls.configs[0]: return cls.generateapache2conf(node, filename, services) - elif filename == cls._configs[1]: + elif filename == cls.configs[1]: return cls.generateenvvars(node, filename, services) - elif filename == cls._configs[2]: + elif filename == cls.configs[2]: return cls.generatehtml(node, filename, services) else: return "" @@ -561,7 +561,7 @@ export LANG

The web server software is running but no content has been added, yet.

""" % node.name for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue body += "
  • %s - %s
    • \n" % (ifc.name, ifc.addrlist) return "%s" % body @@ -571,14 +571,14 @@ class PcapService(UtilService): """ Pcap service for logging packets. """ - _name = "pcap" - _configs = ("pcap.sh",) - _dirs = () - _startindex = 1 - _startup = ("sh pcap.sh start",) - _shutdown = ("sh pcap.sh stop",) - _validate = ("pidof tcpdump",) - _meta = "logs network traffic to pcap packet capture files" + name = "pcap" + configs = ("pcap.sh",) + dirs = () + startindex = 1 + startup = ("sh pcap.sh start",) + shutdown = ("sh pcap.sh stop",) + validate = ("pidof tcpdump",) + meta = "logs network traffic to pcap packet capture files" @classmethod def generateconfig(cls, node, filename, services): @@ -595,7 +595,7 @@ if [ "x$1" = "xstart" ]; then """ for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: cfg += '# ' redir = "< /dev/null" cfg += "tcpdump ${DUMPOPTS} -w %s.%s.pcap -i %s %s &\n" % \ @@ -611,12 +611,12 @@ fi; class RadvdService(UtilService): - _name = "radvd" - _configs = ("/etc/radvd/radvd.conf",) - _dirs = ("/etc/radvd",) - _startup = ("radvd -C /etc/radvd/radvd.conf -m logfile -l /var/log/radvd.log",) - _shutdown = ("pkill radvd",) - _validate = ("pidof radvd",) + name = "radvd" + configs = ("/etc/radvd/radvd.conf",) + dirs = ("/etc/radvd",) + startup = ("radvd -C /etc/radvd/radvd.conf -m logfile -l /var/log/radvd.log",) + shutdown = ("pkill radvd",) + validate = ("pidof radvd",) @classmethod def generateconfig(cls, node, filename, services): @@ -626,7 +626,7 @@ class RadvdService(UtilService): """ cfg = "# auto-generated by RADVD service (utility.py)\n" for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue prefixes = map(cls.subnetentry, ifc.addrlist) if len(prefixes) < 1: @@ -671,11 +671,11 @@ class AtdService(UtilService): """ Atd service for scheduling at jobs """ - _name = "atd" - _configs = ("startatd.sh",) - _dirs = ("/var/spool/cron/atjobs", "/var/spool/cron/atspool") - _startup = ("sh startatd.sh",) - _shutdown = ("pkill atd",) + name = "atd" + configs = ("startatd.sh",) + dirs = ("/var/spool/cron/atjobs", "/var/spool/cron/atspool") + startup = ("sh startatd.sh",) + shutdown = ("pkill atd",) @classmethod def generateconfig(cls, node, filename, services): @@ -692,6 +692,6 @@ class UserDefinedService(UtilService): """ Dummy service allowing customization of anything. """ - _name = "UserDefined" - _startindex = 50 - _meta = "Customize this service to do anything upon startup." + name = "UserDefined" + startindex = 50 + meta = "Customize this service to do anything upon startup." diff --git a/daemon/core/services/xorp.py b/daemon/core/services/xorp.py index bbe58769..a6b85609 100644 --- a/daemon/core/services/xorp.py +++ b/daemon/core/services/xorp.py @@ -11,15 +11,15 @@ class XorpRtrmgr(CoreService): XORP router manager service builds a config.boot file based on other enabled XORP services, and launches necessary daemons upon startup. """ - _name = "xorp_rtrmgr" - _group = "XORP" - _depends = () - _dirs = ("/etc/xorp",) - _configs = ("/etc/xorp/config.boot",) - _startindex = 35 - _startup = ("xorp_rtrmgr -d -b %s -l /var/log/%s.log -P /var/run/%s.pid" % (_configs[0], _name, _name),) - _shutdown = ("killall xorp_rtrmgr",) - _validate = ("pidof xorp_rtrmgr",) + name = "xorp_rtrmgr" + group = "XORP" + depends = () + dirs = ("/etc/xorp",) + configs = ("/etc/xorp/config.boot",) + startindex = 35 + startup = ("xorp_rtrmgr -d -b %s -l /var/log/%s.log -P /var/run/%s.pid" % (configs[0], name, name),) + shutdown = ("killall xorp_rtrmgr",) + validate = ("pidof xorp_rtrmgr",) @classmethod def generateconfig(cls, node, filename, services): @@ -40,10 +40,10 @@ class XorpRtrmgr(CoreService): for s in services: try: - s._depends.index(cls._name) + s.depends.index(cls.name) cfg += s.generatexorpconfig(node) except ValueError: - logger.exception("error getting value from service: %s", cls._name) + logger.exception("error getting value from service: %s", cls.name) return cfg @@ -74,15 +74,15 @@ class XorpService(CoreService): Parent class for XORP services. Defines properties and methods common to XORP's routing daemons. """ - _name = None - _group = "XORP" - _depends = ("xorp_rtrmgr",) - _dirs = () - _configs = () - _startindex = 40 - _startup = () - _shutdown = () - _meta = "The config file for this service can be found in the xorp_rtrmgr service." + name = None + group = "XORP" + depends = ("xorp_rtrmgr",) + dirs = () + configs = () + startindex = 40 + startup = () + shutdown = () + meta = "The config file for this service can be found in the xorp_rtrmgr service." @staticmethod def fea(forwarding): @@ -165,7 +165,7 @@ class XorpOspfv2(XorpService): not build its own configuration file but has hooks for adding to the unified XORP configuration file. """ - _name = "XORP_OSPFv2" + name = "XORP_OSPFv2" @classmethod def generatexorpconfig(cls, node): @@ -200,7 +200,7 @@ class XorpOspfv3(XorpService): not build its own configuration file but has hooks for adding to the unified XORP configuration file. """ - _name = "XORP_OSPFv3" + name = "XORP_OSPFv3" @classmethod def generatexorpconfig(cls, node): @@ -227,8 +227,8 @@ class XorpBgp(XorpService): """ IPv4 inter-domain routing. AS numbers and peers must be customized. """ - _name = "XORP_BGP" - _custom_needed = True + name = "XORP_BGP" + custom_needed = True @classmethod def generatexorpconfig(cls, node): @@ -257,7 +257,7 @@ class XorpRip(XorpService): RIP IPv4 unicast routing. """ - _name = "XORP_RIP" + name = "XORP_RIP" @classmethod def generatexorpconfig(cls, node): @@ -289,7 +289,7 @@ class XorpRipng(XorpService): """ RIP NG IPv6 unicast routing. """ - _name = "XORP_RIPNG" + name = "XORP_RIPNG" @classmethod def generatexorpconfig(cls, node): @@ -324,7 +324,7 @@ class XorpPimSm4(XorpService): """ PIM Sparse Mode IPv4 multicast routing. """ - _name = "XORP_PIMSM4" + name = "XORP_PIMSM4" @classmethod def generatexorpconfig(cls, node): @@ -383,7 +383,7 @@ class XorpPimSm6(XorpService): """ PIM Sparse Mode IPv6 multicast routing. """ - _name = "XORP_PIMSM6" + name = "XORP_PIMSM6" @classmethod def generatexorpconfig(cls, node): @@ -442,7 +442,7 @@ class XorpOlsr(XorpService): """ OLSR IPv4 unicast MANET routing. """ - _name = "XORP_OLSR" + name = "XORP_OLSR" @classmethod def generatexorpconfig(cls, node): diff --git a/daemon/core/xml/xmlparser0.py b/daemon/core/xml/xmlparser0.py index 6ab25912..be9a1f19 100644 --- a/daemon/core/xml/xmlparser0.py +++ b/daemon/core/xml/xmlparser0.py @@ -4,7 +4,7 @@ from core import logger from core.conf import ConfigShim from core.enumerations import NodeTypes from core.misc import nodeutils -from core.service import ServiceManager +from core.service import ServiceManager, ServiceShim from core.xml import xmlutils @@ -316,7 +316,10 @@ class CoreDocumentParser0(object): # associate nodes with services for objid in sorted(svclists.keys()): n = self.session.get_object(objid) - self.session.services.addservicestonode(node=n, nodetype=n.type, services_str=svclists[objid]) + services = svclists[objid] + if services: + services = services.split("|") + self.session.services.addservicestonode(node=n, node_type=n.type, services=services) def parseservice(self, service, n): """ @@ -367,16 +370,20 @@ class CoreDocumentParser0(object): filename = file.getAttribute("name") files.append(filename) data = xmlutils.get_text_child(file) - typestr = "service:%s:%s" % (name, filename) - self.session.services.setservicefile(nodenum=n.objid, type=typestr, - filename=filename, - srcname=None, data=data) + self.session.services.setservicefile(node_id=n.objid, service_name=name, filename=filename, data=data) + if len(files): values.append("files=%s" % files) if not bool(service.getAttribute("custom")): return True - values = ConfigShim.str_to_dict(values) - self.session.services.setcustomservice(n.objid, svc, values) + self.session.services.setcustomservice(n.objid, svc) + # set custom values for custom service + svc = self.session.services.getcustomservice(n.objid, None) + if not svc: + raise ValueError("custom service(%s) for node(%s) does not exist", svc.name, n.objid) + values = ConfigShim.str_to_dict("|".join(values)) + for name, value in values.iteritems(): + ServiceShim.setvalue(svc, name, value) return True def parsehooks(self, hooks): diff --git a/daemon/core/xml/xmlparser1.py b/daemon/core/xml/xmlparser1.py index 46f6bcf9..cef79590 100644 --- a/daemon/core/xml/xmlparser1.py +++ b/daemon/core/xml/xmlparser1.py @@ -8,7 +8,7 @@ from core.conf import ConfigShim from core.enumerations import NodeTypes from core.misc import nodeutils from core.misc.ipaddress import MacAddress -from core.service import ServiceManager +from core.service import ServiceManager, ServiceShim from core.xml import xmlutils @@ -639,17 +639,19 @@ class CoreDocumentParser1(object): custom = service.getAttribute('custom') if custom and custom.lower() == 'true': - values = ConfigShim.str_to_dict(values) - self.session.services.setcustomservice(node.objid, session_service, values) + self.session.services.setcustomservice(node.objid, session_service.name) + values = ConfigShim.str_to_dict("|".join(values)) + for key, value in values.iteritems(): + ServiceShim.setvalue(session_service, key, value) # NOTE: if a custom service is used, setservicefile() must be # called after the custom service exists for typestr, filename, data in files: + svcname = typestr.split(":")[1] self.session.services.setservicefile( - nodenum=node.objid, - type=typestr, + node_id=node.objid, + service_name=svcname, filename=filename, - srcname=None, data=data ) return str(name) @@ -678,10 +680,13 @@ class CoreDocumentParser1(object): services_str = None # default services will be added else: return + if services_str: + services_str = services_str.split("|") + self.session.services.addservicestonode( node=node, - nodetype=node_type, - services_str=services_str + node_type=node_type, + services=services_str ) def set_object_presentation(self, obj, element, node_type): diff --git a/daemon/core/xml/xmlwriter0.py b/daemon/core/xml/xmlwriter0.py index a11e099a..41f097fc 100644 --- a/daemon/core/xml/xmlwriter0.py +++ b/daemon/core/xml/xmlwriter0.py @@ -284,7 +284,7 @@ class CoreDocumentWriter0(Document): for svc in defaults: s = self.createElement("Service") spn.appendChild(s) - s.setAttribute("name", str(svc._name)) + s.setAttribute("name", str(svc.name)) def addservices(self, node): """ @@ -302,17 +302,17 @@ class CoreDocumentWriter0(Document): for svc in node.services: s = self.createElement("Service") spn.appendChild(s) - s.setAttribute("name", str(svc._name)) - s.setAttribute("startup_idx", str(svc._startindex)) - if svc._starttime != "": - s.setAttribute("start_time", str(svc._starttime)) + s.setAttribute("name", str(svc.name)) + s.setAttribute("startup_idx", str(svc.startindex)) + if svc.starttime != "": + s.setAttribute("start_time", str(svc.starttime)) # only record service names if not a customized service - if not svc._custom: + if not svc.custom: continue - s.setAttribute("custom", str(svc._custom)) - xmlutils.add_elements_from_list(self, s, svc._dirs, "Directory", "name") + s.setAttribute("custom", str(svc.custom)) + xmlutils.add_elements_from_list(self, s, svc.dirs, "Directory", "name") - for fn in svc._configs: + for fn in svc.configs: if len(fn) == 0: continue f = self.createElement("File") @@ -327,9 +327,9 @@ class CoreDocumentWriter0(Document): txt = self.createTextNode(data) f.appendChild(txt) - xmlutils.add_text_elements_from_list(self, s, svc._startup, "Command", (("type", "start"),)) - xmlutils.add_text_elements_from_list(self, s, svc._shutdown, "Command", (("type", "stop"),)) - xmlutils.add_text_elements_from_list(self, s, svc._validate, "Command", (("type", "validate"),)) + xmlutils.add_text_elements_from_list(self, s, svc.startup, "Command", (("type", "start"),)) + xmlutils.add_text_elements_from_list(self, s, svc.shutdown, "Command", (("type", "stop"),)) + xmlutils.add_text_elements_from_list(self, s, svc.validate, "Command", (("type", "validate"),)) def addaddresses(self, i, netif): """ diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index 6f5deb50..22c03a23 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -277,7 +277,7 @@ class ScenarioPlan(XmlElement): for svc in defaults: s = self.createElement("service") spn.appendChild(s) - s.setAttribute("name", str(svc._name)) + s.setAttribute("name", str(svc.name)) if defaultservices.hasChildNodes(): self.appendChild(defaultservices) @@ -680,24 +680,24 @@ class DeviceElement(NamedXmlElement): for svc in device_object.services: s = self.createElement("service") spn.appendChild(s) - s.setAttribute("name", str(svc._name)) - s.setAttribute("startup_idx", str(svc._startindex)) - if svc._starttime != "": - s.setAttribute("start_time", str(svc._starttime)) + s.setAttribute("name", str(svc.name)) + s.setAttribute("startup_idx", str(svc.startindex)) + if svc.starttime != "": + s.setAttribute("start_time", str(svc.starttime)) # only record service names if not a customized service - if not svc._custom: + if not svc.custom: continue - s.setAttribute("custom", str(svc._custom)) - xmlutils.add_elements_from_list(self, s, svc._dirs, "directory", "name") + s.setAttribute("custom", str(svc.custom)) + xmlutils.add_elements_from_list(self, s, svc.dirs, "directory", "name") - for fn in svc._configs: + for fn in svc.configs: if len(fn) == 0: continue f = self.createElement("file") f.setAttribute("name", fn) # all file names are added to determine when a file has been deleted s.appendChild(f) - data = self.coreSession.services.getservicefiledata(svc, fn) + data = svc.configtxt.get(fn) if data is None: # this includes only customized file contents and skips # the auto-generated files @@ -705,12 +705,9 @@ class DeviceElement(NamedXmlElement): txt = self.createTextNode("\n" + data) f.appendChild(txt) - xmlutils.add_text_elements_from_list(self, s, svc._startup, "command", - (("type", "start"),)) - xmlutils.add_text_elements_from_list(self, s, svc._shutdown, "command", - (("type", "stop"),)) - xmlutils.add_text_elements_from_list(self, s, svc._validate, "command", - (("type", "validate"),)) + xmlutils.add_text_elements_from_list(self, s, svc.startup, "command", (("type", "start"),)) + xmlutils.add_text_elements_from_list(self, s, svc.shutdown, "command", (("type", "stop"),)) + xmlutils.add_text_elements_from_list(self, s, svc.validate, "command", (("type", "validate"),)) class ChannelElement(NamedXmlElement): diff --git a/daemon/examples/netns/howmanynodes.py b/daemon/examples/netns/howmanynodes.py index 1a20e013..5ca989f6 100755 --- a/daemon/examples/netns/howmanynodes.py +++ b/daemon/examples/netns/howmanynodes.py @@ -159,7 +159,7 @@ def main(): n.newnetif(switch, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) n.cmd([constants.SYSCTL_BIN, "net.ipv4.icmp_echo_ignore_broadcasts=0"]) if options.services is not None: - session.services.addservicestonode(n, "", options.services) + session.services.addservicestonode(n, "", options.services.split("|")) n.boot() nodelist.append(n) if i % 25 == 0: diff --git a/daemon/examples/netns/wlanemanetests.py b/daemon/examples/netns/wlanemanetests.py index ba6f2aa2..d3323f13 100755 --- a/daemon/examples/netns/wlanemanetests.py +++ b/daemon/examples/netns/wlanemanetests.py @@ -429,8 +429,7 @@ class Experiment(object): self.net.link(prev.netif(0), tmp.netif(0)) prev = tmp - def createemanesession(self, numnodes, verbose=False, cls=None, - values=None): + def createemanesession(self, numnodes, verbose=False, cls=None, values=None): """ Build a topology consisting of the given number of LxcNodes connected to an EMANE WLAN. """ diff --git a/daemon/tests/conftest.py b/daemon/tests/conftest.py index 6923807a..d087e2e6 100644 --- a/daemon/tests/conftest.py +++ b/daemon/tests/conftest.py @@ -7,7 +7,6 @@ import os import pytest from mock.mock import MagicMock -from core import services from core.api.coreapi import CoreConfMessage from core.api.coreapi import CoreEventMessage from core.api.coreapi import CoreExecMessage @@ -29,6 +28,7 @@ from core.enumerations import NodeTlvs from core.enumerations import NodeTypes from core.misc import ipaddress from core.misc.ipaddress import MacAddress +from core.service import ServiceManager EMANE_SERVICES = "zebra|OSPFv3MDR|IPForward" @@ -199,6 +199,7 @@ class CoreServerTest(object): self.request_handler.handle_message(message) def shutdown(self): + self.server.coreemu.shutdown() self.server.shutdown() self.server.server_close() @@ -223,6 +224,9 @@ def session(): # shutdown coreemu coreemu.shutdown() + # clear services, since they will be reloaded + ServiceManager.services.clear() + @pytest.fixture(scope="module") def ip_prefixes(): @@ -231,9 +235,6 @@ def ip_prefixes(): @pytest.fixture() def cored(): - # load default services - services.load() - # create and return server server = CoreServerTest() yield server @@ -241,6 +242,11 @@ def cored(): # cleanup server.shutdown() + # + + # cleanup services + ServiceManager.services.clear() + def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) diff --git a/daemon/tests/myservices/sample.py b/daemon/tests/myservices/sample.py index c986cc4a..80545d29 100644 --- a/daemon/tests/myservices/sample.py +++ b/daemon/tests/myservices/sample.py @@ -6,22 +6,22 @@ from core.service import CoreService class MyService(CoreService): - _name = "MyService" - _group = "Utility" - _depends = () - _dirs = () - _configs = ('myservice.sh',) - _startindex = 50 - _startup = ('sh myservice.sh',) - _shutdown = () + name = "MyService" + group = "Utility" + depends = () + dirs = () + configs = ('myservice.sh',) + startindex = 50 + startup = ('sh myservice.sh',) + shutdown = () class MyService2(CoreService): - _name = "MyService2" - _group = "Utility" - _depends = () - _dirs = () - _configs = ('myservice.sh',) - _startindex = 50 - _startup = ('sh myservice.sh',) - _shutdown = () + name = "MyService2" + group = "Utility" + depends = () + dirs = () + configs = ('myservice.sh',) + startindex = 50 + startup = ('sh myservice.sh',) + shutdown = () diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 6b657304..453c7717 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -15,10 +15,8 @@ from core.enumerations import NodeTypes from core.mobility import BasicRangeModel from core.mobility import Ns2ScriptedMobility from core.netns.vnodeclient import VnodeClient -from core.service import ServiceManager _PATH = os.path.abspath(os.path.dirname(__file__)) -_SERVICES_PATH = os.path.join(_PATH, "myservices") _MOBILITY_FILE = os.path.join(_PATH, "mobility.scen") _WIRED = [ NodeTypes.PEER_TO_PEER, @@ -51,16 +49,6 @@ def ping(from_node, to_node, ip_prefixes): class TestCore: - def test_import_service(self): - """ - Test importing a custom service. - - :param conftest.Core core: core fixture to test with - """ - ServiceManager.add_services(_SERVICES_PATH) - assert ServiceManager.get("MyService") - assert ServiceManager.get("MyService2") - @pytest.mark.parametrize("net_type", _WIRED) def test_wired_ping(self, session, net_type, ip_prefixes): """ diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index bf6d3b01..d0fd91fc 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -101,7 +101,7 @@ def run_cmd(node, exec_cmd): class TestGui: - def test_broker(self, session, cored): + def test_broker(self, cored): """ Test session broker creation. @@ -119,6 +119,7 @@ class TestGui: daemon = "localhost" # add server + session = cored.server.coreemu.create_session() session.broker.addserver(daemon, "127.0.0.1", CORE_API_PORT) # setup server diff --git a/daemon/tests/test_services.py b/daemon/tests/test_services.py new file mode 100644 index 00000000..7b60ff1b --- /dev/null +++ b/daemon/tests/test_services.py @@ -0,0 +1,18 @@ +import os + +from core.service import ServiceManager + +_PATH = os.path.abspath(os.path.dirname(__file__)) +_SERVICES_PATH = os.path.join(_PATH, "myservices") + + +class TestServices: + def test_import_service(self): + """ + Test importing a custom service. + + :param conftest.Core core: core fixture to test with + """ + ServiceManager.add_services(_SERVICES_PATH) + assert ServiceManager.get("MyService") + assert ServiceManager.get("MyService2") diff --git a/daemon/tests/test_xml.py b/daemon/tests/test_xml.py index 4d0cfa7f..f3fe8660 100644 --- a/daemon/tests/test_xml.py +++ b/daemon/tests/test_xml.py @@ -6,6 +6,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emulator.emudata import NodeOptions from core.enumerations import NodeTypes from core.mobility import BasicRangeModel +from core.services.utility import SshService _XML_VERSIONS = [ "0.0", @@ -68,6 +69,75 @@ class TestXml: assert session.get_object(n1_id) assert session.get_object(n2_id) + @pytest.mark.parametrize("version", _XML_VERSIONS) + def test_xml_ptp_services(self, session, tmpdir, version, ip_prefixes): + """ + Test xml client methods for a ptp neetwork. + + :param session: session for test + :param tmpdir: tmpdir to create data in + :param str version: xml version to write and parse + :param ip_prefixes: generates ip addresses for nodes + """ + # create ptp + ptp_node = session.add_node(_type=NodeTypes.PEER_TO_PEER) + + # create nodes + node_options = NodeOptions(model="host") + node_one = session.add_node(node_options=node_options) + node_two = session.add_node() + + # link nodes to ptp net + for node in [node_one, node_two]: + interface = ip_prefixes.create_interface(node) + session.add_link(node.objid, ptp_node.objid, interface_one=interface) + + # set custom values for node service\ + custom_start = 50 + session.services.setcustomservice(node_one.objid, SshService.name) + service = session.services.getcustomservice(node_one.objid, SshService.name) + service.startindex = custom_start + service_file = SshService.configs[0] + file_data = "# test" + session.services.setservicefile(node_one.objid, SshService.name, service_file, file_data) + + # instantiate session + session.instantiate() + + # get ids for nodes + n1_id = node_one.objid + n2_id = node_two.objid + + # save xml + xml_file = tmpdir.join("session.xml") + file_path = xml_file.strpath + session.save_xml(file_path, version) + + # verify xml file was created and can be parsed + assert xml_file.isfile() + assert ElementTree.parse(file_path) + + # stop current session, clearing data + session.shutdown() + + # verify nodes have been removed from session + with pytest.raises(KeyError): + assert not session.get_object(n1_id) + with pytest.raises(KeyError): + assert not session.get_object(n2_id) + + # load saved xml + session.open_xml(file_path, start=True) + + # retrieve custom service + service = session.services.getcustomservice(node_one.objid, SshService.name) + + # verify nodes have been recreated + assert session.get_object(n1_id) + assert session.get_object(n2_id) + assert service.startindex == custom_start + assert service.configtxt.get(service_file) == file_data + @pytest.mark.parametrize("version", _XML_VERSIONS) def test_xml_mobility(self, session, tmpdir, version, ip_prefixes): """ diff --git a/ns3/examples/ns3wifirandomwalk.py b/ns3/examples/ns3wifirandomwalk.py index 18f473ad..cf5acee1 100644 --- a/ns3/examples/ns3wifirandomwalk.py +++ b/ns3/examples/ns3wifirandomwalk.py @@ -16,13 +16,13 @@ import sys import ns.core import ns.network +from corens3.obj import Ns3Session +from corens3.obj import Ns3WifiNet from core import logger from core.misc import ipaddress from core.misc import nodemaps from core.misc import nodeutils -from corens3.obj import Ns3Session -from corens3.obj import Ns3WifiNet def add_to_server(session): @@ -60,7 +60,7 @@ def wifisession(opt): node = session.addnode(name="n%d" % i) node.newnetif(wifi, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) nodes.append(node) - session.services.addservicestonode(node, "router", services_str) + session.services.addservicestonode(node, "router", services_str.split("|")) session.services.bootnodeservices(node) session.setuprandomwalkmobility(bounds=(1000.0, 750.0, 0)) From bf47e5fc0d8ae266723ca230261e98b60d6dbe6f Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Tue, 19 Jun 2018 09:19:49 -0700 Subject: [PATCH 49/83] added service executable check, added error message to gui for service load errors --- daemon/core/corehandlers.py | 32 ++++++++++++++++++++++++++-- daemon/core/emulator/coreemu.py | 19 +++++++++++++++-- daemon/core/service.py | 28 +++++++++++++++++++++--- daemon/core/services/__init__.py | 5 +++-- daemon/core/services/bird.py | 2 ++ daemon/core/services/dockersvc.py | 1 + daemon/core/services/sdn.py | 8 ++----- daemon/core/services/xorp.py | 2 ++ daemon/examples/myservices/sample.py | 23 +++++++------------- daemon/scripts/core-daemon | 9 -------- 10 files changed, 90 insertions(+), 39 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 9a220fe0..bd44b24c 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -17,13 +17,13 @@ from core import logger from core.api import coreapi from core.api import dataconversion from core.conf import ConfigShim -from core.data import ConfigData +from core.data import ConfigData, ExceptionData from core.data import EventData from core.data import FileData from core.emulator.emudata import InterfaceData from core.emulator.emudata import LinkOptions from core.emulator.emudata import NodeOptions -from core.enumerations import ConfigDataTypes +from core.enumerations import ConfigDataTypes, ExceptionLevels from core.enumerations import ConfigFlags from core.enumerations import ConfigTlvs from core.enumerations import EventTlvs @@ -551,6 +551,14 @@ class CoreHandler(SocketServer.BaseRequestHandler): # set initial session state self.session.set_state(EventTypes.DEFINITION_STATE) + # send service errors, if present + if self.coreemu.service_errors: + self.send_exception( + ExceptionLevels.ERROR, + "ServiceManager.load()", + "Failed to load services: %s" % " ".join(self.coreemu.service_errors) + ) + while True: try: message = self.receive_message() @@ -579,6 +587,26 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.debug("BROADCAST TO OTHER CLIENT: %s", client) client.sendall(message.raw_message) + def send_exception(self, level, source, text, node=None): + """ + Sends an exception for display within the GUI. + + :param core.enumerations.ExceptionLevel level: level for exception + :param str source: source where exception came from + :param str text: details about exception + :param int node: node id, if related to a specific node + :return: + """ + exception_data = ExceptionData( + session=str(self.session.session_id), + node=node, + date=time.ctime(), + level=level.value, + source=source, + text=text + ) + self.handle_broadcast_exception(exception_data) + def add_session_handlers(self): logger.debug("adding session broadcast handlers") self.session.event_handlers.append(self.handle_broadcast_event) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 7d948552..4e207bf7 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -15,6 +15,7 @@ from core.enumerations import LinkTypes from core.enumerations import NodeTypes from core.misc import nodemaps from core.misc import nodeutils +from core.service import ServiceManager from core.session import Session from core.xml.xmlparser import core_document_parser from core.xml.xmlwriter import core_document_writer @@ -813,12 +814,26 @@ class CoreEmu(object): node_map = nodemaps.NODES nodeutils.set_node_map(node_map) - # load default services - core.services.load() + # load services + self.service_errors = [] + self.load_services() # catch exit event atexit.register(self.shutdown) + def load_services(self): + # load default services + self.service_errors = core.services.load() + + # load custom services + service_paths = self.config.get("custom_services_dir") + logger.debug("custom service paths: %s", service_paths) + if service_paths: + for service_path in service_paths.split(','): + service_path = service_path.strip() + custom_service_errors = ServiceManager.add_services(service_path) + self.service_errors.extend(custom_service_errors) + def update_nodes(self, node_map): """ Updates node map used by core. diff --git a/daemon/core/service.py b/daemon/core/service.py index 21f94a24..4f367dc2 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -7,6 +7,8 @@ a list of available services to the GUI and for configuring individual services. """ +from core.constants import which + from core import CoreCommandError from core import logger from core.data import FileData @@ -126,8 +128,18 @@ class ServiceManager(object): """ logger.info("loading service: %s", service.__name__) name = service.name + + # avoid duplicate services if name in cls.services: raise ValueError("duplicate service being added: %s" % name) + + # validate dependent executables are present + for executable in service.executables: + if not which(executable): + logger.error("service(%s) missing executable: %s", service.name, executable) + raise ValueError("service(%s) missing executable: %s" % (service.name, executable)) + + # make service available cls.services[name] = service @classmethod @@ -147,15 +159,22 @@ class ServiceManager(object): Method for retrieving all CoreServices from a given path. :param str path: path to retrieve services from - :return: list of core services - :rtype: list + :return: list of core services that failed to load + :rtype: list[str] """ + service_errors = [] services = utils.load_classes(path, CoreService) for service in services: if not service.name: continue service.on_load() - cls.add(service) + + try: + cls.add(service) + except ValueError as e: + service_errors.append(service.name) + logger.error("failure loading service: %s", e.message) + return service_errors class CoreServices(object): @@ -623,6 +642,9 @@ class CoreService(object): # service name should not include spaces name = None + # executables that must exist for service to run + executables = () + # group string allows grouping services together group = None diff --git a/daemon/core/services/__init__.py b/daemon/core/services/__init__.py index 2598d105..974cd03e 100644 --- a/daemon/core/services/__init__.py +++ b/daemon/core/services/__init__.py @@ -15,6 +15,7 @@ def load(): """ Loads all services from the modules that reside under core.services. - :return: nothing + :return: list of services that failed to load + :rtype: list[str] """ - ServiceManager.add_services(_PATH) + return ServiceManager.add_services(_PATH) diff --git a/daemon/core/services/bird.py b/daemon/core/services/bird.py index d56bc14c..23a9fcec 100644 --- a/daemon/core/services/bird.py +++ b/daemon/core/services/bird.py @@ -10,6 +10,7 @@ class Bird(CoreService): Bird router support """ name = "bird" + executables = ("bird",) group = "BIRD" depends = () dirs = ("/etc/bird",) @@ -91,6 +92,7 @@ class BirdService(CoreService): """ name = None + executables = ("bird",) group = "BIRD" depends = ("bird",) dirs = () diff --git a/daemon/core/services/dockersvc.py b/daemon/core/services/dockersvc.py index 9699465c..d36ec5a4 100644 --- a/daemon/core/services/dockersvc.py +++ b/daemon/core/services/dockersvc.py @@ -113,6 +113,7 @@ class DockerService(CoreService): node. """ name = "Docker" + executables = ("docker",) group = "Docker" depends = () dirs = ('/var/lib/docker/containers/', '/run/shm', '/run/resolvconf',) diff --git a/daemon/core/services/sdn.py b/daemon/core/services/sdn.py index 33f50109..4e144ea0 100644 --- a/daemon/core/services/sdn.py +++ b/daemon/core/services/sdn.py @@ -11,14 +11,8 @@ class SdnService(CoreService): """ Parent class for SDN services. """ - name = None group = "SDN" - depends = () - dirs = () - configs = () startindex = 50 - startup = () - shutdown = () @classmethod def generateconfig(cls, node, filename, services): @@ -27,6 +21,7 @@ class SdnService(CoreService): class OvsService(SdnService): name = "OvsService" + executables = ("ovs-ofctl", "ovs-vsctl") group = "SDN" depends = () dirs = ("/etc/openvswitch", "/var/run/openvswitch", "/var/log/openvswitch") @@ -101,6 +96,7 @@ class OvsService(SdnService): class RyuService(SdnService): name = "ryuService" + executables = ("ryu-manager",) group = "SDN" depends = () dirs = () diff --git a/daemon/core/services/xorp.py b/daemon/core/services/xorp.py index a6b85609..326c09b5 100644 --- a/daemon/core/services/xorp.py +++ b/daemon/core/services/xorp.py @@ -12,6 +12,7 @@ class XorpRtrmgr(CoreService): enabled XORP services, and launches necessary daemons upon startup. """ name = "xorp_rtrmgr" + executables = ("xorp_rtrmgr",) group = "XORP" depends = () dirs = ("/etc/xorp",) @@ -75,6 +76,7 @@ class XorpService(CoreService): common to XORP's routing daemons. """ name = None + executables = ("xorp_rtrmgr",) group = "XORP" depends = ("xorp_rtrmgr",) dirs = () diff --git a/daemon/examples/myservices/sample.py b/daemon/examples/myservices/sample.py index bd4a05b3..bc5113ee 100644 --- a/daemon/examples/myservices/sample.py +++ b/daemon/examples/myservices/sample.py @@ -4,7 +4,6 @@ Sample user-defined service. from core.misc.ipaddress import Ipv4Prefix from core.service import CoreService -from core.service import ServiceManager class MyService(CoreService): @@ -12,22 +11,22 @@ class MyService(CoreService): This is a sample user-defined service. """ # a unique name is required, without spaces - _name = "MyService" + name = "MyService" # you can create your own group here - _group = "Utility" + group = "Utility" # list of other services this service depends on - _depends = () + depends = () # per-node directories - _dirs = () + dirs = () # generated files (without a full path this file goes in the node's dir, # e.g. /tmp/pycore.12345/n1.conf/) - _configs = ('myservice.sh',) + configs = ('myservice.sh',) # this controls the starting order vs other enabled services - _startindex = 50 + startindex = 50 # list of startup commands, also may be generated during startup - _startup = ('sh myservice.sh',) + startup = ('sh myservice.sh',) # list of shutdown commands - _shutdown = () + shutdown = () @classmethod def generateconfig(cls, node, filename, services): @@ -57,9 +56,3 @@ class MyService(CoreService): else: net = Ipv4Prefix(x) return 'echo " network %s"' % net - - -# this is needed to load desired services when being integrated into core, otherwise this is not needed -def load_services(): - # this line is required to add the above class to the list of available services - ServiceManager.add(MyService) diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 86c5cd47..1943fa10 100644 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -16,7 +16,6 @@ from core import logger from core.corehandlers import CoreHandler from core.coreserver import CoreServer from core.misc.utils import close_onexec -from core.service import ServiceManager def banner(): @@ -116,14 +115,6 @@ def main(): for a in args: logger.error("ignoring command line argument: %s", a) - # attempt load custom services - service_paths = cfg.get("custom_services_dir") - logger.debug("custom service paths: %s", service_paths) - if service_paths: - for service_path in service_paths.split(','): - service_path = service_path.strip() - ServiceManager.add_services(service_path) - banner() # check if ovs flag was provided From 37ce407460021a85f6c05634e2ac6bcfe50e3304 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Tue, 19 Jun 2018 18:36:53 -0700 Subject: [PATCH 50/83] added logic to help provide dependent service ordering --- daemon/core/emulator/coreemu.py | 2 + daemon/core/service.py | 70 ++++++++++++++++++++++++++++ daemon/core/services/nrl.py | 8 ++++ daemon/tests/test_services.py | 82 +++++++++++++++++++++++++++++++-- 4 files changed, 159 insertions(+), 3 deletions(-) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 4e207bf7..5b8d9301 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -804,6 +804,8 @@ class CoreEmu(object): os.umask(0) # configuration + if not config: + config = {} self.config = config # session management diff --git a/daemon/core/service.py b/daemon/core/service.py index 4f367dc2..ab0dbe5d 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -211,6 +211,73 @@ class CoreServices(object): self.defaultservices.clear() self.customservices.clear() + def get_service_startups(self, services): + # generate service map and find starting points + node_services = {service.name: service for service in services} + is_dependency = set() + all_services = set() + for service in services: + all_services.add(service.name) + for service_name in service.dependencies: + # check service needed is valid + if service_name not in node_services: + raise ValueError("service(%s) dependency does not exist: %s" % (service.name, service_name)) + is_dependency.add(service_name) + starting_points = all_services - is_dependency + + # cycles means no starting points + if not starting_points: + raise ValueError("no valid service starting points") + + stack = [iter(starting_points)] + + # information used to traverse dependency graph + visited = set() + path = [] + path_set = set() + + # store startup orderings + startups = [] + startup = [] + + logger.debug("starting points: %s", starting_points) + while stack: + for service_name in stack[-1]: + service = node_services[service_name] + logger.debug("evaluating: %s", service.name) + + # check this is not a cycle + if service.name in path_set: + raise ValueError("service has a cyclic dependency: %s" % service.name) + # check that we have not already visited this node + elif service.name not in visited: + logger.debug("visiting: %s", service.name) + visited.add(service.name) + path.append(service.name) + path_set.add(service.name) + + # retrieve and set dependencies to the stack + stack.append(iter(service.dependencies)) + startup.append(service) + break + # for loop completed without a break + else: + logger.debug("finished a visit: path(%s)", path) + if path: + path_set.remove(path.pop()) + + if not path and startup: + startup.reverse() + startups.append(startup) + startup = [] + + stack.pop() + + if visited != all_services: + raise ValueError("cycle encountered, services are being skipped") + + return startups + def getdefaultservices(self, service_type): """ Get the list of default services that should be enabled for a @@ -645,6 +712,9 @@ class CoreService(object): # executables that must exist for service to run executables = () + # sets service requirements that must be started prior to this service starting + dependencies = () + # group string allows grouping services together group = None diff --git a/daemon/core/services/nrl.py b/daemon/core/services/nrl.py index 07152404..d446ef81 100644 --- a/daemon/core/services/nrl.py +++ b/daemon/core/services/nrl.py @@ -47,6 +47,7 @@ class NrlService(CoreService): class MgenSinkService(NrlService): name = "MGEN_Sink" + executables = ("mgen",) configs = ("sink.mgen",) startindex = 5 startup = ("mgen input sink.mgen",) @@ -73,6 +74,7 @@ class NrlNhdp(NrlService): NeighborHood Discovery Protocol for MANET networks. """ name = "NHDP" + executables = ("nrlnhdp",) startup = ("nrlnhdp",) shutdown = ("killall nrlnhdp",) validate = ("pidof nrlnhdp",) @@ -105,6 +107,7 @@ class NrlSmf(NrlService): Simplified Multicast Forwarding for MANET networks. """ name = "SMF" + executables = ("nrlsmf",) startup = ("sh startsmf.sh",) shutdown = ("killall nrlsmf",) validate = ("pidof nrlsmf",) @@ -156,6 +159,7 @@ class NrlOlsr(NrlService): Optimized Link State Routing protocol for MANET networks. """ name = "OLSR" + executables = ("nrlolsrd",) startup = ("nrlolsrd",) shutdown = ("killall nrlolsrd",) validate = ("pidof nrlolsrd",) @@ -189,6 +193,7 @@ class NrlOlsrv2(NrlService): Optimized Link State Routing protocol version 2 for MANET networks. """ name = "OLSRv2" + executables = ("nrlolsrv2",) startup = ("nrlolsrv2",) shutdown = ("killall nrlolsrv2",) validate = ("pidof nrlolsrv2",) @@ -223,6 +228,7 @@ class OlsrOrg(NrlService): Optimized Link State Routing protocol from olsr.org for MANET networks. """ name = "OLSRORG" + executables = ("olsrd",) configs = ("/etc/olsrd/olsrd.conf",) dirs = ("/etc/olsrd",) startup = ("olsrd",) @@ -572,6 +578,7 @@ class MgenActor(NrlService): # a unique name is required, without spaces name = "MgenActor" + executables = ("mgen",) # you can create your own group here group = "ProtoSvc" # list of other services this service depends on @@ -616,6 +623,7 @@ class Arouted(NrlService): Adaptive Routing """ name = "arouted" + executables = ("arouted",) configs = ("startarouted.sh",) startindex = NrlService.startindex + 10 startup = ("sh startarouted.sh",) diff --git a/daemon/tests/test_services.py b/daemon/tests/test_services.py index 7b60ff1b..3cefd7bf 100644 --- a/daemon/tests/test_services.py +++ b/daemon/tests/test_services.py @@ -1,18 +1,94 @@ import os +import pytest + +from core.service import CoreService from core.service import ServiceManager _PATH = os.path.abspath(os.path.dirname(__file__)) _SERVICES_PATH = os.path.join(_PATH, "myservices") +class ServiceA(CoreService): + name = "A" + dependencies = ("B",) + + +class ServiceB(CoreService): + name = "B" + dependencies = ("C",) + + +class ServiceC(CoreService): + name = "C" + dependencies = () + + +class ServiceD(CoreService): + name = "D" + dependencies = ("A",) + + +class ServiceE(CoreService): + name = "E" + dependencies = ("Z",) + + +class ServiceF(CoreService): + name = "F" + dependencies = () + + class TestServices: - def test_import_service(self): + def test_service_import(self): """ Test importing a custom service. - - :param conftest.Core core: core fixture to test with """ ServiceManager.add_services(_SERVICES_PATH) assert ServiceManager.get("MyService") assert ServiceManager.get("MyService2") + + def test_services_dependencies(self, session): + # given + services = [ + ServiceA(), + ServiceB(), + ServiceC(), + ServiceD(), + ServiceF(), + ] + + # when + startups = session.services.get_service_startups(services) + + # then + assert len(startups) == 2 + + def test_services_dependencies_not_present(self, session): + # given + services = [ + ServiceA(), + ServiceB(), + ServiceC(), + ServiceE() + ] + + # when + with pytest.raises(ValueError): + session.services.get_service_startups(services) + + def test_services_dependencies_cycle(self, session): + # given + service_c = ServiceC() + service_c.dependencies = ("D",) + services = [ + ServiceA(), + ServiceB(), + service_c, + ServiceD(), + ServiceF() + ] + + # when + with pytest.raises(ValueError): + session.services.get_service_startups(services) From c6d2ca6b020f654d870274078084c04e25866f34 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 20 Jun 2018 12:59:07 -0700 Subject: [PATCH 51/83] made use of threadpool for starting services, refactored services to support 3 validation models (blocking, non-blocking, timer) --- daemon/core/corehandlers.py | 6 +- daemon/core/service.py | 201 +++++++++++++++++++++--------------- daemon/core/session.py | 24 ----- daemon/tests/test_nodes.py | 1 - 4 files changed, 121 insertions(+), 111 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index bd44b24c..0237ca56 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1527,15 +1527,15 @@ class CoreHandler(SocketServer.BaseRequestHandler): if event_type == EventTypes.STOP.value or event_type == EventTypes.RESTART.value: status = self.session.services.stopnodeservice(node, service) - if status != "0": + if status: fail += "Stop %s," % service.name if event_type == EventTypes.START.value or event_type == EventTypes.RESTART.value: status = self.session.services.node_service_startup(node, service, services) - if status != "0": + if status: fail += "Start %s(%s)," % service.name if event_type == EventTypes.PAUSE.value: status = self.session.services.validatenodeservice(node, service, services) - if status != 0: + if status: fail += "%s," % service.name if event_type == EventTypes.RECONFIGURE.value: self.session.services.node_service_reconfigure(node, service, services) diff --git a/daemon/core/service.py b/daemon/core/service.py index ab0dbe5d..633a09c0 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -7,6 +7,10 @@ a list of available services to the GUI and for configuring individual services. """ +import time +from multiprocessing.pool import ThreadPool + +import enum from core.constants import which from core import CoreCommandError @@ -17,6 +21,16 @@ from core.enumerations import RegisterTlvs from core.misc import utils +class ServiceBootError(Exception): + pass + + +class ServiceMode(enum.Enum): + BLOCKING = 0 + NON_BLOCKING = 1 + TIMER = 2 + + class ServiceShim(object): keys = ["dirs", "files", "startidx", "cmdup", "cmddown", "cmdval", "meta", "starttime"] @@ -211,7 +225,7 @@ class CoreServices(object): self.defaultservices.clear() self.customservices.clear() - def get_service_startups(self, services): + def node_service_startups(self, services): # generate service map and find starting points node_services = {service.name: service for service in services} is_dependency = set() @@ -400,6 +414,9 @@ class CoreServices(object): :param core.netns.vnode.LxcNode node: node to start services on :return: nothing """ + pool = ThreadPool() + results = [] + services = sorted(node.services, key=lambda x: x.startindex) use_startup_service = any(map(self.is_startup_service, services)) for service in services: @@ -412,7 +429,13 @@ class CoreServices(object): continue except ValueError: logger.exception("error converting start time to float") - self.bootnodeservice(node, service, services, use_startup_service) + result = pool.apply_async(self.bootnodeservice, (node, service, services, use_startup_service)) + results.append(result) + + pool.close() + pool.join() + for result in results: + result.get() def bootnodeservice(self, node, service, services, use_startup_service): """ @@ -425,62 +448,35 @@ class CoreServices(object): :param bool use_startup_service: flag to use startup services or not :return: nothing """ - if service.custom: - self.bootnodecustomservice(node, service, services, use_startup_service) - return - logger.info("starting node(%s) service: %s (%s)", node.name, service.name, service.startindex) + + # create service directories for directory in service.dirs: node.privatedir(directory) - for filename in service.getconfigfilenames(node.objid, services): - cfg = service.generateconfig(node, filename, services) - node.nodefile(filename, cfg) + # create service files + self.node_service_files(node, service, services) + # check for startup service if use_startup_service and not self.is_startup_service(service): return - for args in service.getstartup(node, services): - # TODO: this wait=False can be problematic! - node.cmd(args, wait=False) + # run startup + wait = service.validation_mode == ServiceMode.BLOCKING + status = self.node_service_startup(node, service, services, wait) + if status: + raise ServiceBootError("node(%s) service(%s) error during startup" % (node.name, service.name)) - def bootnodecustomservice(self, node, service, services, use_startup_service): - """ - Start a custom service on a node. Create private dirs, use supplied - config files, and execute supplied startup commands. + # wait for time if provided, default to a time previously used to provide a small buffer + time.sleep(0.125) + if service.validation_timer: + time.sleep(service.validation_timer) - :param core.netns.vnode.LxcNode node: node to boot services on - :param CoreService service: service to start - :param list services: service list - :param bool use_startup_service: flag to use startup services or not - :return: nothing - """ - logger.info("starting node(%s) service(custom): %s (%s)", node.name, service.name, service.startindex) - for directory in service.dirs: - node.privatedir(directory) - - logger.info("service configurations: %s", service.configs) - for filename in service.configs: - logger.info("generating service config: %s", filename) - cfg = service.configtxt.get(filename) - if cfg is None: - cfg = service.generateconfig(node, filename, services) - - # cfg may have a file:/// url for copying from a file - try: - if self.copyservicefile(node, filename, cfg): - continue - except IOError: - logger.exception("error copying service file '%s'", filename) - continue - node.nodefile(filename, cfg) - - if use_startup_service and not self.is_startup_service(service): - return - - for args in service.startup: - # TODO: this wait=False can be problematic! - node.cmd(args, wait=False) + # run validation commands, if present and not timer mode + if service.validation_mode != ServiceMode.TIMER: + status = self.validatenodeservice(node, service, services) + if status: + raise ServiceBootError("node(%s) service(%s) failed validation" % (node.name, service.name)) def copyservicefile(self, node, filename, cfg): """ @@ -510,7 +506,7 @@ class CoreServices(object): :param core.netns.vnode.LxcNode node: node to validate services for :return: nothing """ - services = sorted(node.services, key=lambda service: service.startindex) + services = sorted(node.services, key=lambda x: x.startindex) for service in services: self.validatenodeservice(node, service, services) @@ -524,17 +520,16 @@ class CoreServices(object): :return: service validation status :rtype: int """ - logger.info("validating service for node (%s): %s (%s)", node.name, service.name, service.startindex) - if service.custom: - validate_cmds = service.validate - else: - validate_cmds = service.getvalidate(node, services) + logger.info("validating node(%s) service(%s): %s", node.name, service.name, service.startindex) + cmds = service.validate + if not service.custom: + cmds = service.getvalidate(node, services) status = 0 - for args in validate_cmds: - logger.info("validating service %s using: %s", service.name, args) + for cmd in cmds: + logger.info("validating service %s using: %s", service.name, cmd) try: - node.check_cmd(args) + node.check_cmd(cmd) except CoreCommandError: logger.exception("validate command failed") status = -1 @@ -561,14 +556,13 @@ class CoreServices(object): :return: status for stopping the services :rtype: str """ - status = "0" + status = 0 for args in service.shutdown: try: node.check_cmd(args) except CoreCommandError: logger.exception("error running stop command %s", args) - # TODO: determine if its ok to just return the bad exit status - status = "-1" + status = -1 return status def getservicefile(self, service_name, node, filename, services): @@ -608,7 +602,7 @@ class CoreServices(object): # get the file data data = service.configtxt.get(filename) if data is None: - data = "%s" % service.generateconfig(node, filename, services) + data = "%s" % service.generateconfig(node, filename, node_services) else: data = "%s" % data @@ -637,45 +631,81 @@ class CoreServices(object): self.setcustomservice(node_id, service_name) # retrieve custom service - svc = self.getcustomservice(node_id, service_name) - if svc is None: + service = self.getcustomservice(node_id, service_name) + if service is None: logger.warn("received filename for unknown service: %s", service_name) return # validate file being set is valid - cfgfiles = svc.configs + cfgfiles = service.configs if filename not in cfgfiles: logger.warn("received unknown file '%s' for service '%s'", filename, service_name) return # set custom service file data - svc.configtxt[filename] = data + service.configtxt[filename] = data - def node_service_startup(self, node, service, services): + def node_service_startup(self, node, service, services, wait=False): """ Startup a node service. :param PyCoreNode node: node to reconfigure service for :param CoreService service: service to reconfigure :param list[CoreService] services: node services + :param bool wait: determines if we should wait to validate startup :return: status of startup - :rtype: str + :rtype: int """ - if service.custom: - cmds = service.startup - else: + cmds = service.startup + if not service.custom: cmds = service.getstartup(node, services) - status = "0" - for args in cmds: + status = 0 + for cmd in cmds: try: - node.check_cmd(args) + if wait: + node.check_cmd(cmd) + else: + node.cmd(cmd, wait=False) except CoreCommandError: logger.exception("error starting command") - status = "-1" + status = -1 return status + def node_service_files(self, node, service, services): + """ + Creates node service files. + + :param PyCoreNode node: node to reconfigure service for + :param CoreService service: service to reconfigure + :param list[CoreService] services: node services + :return: nothing + """ + # get values depending on if custom or not + file_names = service.configs + if not service.custom: + file_names = service.getconfigfilenames(node.objid, services) + + for file_name in file_names: + logger.info("generating service config: %s", file_name) + if service.custom: + cfg = service.configtxt.get(file_name) + if cfg is None: + cfg = service.generateconfig(node, file_name, services) + + # cfg may have a file:/// url for copying from a file + try: + if self.copyservicefile(node, file_name, cfg): + continue + except IOError: + logger.exception("error copying service file: %s", file_name) + continue + else: + cfg = service.generateconfig(node, file_name, services) + + node.nodefile(file_name, cfg) + def node_service_reconfigure(self, node, service, services): """ Reconfigure a node service. @@ -685,21 +715,20 @@ class CoreServices(object): :param list[CoreService] services: node services :return: nothing """ - if service.custom: - cfgfiles = service.configs - else: - cfgfiles = service.getconfigfilenames(node.objid, services) + file_names = service.configs + if not service.custom: + file_names = service.getconfigfilenames(node.objid, services) - for filename in cfgfiles: - if filename[:7] == "file:///": + for file_name in file_names: + if file_name[:7] == "file:///": # TODO: implement this raise NotImplementedError - cfg = service.configtxt.get(filename) + cfg = service.configtxt.get(file_name) if cfg is None: - cfg = service.generateconfig(node, filename, services) + cfg = service.generateconfig(node, file_name, services) - node.nodefile(filename, cfg) + node.nodefile(file_name, cfg) class CoreService(object): @@ -742,6 +771,12 @@ class CoreService(object): # list of validate commands validate = () + # validation mode, used to determine startup success + validation_mode = ServiceMode.NON_BLOCKING + + # time to wait for determining if service started successfully + validation_timer = 0 + # metadata associated with this service meta = None diff --git a/daemon/core/session.py b/daemon/core/session.py index 57a007cd..3af1b5fe 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -607,12 +607,6 @@ class Session(object): # boot the services on each node self.boot_nodes() - # allow time for processes to start - time.sleep(0.125) - - # validate nodes - self.validate_nodes() - # set broker local instantiation to complete self.broker.local_instantiation_complete() @@ -732,24 +726,6 @@ class Session(object): self.update_control_interface_hosts() - def validate_nodes(self): - """ - Validate all nodes that are known by the session. - - :return: nothing - """ - with self._objects_lock: - for obj in self.objects.itervalues(): - # TODO: issues with checking PyCoreNode alone, validate is not a method - # such as vnoded process, bridges, etc. - if not isinstance(obj, nodes.PyCoreNode): - continue - - if nodeutils.is_node(obj, NodeTypes.RJ45): - continue - - obj.validate() - def get_control_net_prefixes(self): """ Retrieve control net prefixes. diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index 68872dbe..513856ec 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -39,7 +39,6 @@ class TestNodes: assert node.alive() assert node.up assert node.check_cmd(["ip", "addr", "show", "lo"]) - node.validate() def test_node_update(self, session): # given From 3443937ff2365692fd8536a3d99105a1aca6926a Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 20 Jun 2018 13:04:13 -0700 Subject: [PATCH 52/83] updated service load errors to be warnings --- daemon/core/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/core/service.py b/daemon/core/service.py index 633a09c0..2886bb12 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -150,7 +150,7 @@ class ServiceManager(object): # validate dependent executables are present for executable in service.executables: if not which(executable): - logger.error("service(%s) missing executable: %s", service.name, executable) + logger.warn("service(%s) missing executable: %s", service.name, executable) raise ValueError("service(%s) missing executable: %s" % (service.name, executable)) # make service available @@ -187,7 +187,7 @@ class ServiceManager(object): cls.add(service) except ValueError as e: service_errors.append(service.name) - logger.error("failure loading service: %s", e.message) + logger.warn("not loading service: %s", e.message) return service_errors From ed4e6f0f00b422f993ba202d8699278c77bcad2f Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 20 Jun 2018 13:07:43 -0700 Subject: [PATCH 53/83] removed startup service --- daemon/core/service.py | 14 ++---------- daemon/core/services/startup.py | 38 --------------------------------- 2 files changed, 2 insertions(+), 50 deletions(-) delete mode 100644 daemon/core/services/startup.py diff --git a/daemon/core/service.py b/daemon/core/service.py index 2886bb12..944dbd5f 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -214,10 +214,6 @@ class CoreServices(object): # dict of tuple of service objects, key is node number self.customservices = {} - # TODO: remove need for cyclic import - from core.services import startup - self.is_startup_service = startup.Startup.is_startup_service - def reset(self): """ Called when config message with reset flag is received @@ -418,7 +414,6 @@ class CoreServices(object): results = [] services = sorted(node.services, key=lambda x: x.startindex) - use_startup_service = any(map(self.is_startup_service, services)) for service in services: if len(str(service.starttime)) > 0: try: @@ -429,7 +424,7 @@ class CoreServices(object): continue except ValueError: logger.exception("error converting start time to float") - result = pool.apply_async(self.bootnodeservice, (node, service, services, use_startup_service)) + result = pool.apply_async(self.bootnodeservice, (node, service, services)) results.append(result) pool.close() @@ -437,7 +432,7 @@ class CoreServices(object): for result in results: result.get() - def bootnodeservice(self, node, service, services, use_startup_service): + def bootnodeservice(self, node, service, services): """ Start a service on a node. Create private dirs, generate config files, and execute startup commands. @@ -445,7 +440,6 @@ class CoreServices(object): :param core.netns.vnode.LxcNode node: node to boot services on :param CoreService service: service to start :param list services: service list - :param bool use_startup_service: flag to use startup services or not :return: nothing """ logger.info("starting node(%s) service: %s (%s)", node.name, service.name, service.startindex) @@ -457,10 +451,6 @@ class CoreServices(object): # create service files self.node_service_files(node, service, services) - # check for startup service - if use_startup_service and not self.is_startup_service(service): - return - # run startup wait = service.validation_mode == ServiceMode.BLOCKING status = self.node_service_startup(node, service, services, wait) diff --git a/daemon/core/services/startup.py b/daemon/core/services/startup.py deleted file mode 100644 index 6346c541..00000000 --- a/daemon/core/services/startup.py +++ /dev/null @@ -1,38 +0,0 @@ -from inspect import isclass -from sys import maxint - -from core.service import CoreService - - -class Startup(CoreService): - """ - A CORE service to start other services in order, serially - """ - name = 'startup' - group = 'Utility' - depends = () - dirs = () - configs = ('startup.sh',) - startindex = maxint - startup = ('sh startup.sh',) - shutdown = () - validate = () - - @staticmethod - def is_startup_service(s): - return isinstance(s, Startup) or (isclass(s) and issubclass(s, Startup)) - - @classmethod - def generateconfig(cls, node, filename, services): - if filename != cls.configs[0]: - return '' - script = '#!/bin/sh\n' \ - '# auto-generated by Startup (startup.py)\n\n' \ - 'exec > startup.log 2>&1\n\n' - for s in sorted(services, key=lambda x: x.startindex): - if cls.is_startup_service(s) or len(str(s.starttime)) > 0: - continue - start = '\n'.join(s.getstartup(node, services)) - if start: - script += start + '\n' - return script From 08956e7b93c3b25e7a906139dbe56dbdb1236fa3 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 20 Jun 2018 16:18:30 -0700 Subject: [PATCH 54/83] making use of threadpools to boot each node and boot the services within a node --- daemon/core/service.py | 3 ++- daemon/core/session.py | 14 +++++++++++++- daemon/tests/test_services.py | 6 +++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/daemon/core/service.py b/daemon/core/service.py index 944dbd5f..e20faf00 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -221,7 +221,7 @@ class CoreServices(object): self.defaultservices.clear() self.customservices.clear() - def node_service_startups(self, services): + def node_service_dependencies(self, services): # generate service map and find starting points node_services = {service.name: service for service in services} is_dependency = set() @@ -424,6 +424,7 @@ class CoreServices(object): continue except ValueError: logger.exception("error converting start time to float") + # self.bootnodeservice(node, service, services) result = pool.apply_async(self.bootnodeservice, (node, service, services)) results.append(result) diff --git a/daemon/core/session.py b/daemon/core/session.py index 3af1b5fe..6cae3a62 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -10,6 +10,7 @@ import subprocess import tempfile import threading import time +from multiprocessing.pool import ThreadPool import pwd @@ -716,13 +717,24 @@ class Session(object): request flag. """ with self._objects_lock: + pool = ThreadPool() + results = [] + + start = time.time() for obj in self.objects.itervalues(): # TODO: PyCoreNode is not the type to check if isinstance(obj, nodes.PyCoreNode) and not nodeutils.is_node(obj, NodeTypes.RJ45): # add a control interface if configured logger.info("booting node: %s", obj.name) self.add_remove_control_interface(node=obj, remove=False) - obj.boot() + result = pool.apply_async(obj.boot) + results.append(result) + + pool.close() + pool.join() + for result in results: + result.get() + logger.info("BOOT RUN TIME: %s", time.time() - start) self.update_control_interface_hosts() diff --git a/daemon/tests/test_services.py b/daemon/tests/test_services.py index 3cefd7bf..346fa502 100644 --- a/daemon/tests/test_services.py +++ b/daemon/tests/test_services.py @@ -59,7 +59,7 @@ class TestServices: ] # when - startups = session.services.get_service_startups(services) + startups = session.services.node_service_dependencies(services) # then assert len(startups) == 2 @@ -75,7 +75,7 @@ class TestServices: # when with pytest.raises(ValueError): - session.services.get_service_startups(services) + session.services.node_service_dependencies(services) def test_services_dependencies_cycle(self, session): # given @@ -91,4 +91,4 @@ class TestServices: # when with pytest.raises(ValueError): - session.services.get_service_startups(services) + session.services.node_service_dependencies(services) From b868454b5e80e4c6eda4c35986fd8ac5c2a0bf72 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 21 Jun 2018 11:20:08 -0700 Subject: [PATCH 55/83] updates to use dependency based startup logic --- daemon/core/service.py | 26 +++++++++++--------------- daemon/core/services/bird.py | 1 + daemon/core/services/dockersvc.py | 2 +- daemon/core/services/quagga.py | 1 + daemon/core/services/xorp.py | 1 + 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/daemon/core/service.py b/daemon/core/service.py index e20faf00..41e155df 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -414,18 +414,9 @@ class CoreServices(object): results = [] services = sorted(node.services, key=lambda x: x.startindex) - for service in services: - if len(str(service.starttime)) > 0: - try: - starttime = float(service.starttime) - if starttime > 0.0: - fn = self.bootnodeservice - self.session.event_loop.add_event(starttime, fn, node, service, services, False) - continue - except ValueError: - logger.exception("error converting start time to float") - # self.bootnodeservice(node, service, services) - result = pool.apply_async(self.bootnodeservice, (node, service, services)) + boot_paths = self.node_service_dependencies(services) + for boot_path in boot_paths: + result = pool.apply_async(self.boot_node_dependencies, (node, boot_path, services)) results.append(result) pool.close() @@ -433,6 +424,11 @@ class CoreServices(object): for result in results: result.get() + def boot_node_dependencies(self, node, boot_path, all_services): + logger.info("booting node service dependencies: %s", boot_path) + for service in boot_path: + self.bootnodeservice(node, service, all_services) + def bootnodeservice(self, node, service, services): """ Start a service on a node. Create private dirs, generate config @@ -518,11 +514,11 @@ class CoreServices(object): status = 0 for cmd in cmds: - logger.info("validating service %s using: %s", service.name, cmd) + logger.info("validating service(%s) using: %s", service.name, cmd) try: node.check_cmd(cmd) - except CoreCommandError: - logger.exception("validate command failed") + except CoreCommandError as e: + logger.exception("node(%s) service(%s) validate command failed", node.name, service.name) status = -1 return status diff --git a/daemon/core/services/bird.py b/daemon/core/services/bird.py index 23a9fcec..639b99ed 100644 --- a/daemon/core/services/bird.py +++ b/daemon/core/services/bird.py @@ -94,6 +94,7 @@ class BirdService(CoreService): name = None executables = ("bird",) group = "BIRD" + dependencies = ("bird",) depends = ("bird",) dirs = () configs = () diff --git a/daemon/core/services/dockersvc.py b/daemon/core/services/dockersvc.py index d36ec5a4..0cb9cdf1 100644 --- a/daemon/core/services/dockersvc.py +++ b/daemon/core/services/dockersvc.py @@ -104,7 +104,7 @@ from core.service import ServiceManager try: from docker import Client except ImportError: - logger.error("failure to import docker") + logger.warns("missing python docker bindings") class DockerService(CoreService): diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index 791d892d..d1c9fb74 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -222,6 +222,7 @@ class QuaggaService(CoreService): """ name = None group = "Quagga" + dependencies = ("zebra",) depends = ("zebra",) dirs = () configs = () diff --git a/daemon/core/services/xorp.py b/daemon/core/services/xorp.py index 326c09b5..da81b33b 100644 --- a/daemon/core/services/xorp.py +++ b/daemon/core/services/xorp.py @@ -78,6 +78,7 @@ class XorpService(CoreService): name = None executables = ("xorp_rtrmgr",) group = "XORP" + dependencies = ("xorp_rtrmgr",) depends = ("xorp_rtrmgr",) dirs = () configs = () From 0aca9d7809d7d43bf126068c7facb24ce5eea531 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 21 Jun 2018 14:56:30 -0700 Subject: [PATCH 56/83] changes to remove the need to pass services separately, when the node has access to them directly, also made a couple CoreServices methods more consistent to have the node as first param and service second --- daemon/core/corehandlers.py | 12 ++-- daemon/core/service.py | 107 ++++++++++++------------------ daemon/core/services/bird.py | 8 +-- daemon/core/services/dockersvc.py | 2 +- daemon/core/services/nrl.py | 32 ++++----- daemon/core/services/quagga.py | 23 +++---- daemon/core/services/sdn.py | 10 +-- daemon/core/services/security.py | 8 +-- daemon/core/services/ucarp.py | 18 ++--- daemon/core/services/utility.py | 44 ++++++------ daemon/core/services/xorp.py | 6 +- 11 files changed, 123 insertions(+), 147 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 0237ca56..e5d9cb35 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1151,7 +1151,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): # a file request: e.g. "service:zebra:quagga.conf" file_name = servicesstring[2] service_name = services[0] - file_data = self.session.services.getservicefile(service_name, node, file_name, services) + file_data = self.session.services.getservicefile(node, service_name, file_name) self.session.broadcast_file(file_data) # short circuit this request early to avoid returning response below return replies @@ -1163,7 +1163,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): type_flag = ConfigFlags.UPDATE.value data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(ServiceShim.keys))) service = self.session.services.getcustomservice(node_id, service_name, default_service=True) - values = ServiceShim.tovaluelist(service, node, services) + values = ServiceShim.tovaluelist(node, service) captions = None possible_values = None groups = None @@ -1530,15 +1530,15 @@ class CoreHandler(SocketServer.BaseRequestHandler): if status: fail += "Stop %s," % service.name if event_type == EventTypes.START.value or event_type == EventTypes.RESTART.value: - status = self.session.services.node_service_startup(node, service, services) + status = self.session.services.node_service_startup(node, service) if status: fail += "Start %s(%s)," % service.name if event_type == EventTypes.PAUSE.value: - status = self.session.services.validatenodeservice(node, service, services) + status = self.session.services.validatenodeservice(node, service) if status: fail += "%s," % service.name if event_type == EventTypes.RECONFIGURE.value: - self.session.services.node_service_reconfigure(node, service, services) + self.session.services.node_service_reconfigure(node, service) fail_data = "" if len(fail) > 0: @@ -1724,7 +1724,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): opaque = "service:%s" % service.name data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(ServiceShim.keys))) node = self.session.get_object(node_id) - values = ServiceShim.tovaluelist(service, node, node.services) + values = ServiceShim.tovaluelist(node, service) config_data = ConfigData( message_type=0, node=node_id, diff --git a/daemon/core/service.py b/daemon/core/service.py index 41e155df..fa45f149 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -35,14 +35,13 @@ class ServiceShim(object): keys = ["dirs", "files", "startidx", "cmdup", "cmddown", "cmdval", "meta", "starttime"] @classmethod - def tovaluelist(cls, service, node, services): + def tovaluelist(cls, node, service): """ Convert service properties into a string list of key=value pairs, separated by "|". - :param CoreService service: service to get value list for :param core.netns.nodes.CoreNode node: node to get value list for - :param list[CoreService] services: services for node + :param CoreService service: service to get value list for :return: value list string :rtype: str """ @@ -50,8 +49,8 @@ class ServiceShim(object): service.shutdown, service.validate, service.meta, service.starttime] if not service.custom: # this is always reached due to classmethod - valmap[valmap.index(service.configs)] = service.getconfigfilenames(node.objid, services) - valmap[valmap.index(service.startup)] = service.getstartup(node, services) + valmap[valmap.index(service.configs)] = service.getconfigfilenames(node) + valmap[valmap.index(service.startup)] = service.getstartup(node) vals = map(lambda a, b: "%s=%s" % (a, str(b)), cls.keys, valmap) return "|".join(vals) @@ -112,7 +111,7 @@ class ServiceShim(object): service.starttime = value @classmethod - def servicesfromopaque(self, opaque): + def servicesfromopaque(cls, opaque): """ Build a list of services from an opaque data string. @@ -288,18 +287,18 @@ class CoreServices(object): return startups - def getdefaultservices(self, service_type): + def getdefaultservices(self, node_type): """ Get the list of default services that should be enabled for a node for the given node type. - :param service_type: service type to get default services for + :param node_type: node type to get default services for :return: default services :rtype: list[CoreService] """ - logger.debug("getting default services for type: %s", service_type) + logger.debug("getting default services for type: %s", node_type) results = [] - defaults = self.defaultservices.get(service_type, []) + defaults = self.defaultservices.get(node_type, []) for name in defaults: logger.debug("checking for service with service manager: %s", name) service = ServiceManager.get(name) @@ -351,7 +350,7 @@ class CoreServices(object): :param core.coreobj.PyCoreNode node: node to add services to :param str node_type: node type to add services to - :param list[str] services: services to add to node + :param list[str] services: names of services to add to node :return: nothing """ if not services: @@ -372,7 +371,6 @@ class CoreServices(object): Return (nodenum, service) tuples for all stored configs. Used when reconnecting to a session or opening XML. - :param bool use_clsmap: should a class map be used, default to True :return: list of tuples of node ids and services :rtype: list """ @@ -413,10 +411,9 @@ class CoreServices(object): pool = ThreadPool() results = [] - services = sorted(node.services, key=lambda x: x.startindex) - boot_paths = self.node_service_dependencies(services) + boot_paths = self.node_service_dependencies(node.services) for boot_path in boot_paths: - result = pool.apply_async(self.boot_node_dependencies, (node, boot_path, services)) + result = pool.apply_async(self.boot_node_dependencies, (node, boot_path)) results.append(result) pool.close() @@ -424,19 +421,18 @@ class CoreServices(object): for result in results: result.get() - def boot_node_dependencies(self, node, boot_path, all_services): + def boot_node_dependencies(self, node, boot_path): logger.info("booting node service dependencies: %s", boot_path) for service in boot_path: - self.bootnodeservice(node, service, all_services) + self.bootnodeservice(node, service) - def bootnodeservice(self, node, service, services): + def bootnodeservice(self, node, service): """ Start a service on a node. Create private dirs, generate config files, and execute startup commands. :param core.netns.vnode.LxcNode node: node to boot services on :param CoreService service: service to start - :param list services: service list :return: nothing """ logger.info("starting node(%s) service: %s (%s)", node.name, service.name, service.startindex) @@ -446,11 +442,11 @@ class CoreServices(object): node.privatedir(directory) # create service files - self.node_service_files(node, service, services) + self.node_service_files(node, service) # run startup wait = service.validation_mode == ServiceMode.BLOCKING - status = self.node_service_startup(node, service, services, wait) + status = self.node_service_startup(node, service, wait) if status: raise ServiceBootError("node(%s) service(%s) error during startup" % (node.name, service.name)) @@ -461,7 +457,7 @@ class CoreServices(object): # run validation commands, if present and not timer mode if service.validation_mode != ServiceMode.TIMER: - status = self.validatenodeservice(node, service, services) + status = self.validatenodeservice(node, service) if status: raise ServiceBootError("node(%s) service(%s) failed validation" % (node.name, service.name)) @@ -493,31 +489,29 @@ class CoreServices(object): :param core.netns.vnode.LxcNode node: node to validate services for :return: nothing """ - services = sorted(node.services, key=lambda x: x.startindex) - for service in services: - self.validatenodeservice(node, service, services) + for service in node.services: + self.validatenodeservice(node, service) - def validatenodeservice(self, node, service, services): + def validatenodeservice(self, node, service): """ Run the validation command(s) for a service. :param core.netns.vnode.LxcNode node: node to validate service for :param CoreService service: service to validate - :param list services: services for node :return: service validation status :rtype: int """ logger.info("validating node(%s) service(%s): %s", node.name, service.name, service.startindex) cmds = service.validate if not service.custom: - cmds = service.getvalidate(node, services) + cmds = service.getvalidate(node) status = 0 for cmd in cmds: logger.info("validating service(%s) using: %s", service.name, cmd) try: node.check_cmd(cmd) - except CoreCommandError as e: + except CoreCommandError: logger.exception("node(%s) service(%s) validate command failed", node.name, service.name) status = -1 @@ -552,15 +546,14 @@ class CoreServices(object): status = -1 return status - def getservicefile(self, service_name, node, filename, services): + def getservicefile(self, node, service_name, filename): """ Send a File Message when the GUI has requested a service file. The file data is either auto-generated or comes from an existing config. - :param str service_name: service to get file from :param core.netns.vnode.LxcNode node: node to get service file from + :param str service_name: service to get file from :param str filename: file name to retrieve - :param list[str] services: list of services associated with node :return: file message for node """ # get service to get file from @@ -568,20 +561,11 @@ class CoreServices(object): if not service: raise ValueError("invalid service: %s", service_name) - # get service for node - node_services = [] - for service_name in services: - node_service = self.getcustomservice(node.objid, service_name, default_service=True) - if not node_service: - logger.warn("unknown service: %s", service) - continue - node_services.append(node_service) - # retrieve config files for default/custom service if service.custom: config_files = service.configs else: - config_files = service.getconfigfilenames(node.objid, node_services) + config_files = service.getconfigfilenames(node) if filename not in config_files: raise ValueError("unknown service(%s) config file: %s", service_name, filename) @@ -589,7 +573,7 @@ class CoreServices(object): # get the file data data = service.configtxt.get(filename) if data is None: - data = "%s" % service.generateconfig(node, filename, node_services) + data = "%s" % service.generateconfig(node, filename) else: data = "%s" % data @@ -632,13 +616,12 @@ class CoreServices(object): # set custom service file data service.configtxt[filename] = data - def node_service_startup(self, node, service, services, wait=False): + def node_service_startup(self, node, service, wait=False): """ Startup a node service. :param PyCoreNode node: node to reconfigure service for :param CoreService service: service to reconfigure - :param list[CoreService] services: node services :param bool wait: determines if we should wait to validate startup :return: status of startup :rtype: int @@ -646,7 +629,7 @@ class CoreServices(object): cmds = service.startup if not service.custom: - cmds = service.getstartup(node, services) + cmds = service.getstartup(node) status = 0 for cmd in cmds: @@ -660,26 +643,25 @@ class CoreServices(object): status = -1 return status - def node_service_files(self, node, service, services): + def node_service_files(self, node, service): """ Creates node service files. :param PyCoreNode node: node to reconfigure service for :param CoreService service: service to reconfigure - :param list[CoreService] services: node services :return: nothing """ # get values depending on if custom or not file_names = service.configs if not service.custom: - file_names = service.getconfigfilenames(node.objid, services) + file_names = service.getconfigfilenames(node) for file_name in file_names: logger.info("generating service config: %s", file_name) if service.custom: cfg = service.configtxt.get(file_name) if cfg is None: - cfg = service.generateconfig(node, file_name, services) + cfg = service.generateconfig(node, file_name) # cfg may have a file:/// url for copying from a file try: @@ -689,22 +671,21 @@ class CoreServices(object): logger.exception("error copying service file: %s", file_name) continue else: - cfg = service.generateconfig(node, file_name, services) + cfg = service.generateconfig(node, file_name) node.nodefile(file_name, cfg) - def node_service_reconfigure(self, node, service, services): + def node_service_reconfigure(self, node, service): """ Reconfigure a node service. :param PyCoreNode node: node to reconfigure service for :param CoreService service: service to reconfigure - :param list[CoreService] services: node services :return: nothing """ file_names = service.configs if not service.custom: - file_names = service.getconfigfilenames(node.objid, services) + file_names = service.getconfigfilenames(node) for file_name in file_names: if file_name[:7] == "file:///": @@ -713,7 +694,7 @@ class CoreServices(object): cfg = service.configtxt.get(file_name) if cfg is None: - cfg = service.generateconfig(node, file_name, services) + cfg = service.generateconfig(node, file_name) node.nodefile(file_name, cfg) @@ -794,21 +775,20 @@ class CoreService(object): pass @classmethod - def getconfigfilenames(cls, node_id, services): + def getconfigfilenames(cls, node): """ Return the tuple of configuration file filenames. This default method returns the cls._configs tuple, but this method may be overriden to provide node-specific filenames that may be based on other services. - :param int node_id: node id to get config file names for - :param list services: node services - :return: class configuration files + :param core.netns.vnode.LxcNode node: node to generate config for + :return: configuration files :rtype: tuple """ return cls.configs @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate configuration file given a node object. The filename is provided to allow for multiple config files. The other services are @@ -818,13 +798,12 @@ class CoreService(object): :param core.netns.vnode.LxcNode node: node to generate config for :param str filename: file name to generate config for - :param list services: services for node :return: nothing """ raise NotImplementedError @classmethod - def getstartup(cls, node, services): + def getstartup(cls, node): """ Return the tuple of startup commands. This default method returns the cls._startup tuple, but this method may be @@ -832,14 +811,13 @@ class CoreService(object): based on other services. :param core.netns.vnode.LxcNode node: node to get startup for - :param list services: services for node :return: startup commands :rtype: tuple """ return cls.startup @classmethod - def getvalidate(cls, node, services): + def getvalidate(cls, node): """ Return the tuple of validate commands. This default method returns the cls._validate tuple, but this method may be @@ -847,7 +825,6 @@ class CoreService(object): based on other services. :param core.netns.vnode.LxcNode node: node to validate - :param list services: services for node :return: validation commands :rtype: tuple """ diff --git a/daemon/core/services/bird.py b/daemon/core/services/bird.py index 639b99ed..a869fe7c 100644 --- a/daemon/core/services/bird.py +++ b/daemon/core/services/bird.py @@ -21,12 +21,12 @@ class Bird(CoreService): validate = ("pidof bird",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the bird.conf file contents. """ if filename == cls.configs[0]: - return cls.generateBirdConf(node, services) + return cls.generateBirdConf(node) else: raise ValueError @@ -45,7 +45,7 @@ class Bird(CoreService): return "0.0.0.0" @classmethod - def generateBirdConf(cls, node, services): + def generateBirdConf(cls, node): """ Returns configuration file text. Other services that depend on bird will have generatebirdifcconfig() and generatebirdconfig() @@ -77,7 +77,7 @@ protocol device { """ % (cls.name, cls.routerid(node)) # Generate protocol specific configurations - for s in services: + for s in node.services: if cls.name not in s.depends: continue cfg += s.generatebirdconfig(node) diff --git a/daemon/core/services/dockersvc.py b/daemon/core/services/dockersvc.py index 0cb9cdf1..797a29e6 100644 --- a/daemon/core/services/dockersvc.py +++ b/daemon/core/services/dockersvc.py @@ -125,7 +125,7 @@ class DockerService(CoreService): image = "" @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Returns a string having contents of a docker.sh script that can be modified to start a specific docker image. diff --git a/daemon/core/services/nrl.py b/daemon/core/services/nrl.py index d446ef81..cf5e1d9c 100644 --- a/daemon/core/services/nrl.py +++ b/daemon/core/services/nrl.py @@ -23,7 +23,7 @@ class NrlService(CoreService): shutdown = () @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): return "" @staticmethod @@ -55,7 +55,7 @@ class MgenSinkService(NrlService): shutdown = ("killall mgen",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): cfg = "0.0 LISTEN UDP 5000\n" for ifc in node.netifs(): name = utils.sysctl_devname(ifc.name) @@ -63,7 +63,7 @@ class MgenSinkService(NrlService): return cfg @classmethod - def getstartup(cls, node, services): + def getstartup(cls, node): cmd = cls.startup[0] cmd += " output /tmp/mgen_%s.log" % node.name return cmd, @@ -80,7 +80,7 @@ class NrlNhdp(NrlService): validate = ("pidof nrlnhdp",) @classmethod - def getstartup(cls, node, services): + def getstartup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -88,7 +88,7 @@ class NrlNhdp(NrlService): cmd += " -l /var/log/nrlnhdp.log" cmd += " -rpipe %s_nhdp" % node.name - servicenames = map(lambda x: x.name, services) + servicenames = map(lambda x: x.name, node.services) if "SMF" in servicenames: cmd += " -flooding ecds" cmd += " -smfClient %s_smf" % node.name @@ -114,7 +114,7 @@ class NrlSmf(NrlService): configs = ("startsmf.sh",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename, ): """ Generate a startup script for SMF. Because nrlsmf does not daemonize, it can cause problems in some situations when launched @@ -125,7 +125,7 @@ class NrlSmf(NrlService): comments = "" cmd = "nrlsmf instance %s_smf" % node.name - servicenames = map(lambda x: x.name, services) + servicenames = map(lambda x: x.name, node.services) netifs = filter(lambda x: not getattr(x, 'control', False), node.netifs()) if len(netifs) == 0: return "" @@ -165,7 +165,7 @@ class NrlOlsr(NrlService): validate = ("pidof nrlolsrd",) @classmethod - def getstartup(cls, node, services): + def getstartup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -178,7 +178,7 @@ class NrlOlsr(NrlService): cmd += " -l /var/log/nrlolsrd.log" cmd += " -rpipe %s_olsr" % node.name - servicenames = map(lambda x: x.name, services) + servicenames = map(lambda x: x.name, node.services) if "SMF" in servicenames and not "NHDP" in servicenames: cmd += " -flooding s-mpr" cmd += " -smfClient %s_smf" % node.name @@ -199,7 +199,7 @@ class NrlOlsrv2(NrlService): validate = ("pidof nrlolsrv2",) @classmethod - def getstartup(cls, node, services): + def getstartup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -207,7 +207,7 @@ class NrlOlsrv2(NrlService): cmd += " -l /var/log/nrlolsrv2.log" cmd += " -rpipe %s_olsrv2" % node.name - servicenames = map(lambda x: x.name, services) + servicenames = map(lambda x: x.name, node.services) if "SMF" in servicenames: cmd += " -flooding ecds" cmd += " -smfClient %s_smf" % node.name @@ -236,7 +236,7 @@ class OlsrOrg(NrlService): validate = ("pidof olsrd",) @classmethod - def getstartup(cls, node, services): + def getstartup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -250,7 +250,7 @@ class OlsrOrg(NrlService): return cmd, @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate a default olsrd config file to use the broadcast address of 255.255.255.255. """ @@ -598,7 +598,7 @@ class MgenActor(NrlService): shutdown = ("killall mgen",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate a startup script for MgenActor. Because mgenActor does not daemonize, it can cause problems in some situations when launched @@ -609,7 +609,7 @@ class MgenActor(NrlService): comments = "" cmd = "mgenBasicActor.py -n %s -a 0.0.0.0" % node.name - servicenames = map(lambda x: x.name, services) + servicenames = map(lambda x: x.name, node.services) netifs = filter(lambda x: not getattr(x, 'control', False), node.netifs()) if len(netifs) == 0: return "" @@ -631,7 +631,7 @@ class Arouted(NrlService): validate = ("pidof arouted",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the Quagga.conf or quaggaboot.sh file contents. """ diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index d1c9fb74..79e607f2 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -24,29 +24,28 @@ class Zebra(CoreService): validate = ("pidof zebra",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the Quagga.conf or quaggaboot.sh file contents. """ if filename == cls.configs[0]: - return cls.generateQuaggaConf(node, services) + return cls.generateQuaggaConf(node) elif filename == cls.configs[1]: - return cls.generateQuaggaBoot(node, services) + return cls.generateQuaggaBoot(node) elif filename == cls.configs[2]: - return cls.generateVtyshConf(node, services) + return cls.generateVtyshConf(node) else: - raise ValueError("file name (%s) is not a known configuration: %s", - filename, cls.configs) + raise ValueError("file name (%s) is not a known configuration: %s", filename, cls.configs) @classmethod - def generateVtyshConf(cls, node, services): + def generateVtyshConf(cls, node): """ Returns configuration file text. """ return "service integrated-vtysh-config\n" @classmethod - def generateQuaggaConf(cls, node, services): + def generateQuaggaConf(cls, node): """ Returns configuration file text. Other services that depend on zebra will have generatequaggaifcconfig() and generatequaggaconfig() @@ -66,7 +65,7 @@ class Zebra(CoreService): cfgv6 = "" want_ipv4 = False want_ipv6 = False - for s in services: + for s in node.services: if cls.name not in s.depends: continue ifccfg = s.generatequaggaifcconfig(node, ifc) @@ -92,7 +91,7 @@ class Zebra(CoreService): cfg += cfgv6 cfg += "!\n" - for s in services: + for s in node.services: if cls.name not in s.depends: continue cfg += s.generatequaggaconfig(node) @@ -111,7 +110,7 @@ class Zebra(CoreService): raise ValueError("invalid address: %s", x) @classmethod - def generateQuaggaBoot(cls, node, services): + def generateQuaggaBoot(cls, node): """ Generate a shell script used to boot the Quagga daemons. """ @@ -263,7 +262,7 @@ class QuaggaService(CoreService): return False @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): return "" @classmethod diff --git a/daemon/core/services/sdn.py b/daemon/core/services/sdn.py index 4e144ea0..c9769278 100644 --- a/daemon/core/services/sdn.py +++ b/daemon/core/services/sdn.py @@ -15,7 +15,7 @@ class SdnService(CoreService): startindex = 50 @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): return "" @@ -31,16 +31,16 @@ class OvsService(SdnService): shutdown = ('killall ovs-vswitchd', 'killall ovsdb-server') @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): # Check whether the node is running zebra has_zebra = 0 - for s in services: + for s in node.services: if s.name == "zebra": has_zebra = 1 # Check whether the node is running an SDN controller has_sdn_ctrlr = 0 - for s in services: + for s in node.services: if s.name == "ryuService": has_sdn_ctrlr = 1 @@ -106,7 +106,7 @@ class RyuService(SdnService): shutdown = ('killall ryu-manager',) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return a string that will be written to filename, or sent to the GUI for user customization. diff --git a/daemon/core/services/security.py b/daemon/core/services/security.py index a8318e42..be3db917 100644 --- a/daemon/core/services/security.py +++ b/daemon/core/services/security.py @@ -19,7 +19,7 @@ class VPNClient(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the client.conf and vpnclient.sh file contents to """ @@ -46,7 +46,7 @@ class VPNServer(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the sample server.conf and vpnserver.sh file contents to GUI for user customization. @@ -73,7 +73,7 @@ class IPsec(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the ipsec.conf and racoon.conf file contents to GUI for user customization. @@ -100,7 +100,7 @@ class Firewall(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the firewall rule examples to GUI for user customization. """ diff --git a/daemon/core/services/ucarp.py b/daemon/core/services/ucarp.py index 5009ea6d..80786999 100644 --- a/daemon/core/services/ucarp.py +++ b/daemon/core/services/ucarp.py @@ -20,23 +20,23 @@ class Ucarp(CoreService): validate = ("pidof ucarp",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return the default file contents """ if filename == cls.configs[0]: - return cls.generateUcarpConf(node, services) + return cls.generateUcarpConf(node) elif filename == cls.configs[1]: - return cls.generateVipUp(node, services) + return cls.generateVipUp(node) elif filename == cls.configs[2]: - return cls.generateVipDown(node, services) + return cls.generateVipDown(node) elif filename == cls.configs[3]: - return cls.generateUcarpBoot(node, services) + return cls.generateUcarpBoot(node) else: raise ValueError @classmethod - def generateUcarpConf(cls, node, services): + def generateUcarpConf(cls, node): """ Returns configuration file text. """ @@ -105,7 +105,7 @@ ${UCARP_EXEC} -B ${UCARP_OPTS} """ % (ucarp_bin, UCARP_ETC) @classmethod - def generateUcarpBoot(cls, node, services): + def generateUcarpBoot(cls, node): """ Generate a shell script used to boot the Ucarp daemons. """ @@ -127,7 +127,7 @@ ${UCARP_CFGDIR}/default.sh """ % UCARP_ETC @classmethod - def generateVipUp(cls, node, services): + def generateVipUp(cls, node): """ Generate a shell script used to start the virtual ip """ @@ -154,7 +154,7 @@ fi """ @classmethod - def generateVipDown(cls, node, services): + def generateVipDown(cls, node): """ Generate a shell script used to stop the virtual ip """ diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 2bc11893..79f35c16 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -26,7 +26,7 @@ class UtilService(CoreService): shutdown = () @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): return "" @@ -37,14 +37,14 @@ class IPForwardService(UtilService): startup = ("sh ipforward.sh",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): if os.uname()[0] == "Linux": - return cls.generateconfiglinux(node, filename, services) + return cls.generateconfiglinux(node, filename) else: raise Exception("unknown platform") @classmethod - def generateconfiglinux(cls, node, filename, services): + def generateconfiglinux(cls, node, filename): cfg = """\ #!/bin/sh # auto-generated by IPForward service (utility.py) @@ -72,7 +72,7 @@ class DefaultRouteService(UtilService): startup = ("sh defaultroute.sh",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): cfg = "#!/bin/sh\n" cfg += "# auto-generated by DefaultRoute service (utility.py)\n" for ifc in node.netifs(): @@ -106,7 +106,7 @@ class DefaultMulticastRouteService(UtilService): startup = ("sh defaultmroute.sh",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): cfg = "#!/bin/sh\n" cfg += "# auto-generated by DefaultMulticastRoute service (utility.py)\n" cfg += "# the first interface is chosen below; please change it " @@ -132,7 +132,7 @@ class StaticRouteService(UtilService): custom_needed = True @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): cfg = "#!/bin/sh\n" cfg += "# auto-generated by StaticRoute service (utility.py)\n#\n" cfg += "# NOTE: this service must be customized to be of any use\n" @@ -173,7 +173,7 @@ class SshService(UtilService): validate = () @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Use a startup script for launching sshd in order to wait for host key generation. @@ -241,7 +241,7 @@ class DhcpService(UtilService): validate = ("pidof dhcpd",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate a dhcpd config file using the network address of each interface. @@ -303,7 +303,7 @@ class DhcpClientService(UtilService): validate = ("pidof dhclient",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate a script to invoke dhclient on all interfaces. """ @@ -314,7 +314,7 @@ class DhcpClientService(UtilService): cfg += "#mkdir -p /var/run/resolvconf/interface\n" for ifc in node.netifs(): - if hasattr(ifc, 'control') and ifc.control == True: + if hasattr(ifc, 'control') and ifc.control is True: continue cfg += "#ln -s /var/run/resolvconf/interface/%s.dhclient" % ifc.name cfg += " /var/run/resolvconf/resolv.conf\n" @@ -335,7 +335,7 @@ class FtpService(UtilService): validate = ("pidof vsftpd",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate a vsftpd.conf configuration file. """ @@ -371,16 +371,16 @@ class HttpService(UtilService): APACHEVER22, APACHEVER24 = (22, 24) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate an apache2.conf configuration file. """ if filename == cls.configs[0]: - return cls.generateapache2conf(node, filename, services) + return cls.generateapache2conf(node, filename) elif filename == cls.configs[1]: - return cls.generateenvvars(node, filename, services) + return cls.generateenvvars(node, filename) elif filename == cls.configs[2]: - return cls.generatehtml(node, filename, services) + return cls.generatehtml(node, filename) else: return "" @@ -400,7 +400,7 @@ class HttpService(UtilService): return cls.APACHEVER22 @classmethod - def generateapache2conf(cls, node, filename, services): + def generateapache2conf(cls, node, filename): lockstr = {cls.APACHEVER22: 'LockFile ${APACHE_LOCK_DIR}/accept.lock\n', cls.APACHEVER24: @@ -538,7 +538,7 @@ TraceEnable Off return cfg @classmethod - def generateenvvars(cls, node, filename, services): + def generateenvvars(cls, node, filename): return """\ # this file is used by apache2ctl - generated by utility.py:HttpService # these settings come from a default Ubuntu apache2 installation @@ -553,7 +553,7 @@ export LANG """ @classmethod - def generatehtml(cls, node, filename, services): + def generatehtml(cls, node, filename): body = """\

    %s web server

    @@ -581,7 +581,7 @@ class PcapService(UtilService): meta = "logs network traffic to pcap packet capture files" @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate a startpcap.sh traffic logging script. """ @@ -619,7 +619,7 @@ class RadvdService(UtilService): validate = ("pidof radvd",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Generate a RADVD router advertisement daemon config file using the network address of each interface. @@ -678,7 +678,7 @@ class AtdService(UtilService): shutdown = ("pkill atd",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): return """ #!/bin/sh echo 00001 > /var/spool/cron/atjobs/.SEQ diff --git a/daemon/core/services/xorp.py b/daemon/core/services/xorp.py index da81b33b..58668b1f 100644 --- a/daemon/core/services/xorp.py +++ b/daemon/core/services/xorp.py @@ -23,7 +23,7 @@ class XorpRtrmgr(CoreService): validate = ("pidof xorp_rtrmgr",) @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Returns config.boot configuration file text. Other services that depend on this will have generatexorpconfig() hooks that are @@ -39,7 +39,7 @@ class XorpRtrmgr(CoreService): cfg += " }\n" cfg += "}\n\n" - for s in services: + for s in node.services: try: s.depends.index(cls.name) cfg += s.generatexorpconfig(node) @@ -154,7 +154,7 @@ class XorpService(CoreService): return "0.0.0.0" @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): return "" @classmethod From 37517c45f4b8aad1604c6dbcaf6dfb64cd047c7c Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 22 Jun 2018 08:16:59 -0700 Subject: [PATCH 57/83] changed some logging from info to debug, removed startindex and depends from services --- daemon/core/mobility.py | 4 +-- daemon/core/netns/vnet.py | 6 ++--- daemon/core/service.py | 40 +++++++++------------------- daemon/core/services/bird.py | 6 +---- daemon/core/services/dockersvc.py | 4 +-- daemon/core/services/nrl.py | 8 ------ daemon/core/services/quagga.py | 7 ++--- daemon/core/services/sdn.py | 5 ---- daemon/core/services/security.py | 4 --- daemon/core/services/ucarp.py | 2 -- daemon/core/services/utility.py | 5 ---- daemon/core/services/xorp.py | 6 +---- daemon/core/xml/xmlwriter0.py | 3 --- daemon/core/xml/xmlwriter1.py | 3 --- daemon/data/logging.conf | 2 +- daemon/examples/myservices/sample.py | 12 ++++----- daemon/tests/myservices/sample.py | 16 +++-------- daemon/tests/test_xml.py | 6 +---- 18 files changed, 35 insertions(+), 104 deletions(-) diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 46f1f966..abc8bf55 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -471,12 +471,12 @@ class BasicRangeModel(WirelessModel): a.name, b.name, linked, d, self.range) if d > self.range: if linked: - logger.info("was linked, unlinking") + logger.debug("was linked, unlinking") self.wlan.unlink(a, b) self.sendlinkmsg(a, b, unlink=True) else: if not linked: - logger.info("was not linked, linking") + logger.debug("was not linked, linking") self.wlan.link(a, b) self.sendlinkmsg(a, b) except KeyError: diff --git a/daemon/core/netns/vnet.py b/daemon/core/netns/vnet.py index 7a517d08..af5d8ffc 100644 --- a/daemon/core/netns/vnet.py +++ b/daemon/core/netns/vnet.py @@ -440,7 +440,7 @@ class LxBrNet(PyCoreNet): "burst", str(burst), "limit", str(limit)] if bw > 0: if self.up: - logger.info("linkconfig: %s" % ([tc + parent + ["handle", "1:"] + tbf],)) + logger.debug("linkconfig: %s" % ([tc + parent + ["handle", "1:"] + tbf],)) utils.check_cmd(tc + parent + ["handle", "1:"] + tbf) netif.setparam("has_tbf", True) changed = True @@ -485,12 +485,12 @@ class LxBrNet(PyCoreNet): return tc[2] = "delete" if self.up: - logger.info("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) + logger.debug("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) utils.check_cmd(tc + parent + ["handle", "10:"]) netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: - logger.info("linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],)) + logger.debug("linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],)) utils.check_cmd(tc + parent + ["handle", "10:"] + netem) netif.setparam("has_netem", True) diff --git a/daemon/core/service.py b/daemon/core/service.py index fa45f149..878e8e58 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -45,12 +45,13 @@ class ServiceShim(object): :return: value list string :rtype: str """ - valmap = [service.dirs, service.configs, service.startindex, service.startup, - service.shutdown, service.validate, service.meta, service.starttime] + start_time = 0 + start_index = 0 + valmap = [service.dirs, service.configs, start_index, service.startup, + service.shutdown, service.validate, service.meta, start_time] if not service.custom: - # this is always reached due to classmethod - valmap[valmap.index(service.configs)] = service.getconfigfilenames(node) - valmap[valmap.index(service.startup)] = service.getstartup(node) + valmap[1] = service.getconfigfilenames(node) + valmap[3] = service.getstartup(node) vals = map(lambda a, b: "%s=%s" % (a, str(b)), cls.keys, valmap) return "|".join(vals) @@ -97,8 +98,6 @@ class ServiceShim(object): service.dirs = value elif key == "files": service.configs = value - elif key == "startidx": - service.startindex = value elif key == "cmdup": service.startup = value elif key == "cmddown": @@ -107,8 +106,6 @@ class ServiceShim(object): service.validate = value elif key == "meta": service.meta = value - elif key == "starttime": - service.starttime = value @classmethod def servicesfromopaque(cls, opaque): @@ -422,7 +419,7 @@ class CoreServices(object): result.get() def boot_node_dependencies(self, node, boot_path): - logger.info("booting node service dependencies: %s", boot_path) + logger.debug("booting node service dependencies: %s", boot_path) for service in boot_path: self.bootnodeservice(node, service) @@ -435,7 +432,7 @@ class CoreServices(object): :param CoreService service: service to start :return: nothing """ - logger.info("starting node(%s) service: %s (%s)", node.name, service.name, service.startindex) + logger.info("starting node(%s) service(%s)", node.name, service.name) # create service directories for directory in service.dirs: @@ -501,14 +498,14 @@ class CoreServices(object): :return: service validation status :rtype: int """ - logger.info("validating node(%s) service(%s): %s", node.name, service.name, service.startindex) + logger.info("validating node(%s) service(%s)", node.name, service.name) cmds = service.validate if not service.custom: cmds = service.getvalidate(node) status = 0 for cmd in cmds: - logger.info("validating service(%s) using: %s", service.name, cmd) + logger.debug("validating service(%s) using: %s", service.name, cmd) try: node.check_cmd(cmd) except CoreCommandError: @@ -524,8 +521,7 @@ class CoreServices(object): :param core.netns.nodes.CoreNode node: node to stop services on :return: nothing """ - services = sorted(node.services, key=lambda x: x.startindex) - for service in services: + for service in node.services: self.stopnodeservice(node, service) def stopnodeservice(self, node, service): @@ -651,13 +647,14 @@ class CoreServices(object): :param CoreService service: service to reconfigure :return: nothing """ + logger.info("node(%s) service(%s) creating config files", node.name, service.name) # get values depending on if custom or not file_names = service.configs if not service.custom: file_names = service.getconfigfilenames(node) for file_name in file_names: - logger.info("generating service config: %s", file_name) + logger.debug("generating service config: %s", file_name) if service.custom: cfg = service.configtxt.get(file_name) if cfg is None: @@ -715,21 +712,12 @@ class CoreService(object): # group string allows grouping services together group = None - # list name(s) of services that this service depends upon - depends = () - # private, per-node directories required by this service dirs = () # config files written by this service configs = () - # index used to determine start order with other services - startindex = 0 - - # time in seconds after runtime to run startup commands - starttime = 0 - # list of startup commands startup = () @@ -762,12 +750,10 @@ class CoreService(object): self.custom = True self.dirs = self.__class__.dirs self.configs = self.__class__.configs - self.startindex = self.__class__.startindex self.startup = self.__class__.startup self.shutdown = self.__class__.shutdown self.validate = self.__class__.validate self.meta = self.__class__.meta - self.starttime = self.__class__.starttime self.configtxt = self.__class__.configtxt @classmethod diff --git a/daemon/core/services/bird.py b/daemon/core/services/bird.py index a869fe7c..381a0390 100644 --- a/daemon/core/services/bird.py +++ b/daemon/core/services/bird.py @@ -12,10 +12,8 @@ class Bird(CoreService): name = "bird" executables = ("bird",) group = "BIRD" - depends = () dirs = ("/etc/bird",) configs = ("/etc/bird/bird.conf",) - startindex = 35 startup = ("bird -c %s" % (configs[0]),) shutdown = ("killall bird",) validate = ("pidof bird",) @@ -78,7 +76,7 @@ protocol device { # Generate protocol specific configurations for s in node.services: - if cls.name not in s.depends: + if cls.name not in s.dependencies: continue cfg += s.generatebirdconfig(node) @@ -95,10 +93,8 @@ class BirdService(CoreService): executables = ("bird",) group = "BIRD" dependencies = ("bird",) - depends = ("bird",) dirs = () configs = () - startindex = 40 startup = () shutdown = () meta = "The config file for this service can be found in the bird service." diff --git a/daemon/core/services/dockersvc.py b/daemon/core/services/dockersvc.py index 797a29e6..e4ee597e 100644 --- a/daemon/core/services/dockersvc.py +++ b/daemon/core/services/dockersvc.py @@ -104,7 +104,7 @@ from core.service import ServiceManager try: from docker import Client except ImportError: - logger.warns("missing python docker bindings") + logger.warn("missing python docker bindings") class DockerService(CoreService): @@ -115,10 +115,8 @@ class DockerService(CoreService): name = "Docker" executables = ("docker",) group = "Docker" - depends = () dirs = ('/var/lib/docker/containers/', '/run/shm', '/run/resolvconf',) configs = ('docker.sh',) - startindex = 50 startup = ('sh docker.sh',) shutdown = ('service docker stop',) # Container image to start diff --git a/daemon/core/services/nrl.py b/daemon/core/services/nrl.py index cf5e1d9c..50b17b38 100644 --- a/daemon/core/services/nrl.py +++ b/daemon/core/services/nrl.py @@ -15,10 +15,8 @@ class NrlService(CoreService): """"" name = None group = "ProtoSvc" - depends = () dirs = () configs = () - startindex = 45 startup = () shutdown = () @@ -49,7 +47,6 @@ class MgenSinkService(NrlService): name = "MGEN_Sink" executables = ("mgen",) configs = ("sink.mgen",) - startindex = 5 startup = ("mgen input sink.mgen",) validate = ("pidof mgen",) shutdown = ("killall mgen",) @@ -581,15 +578,11 @@ class MgenActor(NrlService): executables = ("mgen",) # you can create your own group here group = "ProtoSvc" - # list of other services this service depends on - depends = () # per-node directories dirs = () # generated files (without a full path this file goes in the node's dir, # e.g. /tmp/pycore.12345/n1.conf/) configs = ('start_mgen_actor.sh',) - # this controls the starting order vs other enabled services - startindex = 50 # list of startup commands, also may be generated during startup startup = ("sh start_mgen_actor.sh",) # list of validation commands @@ -625,7 +618,6 @@ class Arouted(NrlService): name = "arouted" executables = ("arouted",) configs = ("startarouted.sh",) - startindex = NrlService.startindex + 10 startup = ("sh startarouted.sh",) shutdown = ("pkill arouted",) validate = ("pidof arouted",) diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index 79e607f2..8919832a 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -18,7 +18,6 @@ class Zebra(CoreService): "quaggaboot.sh", "/usr/local/etc/quagga/vtysh.conf" ) - startindex = 35 startup = ("sh quaggaboot.sh zebra",) shutdown = ("killall zebra",) validate = ("pidof zebra",) @@ -66,7 +65,7 @@ class Zebra(CoreService): want_ipv4 = False want_ipv6 = False for s in node.services: - if cls.name not in s.depends: + if cls.name not in s.dependencies: continue ifccfg = s.generatequaggaifcconfig(node, ifc) if s.ipv4_routing: @@ -92,7 +91,7 @@ class Zebra(CoreService): cfg += "!\n" for s in node.services: - if cls.name not in s.depends: + if cls.name not in s.dependencies: continue cfg += s.generatequaggaconfig(node) return cfg @@ -222,10 +221,8 @@ class QuaggaService(CoreService): name = None group = "Quagga" dependencies = ("zebra",) - depends = ("zebra",) dirs = () configs = () - startindex = 40 startup = () shutdown = () meta = "The config file for this service can be found in the Zebra service." diff --git a/daemon/core/services/sdn.py b/daemon/core/services/sdn.py index c9769278..4229175c 100644 --- a/daemon/core/services/sdn.py +++ b/daemon/core/services/sdn.py @@ -12,7 +12,6 @@ class SdnService(CoreService): Parent class for SDN services. """ group = "SDN" - startindex = 50 @classmethod def generateconfig(cls, node, filename): @@ -23,10 +22,8 @@ class OvsService(SdnService): name = "OvsService" executables = ("ovs-ofctl", "ovs-vsctl") group = "SDN" - depends = () dirs = ("/etc/openvswitch", "/var/run/openvswitch", "/var/log/openvswitch") configs = ('OvsService.sh',) - startindex = 50 startup = ('sh OvsService.sh',) shutdown = ('killall ovs-vswitchd', 'killall ovsdb-server') @@ -98,10 +95,8 @@ class RyuService(SdnService): name = "ryuService" executables = ("ryu-manager",) group = "SDN" - depends = () dirs = () configs = ('ryuService.sh',) - startindex = 50 startup = ('sh ryuService.sh',) shutdown = ('killall ryu-manager',) diff --git a/daemon/core/services/security.py b/daemon/core/services/security.py index be3db917..177ce71d 100644 --- a/daemon/core/services/security.py +++ b/daemon/core/services/security.py @@ -12,7 +12,6 @@ class VPNClient(CoreService): name = "VPNClient" group = "Security" configs = ('vpnclient.sh',) - startindex = 60 startup = ('sh vpnclient.sh',) shutdown = ("killall openvpn",) validate = ("pidof openvpn",) @@ -39,7 +38,6 @@ class VPNServer(CoreService): name = "VPNServer" group = "Security" configs = ('vpnserver.sh',) - startindex = 50 startup = ('sh vpnserver.sh',) shutdown = ("killall openvpn",) validate = ("pidof openvpn",) @@ -67,7 +65,6 @@ class IPsec(CoreService): name = "IPsec" group = "Security" configs = ('ipsec.sh',) - startindex = 60 startup = ('sh ipsec.sh',) shutdown = ("killall racoon",) custom_needed = True @@ -95,7 +92,6 @@ class Firewall(CoreService): name = "Firewall" group = "Security" configs = ('firewall.sh',) - startindex = 20 startup = ('sh firewall.sh',) custom_needed = True diff --git a/daemon/core/services/ucarp.py b/daemon/core/services/ucarp.py index 80786999..686f3646 100644 --- a/daemon/core/services/ucarp.py +++ b/daemon/core/services/ucarp.py @@ -10,11 +10,9 @@ UCARP_ETC = "/usr/local/etc/ucarp" class Ucarp(CoreService): name = "ucarp" group = "Utility" - depends = ( ) dirs = (UCARP_ETC,) configs = ( UCARP_ETC + "/default.sh", UCARP_ETC + "/default-up.sh", UCARP_ETC + "/default-down.sh", "ucarpboot.sh",) - startindex = 65 startup = ("sh ucarpboot.sh",) shutdown = ("killall ucarp",) validate = ("pidof ucarp",) diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 79f35c16..ff770a64 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -18,10 +18,8 @@ class UtilService(CoreService): """ name = None group = "Utility" - depends = () dirs = () configs = () - startindex = 80 startup = () shutdown = () @@ -33,7 +31,6 @@ class UtilService(CoreService): class IPForwardService(UtilService): name = "IPForward" configs = ("ipforward.sh",) - startindex = 5 startup = ("sh ipforward.sh",) @classmethod @@ -574,7 +571,6 @@ class PcapService(UtilService): name = "pcap" configs = ("pcap.sh",) dirs = () - startindex = 1 startup = ("sh pcap.sh start",) shutdown = ("sh pcap.sh stop",) validate = ("pidof tcpdump",) @@ -693,5 +689,4 @@ class UserDefinedService(UtilService): Dummy service allowing customization of anything. """ name = "UserDefined" - startindex = 50 meta = "Customize this service to do anything upon startup." diff --git a/daemon/core/services/xorp.py b/daemon/core/services/xorp.py index 58668b1f..7bd5b963 100644 --- a/daemon/core/services/xorp.py +++ b/daemon/core/services/xorp.py @@ -14,10 +14,8 @@ class XorpRtrmgr(CoreService): name = "xorp_rtrmgr" executables = ("xorp_rtrmgr",) group = "XORP" - depends = () dirs = ("/etc/xorp",) configs = ("/etc/xorp/config.boot",) - startindex = 35 startup = ("xorp_rtrmgr -d -b %s -l /var/log/%s.log -P /var/run/%s.pid" % (configs[0], name, name),) shutdown = ("killall xorp_rtrmgr",) validate = ("pidof xorp_rtrmgr",) @@ -41,7 +39,7 @@ class XorpRtrmgr(CoreService): for s in node.services: try: - s.depends.index(cls.name) + s.dependencies.index(cls.name) cfg += s.generatexorpconfig(node) except ValueError: logger.exception("error getting value from service: %s", cls.name) @@ -79,10 +77,8 @@ class XorpService(CoreService): executables = ("xorp_rtrmgr",) group = "XORP" dependencies = ("xorp_rtrmgr",) - depends = ("xorp_rtrmgr",) dirs = () configs = () - startindex = 40 startup = () shutdown = () meta = "The config file for this service can be found in the xorp_rtrmgr service." diff --git a/daemon/core/xml/xmlwriter0.py b/daemon/core/xml/xmlwriter0.py index 41f097fc..57085c5e 100644 --- a/daemon/core/xml/xmlwriter0.py +++ b/daemon/core/xml/xmlwriter0.py @@ -303,9 +303,6 @@ class CoreDocumentWriter0(Document): s = self.createElement("Service") spn.appendChild(s) s.setAttribute("name", str(svc.name)) - s.setAttribute("startup_idx", str(svc.startindex)) - if svc.starttime != "": - s.setAttribute("start_time", str(svc.starttime)) # only record service names if not a customized service if not svc.custom: continue diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index 22c03a23..452a395e 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -681,9 +681,6 @@ class DeviceElement(NamedXmlElement): s = self.createElement("service") spn.appendChild(s) s.setAttribute("name", str(svc.name)) - s.setAttribute("startup_idx", str(svc.startindex)) - if svc.starttime != "": - s.setAttribute("start_time", str(svc.starttime)) # only record service names if not a customized service if not svc.custom: continue diff --git a/daemon/data/logging.conf b/daemon/data/logging.conf index 46de6e92..7f3d496f 100644 --- a/daemon/data/logging.conf +++ b/daemon/data/logging.conf @@ -14,7 +14,7 @@ } }, "root": { - "level": "DEBUG", + "level": "INFO", "handlers": ["console"] } } diff --git a/daemon/examples/myservices/sample.py b/daemon/examples/myservices/sample.py index bc5113ee..a7def96a 100644 --- a/daemon/examples/myservices/sample.py +++ b/daemon/examples/myservices/sample.py @@ -14,22 +14,22 @@ class MyService(CoreService): name = "MyService" # you can create your own group here group = "Utility" + # list executables that this service requires + executables = () # list of other services this service depends on - depends = () + dependencies = () # per-node directories dirs = () # generated files (without a full path this file goes in the node's dir, # e.g. /tmp/pycore.12345/n1.conf/) - configs = ('myservice.sh',) - # this controls the starting order vs other enabled services - startindex = 50 + configs = ("myservice.sh",) # list of startup commands, also may be generated during startup - startup = ('sh myservice.sh',) + startup = ("sh myservice.sh",) # list of shutdown commands shutdown = () @classmethod - def generateconfig(cls, node, filename, services): + def generateconfig(cls, node, filename): """ Return a string that will be written to filename, or sent to the GUI for user customization. diff --git a/daemon/tests/myservices/sample.py b/daemon/tests/myservices/sample.py index 80545d29..633670bc 100644 --- a/daemon/tests/myservices/sample.py +++ b/daemon/tests/myservices/sample.py @@ -8,20 +8,12 @@ from core.service import CoreService class MyService(CoreService): name = "MyService" group = "Utility" - depends = () - dirs = () - configs = ('myservice.sh',) - startindex = 50 - startup = ('sh myservice.sh',) - shutdown = () + configs = ("myservice.sh",) + startup = ("sh myservice.sh",) class MyService2(CoreService): name = "MyService2" group = "Utility" - depends = () - dirs = () - configs = ('myservice.sh',) - startindex = 50 - startup = ('sh myservice.sh',) - shutdown = () + configs = ("myservice.sh",) + startup = ("sh myservice.sh",) diff --git a/daemon/tests/test_xml.py b/daemon/tests/test_xml.py index f3fe8660..212f2de9 100644 --- a/daemon/tests/test_xml.py +++ b/daemon/tests/test_xml.py @@ -92,11 +92,8 @@ class TestXml: interface = ip_prefixes.create_interface(node) session.add_link(node.objid, ptp_node.objid, interface_one=interface) - # set custom values for node service\ - custom_start = 50 + # set custom values for node service session.services.setcustomservice(node_one.objid, SshService.name) - service = session.services.getcustomservice(node_one.objid, SshService.name) - service.startindex = custom_start service_file = SshService.configs[0] file_data = "# test" session.services.setservicefile(node_one.objid, SshService.name, service_file, file_data) @@ -135,7 +132,6 @@ class TestXml: # verify nodes have been recreated assert session.get_object(n1_id) assert session.get_object(n2_id) - assert service.startindex == custom_start assert service.configtxt.get(service_file) == file_data @pytest.mark.parametrize("version", _XML_VERSIONS) From 0efcd910dbde8493dcc12ead2173c00302646c61 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 22 Jun 2018 11:59:16 -0700 Subject: [PATCH 58/83] removed node boot/validate methods, since it was using a circular reference to run a CoreService method --- daemon/core/emulator/coreemu.py | 4 +--- daemon/core/netns/vnode.py | 25 ------------------------- daemon/core/phys/pnodes.py | 6 ------ daemon/core/service.py | 11 ----------- daemon/core/session.py | 2 +- daemon/examples/netns/howmanynodes.py | 2 +- 6 files changed, 3 insertions(+), 47 deletions(-) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 5b8d9301..df144f1e 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -515,9 +515,7 @@ class EmuSession(Session): if self.state == EventTypes.RUNTIME_STATE.value and is_boot_node: self.write_objects() self.add_remove_control_interface(node=node, remove=False) - - # TODO: common method to both Physical and LxcNodes, but not the common PyCoreNode - node.boot() + self.services.bootnodeservices(node) return node diff --git a/daemon/core/netns/vnode.py b/daemon/core/netns/vnode.py index c0013092..4406d315 100644 --- a/daemon/core/netns/vnode.py +++ b/daemon/core/netns/vnode.py @@ -156,14 +156,6 @@ class SimpleLxcNode(PyCoreNode): self.client.close() self.up = False - def boot(self): - """ - Boot logic. - - :return: nothing - """ - return None - def cmd(self, args, wait=True): """ Runs shell command on node, with option to not wait for a result. @@ -222,7 +214,6 @@ class SimpleLxcNode(PyCoreNode): raise CoreCommandError(status, cmd, output) self._mounts.append((source, target)) - def newifindex(self): """ Retrieve a new interface index. @@ -511,22 +502,6 @@ class LxcNode(SimpleLxcNode): if start: self.startup() - def boot(self): - """ - Boot the node. - - :return: nothing - """ - self.session.services.bootnodeservices(self) - - def validate(self): - """ - Validate the node. - - :return: nothing - """ - self.session.services.validatenodeservices(self) - def startup(self): """ Startup logic for the node. diff --git a/daemon/core/phys/pnodes.py b/daemon/core/phys/pnodes.py index eb3d9bbb..63db48bb 100644 --- a/daemon/core/phys/pnodes.py +++ b/daemon/core/phys/pnodes.py @@ -25,12 +25,6 @@ class PhysicalNode(PyCoreNode): if start: self.startup() - def boot(self): - self.session.services.bootnodeservices(self) - - def validate(self): - self.session.services.validatenodeservices(self) - def startup(self): with self.lock: self.makenodedir() diff --git a/daemon/core/service.py b/daemon/core/service.py index 878e8e58..ae114b97 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -386,7 +386,6 @@ class CoreServices(object): :return: """ files = [] - if not service.custom: return files @@ -479,16 +478,6 @@ class CoreServices(object): return True return False - def validatenodeservices(self, node): - """ - Run validation commands for all services on a node. - - :param core.netns.vnode.LxcNode node: node to validate services for - :return: nothing - """ - for service in node.services: - self.validatenodeservice(node, service) - def validatenodeservice(self, node, service): """ Run the validation command(s) for a service. diff --git a/daemon/core/session.py b/daemon/core/session.py index 6cae3a62..b9dc99aa 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -727,7 +727,7 @@ class Session(object): # add a control interface if configured logger.info("booting node: %s", obj.name) self.add_remove_control_interface(node=obj, remove=False) - result = pool.apply_async(obj.boot) + result = pool.apply_async(self.services.bootnodeservices, (obj,)) results.append(result) pool.close() diff --git a/daemon/examples/netns/howmanynodes.py b/daemon/examples/netns/howmanynodes.py index 5ca989f6..9d5d61c0 100755 --- a/daemon/examples/netns/howmanynodes.py +++ b/daemon/examples/netns/howmanynodes.py @@ -160,7 +160,7 @@ def main(): n.cmd([constants.SYSCTL_BIN, "net.ipv4.icmp_echo_ignore_broadcasts=0"]) if options.services is not None: session.services.addservicestonode(n, "", options.services.split("|")) - n.boot() + session.services.bootnodeservices(n) nodelist.append(n) if i % 25 == 0: print "\n%s nodes created " % i, From 8186f3716cc268e61e46bb14f1d3a6f3e4c20b80 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 22 Jun 2018 14:41:06 -0700 Subject: [PATCH 59/83] refactored service function names --- daemon/core/corehandlers.py | 22 ++-- daemon/core/emulator/coreemu.py | 6 +- daemon/core/service.py | 117 ++++++++++++---------- daemon/core/session.py | 4 +- daemon/core/xml/xmlparser0.py | 10 +- daemon/core/xml/xmlparser1.py | 8 +- daemon/core/xml/xmlwriter0.py | 8 +- daemon/core/xml/xmlwriter1.py | 6 +- daemon/examples/netns/howmanynodes.py | 4 +- daemon/examples/netns/ospfmanetmdrtest.py | 2 +- daemon/examples/netns/wlanemanetests.py | 8 +- daemon/tests/test_services.py | 35 ++++--- daemon/tests/test_xml.py | 8 +- ns3/examples/ns3wifirandomwalk.py | 4 +- 14 files changed, 127 insertions(+), 115 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index e5d9cb35..a8c86e63 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1151,7 +1151,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): # a file request: e.g. "service:zebra:quagga.conf" file_name = servicesstring[2] service_name = services[0] - file_data = self.session.services.getservicefile(node, service_name, file_name) + file_data = self.session.services.get_service_file(node, service_name, file_name) self.session.broadcast_file(file_data) # short circuit this request early to avoid returning response below return replies @@ -1162,7 +1162,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): # dirs, configs, startindex, startup, shutdown, metadata, config type_flag = ConfigFlags.UPDATE.value data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(ServiceShim.keys))) - service = self.session.services.getcustomservice(node_id, service_name, default_service=True) + service = self.session.services.get_service(node_id, service_name, default_service=True) values = ServiceShim.tovaluelist(node, service) captions = None possible_values = None @@ -1199,7 +1199,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): logger.info(error_message) return None key = values.pop(0) - self.session.services.defaultservices[key] = values + self.session.services.default_services[key] = values logger.debug("default services for type %s set to %s", key, values) elif node_id: services = ServiceShim.servicesfromopaque(opaque) @@ -1207,10 +1207,10 @@ class CoreHandler(SocketServer.BaseRequestHandler): service_name = services[0] # set custom service for node - self.session.services.setcustomservice(node_id, service_name) + self.session.services.set_service(node_id, service_name) # set custom values for custom service - service = self.session.services.getcustomservice(node_id, service_name) + service = self.session.services.get_service(node_id, service_name) if not service: raise ValueError("custom service(%s) for node(%s) does not exist", service_name, node_id) @@ -1359,7 +1359,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): if file_type is not None: if file_type.startswith("service:"): _, service_name = file_type.split(':')[:2] - self.session.services.setservicefile(node_num, service_name, file_name, data) + self.session.services.set_service_file(node_num, service_name, file_name, data) return () elif file_type.startswith("hook:"): _, state = file_type.split(':')[:2] @@ -1520,13 +1520,13 @@ class CoreHandler(SocketServer.BaseRequestHandler): unknown = [] services = ServiceShim.servicesfromopaque(name) for service_name in services: - service = self.session.services.getcustomservice(node_id, service_name, default_service=True) + service = self.session.services.get_service(node_id, service_name, default_service=True) if not service: unknown.append(service_name) continue if event_type == EventTypes.STOP.value or event_type == EventTypes.RESTART.value: - status = self.session.services.stopnodeservice(node, service) + status = self.session.services.stop_node_service(node, service) if status: fail += "Stop %s," % service.name if event_type == EventTypes.START.value or event_type == EventTypes.RESTART.value: @@ -1534,7 +1534,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): if status: fail += "Start %s(%s)," % service.name if event_type == EventTypes.PAUSE.value: - status = self.session.services.validatenodeservice(node, service) + status = self.session.services.validate_node_service(node, service) if status: fail += "%s," % service.name if event_type == EventTypes.RECONFIGURE.value: @@ -1719,7 +1719,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): self.session.broadcast_config(config_data) # service customizations - service_configs = self.session.services.getallconfigs() + service_configs = self.session.services.all_configs() for node_id, service in service_configs: opaque = "service:%s" % service.name data_types = tuple(repeat(ConfigDataTypes.STRING.value, len(ServiceShim.keys))) @@ -1737,7 +1737,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): ) self.session.broadcast_config(config_data) - for file_name, config_data in self.session.services.getallfiles(service): + for file_name, config_data in self.session.services.all_files(service): file_data = FileData( message_type=MessageFlags.ADD.value, node=node_id, diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index df144f1e..6823ae08 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -126,7 +126,7 @@ class EmuSession(Session): self.node_id_gen = IdGen() # set default services - self.services.defaultservices = { + self.services.default_services = { "mdr": ("zebra", "OSPFv3MDR", "IPForward"), "PC": ("DefaultRoute",), "prouter": ("zebra", "OSPFv2", "OSPFv3", "IPForward"), @@ -508,14 +508,14 @@ class EmuSession(Session): if _type in [NodeTypes.DEFAULT, NodeTypes.PHYSICAL]: node.type = node_options.model logger.debug("set node type: %s", node.type) - self.services.addservicestonode(node, node.type, node_options.services) + self.services.add_services(node, node.type, node_options.services) # boot nodes if created after runtime, LcxNodes, Physical, and RJ45 are all PyCoreNodes is_boot_node = isinstance(node, PyCoreNode) and not nodeutils.is_node(node, NodeTypes.RJ45) if self.state == EventTypes.RUNTIME_STATE.value and is_boot_node: self.write_objects() self.add_remove_control_interface(node=node, remove=False) - self.services.bootnodeservices(node) + self.services.boot_node_services(node) return node diff --git a/daemon/core/service.py b/daemon/core/service.py index ae114b97..f9da6bd2 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -206,18 +206,18 @@ class CoreServices(object): """ self.session = session # dict of default services tuples, key is node type - self.defaultservices = {} - # dict of tuple of service objects, key is node number - self.customservices = {} + self.default_services = {} + # dict of node ids to dict of custom services by name + self.custom_services = {} def reset(self): """ Called when config message with reset flag is received """ - self.defaultservices.clear() - self.customservices.clear() + self.default_services.clear() + self.custom_services.clear() - def node_service_dependencies(self, services): + def node_boot_paths(self, services): # generate service map and find starting points node_services = {service.name: service for service in services} is_dependency = set() @@ -280,11 +280,11 @@ class CoreServices(object): stack.pop() if visited != all_services: - raise ValueError("cycle encountered, services are being skipped") + raise ValueError("failure to visit all services for boot path") return startups - def getdefaultservices(self, node_type): + def get_default_services(self, node_type): """ Get the list of default services that should be enabled for a node for the given node type. @@ -295,7 +295,7 @@ class CoreServices(object): """ logger.debug("getting default services for type: %s", node_type) results = [] - defaults = self.defaultservices.get(node_type, []) + defaults = self.default_services.get(node_type, []) for name in defaults: logger.debug("checking for service with service manager: %s", name) service = ServiceManager.get(name) @@ -305,7 +305,7 @@ class CoreServices(object): results.append(service) return results - def getcustomservice(self, node_id, service_name, default_service=False): + def get_service(self, node_id, service_name, default_service=False): """ Get any custom service configured for the given node that matches the specified service name. If no custom service is found, return the specified service. @@ -316,13 +316,13 @@ class CoreServices(object): :return: custom service from the node :rtype: CoreService """ - node_services = self.customservices.setdefault(node_id, {}) + node_services = self.custom_services.setdefault(node_id, {}) default = None if default_service: default = ServiceManager.get(service_name) return node_services.get(service_name, default) - def setcustomservice(self, node_id, service_name): + def set_service(self, node_id, service_name): """ Store service customizations in an instantiated service object using a list of values that came from a config message. @@ -332,16 +332,16 @@ class CoreServices(object): :return: nothing """ logger.debug("setting custom service(%s) for node: %s", node_id, service_name) - service = self.getcustomservice(node_id, service_name) + service = self.get_service(node_id, service_name) if not service: service_class = ServiceManager.get(service_name) service = service_class() # add the custom service to dict - node_services = self.customservices.setdefault(node_id, {}) + node_services = self.custom_services.setdefault(node_id, {}) node_services[service.name] = service - def addservicestonode(self, node, node_type, services=None): + def add_services(self, node, node_type, services=None): """ Add services to a node. @@ -352,52 +352,53 @@ class CoreServices(object): """ if not services: logger.info("using default services for node(%s) type(%s)", node.name, node_type) - services = self.defaultservices.get(node_type, []) + services = self.default_services.get(node_type, []) logger.info("setting services for node(%s): %s", node.name, services) for service_name in services: - service = self.getcustomservice(node.objid, service_name, default_service=True) + service = self.get_service(node.objid, service_name, default_service=True) if not service: logger.warn("unknown service(%s) for node(%s)", service_name, node.name) continue logger.info("adding service to node(%s): %s", node.name, service_name) node.addservice(service) - def getallconfigs(self): + def all_configs(self): """ - Return (nodenum, service) tuples for all stored configs. Used when reconnecting to a + Return (node_id, service) tuples for all stored configs. Used when reconnecting to a session or opening XML. :return: list of tuples of node ids and services - :rtype: list + :rtype: list[tuple] """ configs = [] - for node_id in self.customservices.iterkeys(): - for service in self.customservices[node_id].itervalues(): + for node_id in self.custom_services.iterkeys(): + for service in self.custom_services[node_id].itervalues(): configs.append((node_id, service)) return configs - def getallfiles(self, service): + def all_files(self, service): """ Return all customized files stored with a service. Used when reconnecting to a session or opening XML. :param CoreService service: service to get files for - :return: + :return: list of all custom service files + :rtype: list[tuple] """ files = [] if not service.custom: return files for filename in service.configs: - data = service.configtxt.get(filename) + data = service.config_data.get(filename) if data is None: continue files.append((filename, data)) return files - def bootnodeservices(self, node): + def boot_node_services(self, node): """ Start all services on a node. @@ -407,9 +408,9 @@ class CoreServices(object): pool = ThreadPool() results = [] - boot_paths = self.node_service_dependencies(node.services) + boot_paths = self.node_boot_paths(node.services) for boot_path in boot_paths: - result = pool.apply_async(self.boot_node_dependencies, (node, boot_path)) + result = pool.apply_async(self._boot_node_paths, (node, boot_path)) results.append(result) pool.close() @@ -417,12 +418,19 @@ class CoreServices(object): for result in results: result.get() - def boot_node_dependencies(self, node, boot_path): + def _boot_node_paths(self, node, boot_path): + """ + Start all service boot paths found, based on dependencies. + + :param core.netns.vnode.LxcNode node: node to start services on + :param list[CoreService] boot_path: service to start in dependent order + :return: nothing + """ logger.debug("booting node service dependencies: %s", boot_path) for service in boot_path: - self.bootnodeservice(node, service) + self.boot_node_service(node, service) - def bootnodeservice(self, node, service): + def boot_node_service(self, node, service): """ Start a service on a node. Create private dirs, generate config files, and execute startup commands. @@ -453,11 +461,11 @@ class CoreServices(object): # run validation commands, if present and not timer mode if service.validation_mode != ServiceMode.TIMER: - status = self.validatenodeservice(node, service) + status = self.validate_node_service(node, service) if status: raise ServiceBootError("node(%s) service(%s) failed validation" % (node.name, service.name)) - def copyservicefile(self, node, filename, cfg): + def copy_service_file(self, node, filename, cfg): """ Given a configured service filename and config, determine if the config references an existing file that should be copied. @@ -478,7 +486,7 @@ class CoreServices(object): return True return False - def validatenodeservice(self, node, service): + def validate_node_service(self, node, service): """ Run the validation command(s) for a service. @@ -503,7 +511,7 @@ class CoreServices(object): return status - def stopnodeservices(self, node): + def stop_node_services(self, node): """ Stop all services on a node. @@ -511,9 +519,9 @@ class CoreServices(object): :return: nothing """ for service in node.services: - self.stopnodeservice(node, service) + self.stop_node_service(node, service) - def stopnodeservice(self, node, service): + def stop_node_service(self, node, service): """ Stop a service on a node. @@ -531,7 +539,7 @@ class CoreServices(object): status = -1 return status - def getservicefile(self, node, service_name, filename): + def get_service_file(self, node, service_name, filename): """ Send a File Message when the GUI has requested a service file. The file data is either auto-generated or comes from an existing config. @@ -542,7 +550,7 @@ class CoreServices(object): :return: file message for node """ # get service to get file from - service = self.getcustomservice(node.objid, service_name, default_service=True) + service = self.get_service(node.objid, service_name, default_service=True) if not service: raise ValueError("invalid service: %s", service_name) @@ -556,7 +564,7 @@ class CoreServices(object): raise ValueError("unknown service(%s) config file: %s", service_name, filename) # get the file data - data = service.configtxt.get(filename) + data = service.config_data.get(filename) if data is None: data = "%s" % service.generateconfig(node, filename) else: @@ -571,7 +579,7 @@ class CoreServices(object): data=data ) - def setservicefile(self, node_id, service_name, filename, data): + def set_service_file(self, node_id, service_name, filename, data): """ Receive a File Message from the GUI and store the customized file in the service config. The filename must match one from the list of @@ -584,10 +592,10 @@ class CoreServices(object): :return: nothing """ # attempt to set custom service, if needed - self.setcustomservice(node_id, service_name) + self.set_service(node_id, service_name) # retrieve custom service - service = self.getcustomservice(node_id, service_name) + service = self.get_service(node_id, service_name) if service is None: logger.warn("received filename for unknown service: %s", service_name) return @@ -599,7 +607,7 @@ class CoreServices(object): return # set custom service file data - service.configtxt[filename] = data + service.config_data[filename] = data def node_service_startup(self, node, service, wait=False): """ @@ -645,13 +653,13 @@ class CoreServices(object): for file_name in file_names: logger.debug("generating service config: %s", file_name) if service.custom: - cfg = service.configtxt.get(file_name) + cfg = service.config_data.get(file_name) if cfg is None: cfg = service.generateconfig(node, file_name) # cfg may have a file:/// url for copying from a file try: - if self.copyservicefile(node, file_name, cfg): + if self.copy_service_file(node, file_name, cfg): continue except IOError: logger.exception("error copying service file: %s", file_name) @@ -678,7 +686,7 @@ class CoreServices(object): # TODO: implement this raise NotImplementedError - cfg = service.configtxt.get(file_name) + cfg = service.config_data.get(file_name) if cfg is None: cfg = service.generateconfig(node, file_name) @@ -707,6 +715,9 @@ class CoreService(object): # config files written by this service configs = () + # config file data + config_data = {} + # list of startup commands startup = () @@ -726,7 +737,6 @@ class CoreService(object): meta = None # custom configuration text - configtxt = {} custom = False custom_needed = False @@ -743,7 +753,7 @@ class CoreService(object): self.shutdown = self.__class__.shutdown self.validate = self.__class__.validate self.meta = self.__class__.meta - self.configtxt = self.__class__.configtxt + self.config_data = self.__class__.config_data @classmethod def on_load(cls): @@ -766,8 +776,7 @@ class CoreService(object): def generateconfig(cls, node, filename): """ Generate configuration file given a node object. The filename is - provided to allow for multiple config files. The other services are - provided to allow interdependencies (e.g. zebra and OSPF). + provided to allow for multiple config files. Return the configuration string to be written to a file or sent to the GUI for customization. @@ -781,7 +790,7 @@ class CoreService(object): def getstartup(cls, node): """ Return the tuple of startup commands. This default method - returns the cls._startup tuple, but this method may be + returns the cls.startup tuple, but this method may be overridden to provide node-specific commands that may be based on other services. @@ -795,8 +804,8 @@ class CoreService(object): def getvalidate(cls, node): """ Return the tuple of validate commands. This default method - returns the cls._validate tuple, but this method may be - overriden to provide node-specific commands that may be + returns the cls.validate tuple, but this method may be + overridden to provide node-specific commands that may be based on other services. :param core.netns.vnode.LxcNode node: node to validate diff --git a/daemon/core/session.py b/daemon/core/session.py index b9dc99aa..3f7c17aa 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -673,7 +673,7 @@ class Session(object): for obj in self.objects.itervalues(): # TODO: determine if checking for CoreNode alone is ok if isinstance(obj, nodes.PyCoreNode): - self.services.stopnodeservices(obj) + self.services.stop_node_services(obj) # shutdown emane self.emane.shutdown() @@ -727,7 +727,7 @@ class Session(object): # add a control interface if configured logger.info("booting node: %s", obj.name) self.add_remove_control_interface(node=obj, remove=False) - result = pool.apply_async(self.services.bootnodeservices, (obj,)) + result = pool.apply_async(self.services.boot_node_services, (obj,)) results.append(result) pool.close() diff --git a/daemon/core/xml/xmlparser0.py b/daemon/core/xml/xmlparser0.py index be9a1f19..ed1bc5cf 100644 --- a/daemon/core/xml/xmlparser0.py +++ b/daemon/core/xml/xmlparser0.py @@ -279,7 +279,7 @@ class CoreDocumentParser0(object): services = [] for service in node.getElementsByTagName("Service"): services.append(str(service.getAttribute("name"))) - self.session.services.defaultservices[type] = services + self.session.services.default_services[type] = services logger.info("default services for type %s set to %s" % (type, services)) def parseservices(self): @@ -319,7 +319,7 @@ class CoreDocumentParser0(object): services = svclists[objid] if services: services = services.split("|") - self.session.services.addservicestonode(node=n, node_type=n.type, services=services) + self.session.services.add_services(node=n, node_type=n.type, services=services) def parseservice(self, service, n): """ @@ -370,15 +370,15 @@ class CoreDocumentParser0(object): filename = file.getAttribute("name") files.append(filename) data = xmlutils.get_text_child(file) - self.session.services.setservicefile(node_id=n.objid, service_name=name, filename=filename, data=data) + self.session.services.set_service_file(node_id=n.objid, service_name=name, filename=filename, data=data) if len(files): values.append("files=%s" % files) if not bool(service.getAttribute("custom")): return True - self.session.services.setcustomservice(n.objid, svc) + self.session.services.set_service(n.objid, svc) # set custom values for custom service - svc = self.session.services.getcustomservice(n.objid, None) + svc = self.session.services.get_service(n.objid, None) if not svc: raise ValueError("custom service(%s) for node(%s) does not exist", svc.name, n.objid) values = ConfigShim.str_to_dict("|".join(values)) diff --git a/daemon/core/xml/xmlparser1.py b/daemon/core/xml/xmlparser1.py index cef79590..0b5c61da 100644 --- a/daemon/core/xml/xmlparser1.py +++ b/daemon/core/xml/xmlparser1.py @@ -639,7 +639,7 @@ class CoreDocumentParser1(object): custom = service.getAttribute('custom') if custom and custom.lower() == 'true': - self.session.services.setcustomservice(node.objid, session_service.name) + self.session.services.set_service(node.objid, session_service.name) values = ConfigShim.str_to_dict("|".join(values)) for key, value in values.iteritems(): ServiceShim.setvalue(session_service, key, value) @@ -648,7 +648,7 @@ class CoreDocumentParser1(object): # called after the custom service exists for typestr, filename, data in files: svcname = typestr.split(":")[1] - self.session.services.setservicefile( + self.session.services.set_service_file( node_id=node.objid, service_name=svcname, filename=filename, @@ -683,7 +683,7 @@ class CoreDocumentParser1(object): if services_str: services_str = services_str.split("|") - self.session.services.addservicestonode( + self.session.services.add_services( node=node, node_type=node_type, services=services_str @@ -885,5 +885,5 @@ class CoreDocumentParser1(object): self.default_services[device_type] = services # store default services for the session for t, s in self.default_services.iteritems(): - self.session.services.defaultservices[t] = s + self.session.services.default_services[t] = s logger.info('default services for node type \'%s\' set to: %s' % (t, s)) diff --git a/daemon/core/xml/xmlwriter0.py b/daemon/core/xml/xmlwriter0.py index 57085c5e..b97e361a 100644 --- a/daemon/core/xml/xmlwriter0.py +++ b/daemon/core/xml/xmlwriter0.py @@ -276,8 +276,8 @@ class CoreDocumentWriter0(Document): """ Add default services and node types to the ServicePlan. """ - for type in self.session.services.defaultservices: - defaults = self.session.services.getdefaultservices(type) + for type in self.session.services.default_services: + defaults = self.session.services.get_default_services(type) spn = self.createElement("Node") spn.setAttribute("type", type) self.sp.appendChild(spn) @@ -292,7 +292,7 @@ class CoreDocumentWriter0(Document): """ if len(node.services) == 0: return - defaults = self.session.services.getdefaultservices(node.type) + defaults = self.session.services.get_default_services(node.type) if node.services == defaults: return spn = self.createElement("Node") @@ -316,7 +316,7 @@ class CoreDocumentWriter0(Document): f.setAttribute("name", fn) # all file names are added to determine when a file has been deleted s.appendChild(f) - data = self.session.services.getservicefiledata(svc, fn) + data = svc.config_data.get(fn) if data is None: # this includes only customized file contents and skips # the auto-generated files diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index 452a395e..27246859 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -269,8 +269,8 @@ class ScenarioPlan(XmlElement): Add default services and node types to the ServicePlan. """ defaultservices = self.createElement("CORE:defaultservices") - for type in self.coreSession.services.defaultservices: - defaults = self.coreSession.services.getdefaultservices(type) + for type in self.coreSession.services.default_services: + defaults = self.coreSession.services.get_default_services(type) spn = self.createElement("device") spn.setAttribute("type", type) defaultservices.appendChild(spn) @@ -670,7 +670,7 @@ class DeviceElement(NamedXmlElement): if len(device_object.services) == 0: return - defaults = self.coreSession.services.getdefaultservices(device_object.type) + defaults = self.coreSession.services.get_default_services(device_object.type) if device_object.services == defaults: return spn = self.createElement("CORE:services") diff --git a/daemon/examples/netns/howmanynodes.py b/daemon/examples/netns/howmanynodes.py index 9d5d61c0..7a5035f6 100755 --- a/daemon/examples/netns/howmanynodes.py +++ b/daemon/examples/netns/howmanynodes.py @@ -159,8 +159,8 @@ def main(): n.newnetif(switch, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) n.cmd([constants.SYSCTL_BIN, "net.ipv4.icmp_echo_ignore_broadcasts=0"]) if options.services is not None: - session.services.addservicestonode(n, "", options.services.split("|")) - session.services.bootnodeservices(n) + session.services.add_services(n, "", options.services.split("|")) + session.services.boot_node_services(n) nodelist.append(n) if i % 25 == 0: print "\n%s nodes created " % i, diff --git a/daemon/examples/netns/ospfmanetmdrtest.py b/daemon/examples/netns/ospfmanetmdrtest.py index 7c55fe81..cdc0efb8 100755 --- a/daemon/examples/netns/ospfmanetmdrtest.py +++ b/daemon/examples/netns/ospfmanetmdrtest.py @@ -88,7 +88,7 @@ ip forwarding def boot(self): self.config() - self.session.services.bootnodeservices(self) + self.session.services.boot_node_services(self) def bootscript(self): return """\ diff --git a/daemon/examples/netns/wlanemanetests.py b/daemon/examples/netns/wlanemanetests.py index d3323f13..0783e8cc 100755 --- a/daemon/examples/netns/wlanemanetests.py +++ b/daemon/examples/netns/wlanemanetests.py @@ -420,8 +420,8 @@ class Experiment(object): tmp = self.session.add_object(cls=nodes.CoreNode, objid=i, name="n%d" % i) tmp.newnetif(self.net, [addr]) self.nodes.append(tmp) - self.session.services.addservicestonode(tmp, "router", "IPForward") - self.session.services.bootnodeservices(tmp) + self.session.services.add_services(tmp, "router", "IPForward") + self.session.services.boot_node_services(tmp) self.staticroutes(i, prefix, numnodes) # link each node in a chain, with the previous node @@ -451,7 +451,7 @@ class Experiment(object): tmp.setposition(50, 50, None) tmp.newnetif(self.net, [addr]) self.nodes.append(tmp) - self.session.services.addservicestonode(tmp, "router", "IPForward") + self.session.services.add_services(tmp, "router", "IPForward") if values is None: values = cls.getdefaultvalues() @@ -463,7 +463,7 @@ class Experiment(object): for i in xrange(1, numnodes + 1): tmp = self.nodes[i - 1] - self.session.services.bootnodeservices(tmp) + self.session.services.boot_node_services(tmp) self.staticroutes(i, prefix, numnodes) def setnodes(self): diff --git a/daemon/tests/test_services.py b/daemon/tests/test_services.py index 346fa502..2e6afe4b 100644 --- a/daemon/tests/test_services.py +++ b/daemon/tests/test_services.py @@ -48,18 +48,21 @@ class TestServices: assert ServiceManager.get("MyService") assert ServiceManager.get("MyService2") + def test_service_defaults(self): + pass + def test_services_dependencies(self, session): # given services = [ - ServiceA(), - ServiceB(), - ServiceC(), - ServiceD(), - ServiceF(), + ServiceA, + ServiceB, + ServiceC, + ServiceD, + ServiceF, ] # when - startups = session.services.node_service_dependencies(services) + startups = session.services.node_boot_paths(services) # then assert len(startups) == 2 @@ -67,28 +70,28 @@ class TestServices: def test_services_dependencies_not_present(self, session): # given services = [ - ServiceA(), - ServiceB(), - ServiceC(), - ServiceE() + ServiceA, + ServiceB, + ServiceC, + ServiceE ] # when with pytest.raises(ValueError): - session.services.node_service_dependencies(services) + session.services.node_boot_paths(services) def test_services_dependencies_cycle(self, session): # given service_c = ServiceC() service_c.dependencies = ("D",) services = [ - ServiceA(), - ServiceB(), + ServiceA, + ServiceB, service_c, - ServiceD(), - ServiceF() + ServiceD, + ServiceF ] # when with pytest.raises(ValueError): - session.services.node_service_dependencies(services) + session.services.node_boot_paths(services) diff --git a/daemon/tests/test_xml.py b/daemon/tests/test_xml.py index 212f2de9..b6d91c75 100644 --- a/daemon/tests/test_xml.py +++ b/daemon/tests/test_xml.py @@ -93,10 +93,10 @@ class TestXml: session.add_link(node.objid, ptp_node.objid, interface_one=interface) # set custom values for node service - session.services.setcustomservice(node_one.objid, SshService.name) + session.services.set_service(node_one.objid, SshService.name) service_file = SshService.configs[0] file_data = "# test" - session.services.setservicefile(node_one.objid, SshService.name, service_file, file_data) + session.services.set_service_file(node_one.objid, SshService.name, service_file, file_data) # instantiate session session.instantiate() @@ -127,12 +127,12 @@ class TestXml: session.open_xml(file_path, start=True) # retrieve custom service - service = session.services.getcustomservice(node_one.objid, SshService.name) + service = session.services.get_service(node_one.objid, SshService.name) # verify nodes have been recreated assert session.get_object(n1_id) assert session.get_object(n2_id) - assert service.configtxt.get(service_file) == file_data + assert service.config_data.get(service_file) == file_data @pytest.mark.parametrize("version", _XML_VERSIONS) def test_xml_mobility(self, session, tmpdir, version, ip_prefixes): diff --git a/ns3/examples/ns3wifirandomwalk.py b/ns3/examples/ns3wifirandomwalk.py index cf5acee1..e0529c2e 100644 --- a/ns3/examples/ns3wifirandomwalk.py +++ b/ns3/examples/ns3wifirandomwalk.py @@ -60,8 +60,8 @@ def wifisession(opt): node = session.addnode(name="n%d" % i) node.newnetif(wifi, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) nodes.append(node) - session.services.addservicestonode(node, "router", services_str.split("|")) - session.services.bootnodeservices(node) + session.services.add_services(node, "router", services_str.split("|")) + session.services.boot_node_services(node) session.setuprandomwalkmobility(bounds=(1000.0, 750.0, 0)) # PHY tracing From bb533406a66e8577ec502d3c88bc7532922c4bf2 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 22 Jun 2018 15:47:02 -0700 Subject: [PATCH 60/83] refactored service methods to be shorter, updated some logging to debug, added some more service tests --- daemon/core/corehandlers.py | 18 ++-- daemon/core/emulator/coreemu.py | 2 +- daemon/core/service.py | 94 +++++++++--------- daemon/core/services/bird.py | 2 +- daemon/core/services/dockersvc.py | 2 +- daemon/core/services/nrl.py | 22 ++--- daemon/core/services/quagga.py | 4 +- daemon/core/services/sdn.py | 6 +- daemon/core/services/security.py | 8 +- daemon/core/services/ucarp.py | 2 +- daemon/core/services/utility.py | 26 ++--- daemon/core/services/xorp.py | 4 +- daemon/core/session.py | 4 +- daemon/core/xml/xmlparser0.py | 2 +- daemon/core/xml/xmlparser1.py | 2 +- daemon/core/xml/xmlwriter1.py | 2 +- daemon/examples/myservices/sample.py | 2 +- daemon/examples/netns/howmanynodes.py | 2 +- daemon/examples/netns/ospfmanetmdrtest.py | 2 +- daemon/examples/netns/wlanemanetests.py | 4 +- daemon/tests/myservices/sample.py | 16 +++- daemon/tests/test_services.py | 110 +++++++++++++++++++++- ns3/examples/ns3wifirandomwalk.py | 2 +- 23 files changed, 226 insertions(+), 112 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index a8c86e63..3696ae5a 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -5,7 +5,6 @@ socket server request handlers leveraged by core servers. import Queue import SocketServer import os -import pprint import shlex import shutil import sys @@ -1526,19 +1525,19 @@ class CoreHandler(SocketServer.BaseRequestHandler): continue if event_type == EventTypes.STOP.value or event_type == EventTypes.RESTART.value: - status = self.session.services.stop_node_service(node, service) + status = self.session.services.stop_service(node, service) if status: fail += "Stop %s," % service.name if event_type == EventTypes.START.value or event_type == EventTypes.RESTART.value: - status = self.session.services.node_service_startup(node, service) + status = self.session.services.startup_service(node, service) if status: fail += "Start %s(%s)," % service.name if event_type == EventTypes.PAUSE.value: - status = self.session.services.validate_node_service(node, service) + status = self.session.services.validate_service(node, service) if status: fail += "%s," % service.name if event_type == EventTypes.RECONFIGURE.value: - self.session.services.node_service_reconfigure(node, service) + self.session.services.service_reconfigure(node, service) fail_data = "" if len(fail) > 0: @@ -1679,6 +1678,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): Return API messages that describe the current session. """ # find all nodes and links + nodes_data = [] links_data = [] with self.session._objects_lock: @@ -1692,21 +1692,17 @@ class CoreHandler(SocketServer.BaseRequestHandler): links_data.append(link_data) # send all nodes first, so that they will exist for any links - logger.info("sending nodes:") for node_data in nodes_data: - logger.info(pprint.pformat(dict(node_data._asdict()))) self.session.broadcast_node(node_data) - logger.info("sending links:") for link_data in links_data: - logger.info(pprint.pformat(dict(link_data._asdict()))) self.session.broadcast_link(link_data) # send mobility model info for node_id in self.session.mobility.nodes(): node = self.session.get_object(node_id) for model_class, config in self.session.mobility.get_models(node): - logger.info("mobility config: node(%s) class(%s) values(%s)", node_id, model_class, config) + logger.debug("mobility config: node(%s) class(%s) values(%s)", node_id, model_class, config) config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) self.session.broadcast_config(config_data) @@ -1714,7 +1710,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): for node_id in self.session.emane.nodes(): node = self.session.get_object(node_id) for model_class, config in self.session.emane.get_models(node): - logger.info("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) + logger.debug("emane config: node(%s) class(%s) values(%s)", node_id, model_class, config) config_data = ConfigShim.config_data(0, node_id, ConfigFlags.UPDATE.value, model_class, config) self.session.broadcast_config(config_data) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 6823ae08..6dcbbeb3 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -515,7 +515,7 @@ class EmuSession(Session): if self.state == EventTypes.RUNTIME_STATE.value and is_boot_node: self.write_objects() self.add_remove_control_interface(node=node, remove=False) - self.services.boot_node_services(node) + self.services.boot_services(node) return node diff --git a/daemon/core/service.py b/daemon/core/service.py index f9da6bd2..901a79ca 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -50,8 +50,8 @@ class ServiceShim(object): valmap = [service.dirs, service.configs, start_index, service.startup, service.shutdown, service.validate, service.meta, start_time] if not service.custom: - valmap[1] = service.getconfigfilenames(node) - valmap[3] = service.getstartup(node) + valmap[1] = service.get_configs(node) + valmap[3] = service.get_startup(node) vals = map(lambda a, b: "%s=%s" % (a, str(b)), cls.keys, valmap) return "|".join(vals) @@ -217,7 +217,15 @@ class CoreServices(object): self.default_services.clear() self.custom_services.clear() - def node_boot_paths(self, services): + def create_boot_paths(self, services): + """ + Create boot paths for starting up services based on dependencies. All services provided and their dependencies + must exist within this set of services, to be valid. + + :param list[CoreService] services: service to create boot paths for + :return: list of boot paths for services + :rtype: list[list[CoreService]] + """ # generate service map and find starting points node_services = {service.name: service for service in services} is_dependency = set() @@ -398,7 +406,7 @@ class CoreServices(object): return files - def boot_node_services(self, node): + def boot_services(self, node): """ Start all services on a node. @@ -408,9 +416,9 @@ class CoreServices(object): pool = ThreadPool() results = [] - boot_paths = self.node_boot_paths(node.services) + boot_paths = self.create_boot_paths(node.services) for boot_path in boot_paths: - result = pool.apply_async(self._boot_node_paths, (node, boot_path)) + result = pool.apply_async(self._start_boot_paths, (node, boot_path)) results.append(result) pool.close() @@ -418,7 +426,7 @@ class CoreServices(object): for result in results: result.get() - def _boot_node_paths(self, node, boot_path): + def _start_boot_paths(self, node, boot_path): """ Start all service boot paths found, based on dependencies. @@ -428,9 +436,9 @@ class CoreServices(object): """ logger.debug("booting node service dependencies: %s", boot_path) for service in boot_path: - self.boot_node_service(node, service) + self.boot_service(node, service) - def boot_node_service(self, node, service): + def boot_service(self, node, service): """ Start a service on a node. Create private dirs, generate config files, and execute startup commands. @@ -446,11 +454,11 @@ class CoreServices(object): node.privatedir(directory) # create service files - self.node_service_files(node, service) + self.create_service_files(node, service) # run startup wait = service.validation_mode == ServiceMode.BLOCKING - status = self.node_service_startup(node, service, wait) + status = self.startup_service(node, service, wait) if status: raise ServiceBootError("node(%s) service(%s) error during startup" % (node.name, service.name)) @@ -461,7 +469,7 @@ class CoreServices(object): # run validation commands, if present and not timer mode if service.validation_mode != ServiceMode.TIMER: - status = self.validate_node_service(node, service) + status = self.validate_service(node, service) if status: raise ServiceBootError("node(%s) service(%s) failed validation" % (node.name, service.name)) @@ -486,7 +494,7 @@ class CoreServices(object): return True return False - def validate_node_service(self, node, service): + def validate_service(self, node, service): """ Run the validation command(s) for a service. @@ -498,7 +506,7 @@ class CoreServices(object): logger.info("validating node(%s) service(%s)", node.name, service.name) cmds = service.validate if not service.custom: - cmds = service.getvalidate(node) + cmds = service.get_validate(node) status = 0 for cmd in cmds: @@ -511,7 +519,7 @@ class CoreServices(object): return status - def stop_node_services(self, node): + def stop_services(self, node): """ Stop all services on a node. @@ -519,9 +527,9 @@ class CoreServices(object): :return: nothing """ for service in node.services: - self.stop_node_service(node, service) + self.stop_service(node, service) - def stop_node_service(self, node, service): + def stop_service(self, node, service): """ Stop a service on a node. @@ -558,7 +566,7 @@ class CoreServices(object): if service.custom: config_files = service.configs else: - config_files = service.getconfigfilenames(node) + config_files = service.get_configs(node) if filename not in config_files: raise ValueError("unknown service(%s) config file: %s", service_name, filename) @@ -566,7 +574,7 @@ class CoreServices(object): # get the file data data = service.config_data.get(filename) if data is None: - data = "%s" % service.generateconfig(node, filename) + data = "%s" % service.generate_config(node, filename) else: data = "%s" % data @@ -579,7 +587,7 @@ class CoreServices(object): data=data ) - def set_service_file(self, node_id, service_name, filename, data): + def set_service_file(self, node_id, service_name, file_name, data): """ Receive a File Message from the GUI and store the customized file in the service config. The filename must match one from the list of @@ -587,7 +595,7 @@ class CoreServices(object): :param int node_id: node id to set service file :param str service_name: service name to set file for - :param str filename: file name to set + :param str file_name: file name to set :param data: data for file to set :return: nothing """ @@ -597,19 +605,19 @@ class CoreServices(object): # retrieve custom service service = self.get_service(node_id, service_name) if service is None: - logger.warn("received filename for unknown service: %s", service_name) + logger.warn("received file name for unknown service: %s", service_name) return # validate file being set is valid - cfgfiles = service.configs - if filename not in cfgfiles: - logger.warn("received unknown file '%s' for service '%s'", filename, service_name) + config_files = service.configs + if file_name not in config_files: + logger.warn("received unknown file(%s) for service(%s)", file_name, service_name) return # set custom service file data - service.config_data[filename] = data + service.config_data[file_name] = data - def node_service_startup(self, node, service, wait=False): + def startup_service(self, node, service, wait=False): """ Startup a node service. @@ -622,7 +630,7 @@ class CoreServices(object): cmds = service.startup if not service.custom: - cmds = service.getstartup(node) + cmds = service.get_startup(node) status = 0 for cmd in cmds: @@ -636,7 +644,7 @@ class CoreServices(object): status = -1 return status - def node_service_files(self, node, service): + def create_service_files(self, node, service): """ Creates node service files. @@ -646,16 +654,16 @@ class CoreServices(object): """ logger.info("node(%s) service(%s) creating config files", node.name, service.name) # get values depending on if custom or not - file_names = service.configs + config_files = service.configs if not service.custom: - file_names = service.getconfigfilenames(node) + config_files = service.get_configs(node) - for file_name in file_names: + for file_name in config_files: logger.debug("generating service config: %s", file_name) if service.custom: cfg = service.config_data.get(file_name) if cfg is None: - cfg = service.generateconfig(node, file_name) + cfg = service.generate_config(node, file_name) # cfg may have a file:/// url for copying from a file try: @@ -665,11 +673,11 @@ class CoreServices(object): logger.exception("error copying service file: %s", file_name) continue else: - cfg = service.generateconfig(node, file_name) + cfg = service.generate_config(node, file_name) node.nodefile(file_name, cfg) - def node_service_reconfigure(self, node, service): + def service_reconfigure(self, node, service): """ Reconfigure a node service. @@ -677,18 +685,18 @@ class CoreServices(object): :param CoreService service: service to reconfigure :return: nothing """ - file_names = service.configs + config_files = service.configs if not service.custom: - file_names = service.getconfigfilenames(node) + config_files = service.get_configs(node) - for file_name in file_names: + for file_name in config_files: if file_name[:7] == "file:///": # TODO: implement this raise NotImplementedError cfg = service.config_data.get(file_name) if cfg is None: - cfg = service.generateconfig(node, file_name) + cfg = service.generate_config(node, file_name) node.nodefile(file_name, cfg) @@ -760,7 +768,7 @@ class CoreService(object): pass @classmethod - def getconfigfilenames(cls, node): + def get_configs(cls, node): """ Return the tuple of configuration file filenames. This default method returns the cls._configs tuple, but this method may be overriden to @@ -773,7 +781,7 @@ class CoreService(object): return cls.configs @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate configuration file given a node object. The filename is provided to allow for multiple config files. @@ -787,7 +795,7 @@ class CoreService(object): raise NotImplementedError @classmethod - def getstartup(cls, node): + def get_startup(cls, node): """ Return the tuple of startup commands. This default method returns the cls.startup tuple, but this method may be @@ -801,7 +809,7 @@ class CoreService(object): return cls.startup @classmethod - def getvalidate(cls, node): + def get_validate(cls, node): """ Return the tuple of validate commands. This default method returns the cls.validate tuple, but this method may be diff --git a/daemon/core/services/bird.py b/daemon/core/services/bird.py index 381a0390..d0b3d2e9 100644 --- a/daemon/core/services/bird.py +++ b/daemon/core/services/bird.py @@ -19,7 +19,7 @@ class Bird(CoreService): validate = ("pidof bird",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the bird.conf file contents. """ diff --git a/daemon/core/services/dockersvc.py b/daemon/core/services/dockersvc.py index e4ee597e..308fe789 100644 --- a/daemon/core/services/dockersvc.py +++ b/daemon/core/services/dockersvc.py @@ -123,7 +123,7 @@ class DockerService(CoreService): image = "" @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Returns a string having contents of a docker.sh script that can be modified to start a specific docker image. diff --git a/daemon/core/services/nrl.py b/daemon/core/services/nrl.py index 50b17b38..71d088d3 100644 --- a/daemon/core/services/nrl.py +++ b/daemon/core/services/nrl.py @@ -21,7 +21,7 @@ class NrlService(CoreService): shutdown = () @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): return "" @staticmethod @@ -52,7 +52,7 @@ class MgenSinkService(NrlService): shutdown = ("killall mgen",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): cfg = "0.0 LISTEN UDP 5000\n" for ifc in node.netifs(): name = utils.sysctl_devname(ifc.name) @@ -60,7 +60,7 @@ class MgenSinkService(NrlService): return cfg @classmethod - def getstartup(cls, node): + def get_startup(cls, node): cmd = cls.startup[0] cmd += " output /tmp/mgen_%s.log" % node.name return cmd, @@ -77,7 +77,7 @@ class NrlNhdp(NrlService): validate = ("pidof nrlnhdp",) @classmethod - def getstartup(cls, node): + def get_startup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -111,7 +111,7 @@ class NrlSmf(NrlService): configs = ("startsmf.sh",) @classmethod - def generateconfig(cls, node, filename, ): + def generate_config(cls, node, filename, ): """ Generate a startup script for SMF. Because nrlsmf does not daemonize, it can cause problems in some situations when launched @@ -162,7 +162,7 @@ class NrlOlsr(NrlService): validate = ("pidof nrlolsrd",) @classmethod - def getstartup(cls, node): + def get_startup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -196,7 +196,7 @@ class NrlOlsrv2(NrlService): validate = ("pidof nrlolsrv2",) @classmethod - def getstartup(cls, node): + def get_startup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -233,7 +233,7 @@ class OlsrOrg(NrlService): validate = ("pidof olsrd",) @classmethod - def getstartup(cls, node): + def get_startup(cls, node): """ Generate the appropriate command-line based on node interfaces. """ @@ -247,7 +247,7 @@ class OlsrOrg(NrlService): return cmd, @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate a default olsrd config file to use the broadcast address of 255.255.255.255. """ @@ -591,7 +591,7 @@ class MgenActor(NrlService): shutdown = ("killall mgen",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate a startup script for MgenActor. Because mgenActor does not daemonize, it can cause problems in some situations when launched @@ -623,7 +623,7 @@ class Arouted(NrlService): validate = ("pidof arouted",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the Quagga.conf or quaggaboot.sh file contents. """ diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index 8919832a..de25ecda 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -23,7 +23,7 @@ class Zebra(CoreService): validate = ("pidof zebra",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the Quagga.conf or quaggaboot.sh file contents. """ @@ -259,7 +259,7 @@ class QuaggaService(CoreService): return False @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): return "" @classmethod diff --git a/daemon/core/services/sdn.py b/daemon/core/services/sdn.py index 4229175c..04cb62fc 100644 --- a/daemon/core/services/sdn.py +++ b/daemon/core/services/sdn.py @@ -14,7 +14,7 @@ class SdnService(CoreService): group = "SDN" @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): return "" @@ -28,7 +28,7 @@ class OvsService(SdnService): shutdown = ('killall ovs-vswitchd', 'killall ovsdb-server') @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): # Check whether the node is running zebra has_zebra = 0 for s in node.services: @@ -101,7 +101,7 @@ class RyuService(SdnService): shutdown = ('killall ryu-manager',) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return a string that will be written to filename, or sent to the GUI for user customization. diff --git a/daemon/core/services/security.py b/daemon/core/services/security.py index 177ce71d..926376f8 100644 --- a/daemon/core/services/security.py +++ b/daemon/core/services/security.py @@ -18,7 +18,7 @@ class VPNClient(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the client.conf and vpnclient.sh file contents to """ @@ -44,7 +44,7 @@ class VPNServer(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the sample server.conf and vpnserver.sh file contents to GUI for user customization. @@ -70,7 +70,7 @@ class IPsec(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the ipsec.conf and racoon.conf file contents to GUI for user customization. @@ -96,7 +96,7 @@ class Firewall(CoreService): custom_needed = True @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the firewall rule examples to GUI for user customization. """ diff --git a/daemon/core/services/ucarp.py b/daemon/core/services/ucarp.py index 686f3646..79d1029b 100644 --- a/daemon/core/services/ucarp.py +++ b/daemon/core/services/ucarp.py @@ -18,7 +18,7 @@ class Ucarp(CoreService): validate = ("pidof ucarp",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return the default file contents """ diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index ff770a64..6621ffa3 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -24,7 +24,7 @@ class UtilService(CoreService): shutdown = () @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): return "" @@ -34,7 +34,7 @@ class IPForwardService(UtilService): startup = ("sh ipforward.sh",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): if os.uname()[0] == "Linux": return cls.generateconfiglinux(node, filename) else: @@ -69,7 +69,7 @@ class DefaultRouteService(UtilService): startup = ("sh defaultroute.sh",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): cfg = "#!/bin/sh\n" cfg += "# auto-generated by DefaultRoute service (utility.py)\n" for ifc in node.netifs(): @@ -103,7 +103,7 @@ class DefaultMulticastRouteService(UtilService): startup = ("sh defaultmroute.sh",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): cfg = "#!/bin/sh\n" cfg += "# auto-generated by DefaultMulticastRoute service (utility.py)\n" cfg += "# the first interface is chosen below; please change it " @@ -129,7 +129,7 @@ class StaticRouteService(UtilService): custom_needed = True @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): cfg = "#!/bin/sh\n" cfg += "# auto-generated by StaticRoute service (utility.py)\n#\n" cfg += "# NOTE: this service must be customized to be of any use\n" @@ -170,7 +170,7 @@ class SshService(UtilService): validate = () @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Use a startup script for launching sshd in order to wait for host key generation. @@ -238,7 +238,7 @@ class DhcpService(UtilService): validate = ("pidof dhcpd",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate a dhcpd config file using the network address of each interface. @@ -300,7 +300,7 @@ class DhcpClientService(UtilService): validate = ("pidof dhclient",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate a script to invoke dhclient on all interfaces. """ @@ -332,7 +332,7 @@ class FtpService(UtilService): validate = ("pidof vsftpd",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate a vsftpd.conf configuration file. """ @@ -368,7 +368,7 @@ class HttpService(UtilService): APACHEVER22, APACHEVER24 = (22, 24) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate an apache2.conf configuration file. """ @@ -577,7 +577,7 @@ class PcapService(UtilService): meta = "logs network traffic to pcap packet capture files" @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate a startpcap.sh traffic logging script. """ @@ -615,7 +615,7 @@ class RadvdService(UtilService): validate = ("pidof radvd",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Generate a RADVD router advertisement daemon config file using the network address of each interface. @@ -674,7 +674,7 @@ class AtdService(UtilService): shutdown = ("pkill atd",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): return """ #!/bin/sh echo 00001 > /var/spool/cron/atjobs/.SEQ diff --git a/daemon/core/services/xorp.py b/daemon/core/services/xorp.py index 7bd5b963..9020fc93 100644 --- a/daemon/core/services/xorp.py +++ b/daemon/core/services/xorp.py @@ -21,7 +21,7 @@ class XorpRtrmgr(CoreService): validate = ("pidof xorp_rtrmgr",) @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Returns config.boot configuration file text. Other services that depend on this will have generatexorpconfig() hooks that are @@ -150,7 +150,7 @@ class XorpService(CoreService): return "0.0.0.0" @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): return "" @classmethod diff --git a/daemon/core/session.py b/daemon/core/session.py index 3f7c17aa..d05575bf 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -673,7 +673,7 @@ class Session(object): for obj in self.objects.itervalues(): # TODO: determine if checking for CoreNode alone is ok if isinstance(obj, nodes.PyCoreNode): - self.services.stop_node_services(obj) + self.services.stop_services(obj) # shutdown emane self.emane.shutdown() @@ -727,7 +727,7 @@ class Session(object): # add a control interface if configured logger.info("booting node: %s", obj.name) self.add_remove_control_interface(node=obj, remove=False) - result = pool.apply_async(self.services.boot_node_services, (obj,)) + result = pool.apply_async(self.services.boot_services, (obj,)) results.append(result) pool.close() diff --git a/daemon/core/xml/xmlparser0.py b/daemon/core/xml/xmlparser0.py index ed1bc5cf..0d759b9f 100644 --- a/daemon/core/xml/xmlparser0.py +++ b/daemon/core/xml/xmlparser0.py @@ -370,7 +370,7 @@ class CoreDocumentParser0(object): filename = file.getAttribute("name") files.append(filename) data = xmlutils.get_text_child(file) - self.session.services.set_service_file(node_id=n.objid, service_name=name, filename=filename, data=data) + self.session.services.set_service_file(node_id=n.objid, service_name=name, file_name=filename, data=data) if len(files): values.append("files=%s" % files) diff --git a/daemon/core/xml/xmlparser1.py b/daemon/core/xml/xmlparser1.py index 0b5c61da..42d01cf5 100644 --- a/daemon/core/xml/xmlparser1.py +++ b/daemon/core/xml/xmlparser1.py @@ -651,7 +651,7 @@ class CoreDocumentParser1(object): self.session.services.set_service_file( node_id=node.objid, service_name=svcname, - filename=filename, + file_name=filename, data=data ) return str(name) diff --git a/daemon/core/xml/xmlwriter1.py b/daemon/core/xml/xmlwriter1.py index 27246859..4641d5c7 100644 --- a/daemon/core/xml/xmlwriter1.py +++ b/daemon/core/xml/xmlwriter1.py @@ -694,7 +694,7 @@ class DeviceElement(NamedXmlElement): f.setAttribute("name", fn) # all file names are added to determine when a file has been deleted s.appendChild(f) - data = svc.configtxt.get(fn) + data = svc.config_data.get(fn) if data is None: # this includes only customized file contents and skips # the auto-generated files diff --git a/daemon/examples/myservices/sample.py b/daemon/examples/myservices/sample.py index a7def96a..bd5d4c2d 100644 --- a/daemon/examples/myservices/sample.py +++ b/daemon/examples/myservices/sample.py @@ -29,7 +29,7 @@ class MyService(CoreService): shutdown = () @classmethod - def generateconfig(cls, node, filename): + def generate_config(cls, node, filename): """ Return a string that will be written to filename, or sent to the GUI for user customization. diff --git a/daemon/examples/netns/howmanynodes.py b/daemon/examples/netns/howmanynodes.py index 7a5035f6..851baeb6 100755 --- a/daemon/examples/netns/howmanynodes.py +++ b/daemon/examples/netns/howmanynodes.py @@ -160,7 +160,7 @@ def main(): n.cmd([constants.SYSCTL_BIN, "net.ipv4.icmp_echo_ignore_broadcasts=0"]) if options.services is not None: session.services.add_services(n, "", options.services.split("|")) - session.services.boot_node_services(n) + session.services.boot_services(n) nodelist.append(n) if i % 25 == 0: print "\n%s nodes created " % i, diff --git a/daemon/examples/netns/ospfmanetmdrtest.py b/daemon/examples/netns/ospfmanetmdrtest.py index cdc0efb8..7d0800a7 100755 --- a/daemon/examples/netns/ospfmanetmdrtest.py +++ b/daemon/examples/netns/ospfmanetmdrtest.py @@ -88,7 +88,7 @@ ip forwarding def boot(self): self.config() - self.session.services.boot_node_services(self) + self.session.services.boot_services(self) def bootscript(self): return """\ diff --git a/daemon/examples/netns/wlanemanetests.py b/daemon/examples/netns/wlanemanetests.py index 0783e8cc..9befcc23 100755 --- a/daemon/examples/netns/wlanemanetests.py +++ b/daemon/examples/netns/wlanemanetests.py @@ -421,7 +421,7 @@ class Experiment(object): tmp.newnetif(self.net, [addr]) self.nodes.append(tmp) self.session.services.add_services(tmp, "router", "IPForward") - self.session.services.boot_node_services(tmp) + self.session.services.boot_services(tmp) self.staticroutes(i, prefix, numnodes) # link each node in a chain, with the previous node @@ -463,7 +463,7 @@ class Experiment(object): for i in xrange(1, numnodes + 1): tmp = self.nodes[i - 1] - self.session.services.boot_node_services(tmp) + self.session.services.boot_services(tmp) self.staticroutes(i, prefix, numnodes) def setnodes(self): diff --git a/daemon/tests/myservices/sample.py b/daemon/tests/myservices/sample.py index 633670bc..0b6a579e 100644 --- a/daemon/tests/myservices/sample.py +++ b/daemon/tests/myservices/sample.py @@ -10,10 +10,20 @@ class MyService(CoreService): group = "Utility" configs = ("myservice.sh",) startup = ("sh myservice.sh",) + shutdown = ("sh myservice.sh",) + + @classmethod + def generate_config(cls, node, filename): + return "# test file" -class MyService2(CoreService): +class MyService2(MyService): name = "MyService2" group = "Utility" - configs = ("myservice.sh",) - startup = ("sh myservice.sh",) + configs = ("myservice2.sh",) + startup = ("sh myservice2.sh",) + shutdown = ("sh myservice2.sh",) + + @classmethod + def generate_config(cls, node, filename): + return "exit 1" diff --git a/daemon/tests/test_services.py b/daemon/tests/test_services.py index 2e6afe4b..b6120bb4 100644 --- a/daemon/tests/test_services.py +++ b/daemon/tests/test_services.py @@ -40,6 +40,91 @@ class ServiceF(CoreService): class TestServices: + def test_service_file(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get("MyService") + node = session.add_node() + file_name = my_service.configs[0] + file_path = node.hostfilename(file_name) + + # when + session.services.create_service_files(node, my_service) + + # then + assert os.path.exists(file_path) + + def test_service_startup(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get("MyService") + node = session.add_node() + session.services.create_service_files(node, my_service) + + # when + status = session.services.startup_service(node, my_service, wait=True) + + # then + assert not status + + def test_service_startup_error(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get("MyService2") + node = session.add_node() + session.services.create_service_files(node, my_service) + + # when + status = session.services.startup_service(node, my_service, wait=True) + + # then + assert status + + def test_service_stop(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get("MyService") + node = session.add_node() + session.services.create_service_files(node, my_service) + + # when + status = session.services.stop_service(node, my_service) + + # then + assert not status + + def test_service_stop_error(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get("MyService2") + node = session.add_node() + session.services.create_service_files(node, my_service) + + # when + status = session.services.stop_service(node, my_service) + + # then + assert status + + def test_service_set_file(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get("MyService") + node = session.add_node() + file_name = my_service.configs[0] + file_path = node.hostfilename(file_name) + file_data = "# custom file" + session.services.set_service_file(node.objid, my_service.name, file_name, file_data) + custom_service = session.services.get_service(node.objid, my_service.name) + + # when + session.services.create_service_files(node, custom_service) + + # then + assert os.path.exists(file_path) + with open(file_path, "r") as custom_file: + assert custom_file.read() == file_data + def test_service_import(self): """ Test importing a custom service. @@ -48,8 +133,23 @@ class TestServices: assert ServiceManager.get("MyService") assert ServiceManager.get("MyService2") - def test_service_defaults(self): - pass + def test_service_setget(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + service_name = "MyService" + my_service = ServiceManager.get(service_name) + node = session.add_node() + + # when + no_service = session.services.get_service(node.objid, service_name) + default_service = session.services.get_service(node.objid, service_name, default_service=True) + session.services.set_service(node.objid, service_name) + custom_service = session.services.get_service(node.objid, service_name, default_service=True) + + # then + assert no_service is None + assert default_service == my_service + assert custom_service and custom_service != my_service def test_services_dependencies(self, session): # given @@ -62,7 +162,7 @@ class TestServices: ] # when - startups = session.services.node_boot_paths(services) + startups = session.services.create_boot_paths(services) # then assert len(startups) == 2 @@ -78,7 +178,7 @@ class TestServices: # when with pytest.raises(ValueError): - session.services.node_boot_paths(services) + session.services.create_boot_paths(services) def test_services_dependencies_cycle(self, session): # given @@ -94,4 +194,4 @@ class TestServices: # when with pytest.raises(ValueError): - session.services.node_boot_paths(services) + session.services.create_boot_paths(services) diff --git a/ns3/examples/ns3wifirandomwalk.py b/ns3/examples/ns3wifirandomwalk.py index e0529c2e..bab30a6b 100644 --- a/ns3/examples/ns3wifirandomwalk.py +++ b/ns3/examples/ns3wifirandomwalk.py @@ -61,7 +61,7 @@ def wifisession(opt): node.newnetif(wifi, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) nodes.append(node) session.services.add_services(node, "router", services_str.split("|")) - session.services.boot_node_services(node) + session.services.boot_services(node) session.setuprandomwalkmobility(bounds=(1000.0, 750.0, 0)) # PHY tracing From 1d355d98ec5dd49d970a7d0898611cc8024e4430 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Mon, 25 Jun 2018 08:41:23 -0700 Subject: [PATCH 61/83] added more services tests --- daemon/tests/myservices/sample.py | 3 +- daemon/tests/test_services.py | 98 ++++++++++++++++++++++++++----- 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/daemon/tests/myservices/sample.py b/daemon/tests/myservices/sample.py index 0b6a579e..e6673670 100644 --- a/daemon/tests/myservices/sample.py +++ b/daemon/tests/myservices/sample.py @@ -22,7 +22,8 @@ class MyService2(MyService): group = "Utility" configs = ("myservice2.sh",) startup = ("sh myservice2.sh",) - shutdown = ("sh myservice2.sh",) + shutdown = startup + validate = startup @classmethod def generate_config(cls, node, filename): diff --git a/daemon/tests/test_services.py b/daemon/tests/test_services.py index b6120bb4..4c20d031 100644 --- a/daemon/tests/test_services.py +++ b/daemon/tests/test_services.py @@ -8,6 +8,9 @@ from core.service import ServiceManager _PATH = os.path.abspath(os.path.dirname(__file__)) _SERVICES_PATH = os.path.join(_PATH, "myservices") +SERVICE_ONE = "MyService" +SERVICE_TWO = "MyService2" + class ServiceA(CoreService): name = "A" @@ -40,10 +43,52 @@ class ServiceF(CoreService): class TestServices: + def test_service_all_files(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + file_name = "myservice.sh" + node = session.add_node() + + # when + session.services.set_service_file(node.objid, SERVICE_ONE, file_name, "# test") + + # then + service = session.services.get_service(node.objid, SERVICE_ONE) + all_files = session.services.all_files(service) + assert service + assert all_files and len(all_files) == 1 + + def test_service_all_configs(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + node = session.add_node() + + # when + session.services.set_service(node.objid, SERVICE_ONE) + session.services.set_service(node.objid, SERVICE_TWO) + + # then + all_configs = session.services.all_configs() + assert all_configs + assert len(all_configs) == 2 + + def test_service_add_services(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + node = session.add_node() + total_service = len(node.services) + + # when + session.services.add_services(node, node.type, [SERVICE_ONE, SERVICE_TWO]) + + # then + assert node.services + assert len(node.services) == total_service + 2 + def test_service_file(self, session): # given ServiceManager.add_services(_SERVICES_PATH) - my_service = ServiceManager.get("MyService") + my_service = ServiceManager.get(SERVICE_ONE) node = session.add_node() file_name = my_service.configs[0] file_path = node.hostfilename(file_name) @@ -54,10 +99,36 @@ class TestServices: # then assert os.path.exists(file_path) + def test_service_validate(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get(SERVICE_ONE) + node = session.add_node() + session.services.create_service_files(node, my_service) + + # when + status = session.services.validate_service(node, my_service) + + # then + assert not status + + def test_service_validate_error(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get(SERVICE_TWO) + node = session.add_node() + session.services.create_service_files(node, my_service) + + # when + status = session.services.validate_service(node, my_service) + + # then + assert status + def test_service_startup(self, session): # given ServiceManager.add_services(_SERVICES_PATH) - my_service = ServiceManager.get("MyService") + my_service = ServiceManager.get(SERVICE_ONE) node = session.add_node() session.services.create_service_files(node, my_service) @@ -70,7 +141,7 @@ class TestServices: def test_service_startup_error(self, session): # given ServiceManager.add_services(_SERVICES_PATH) - my_service = ServiceManager.get("MyService2") + my_service = ServiceManager.get(SERVICE_TWO) node = session.add_node() session.services.create_service_files(node, my_service) @@ -83,7 +154,7 @@ class TestServices: def test_service_stop(self, session): # given ServiceManager.add_services(_SERVICES_PATH) - my_service = ServiceManager.get("MyService") + my_service = ServiceManager.get(SERVICE_ONE) node = session.add_node() session.services.create_service_files(node, my_service) @@ -96,7 +167,7 @@ class TestServices: def test_service_stop_error(self, session): # given ServiceManager.add_services(_SERVICES_PATH) - my_service = ServiceManager.get("MyService2") + my_service = ServiceManager.get(SERVICE_TWO) node = session.add_node() session.services.create_service_files(node, my_service) @@ -109,7 +180,7 @@ class TestServices: def test_service_set_file(self, session): # given ServiceManager.add_services(_SERVICES_PATH) - my_service = ServiceManager.get("MyService") + my_service = ServiceManager.get(SERVICE_ONE) node = session.add_node() file_name = my_service.configs[0] file_path = node.hostfilename(file_name) @@ -130,21 +201,20 @@ class TestServices: Test importing a custom service. """ ServiceManager.add_services(_SERVICES_PATH) - assert ServiceManager.get("MyService") - assert ServiceManager.get("MyService2") + assert ServiceManager.get(SERVICE_ONE) + assert ServiceManager.get(SERVICE_TWO) def test_service_setget(self, session): # given ServiceManager.add_services(_SERVICES_PATH) - service_name = "MyService" - my_service = ServiceManager.get(service_name) + my_service = ServiceManager.get(SERVICE_ONE) node = session.add_node() # when - no_service = session.services.get_service(node.objid, service_name) - default_service = session.services.get_service(node.objid, service_name, default_service=True) - session.services.set_service(node.objid, service_name) - custom_service = session.services.get_service(node.objid, service_name, default_service=True) + no_service = session.services.get_service(node.objid, SERVICE_ONE) + default_service = session.services.get_service(node.objid, SERVICE_ONE, default_service=True) + session.services.set_service(node.objid, SERVICE_ONE) + custom_service = session.services.get_service(node.objid, SERVICE_ONE, default_service=True) # then assert no_service is None From d05bc9240ac4fe3e55faec0f8cce81fa86be8d6b Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 28 Jun 2018 16:30:55 -0700 Subject: [PATCH 62/83] initial code to leverage lxml to create xml for saving and loading core sessions --- daemon/core/conf.py | 5 +- daemon/core/corehandlers.py | 2 +- daemon/core/emane/emanemanager.py | 9 +- daemon/core/emane/emanemanifest.py | 2 +- daemon/core/mobility.py | 3 +- daemon/core/service.py | 2 +- daemon/core/xml/corexml.py | 579 +++++++++++++++++++++++++++++ daemon/core/xml/xmlsession.py | 2 + daemon/data/logging.conf | 2 +- 9 files changed, 597 insertions(+), 9 deletions(-) create mode 100644 daemon/core/xml/corexml.py diff --git a/daemon/core/conf.py b/daemon/core/conf.py index 1e579804..ffe71541 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -269,7 +269,6 @@ class ConfigurableOptions(object): name = None bitmap = None options = [] - _default_node = -1 @classmethod def configurations(cls): @@ -380,9 +379,11 @@ class ModelManager(ConfigurableManager): all_configs = self.get_all_configs(node_id=node.objid) for model_name in all_configs.iterkeys(): + if model_name == ModelManager._default_node: + continue model_class = self.models[model_name] config = self.get_configs(node_id=node.objid, config_type=model_name) models.append((model_class, config)) - logger.debug("mobility models for node(%s): %s", node.objid, models) + logger.debug("models for node(%s): %s", node.objid, models) return models diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 3696ae5a..ec092e41 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1284,7 +1284,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): if values_str: config = ConfigShim.str_to_dict(values_str) - self.session.emane.set_configs(config) + self.session.emane.set_configs(config, node_id=node_id) # extra logic to start slave Emane object after nemid has been configured from the master if message_type == ConfigFlags.UPDATE and self.session.master is False: diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 29de1be2..2f85760b 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -11,6 +11,7 @@ from core import constants from core import logger from core.api import coreapi from core.api import dataconversion +from core.conf import ConfigGroup from core.conf import ConfigShim from core.conf import Configuration from core.conf import ModelManager @@ -1016,8 +1017,12 @@ class EmaneGlobalModel(EmaneModel): @classmethod def config_groups(cls): - return "Platform Attributes:1-%d|NEM Parameters:%d-%d" % ( - len(cls.emulator_config), len(cls.emulator_config) + 1, len(cls.configurations())) + emulator_len = len(cls.emulator_config) + config_len = len(cls.configurations()) + return [ + ConfigGroup("Platform Attributes", 1, emulator_len), + ConfigGroup("NEM Parameters", emulator_len + 1, config_len) + ] def __init__(self, session, object_id=None): super(EmaneGlobalModel, self).__init__(session, object_id) diff --git a/daemon/core/emane/emanemanifest.py b/daemon/core/emane/emanemanifest.py index ed27f445..d3a5a2af 100644 --- a/daemon/core/emane/emanemanifest.py +++ b/daemon/core/emane/emanemanifest.py @@ -34,7 +34,7 @@ def _get_possible(config_type, config_regex): :param str config_type: emane configuration type :param str config_regex: emane configuration regex :return: a string listing comma delimited values, if needed, empty string otherwise - :rtype: str + :rtype: list """ if config_type == "bool": return ["On", "Off"] diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index abc8bf55..30436eec 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -73,7 +73,8 @@ class MobilityManager(ModelManager): continue for model_name in self.models.iterkeys(): - config = self.get_configs(node_id=node_id, config_type=model_name) + all_configs = self.get_all_configs(node_id) + config = all_configs.get(model_name) if not config: continue model_class = self.models[model_name] diff --git a/daemon/core/service.py b/daemon/core/service.py index 901a79ca..7e9cade6 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -339,7 +339,7 @@ class CoreServices(object): :param str service_name: name of service to set :return: nothing """ - logger.debug("setting custom service(%s) for node: %s", node_id, service_name) + logger.debug("setting custom service(%s) for node: %s", service_name, node_id) service = self.get_service(node_id, service_name) if not service: service_class = ServiceManager.get(service_name) diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py new file mode 100644 index 00000000..b061aa36 --- /dev/null +++ b/daemon/core/xml/corexml.py @@ -0,0 +1,579 @@ +from lxml import etree + +from core import coreobj +from core.enumerations import NodeTypes +from core.misc import ipaddress +from core.misc import nodeutils +from core.netns import nodes + + +def add_attribute(element, name, value): + if value is not None: + element.set(name, str(value)) + + +def create_emane_config(node_id, emane_config, config): + emane_configuration = etree.Element("emane_configuration") + add_attribute(emane_configuration, "node", node_id) + add_attribute(emane_configuration, "model", "emane") + emulator_element = etree.SubElement(emane_configuration, "emulator") + for emulator_config in emane_config.emulator_config: + config_element = etree.SubElement(emulator_element, "configuration") + value = config[emulator_config.id] + add_attribute(config_element, "name", emulator_config.id) + add_attribute(config_element, "value", value) + + nem_element = etree.SubElement(emane_configuration, "nem") + for nem_config in emane_config.nem_config: + config_element = etree.SubElement(nem_element, "configuration") + value = config[nem_config.id] + add_attribute(config_element, "name", nem_config.id) + add_attribute(config_element, "value", value) + + return emane_configuration + + +def create_emane_model_config(node_id, model, config): + emane_element = etree.Element("emane_configuration") + add_attribute(emane_element, "node", node_id) + add_attribute(emane_element, "model", model.name) + + mac_element = etree.SubElement(emane_element, "mac") + for mac_config in model.mac_config: + config_element = etree.SubElement(mac_element, "configuration") + value = config[mac_config.id] + add_attribute(config_element, "name", mac_config.id) + add_attribute(config_element, "value", value) + + phy_element = etree.SubElement(emane_element, "phy") + for phy_config in model.phy_config: + config_element = etree.SubElement(phy_element, "configuration") + value = config[phy_config.id] + add_attribute(config_element, "name", phy_config.id) + add_attribute(config_element, "value", value) + + return emane_element + + +def get_endpoints(node): + endpoints = [] + for interface in node.netifs(sort=True): + endpoint = get_endpoint(node, interface) + endpoints.append(endpoint) + return endpoints + + +def get_endpoint(node, interface): + l2devport = None + othernet = getattr(interface, "othernet", None) + + # reference interface of node that is part of this network + if interface.net.objid == node.objid and interface.node: + params = interface.getparams() + if nodeutils.is_node(interface.net, (NodeTypes.HUB, NodeTypes.SWITCH)): + l2devport = "%s/e%s" % (interface.net.name, interface.netindex) + endpoint_id = "%s/%s" % (interface.node.name, interface.name) + endpoint = Endpoint( + node, + interface, + "interface", + endpoint_id, + l2devport, + params + ) + # references another node connected to this network + elif othernet and othernet.objid == node.objid: + interface.swapparams("_params_up") + params = interface.getparams() + interface.swapparams("_params_up") + l2devport = "%s/e%s" % (othernet.name, interface.netindex) + endpoint_id = "%s/%s/%s" % (node.name, interface.node.name, interface.netindex) + endpoint = Endpoint( + interface.net, + interface, + "interface", + endpoint_id, + l2devport, + params + ) + else: + endpoint = Endpoint( + node, + interface, + ) + + return endpoint + + +def get_downstream_l2_devices(node): + all_endpoints = [] + l2_devices = [node] + current_endpoint = get_endpoints(node) + all_endpoints.extend(current_endpoint) + for endpoint in current_endpoint: + if endpoint.type and endpoint.network.objid != node.objid: + new_l2_devices, new_endpoints = get_downstream_l2_devices(endpoint.network) + l2_devices.extend(new_l2_devices) + all_endpoints.extend(new_endpoints) + return l2_devices, all_endpoints + + +def create_link_element(link_data): + link_element = etree.Element("link") + add_attribute(link_element, "node_one", link_data.node1_id) + add_attribute(link_element, "node_two", link_data.node2_id) + + # check for interface one + interface_one = etree.Element("interface_one") + add_attribute(interface_one, "id", link_data.interface1_id) + add_attribute(interface_one, "name", link_data.interface1_name) + add_attribute(interface_one, "mac", link_data.interface1_mac) + add_attribute(interface_one, "ip4", link_data.interface1_ip4) + add_attribute(interface_one, "ip4_mask", link_data.interface1_ip4_mask) + add_attribute(interface_one, "ip6", link_data.interface1_ip6) + add_attribute(interface_one, "ip6_mask", link_data.interface1_ip6_mask) + if interface_one.items(): + link_element.append(interface_one) + + # check for interface two + interface_two = etree.Element("interface_two") + add_attribute(interface_two, "id", link_data.interface2_id) + add_attribute(interface_two, "name", link_data.interface2_name) + add_attribute(interface_two, "mac", link_data.interface2_mac) + add_attribute(interface_two, "ip4", link_data.interface2_ip4) + add_attribute(interface_two, "ip4_mask", link_data.interface2_ip4_mask) + add_attribute(interface_two, "ip6", link_data.interface2_ip6) + add_attribute(interface_two, "ip6_mask", link_data.interface2_ip6_mask) + if interface_two.items(): + link_element.append(interface_two) + + # check for options + options = etree.Element("options") + add_attribute(options, "delay", link_data.delay) + add_attribute(options, "bandwidth", link_data.bandwidth) + add_attribute(options, "per", link_data.per) + add_attribute(options, "dup", link_data.dup) + add_attribute(options, "jitter", link_data.jitter) + add_attribute(options, "mer", link_data.mer) + add_attribute(options, "burst", link_data.burst) + add_attribute(options, "mburst", link_data.mburst) + add_attribute(options, "type", link_data.link_type) + add_attribute(options, "gui_attributes", link_data.gui_attributes) + add_attribute(options, "unidirectional", link_data.unidirectional) + add_attribute(options, "emulation_id", link_data.emulation_id) + add_attribute(options, "network_id", link_data.network_id) + add_attribute(options, "key", link_data.key) + add_attribute(options, "opaque", link_data.opaque) + if options.items(): + link_element.append(options) + + return link_element + + +class Endpoint(object): + def __init__(self, network, interface, _type=None, _id=None, l2devport=None, params=None): + self.network = network + self.interface = interface + self.type = _type + self.id = _id + self.l2devport = l2devport + self.params = params + + +class NodeElement(object): + def __init__(self, session, node, element_name): + self.session = session + self.node = node + self.element = etree.Element(element_name) + add_attribute(self.element, "id", node.objid) + add_attribute(self.element, "name", node.name) + add_attribute(self.element, "icon", node.icon) + add_attribute(self.element, "canvas", node.canvas) + self.add_position() + + def add_position(self): + x = self.node.position.x + y = self.node.position.y + z = self.node.position.z + lat, lon, alt = None, None, None + if x is not None and y is not None: + lat, lon, alt = self.session.location.getgeo(x, y, z) + position = etree.SubElement(self.element, "position") + add_attribute(position, "x", x) + add_attribute(position, "y", y) + add_attribute(position, "z", z) + add_attribute(position, "lat", lat) + add_attribute(position, "lon", lon) + add_attribute(position, "alt", alt) + + +class InterfaceElement(object): + def __init__(self, session, node, interface): + self.session = session + self.node = node + self.interface = interface + self.element = etree.Element("interface") + add_attribute(self.element, "id", interface.netindex) + add_attribute(self.element, "name", interface.name) + mac = etree.SubElement(self.element, "mac") + mac.text = str(interface.hwaddr) + self.add_mtu() + self.addresses = etree.SubElement(self.element, "addresses") + self.add_addresses() + self.add_model() + + def add_mtu(self): + # check to add mtu + if self.interface.mtu and self.interface.mtu != 1500: + add_attribute(self.element, "mtu", self.interface.mtu) + + def add_model(self): + # check for emane specific interface configuration + net_model = None + if self.interface.net and hasattr(self.interface.net, "model"): + net_model = self.interface.net.model + + if net_model and net_model.name.startswith("emane_"): + config = self.session.emane.getifcconfig(self.node.objid, self.interface, net_model.name) + if config: + emane_element = create_emane_model_config(net_model, config) + self.element.append(emane_element) + + def add_addresses(self): + for address in self.interface.addrlist: + ip, mask = address.split("/") + if ipaddress.is_ipv4_address(ip): + address_type = "IPv4" + else: + address_type = "IPv6" + address_element = etree.SubElement(self.addresses, "address") + add_attribute(address_element, "type", address_type) + address_element.text = str(address) + + +class ServiceElement(object): + def __init__(self, service): + self.service = service + self.element = etree.Element("service") + add_attribute(self.element, "name", service.name) + self.add_directories() + self.add_startup() + self.add_validate() + self.add_shutdown() + self.add_files() + + def add_directories(self): + # get custom directories + directories = etree.Element("directories") + for directory in self.service.dirs: + directory_element = etree.SubElement(directories, "directory") + directory_element.text = directory + + if directories.getchildren(): + self.element.append(directories) + + def add_files(self): + # get custom files + file_elements = etree.Element("files") + for file_name, data in self.service.config_data.iteritems(): + file_element = etree.SubElement(file_elements, "file") + add_attribute(file_element, "name", file_name) + file_element.text = data + + if file_elements.getchildren(): + self.element.append(file_elements) + + def add_startup(self): + # get custom startup + startup_elements = etree.Element("startups") + for startup in self.service.startup: + startup_element = etree.SubElement(startup_elements, "startup") + startup_element.text = startup + + if startup_elements.getchildren(): + self.element.append(startup_elements) + + def add_validate(self): + # get custom validate + validate_elements = etree.Element("validates") + for validate in self.service.validate: + validate_element = etree.SubElement(validate_elements, "validate") + validate_element.text = validate + + if validate_elements.getchildren(): + self.element.append(validate_elements) + + def add_shutdown(self): + # get custom shutdown + shutdown_elements = etree.Element("shutdowns") + for shutdown in self.service.shutdown: + shutdown_element = etree.SubElement(shutdown_elements, "shutdown") + shutdown_element.text = shutdown + + if shutdown_elements.getchildren(): + self.element.append(shutdown_elements) + + +class DeviceElement(NodeElement): + def __init__(self, session, node): + super(DeviceElement, self).__init__(session, node, "device") + add_attribute(self.element, "type", node.type) + # self.add_interfaces() + self.add_services() + + def add_services(self): + service_elements = etree.Element("services") + for service in self.node.services: + etree.SubElement(service_elements, "service", name=service.name) + + if service_elements.getchildren(): + self.element.append(service_elements) + + def add_interfaces(self): + interfaces = etree.Element("interfaces") + for interface in self.node.netifs(sort=True): + interface_element = InterfaceElement(self.session, self.node, interface) + interfaces.append(interface_element.element) + + if interfaces.getchildren(): + self.element.append(interfaces) + + +class NetworkElement(NodeElement): + def __init__(self, session, node): + super(NetworkElement, self).__init__(session, node, "network") + model = getattr(self.node, "model", None) + if model: + add_attribute(self.element, "model", model.name) + mobility = getattr(self.node, "mobility", None) + if mobility: + add_attribute(self.element, "mobility", mobility.name) + grekey = getattr(self.node, "grekey", None) + if grekey and grekey is not None: + add_attribute(self.element, "grekey", grekey) + self.add_type() + # self.endpoints = get_endpoints(self.node) + # self.l2_devices = self.get_l2_devices() + # self.add_configs() + + def add_type(self): + if self.node.apitype: + node_type = NodeTypes(self.node.apitype).name + else: + node_type = self.node.__class__.__name__ + add_attribute(self.element, "type", node_type) + + def get_l2_devices(self): + l2_devices = [] + found_l2_devices = [] + found_endpoints = [] + if nodeutils.is_node(self.node, (NodeTypes.SWITCH, NodeTypes.HUB)): + for endpoint in self.endpoints: + if endpoint.type and endpoint.network.objid != self.node.objid: + downstream_l2_devices, downstream_endpoints = get_downstream_l2_devices(endpoint.network) + found_l2_devices.extend(downstream_l2_devices) + found_endpoints.extend(downstream_endpoints) + + for l2_device in found_l2_devices: + pass + + self.endpoints.extend(found_endpoints) + return l2_devices + + def add_peer_to_peer_config(self): + pass + + def add_switch_hub_tunnel_config(self): + pass + + def add_configs(self): + if nodeutils.is_node(self.node, NodeTypes.PEER_TO_PEER): + self.add_peer_to_peer_config() + elif nodeutils.is_node(self.node, (NodeTypes.SWITCH, NodeTypes.HUB, NodeTypes.TUNNEL)): + self.add_switch_hub_tunnel_config() + + +class CoreXmlWriter(object): + def __init__(self, session): + self.session = session + self.scenario = None + self.networks = None + self.devices = None + + def write(self, file_name): + self.scenario = etree.Element("scenario", name=file_name) + self.networks = etree.SubElement(self.scenario, "networks") + self.devices = etree.SubElement(self.scenario, "devices") + links = self.write_nodes() + self.write_links(links) + self.write_mobility_configs() + self.write_emane_configs() + self.write_service_configs() + self.write_session_origin() + self.write_session_hooks() + self.write_session_options() + self.write_session_metadata() + self.write_default_services() + + with open(file_name, "w") as xml_file: + data = etree.tostring(self.scenario, xml_declaration=True, pretty_print=True, encoding="UTF-8") + xml_file.write(data) + + def write_session_origin(self): + # origin: geolocation of cartesian coordinate 0,0,0 + lat, lon, alt = self.session.location.refgeo + origin = etree.Element("session_origin") + add_attribute(origin, "lat", lat) + add_attribute(origin, "lon", lon) + add_attribute(origin, "alt", alt) + has_origin = len(origin.items()) > 0 + + if has_origin: + self.scenario.append(origin) + refscale = self.session.location.refscale + if refscale != 1.0: + add_attribute(origin, "scale", refscale) + if self.session.location.refxyz != (0.0, 0.0, 0.0): + x, y, z = self.session.location.refxyz + add_attribute(origin, "x", x) + add_attribute(origin, "y", y) + add_attribute(origin, "z", z) + + def write_session_hooks(self): + # hook scripts + hooks = etree.Element("session_hooks") + for state in sorted(self.session._hooks.keys()): + for file_name, data in self.session._hooks[state]: + hook = etree.SubElement(hooks, "hook") + add_attribute(hook, "name", file_name) + add_attribute(hook, "state", state) + hook.text = data + + if hooks.getchildren(): + self.scenario.append(hooks) + + def write_session_options(self): + # options + options = etree.Element("session_options") + # TODO: should we just save the current config regardless, since it may change? + options_config = self.session.options.get_configs() + for _id, default_value in self.session.options.default_values().iteritems(): + value = options_config[_id] + if value != default_value: + option = etree.SubElement(options, "option") + add_attribute(option, "name", _id) + add_attribute(option, "value", value) + + if options.getchildren(): + self.scenario.append(options) + + def write_session_metadata(self): + # metadata + metadata = etree.Element("session_metadata") + for _id, value in self.session.metadata.get_configs().iteritems(): + data = etree.SubElement(metadata, "data") + add_attribute(data, "name", _id) + add_attribute(data, "value", value) + + if metadata.getchildren(): + self.scenario.append(metadata) + + def write_emane_configs(self): + emane_configurations = etree.Element("emane_configurations") + for node_id in self.session.emane.nodes(): + all_configs = self.session.emane.get_all_configs(node_id) + for model_name, config in all_configs.iteritems(): + if model_name == -1: + emane_configuration = create_emane_config(node_id, self.session.emane_config, config) + else: + model = self.session.emane.models[model_name] + emane_configuration = create_emane_model_config(node_id, model, config) + emane_configurations.append(emane_configuration) + + if emane_configurations.getchildren(): + self.scenario.append(emane_configurations) + + def write_mobility_configs(self): + mobility_configurations = etree.Element("mobility_configurations") + for node_id in self.session.mobility.nodes(): + all_configs = self.session.emane.get_all_configs(node_id) + for model_name, config in all_configs.iteritems(): + mobility_configuration = etree.SubElement(mobility_configurations, "mobility_configuration") + add_attribute(mobility_configuration, "node", node_id) + add_attribute(mobility_configuration, "model", model_name) + for name, value in config.iteritems(): + config_element = etree.SubElement(mobility_configuration, "configuration") + add_attribute(config_element, "name", name) + add_attribute(config_element, "value", value) + + if mobility_configurations.getchildren(): + self.scenario.append(mobility_configurations) + + def write_service_configs(self): + service_configurations = etree.Element("service_configurations") + service_configs = self.session.services.all_configs() + for node_id, service in service_configs: + service_element = ServiceElement(service) + add_attribute(service_element.element, "node", node_id) + service_configurations.append(service_element.element) + + if service_configurations.getchildren(): + self.scenario.append(service_configurations) + + def write_default_services(self): + node_types = etree.Element("default_services") + for node_type, services in self.session.services.default_services.iteritems(): + node_type = etree.SubElement(node_types, "node", type=node_type) + for service in services: + etree.SubElement(node_type, "service", name=service) + + if node_types.getchildren(): + self.scenario.append(node_types) + + def write_nodes(self): + links = [] + for node in self.session.objects.itervalues(): + # network node + if isinstance(node, coreobj.PyCoreNet) and not nodeutils.is_node(node, NodeTypes.CONTROL_NET): + self.write_network(node) + # device node + elif isinstance(node, nodes.PyCoreNode): + self.write_device(node) + + # add known links + links.extend(node.all_link_data(0)) + + return links + + def write_network(self, node): + # ignore p2p and other nodes that are not part of the api + if not node.apitype: + return + + # ignore nodes tied to a different network + if nodeutils.is_node(node, (NodeTypes.SWITCH, NodeTypes.HUB)): + for netif in node.netifs(sort=True): + othernet = getattr(netif, "othernet", None) + if othernet and othernet.objid != node.objid: + return + + network = NetworkElement(self.session, node) + self.networks.append(network.element) + + def write_links(self, links): + link_elements = etree.Element("links") + # add link data + for link_data in links: + # skip basic range links + if not link_data.interface1_id and not link_data.interface2_id: + continue + + link_element = create_link_element(link_data) + link_elements.append(link_element) + + if link_elements.getchildren(): + self.scenario.append(link_elements) + + def write_device(self, node): + device = DeviceElement(self.session, node) + self.devices.append(device.element) diff --git a/daemon/core/xml/xmlsession.py b/daemon/core/xml/xmlsession.py index 80f0380e..1fbf8d04 100644 --- a/daemon/core/xml/xmlsession.py +++ b/daemon/core/xml/xmlsession.py @@ -7,6 +7,7 @@ import os.path from core.enumerations import NodeTypes from core.misc import nodeutils +from core.xml import corexml from core.xml.xmlparser import core_document_parser from core.xml.xmlwriter import core_document_writer @@ -34,3 +35,4 @@ def save_session_xml(session, filename, version): """ doc = core_document_writer(session, version) doc.writexml(filename) + corexml.CoreXmlWriter(session).write(filename + ".test") diff --git a/daemon/data/logging.conf b/daemon/data/logging.conf index 7f3d496f..46de6e92 100644 --- a/daemon/data/logging.conf +++ b/daemon/data/logging.conf @@ -14,7 +14,7 @@ } }, "root": { - "level": "INFO", + "level": "DEBUG", "handlers": ["console"] } } From 4ccb1ed9f35bcd8a8b67a91c63c2664a4f3cdd5c Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Tue, 3 Jul 2018 09:50:14 -0700 Subject: [PATCH 63/83] working lxml based save/load --- daemon/core/corehandlers.py | 2 +- daemon/core/emulator/coreemu.py | 16 +- daemon/core/emulator/emudata.py | 6 +- daemon/core/session.py | 4 +- daemon/core/xml/corexml.py | 378 +++++++++++++++++++++++++++++--- daemon/core/xml/xmlsession.py | 4 +- daemon/requirements.txt | 1 + daemon/setup.py | 1 + 8 files changed, 360 insertions(+), 52 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index ec092e41..c788f885 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1728,7 +1728,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): type=ConfigFlags.UPDATE.value, data_types=data_types, data_values=values, - session=self.session.session_id, + session=str(self.session.session_id), opaque=opaque ) self.session.broadcast_config(config_data) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 6dcbbeb3..c805a155 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -17,8 +17,7 @@ from core.misc import nodemaps from core.misc import nodeutils from core.service import ServiceManager from core.session import Session -from core.xml.xmlparser import core_document_parser -from core.xml.xmlwriter import core_document_writer +from core.xml.corexml import CoreXmlReader, CoreXmlWriter def signal_handler(signal_number, _): @@ -199,7 +198,7 @@ class EmuSession(Session): objects = [x for x in objects if x] if len(objects) < 2: raise ValueError("wireless link failure: %s", objects) - logger.debug("handling wireless linking objects(%) connect(%s)", objects, connect) + logger.debug("handling wireless linking objects(%s) connect(%s)", objects, connect) common_networks = objects[0].commonnets(objects[1]) if not common_networks: raise ValueError("no common network found for wireless link/unlink") @@ -661,10 +660,10 @@ class EmuSession(Session): # clear out existing session self.clear() - # set default node class when one is not provided - node_class = nodeutils.get_node_class(NodeTypes.DEFAULT) - options = {"start": start, "nodecls": node_class} - core_document_parser(self, file_name, options) + # write out xml file + CoreXmlReader(self).read(file_name) + + # start session if needed if start: self.name = os.path.basename(file_name) self.file_name = file_name @@ -678,8 +677,7 @@ class EmuSession(Session): :param str version: xml version type :return: nothing """ - doc = core_document_writer(self, version) - doc.writexml(file_name) + CoreXmlWriter(self).write(file_name) def add_hook(self, state, file_name, source_name, data): """ diff --git a/daemon/core/emulator/emudata.py b/daemon/core/emulator/emudata.py index c231b743..e224209d 100644 --- a/daemon/core/emulator/emudata.py +++ b/daemon/core/emulator/emudata.py @@ -34,8 +34,8 @@ class NodeOptions(object): """ Convenience method for setting position. - :param int x: x position - :param int y: y position + :param float x: x position + :param float y: y position :return: nothing """ self.x = x @@ -161,7 +161,7 @@ class IpPrefixes(object): # random mac if not mac: - mac = str(MacAddress.random()) + mac = MacAddress.random() return InterfaceData( _id=inteface_id, diff --git a/daemon/core/session.py b/daemon/core/session.py index d05575bf..8424bfe3 100644 --- a/daemon/core/session.py +++ b/daemon/core/session.py @@ -38,7 +38,7 @@ from core.mobility import MobilityManager from core.netns import nodes from core.sdt import Sdt from core.service import CoreServices -from core.xml.xmlsession import save_session_xml +from core.xml import corexml class Session(object): @@ -381,7 +381,7 @@ class Session(object): xml_file_version = self.options.get_config("xmlfilever") if xml_file_version in ("1.0",): xml_file_name = os.path.join(self.session_dir, "session-deployed.xml") - save_session_xml(self, xml_file_name, xml_file_version) + corexml.CoreXmlWriter(self).write(xml_file_name) def get_environment(self, state=True): """ diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index b061aa36..07516762 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -1,34 +1,64 @@ from lxml import etree from core import coreobj +from core import logger +from core.emulator.emudata import InterfaceData +from core.emulator.emudata import LinkOptions +from core.emulator.emudata import NodeOptions from core.enumerations import NodeTypes from core.misc import ipaddress from core.misc import nodeutils +from core.misc.ipaddress import MacAddress from core.netns import nodes +def get_type(element, name, _type): + value = element.get(name) + if value is not None: + value = _type(value) + return value + + +def get_float(element, name): + return get_type(element, name, float) + + +def get_int(element, name): + return get_type(element, name, int) + + def add_attribute(element, name, value): if value is not None: element.set(name, str(value)) +def create_interface_data(interface_element): + interface_id = int(interface_element.get("id")) + name = interface_element.get("name") + mac = interface_element.get("mac") + if mac: + mac = MacAddress.from_string(mac) + ip4 = interface_element.get("ip4") + ip4_mask = get_int(interface_element, "ip4_mask") + ip6 = interface_element.get("ip6") + ip6_mask = get_int(interface_element, "ip6_mask") + return InterfaceData(interface_id, name, mac, ip4, ip4_mask, ip6, ip6_mask) + + def create_emane_config(node_id, emane_config, config): emane_configuration = etree.Element("emane_configuration") add_attribute(emane_configuration, "node", node_id) add_attribute(emane_configuration, "model", "emane") + emulator_element = etree.SubElement(emane_configuration, "emulator") for emulator_config in emane_config.emulator_config: - config_element = etree.SubElement(emulator_element, "configuration") value = config[emulator_config.id] - add_attribute(config_element, "name", emulator_config.id) - add_attribute(config_element, "value", value) + add_configuration(emulator_element, emane_config.id, value) nem_element = etree.SubElement(emane_configuration, "nem") for nem_config in emane_config.nem_config: - config_element = etree.SubElement(nem_element, "configuration") value = config[nem_config.id] - add_attribute(config_element, "name", nem_config.id) - add_attribute(config_element, "value", value) + add_configuration(nem_element, nem_config.id, value) return emane_configuration @@ -40,21 +70,23 @@ def create_emane_model_config(node_id, model, config): mac_element = etree.SubElement(emane_element, "mac") for mac_config in model.mac_config: - config_element = etree.SubElement(mac_element, "configuration") value = config[mac_config.id] - add_attribute(config_element, "name", mac_config.id) - add_attribute(config_element, "value", value) + add_configuration(mac_element, mac_config.id, value) phy_element = etree.SubElement(emane_element, "phy") for phy_config in model.phy_config: - config_element = etree.SubElement(phy_element, "configuration") value = config[phy_config.id] - add_attribute(config_element, "name", phy_config.id) - add_attribute(config_element, "value", value) + add_configuration(phy_element, phy_config.id, value) return emane_element +def add_configuration(parent, name, value): + config_element = etree.SubElement(parent, "configuration") + add_attribute(config_element, "name", name) + add_attribute(config_element, "value", value) + + def get_endpoints(node): endpoints = [] for interface in node.netifs(sort=True): @@ -164,6 +196,7 @@ def create_link_element(link_data): add_attribute(options, "network_id", link_data.network_id) add_attribute(options, "key", link_data.key) add_attribute(options, "opaque", link_data.opaque) + add_attribute(options, "session", link_data.session) if options.items(): link_element.append(options) @@ -401,9 +434,8 @@ class CoreXmlWriter(object): self.devices = None def write(self, file_name): + # generate xml content self.scenario = etree.Element("scenario", name=file_name) - self.networks = etree.SubElement(self.scenario, "networks") - self.devices = etree.SubElement(self.scenario, "devices") links = self.write_nodes() self.write_links(links) self.write_mobility_configs() @@ -415,9 +447,9 @@ class CoreXmlWriter(object): self.write_session_metadata() self.write_default_services() - with open(file_name, "w") as xml_file: - data = etree.tostring(self.scenario, xml_declaration=True, pretty_print=True, encoding="UTF-8") - xml_file.write(data) + # write out generated xml + xml_tree = etree.ElementTree(self.scenario) + xml_tree.write(file_name, xml_declaration=True, pretty_print=True, encoding="UTF-8") def write_session_origin(self): # origin: geolocation of cartesian coordinate 0,0,0 @@ -454,35 +486,32 @@ class CoreXmlWriter(object): def write_session_options(self): # options - options = etree.Element("session_options") + option_elements = etree.Element("session_options") # TODO: should we just save the current config regardless, since it may change? options_config = self.session.options.get_configs() for _id, default_value in self.session.options.default_values().iteritems(): value = options_config[_id] if value != default_value: - option = etree.SubElement(options, "option") - add_attribute(option, "name", _id) - add_attribute(option, "value", value) + add_configuration(option_elements, _id, value) - if options.getchildren(): - self.scenario.append(options) + if option_elements.getchildren(): + self.scenario.append(option_elements) def write_session_metadata(self): # metadata - metadata = etree.Element("session_metadata") + metadata_elements = etree.Element("session_metadata") for _id, value in self.session.metadata.get_configs().iteritems(): - data = etree.SubElement(metadata, "data") - add_attribute(data, "name", _id) - add_attribute(data, "value", value) + add_configuration(metadata_elements, _id, value) - if metadata.getchildren(): - self.scenario.append(metadata) + if metadata_elements.getchildren(): + self.scenario.append(metadata_elements) def write_emane_configs(self): emane_configurations = etree.Element("emane_configurations") for node_id in self.session.emane.nodes(): all_configs = self.session.emane.get_all_configs(node_id) for model_name, config in all_configs.iteritems(): + logger.info("writing emane config: %s", config) if model_name == -1: emane_configuration = create_emane_config(node_id, self.session.emane_config, config) else: @@ -496,15 +525,13 @@ class CoreXmlWriter(object): def write_mobility_configs(self): mobility_configurations = etree.Element("mobility_configurations") for node_id in self.session.mobility.nodes(): - all_configs = self.session.emane.get_all_configs(node_id) + all_configs = self.session.mobility.get_all_configs(node_id) for model_name, config in all_configs.iteritems(): mobility_configuration = etree.SubElement(mobility_configurations, "mobility_configuration") add_attribute(mobility_configuration, "node", node_id) add_attribute(mobility_configuration, "model", model_name) for name, value in config.iteritems(): - config_element = etree.SubElement(mobility_configuration, "configuration") - add_attribute(config_element, "name", name) - add_attribute(config_element, "value", value) + add_configuration(mobility_configuration, name, value) if mobility_configurations.getchildren(): self.scenario.append(mobility_configurations) @@ -531,8 +558,13 @@ class CoreXmlWriter(object): self.scenario.append(node_types) def write_nodes(self): + self.networks = etree.SubElement(self.scenario, "networks") + self.devices = etree.SubElement(self.scenario, "devices") + links = [] for node in self.session.objects.itervalues(): + logger.info("writer adding node(%s)", node.name) + # network node if isinstance(node, coreobj.PyCoreNet) and not nodeutils.is_node(node, NodeTypes.CONTROL_NET): self.write_network(node) @@ -555,6 +587,7 @@ class CoreXmlWriter(object): for netif in node.netifs(sort=True): othernet = getattr(netif, "othernet", None) if othernet and othernet.objid != node.objid: + logger.info("writer ignoring node(%s) othernet(%s)", node.name, othernet.name) return network = NetworkElement(self.session, node) @@ -565,7 +598,7 @@ class CoreXmlWriter(object): # add link data for link_data in links: # skip basic range links - if not link_data.interface1_id and not link_data.interface2_id: + if link_data.interface1_id is None and link_data.interface2_id is None: continue link_element = create_link_element(link_data) @@ -577,3 +610,280 @@ class CoreXmlWriter(object): def write_device(self, node): device = DeviceElement(self.session, node) self.devices.append(device.element) + + +class CoreXmlReader(object): + def __init__(self, session): + self.session = session + self.scenario = None + + def read(self, file_name): + xml_tree = etree.parse(file_name) + self.scenario = xml_tree.getroot() + + # read xml session content + self.read_default_services() + self.read_session_metadata() + self.read_session_options() + self.read_session_hooks() + self.read_session_origin() + self.read_service_configs() + self.read_mobility_configs() + self.read_emane_configs() + self.read_nodes() + self.read_links() + + def read_default_services(self): + default_services = self.scenario.find("default_services") + if default_services is None: + return + + for node in default_services.iterchildren(): + node_type = node.get("type") + services = [] + for service in node.iterchildren(): + services.append(service.get("name")) + logger.info("reading default services for nodes(%s): %s", node_type, services) + self.session.services.default_services[node_type] = services + + def read_session_metadata(self): + session_metadata = self.scenario.find("session_metadata") + if session_metadata is None: + return + + configs = {} + for data in session_metadata.iterchildren(): + name = data.get("name") + value = data.get("value") + configs[name] = value + logger.info("reading session metadata: %s", configs) + self.session.metadata.set_configs(configs) + + def read_session_options(self): + session_options = self.scenario.find("session_options") + if session_options is None: + return + + configs = {} + for config in session_options.iterchildren(): + name = config.get("name") + value = config.get("value") + configs[name] = value + logger.info("reading session options: %s", configs) + self.session.options.set_configs(configs) + + def read_session_hooks(self): + session_hooks = self.scenario.find("session_hooks") + if session_hooks is None: + return + + for hook in session_hooks.iterchildren(): + name = hook.get("name") + state = hook.get("state") + data = hook.text + self.session.add_state_hook() + hook_type = "hook:%s" % state + logger.info("reading hook: state(%s) name(%s)", state, name) + self.session.set_hook(hook_type, file_name=name, source_name=None, data=data) + + def read_session_origin(self): + session_origin = self.scenario.find("session_origin") + if session_origin is None: + return + + lat = get_float(session_origin, "lat") + lon = get_float(session_origin, "lon") + alt = get_float(session_origin, "alt") + if all([lat, lon, alt]): + logger.info("reading session reference geo: %s, %s, %s", lat, lon, alt) + self.session.location.setrefgeo(lat, lon, alt) + + scale = get_float(session_origin, "scale") + if scale: + logger.info("reading session reference scale: %s", scale) + self.session.location.refscale = scale + + x = get_float(session_origin, "x") + y = get_float(session_origin, "y") + z = get_float(session_origin, "z") + if all([x, y]): + logger.info("reading session reference xyz: %s, %s, %s", x, y, z) + self.session.location.refxyz = (x, y, z) + + def read_service_configs(self): + service_configurations = self.scenario.find("service_configurations") + if service_configurations is None: + return + + for service_configuration in service_configurations.iterchildren(): + node_id = get_int(service_configuration, "node") + service_name = service_configuration.get("name") + logger.info("reading custom service(%s) for node(%s)", service_name, node_id) + self.session.services.set_service(node_id, service_name) + service = self.session.services.get_service(node_id, service_name) + + directory_elements = service_configuration.find("directories") + if directory_elements is not None: + service.directories = tuple(x.text for x in directory_elements.iterchildren()) + + startup_elements = service_configuration.find("startups") + if startup_elements is not None: + service.startup = tuple(x.text for x in startup_elements.iterchildren()) + + validate_elements = service_configuration.find("validates") + if validate_elements is not None: + service.validate = tuple(x.text for x in validate_elements.iterchildren()) + + shutdown_elements = service_configuration.find("shutdowns") + if shutdown_elements is not None: + service.shutdown = tuple(x.text for x in shutdown_elements.iterchildren()) + + file_elements = service_configuration.find("files") + if file_elements is not None: + for file_element in file_elements.iterchildren(): + name = file_element.get("name") + data = file_element.text + service.config_data[name] = data + + def read_emane_configs(self): + emane_configurations = self.scenario.find("emane_configurations") + if emane_configurations is None: + return + + for emane_configuration in emane_configurations.iterchildren(): + node_id = get_int(emane_configuration, "node") + model_name = emane_configuration.get("model") + configs = {} + + mac_configuration = emane_configuration.find("mac") + for config in mac_configuration.iterchildren(): + name = config.get("name") + value = config.get("value") + configs[name] = value + + phy_configuration = emane_configuration.find("phy") + for config in phy_configuration.iterchildren(): + name = config.get("name") + value = config.get("value") + configs[name] = value + + logger.info("reading emane configuration node(%s) model(%s)", node_id, model_name) + self.session.emane.set_model_config(node_id, model_name, configs) + + def read_mobility_configs(self): + mobility_configurations = self.scenario.find("mobility_configurations") + if mobility_configurations is None: + return + + for mobility_configuration in mobility_configurations.iterchildren(): + node_id = get_int(mobility_configuration, "node") + model_name = mobility_configuration.get("model") + configs = {} + + for config in mobility_configuration.iterchildren(): + name = config.get("name") + value = config.get("value") + configs[name] = value + + logger.info("reading mobility configuration node(%s) model(%s)", node_id, model_name) + self.session.mobility.set_model_config(node_id, model_name, configs) + + def read_nodes(self): + device_elements = self.scenario.find("devices") + if device_elements is not None: + for device_element in device_elements.iterchildren(): + self.read_device(device_element) + + network_elements = self.scenario.find("networks") + if network_elements is not None: + for network_element in network_elements.iterchildren(): + self.read_network(network_element) + + def read_device(self, device_element): + node_id = get_int(device_element, "id") + name = device_element.get("name") + model = device_element.get("type") + node_options = NodeOptions(name, model) + + service_elements = device_element.find("services") + if service_elements is not None: + node_options.services = [x.get("name") for x in service_elements.iterchildren()] + + position_element = device_element.find("position") + if position_element is not None: + x = get_float(position_element, "x") + y = get_float(position_element, "y") + if all([x, y]): + node_options.set_position(x, y) + + lat = get_float(position_element, "lat") + lon = get_float(position_element, "lon") + alt = get_float(position_element, "alt") + if all([lat, lon, alt]): + node_options.set_location(lat, lon, alt) + + logger.info("reading node id(%s) model(%s) name(%s)", node_id, model, name) + self.session.add_node(_id=node_id, node_options=node_options) + + def read_network(self, network_element): + node_id = get_int(network_element, "id") + name = network_element.get("name") + node_type = NodeTypes[network_element.get("type")] + node_options = NodeOptions(name) + + position_element = network_element.find("position") + if position_element is not None: + x = get_float(position_element, "x") + y = get_float(position_element, "y") + if all([x, y]): + node_options.set_position(x, y) + + lat = get_float(position_element, "lat") + lon = get_float(position_element, "lon") + alt = get_float(position_element, "alt") + if all([lat, lon, alt]): + node_options.set_location(lat, lon, alt) + + logger.info("reading node id(%s) node_type(%s) name(%s)", node_id, node_type, name) + self.session.add_node(_type=node_type, _id=node_id, node_options=node_options) + + def read_links(self): + link_elements = self.scenario.find("links") + if link_elements is None: + return + + for link_element in link_elements.iterchildren(): + node_one = get_int(link_element, "node_one") + node_two = get_int(link_element, "node_two") + + interface_one_element = link_element.find("interface_one") + interface_one = None + if interface_one_element is not None: + interface_one = create_interface_data(interface_one_element) + + interface_two_element = link_element.find("interface_two") + interface_two = None + if interface_two_element is not None: + interface_two = create_interface_data(interface_two_element) + + options_element = link_element.find("options") + link_options = LinkOptions() + if options_element is not None: + link_options.bandwidth = get_float(options_element, "bandwidth") + link_options.burst = get_float(options_element, "burst") + link_options.delay = get_float(options_element, "delay") + link_options.dup = get_float(options_element, "dup") + link_options.mer = get_float(options_element, "mer") + link_options.mburst = get_float(options_element, "mburst") + link_options.jitter = get_float(options_element, "jitter") + link_options.key = get_float(options_element, "key") + link_options.per = get_float(options_element, "per") + link_options.unidirectional = get_int(options_element, "unidirectional") + link_options.session = options_element.get("session") + link_options.emulation_id = get_int(options_element, "emulation_id") + link_options.network_id = get_int(options_element, "network_id") + link_options.opaque = options_element.get("opaque") + link_options.gui_attributes = options_element.get("gui_attributes") + + logger.info("reading link node_one(%s) node_two(%s)", node_one, node_two) + self.session.add_link(node_one, node_two, interface_one, interface_two, link_options) diff --git a/daemon/core/xml/xmlsession.py b/daemon/core/xml/xmlsession.py index 1fbf8d04..0411a232 100644 --- a/daemon/core/xml/xmlsession.py +++ b/daemon/core/xml/xmlsession.py @@ -33,6 +33,4 @@ def save_session_xml(session, filename, version): """ Export a session to the EmulationScript XML format. """ - doc = core_document_writer(session, version) - doc.writexml(filename) - corexml.CoreXmlWriter(session).write(filename + ".test") + corexml.CoreXmlWriter(session).write(filename) diff --git a/daemon/requirements.txt b/daemon/requirements.txt index 977eb416..ba137563 100644 --- a/daemon/requirements.txt +++ b/daemon/requirements.txt @@ -1,4 +1,5 @@ enum34==1.1.6 +lxml==3.5.0 mock==1.3.0 pycco==0.5.1 pytest==3.0.7 diff --git a/daemon/setup.py b/daemon/setup.py index 9a3cbefd..1355ad8b 100644 --- a/daemon/setup.py +++ b/daemon/setup.py @@ -43,6 +43,7 @@ setup( packages=find_packages(), install_requires=[ "enum34", + "lxml" ], tests_require=[ "pytest", From ae94c78fbbe73e21a312bbe7052203f63f6ab246 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Tue, 3 Jul 2018 12:48:54 -0700 Subject: [PATCH 64/83] updated emane model xml to leverage lxml --- daemon/core/emane/bypass.py | 5 +- daemon/core/emane/emanemanager.py | 35 +++++ daemon/core/emane/emanemodel.py | 208 +++++++++++++++++------------- daemon/core/xml/corexml.py | 6 + 4 files changed, 166 insertions(+), 88 deletions(-) diff --git a/daemon/core/emane/bypass.py b/daemon/core/emane/bypass.py index 8f60f238..42340f2b 100644 --- a/daemon/core/emane/bypass.py +++ b/daemon/core/emane/bypass.py @@ -1,6 +1,7 @@ """ EMANE Bypass model for CORE """ +from core.conf import ConfigGroup from core.conf import Configuration from core.emane import emanemodel from core.enumerations import ConfigDataTypes @@ -31,4 +32,6 @@ class EmaneBypassModel(emanemodel.EmaneModel): # override config groups @classmethod def config_groups(cls): - return "Bypass Parameters:1-1" + return [ + ConfigGroup("Bypass Parameters", 1, 1), + ] diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 2f85760b..62acb800 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -4,6 +4,7 @@ emane.py: definition of an Emane class for implementing configuration control of import os import threading +from lxml import etree from xml.dom.minidom import parseString from core import CoreCommandError @@ -55,6 +56,39 @@ EMANE_MODELS = [ ] +def build_node_platform_xml(node): + platform_element = etree.Element("platform") + + nem_entries = {} + + if node.model is None: + logger.info("warning: EmaneNode %s has no associated model", node.name) + return nem_entries + + for netif in node.netifs(): + # build nem xml + # nementry = node.model.build_nem_xml(doc, node, netif) + nem_name = node.model.nem_name(netif) + nem_element = etree.Element("nem", name=netif.localname, definition=nem_name) + + # build transport xml + # trans = node.model.build_transport_xml(doc, node, netif) + transport_type = netif.transport_type + if not transport_type: + logger.info("warning: %s interface type unsupported!", netif.name) + transport_type = "raw" + transport_name = node.transportxmlname(transport_type) + transport_element = etree.SubElement(nem_element, "transport", definition=transport_name) + + # add transport parameter + etree.SubElement(transport_element, "param", name="device", value=netif.name) + + # add nem entry + nem_entries[netif] = nem_element + + return nem_entries + + class EmaneManager(ModelManager): """ EMANE controller object. Lives in a Session instance and is used for @@ -637,6 +671,7 @@ class EmaneManager(ModelManager): for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] nems = emane_node.buildplatformxmlentry(self.xmldoc("platform")) + for netif in sorted(nems, key=lambda x: x.node.objid): nementry = nems[netif] nementry.setAttribute("id", "%d" % nemid) diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 77aafcec..e69f377b 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -1,38 +1,102 @@ """ Defines Emane Models used within CORE. """ +import os + +from lxml import etree from core import logger from core.conf import ConfigGroup from core.emane import emanemanifest from core.misc import utils from core.mobility import WirelessModel -from core.xml import xmlutils +from core.xml import corexml -def value_to_params(doc, name, value): +def emane_xml_doctype(name): """ - Helper to convert a parameter to a paramlist. Returns an XML paramlist, or None if the value does not expand to - multiple values. + Create emane xml doctype for xml files. + + :param str name: name for xml doctype + :return: emane xml doctype + :rtype: str + """ + return '' % (name, name) + + +def add_emane_configurations(xml_element, configurations, config, config_ignore): + """ + + + :param lxml.etree.Element xml_element: xml element to add emane configurations to + :param list[core.config.Configuration] configurations: configurations to add to xml + :param dict config: configuration values + :param set config_ignore: configuration options to ignore + :return: + """ + for configuration in configurations: + # ignore custom configurations + name = configuration.id + if name in config_ignore: + continue + + # check if value is a multi param + value = str(config[name]) + params = lxml_value_to_params(value) + if params: + params_element = etree.SubElement(xml_element, "paramlist", name=name) + for param in params: + etree.SubElement(params_element, "item", value=param) + else: + etree.SubElement(xml_element, "param", name=name, value=value) + + +def lxml_value_to_params(value): + """ + Helper to convert a parameter to a parameter tuple. - :param xml.dom.minidom.Document doc: xml document - :param name: name of element for params :param str value: value string to convert to tuple - :return: xml document with added params or None, when an invalid value has been provided + :return: parameter tuple, None otherwise """ try: values = utils.make_tuple_fromstr(value, str) + + if not hasattr(values, "__iter__"): + return None + + if len(values) < 2: + return None + + return values + except SyntaxError: logger.exception("error in value string to param list") - return None + return None - if not hasattr(values, "__iter__"): - return None - if len(values) < 2: - return None - - return xmlutils.add_param_list_to_parent(doc, parent=None, name=name, values=values) +# def value_to_params(doc, name, value): +# """ +# Helper to convert a parameter to a paramlist. Returns an XML paramlist, or None if the value does not expand to +# multiple values. +# +# :param xml.dom.minidom.Document doc: xml document +# :param name: name of element for params +# :param str value: value string to convert to tuple +# :return: xml document with added params or None, when an invalid value has been provided +# """ +# try: +# values = utils.make_tuple_fromstr(value, str) +# except SyntaxError: +# logger.exception("error in value string to param list") +# return None +# +# if not hasattr(values, "__iter__"): +# return None +# +# if len(values) < 2: +# return None +# +# return xmlutils.add_param_list_to_parent(doc, parent=None, name=name, values=values) class EmaneModel(WirelessModel): @@ -88,112 +152,82 @@ class EmaneModel(WirelessModel): return # create document and write to disk - nem_name = self.nem_name(interface) - nem_document = self.create_nem_doc(emane_manager, interface) - emane_manager.xmlwrite(nem_document, nem_name) + self.create_nem_doc(interface) # create mac document and write to disk - mac_name = self.mac_name(interface) - mac_document = self.create_mac_doc(emane_manager, config) - emane_manager.xmlwrite(mac_document, mac_name) + self.create_mac_doc(interface, config) # create phy document and write to disk - phy_name = self.phy_name(interface) - phy_document = self.create_phy_doc(emane_manager, config) - emane_manager.xmlwrite(phy_document, phy_name) + self.create_phy_doc(interface, config) - def create_nem_doc(self, emane_manager, interface): + def create_nem_doc(self, interface): """ Create the nem xml document. - :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager :param interface: interface for the emane node - :return: nem document - :rtype: xml.dom.minidom.Document + :return: nothing """ + nem_element = etree.Element("nem", name="%s NEM" % self.name) + + # add transport + transport_type = "virtual" + if interface and interface.transport_type == "raw": + transport_type = "raw" + transport_type = "n%strans%s.xml" % (self.object_id, transport_type) + etree.SubElement(nem_element, "transport", definition=transport_type) + + # create mac mac_name = self.mac_name(interface) + etree.SubElement(nem_element, "mac", definition=mac_name) + + # create phy phy_name = self.phy_name(interface) + etree.SubElement(nem_element, "phy", definition=phy_name) - nem_document = emane_manager.xmldoc("nem") - nem_element = nem_document.getElementsByTagName("nem").pop() - nem_element.setAttribute("name", "%s NEM" % self.name) - emane_manager.appendtransporttonem(nem_document, nem_element, self.object_id, interface) + # write out xml + nem_name = self.nem_name(interface) + self.create_file(nem_element, nem_name, "nem") - mac_element = nem_document.createElement("mac") - mac_element.setAttribute("definition", mac_name) - nem_element.appendChild(mac_element) - - phy_element = nem_document.createElement("phy") - phy_element.setAttribute("definition", phy_name) - nem_element.appendChild(phy_element) - - return nem_document - - def create_mac_doc(self, emane_manager, config): + def create_mac_doc(self, interface, config): """ Create the mac xml document. - :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager + :param interface: interface for the emane node :param dict config: all current configuration values, mac + phy - :return: nem document - :rtype: xml.dom.minidom.Document + :return: nothing """ - mac_document = emane_manager.xmldoc("mac") - mac_element = mac_document.getElementsByTagName("mac").pop() - mac_element.setAttribute("name", "%s MAC" % self.name) - if not self.mac_library: raise ValueError("must define emane model library") - mac_element.setAttribute("library", self.mac_library) - for mac_configuration in self.mac_config: - # ignore custom configurations - name = mac_configuration.id - if name in self.config_ignore: - continue + mac_element = etree.Element("mac", name="%s MAC" % self.name, library=self.mac_library) + add_emane_configurations(mac_element, self.mac_config, config, self.config_ignore) - # check if value is a multi param - value = str(config[name]) - param = value_to_params(mac_document, name, value) - if not param: - param = emane_manager.xmlparam(mac_document, name, value) + # write out xml + mac_name = self.mac_name(interface) + self.create_file(mac_element, mac_name, "mac") - mac_element.appendChild(param) - - return mac_document - - def create_phy_doc(self, emane_manager, config): + def create_phy_doc(self, interface, config): """ Create the phy xml document. - :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager + :param interface: interface for the emane node :param dict config: all current configuration values, mac + phy - :return: nem document - :rtype: xml.dom.minidom.Document + :return: nothing """ - phy_document = emane_manager.xmldoc("phy") - phy_element = phy_document.getElementsByTagName("phy").pop() - phy_element.setAttribute("name", "%s PHY" % self.name) - + phy_element = etree.Element("phy", name="%s PHY" % self.name) if self.phy_library: - phy_element.setAttribute("library", self.phy_library) + phy_element.set("library", self.phy_library) - # append all phy options - for phy_configuration in self.phy_config: - # ignore custom configurations - name = phy_configuration.id - if name in self.config_ignore: - continue + add_emane_configurations(phy_element, self.phy_config, config, self.config_ignore) - # check if value is a multi param - value = str(config[name]) - param = value_to_params(phy_document, name, value) - if not param: - param = emane_manager.xmlparam(phy_document, name, value) + # write out xml + phy_name = self.phy_name(interface) + self.create_file(phy_element, phy_name, "phy") - phy_element.appendChild(param) - - return phy_document + def create_file(self, xml_element, file_name, doc_name): + file_path = os.path.join(self.session.session_dir, file_name) + doctype = emane_xml_doctype(doc_name) + corexml.write_xml_file(xml_element, file_path, doctype=doctype) def post_startup(self): """ diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 07516762..551a75e0 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -12,6 +12,12 @@ from core.misc.ipaddress import MacAddress from core.netns import nodes +def write_xml_file(xml_element, file_path, doctype=None): + xml_data = etree.tostring(xml_element, xml_declaration=True, pretty_print=True, encoding="UTF-8", doctype=doctype) + with open(file_path, "w") as xml_file: + xml_file.write(xml_data) + + def get_type(element, name, _type): value = element.get(name) if value is not None: From f115b1a847b1029738c4b2027973bc074004f91a Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Tue, 3 Jul 2018 18:49:36 -0700 Subject: [PATCH 65/83] emane xml fully generated from lxml apis, removed xml functions embedded within emane nodes, emane manager, and emane models. Started consolidating emanexml logic into its own module, when it makes sense --- daemon/core/corehandlers.py | 2 +- daemon/core/emane/commeffect.py | 47 ++-- daemon/core/emane/emanemanager.py | 281 ++++++++++++------------ daemon/core/emane/emanemodel.py | 180 ++------------- daemon/core/emane/nodes.py | 85 ------- daemon/core/xml/corexml.py | 4 +- daemon/core/xml/emanexml.py | 64 ++++++ daemon/examples/myemane/examplemodel.py | 10 +- 8 files changed, 249 insertions(+), 424 deletions(-) create mode 100644 daemon/core/xml/emanexml.py diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index c788f885..0ba3a61b 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -1284,7 +1284,7 @@ class CoreHandler(SocketServer.BaseRequestHandler): if values_str: config = ConfigShim.str_to_dict(values_str) - self.session.emane.set_configs(config, node_id=node_id) + self.session.emane.set_configs(config) # extra logic to start slave Emane object after nemid has been configured from the master if message_type == ConfigFlags.UPDATE and self.session.master is False: diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index bc699099..e2011e15 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -2,10 +2,13 @@ commeffect.py: EMANE CommEffect model for CORE """ +from lxml import etree + from core import logger from core.conf import ConfigGroup from core.emane import emanemanifest from core.emane import emanemodel +from core.xml import emanexml try: from emane.events.commeffectevent import CommEffectEvent @@ -36,6 +39,9 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): shim_defaults = {} config_shim = emanemanifest.parse(shim_xml, shim_defaults) + # comm effect does not need the default phy configuration + phy_config = () + @classmethod def configurations(cls): return cls.config_shim @@ -46,39 +52,36 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): ConfigGroup("CommEffect SHIM Parameters", 1, len(cls.configurations())) ] - def build_xml_files(self, emane_manager, interface): + def build_xml_files(self, config, interface=None): """ Build the necessary nem and commeffect XMLs in the given path. If an individual NEM has a nonstandard config, we need to build that file also. Otherwise the WLAN-wide nXXemane_commeffectnem.xml, nXXemane_commeffectshim.xml are used. - :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager + :param dict config: emane model configuration for the node and interface :param interface: interface for the emane node :return: nothing """ - config = emane_manager.getifcconfig(self.object_id, interface, self.name) - if not config: - return - # retrieve xml names nem_name = self.nem_name(interface) shim_name = self.shim_name(interface) - nem_document = emane_manager.xmldoc("nem") - nem_element = nem_document.getElementsByTagName("nem").pop() - nem_element.setAttribute("name", "%s NEM" % self.name) - nem_element.setAttribute("type", "unstructured") - emane_manager.appendtransporttonem(nem_document, nem_element, self.object_id, interface) + # create and write nem document + nem_element = etree.Element("nem", name="%s NEM" % self.name, type="unstructured") + transport_type = "virtual" + if interface and interface.transport_type == "raw": + transport_type = "raw" + transport_name = "n%strans%s.xml" % (self.object_id, transport_type) + etree.SubElement(nem_element, "transport", definition=transport_name) - shim_xml = emane_manager.xmlshimdefinition(nem_document, shim_name) - nem_element.appendChild(shim_xml) - emane_manager.xmlwrite(nem_document, nem_name) + # set shim configuration + etree.SubElement(nem_element, "shim", definition=shim_name) - shim_document = emane_manager.xmldoc("shim") - shim_element = shim_document.getElementsByTagName("shim").pop() - shim_element.setAttribute("name", "%s SHIM" % self.name) - shim_element.setAttribute("library", self.shim_library) + self.create_file(nem_element, nem_name, "nem") + + # create and write shim document + shim_element = etree.Element("shim", name="%s SHIM" % self.name, library=self.shim_library) # append all shim options (except filterfile) to shimdoc for configuration in self.config_shim: @@ -86,14 +89,14 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): if name == "filterfile": continue value = config[name] - param = emane_manager.xmlparam(shim_document, name, value) - shim_element.appendChild(param) + emanexml.add_param(shim_element, name, value) # empty filterfile is not allowed ff = config["filterfile"] if ff.strip() != "": - shim_element.appendChild(emane_manager.xmlparam(shim_document, "filterfile", ff)) - emane_manager.xmlwrite(shim_document, shim_name) + emanexml.add_param(shim_element, "filterfile", ff) + + self.create_file(shim_element, shim_name, "shim") def linkconfig(self, netif, bw=None, delay=None, loss=None, duplicate=None, jitter=None, netif2=None): """ diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 62acb800..11c58b73 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -4,8 +4,8 @@ emane.py: definition of an Emane class for implementing configuration control of import os import threading + from lxml import etree -from xml.dom.minidom import parseString from core import CoreCommandError from core import constants @@ -33,7 +33,7 @@ from core.enumerations import RegisterTlvs from core.misc import nodeutils from core.misc import utils from core.misc.ipaddress import MacAddress -from core.xml import xmlutils +from core.xml import emanexml try: from emane.events import EventService @@ -56,10 +56,9 @@ EMANE_MODELS = [ ] -def build_node_platform_xml(node): - platform_element = etree.Element("platform") - +def build_node_platform_xml(emane_manager, control_net, node, nem_id): nem_entries = {} + platform_xmls = {} if node.model is None: logger.info("warning: EmaneNode %s has no associated model", node.name) @@ -67,26 +66,147 @@ def build_node_platform_xml(node): for netif in node.netifs(): # build nem xml - # nementry = node.model.build_nem_xml(doc, node, netif) nem_name = node.model.nem_name(netif) - nem_element = etree.Element("nem", name=netif.localname, definition=nem_name) + nem_element = etree.Element("nem", id=str(nem_id), name=netif.localname, definition=nem_name) # build transport xml - # trans = node.model.build_transport_xml(doc, node, netif) transport_type = netif.transport_type if not transport_type: logger.info("warning: %s interface type unsupported!", netif.name) transport_type = "raw" - transport_name = node.transportxmlname(transport_type) + transport_name = "n%strans%s.xml" % (node.objid, transport_type.lower()) transport_element = etree.SubElement(nem_element, "transport", definition=transport_name) # add transport parameter - etree.SubElement(transport_element, "param", name="device", value=netif.name) + emanexml.add_param(transport_element, "device", netif.name) # add nem entry nem_entries[netif] = nem_element - return nem_entries + # merging code + key = netif.node.objid + if netif.transport_type == "raw": + key = "host" + otadev = control_net.brname + eventdev = control_net.brname + else: + otadev = None + eventdev = None + + platform_element = platform_xmls.get(key) + if not platform_element: + platform_element = etree.Element("platform") + + if otadev: + emane_manager.set_config("otamanagerdevice", otadev) + + if eventdev: + emane_manager.set_config("eventservicedevice", eventdev) + + # append all platform options (except starting id) to doc + for configuration in emane_manager.emane_config.emulator_config: + name = configuration.id + if name == "platform_id_start": + continue + + value = emane_manager.get_config(name) + emanexml.add_param(platform_element, name, value) + + # add platform xml + platform_xmls[key] = platform_element + + platform_element.append(nem_element) + + node.setnemid(netif, nem_id) + macstr = emane_manager._hwaddr_prefix + ":00:00:" + macstr += "%02X:%02X" % ((nem_id >> 8) & 0xFF, nem_id & 0xFF) + netif.sethwaddr(MacAddress.from_string(macstr)) + + # increment nem id + nem_id += 1 + + for key in sorted(platform_xmls.keys()): + if key == "host": + file_name = "platform.xml" + else: + file_name = "platform%d.xml" % key + + platform_element = platform_xmls[key] + + doc_name = "platform" + file_path = os.path.join(emane_manager.session.session_dir, file_name) + emanexml.create_file(platform_element, doc_name, file_path) + + return nem_id + + +def build_transport_xml(emane_manager, node, transport_type): + transport_element = etree.Element( + "transport", + name="%s Transport" % transport_type.capitalize(), + library="trans%s" % transport_type.lower() + ) + + # add bitrate + emanexml.add_param(transport_element, "bitrate", "0") + + # get emane model cnfiguration + config = emane_manager.get_configs(node_id=node.objid, config_type=node.model.name) + flowcontrol = config.get("flowcontrolenable", "0") == "1" + + if "virtual" in transport_type.lower(): + device_path = "/dev/net/tun_flowctl" + if not os.path.exists(device_path): + device_path = "/dev/net/tun" + emanexml.add_param(transport_element, "devicepath", device_path) + + if flowcontrol: + emanexml.add_param(transport_element, "flowcontrolenable", "on") + + doc_name = "transport" + file_name = "n%strans%s.xml" % (node.objid, transport_type.lower()) + file_path = os.path.join(emane_manager.session.session_dir, file_name) + emanexml.create_file(transport_element, doc_name, file_path) + + +def build_xml_files(emane_manager, node): + if node.model is None: + return + + # get model configurations + config = emane_manager.getifcconfig(node.model.object_id, None, node.model.name) + if not config: + return + + # build XML for overall network (EmaneNode) configs + node.model.build_xml_files(config) + + # build XML for specific interface (NEM) configs + need_virtual = False + need_raw = False + vtype = "virtual" + rtype = "raw" + + for netif in node.netifs(): + # check for interface specific emane configuration and write xml files, if needed + config = emane_manager.getifcconfig(node.model.object_id, netif, node.model.name) + if config: + node.model.build_xml_files(config, netif) + + # check transport type needed for interface + if "virtual" in netif.transport_type: + need_virtual = True + vtype = netif.transport_type + else: + need_raw = True + rtype = netif.transport_type + + # build transport XML files depending on type of interfaces involved + if need_virtual: + build_transport_xml(emane_manager, node, vtype) + + if need_raw: + build_transport_xml(emane_manager, node, rtype) class EmaneManager(ModelManager): @@ -541,46 +661,6 @@ class EmaneManager(ModelManager): msg = coreapi.CoreConfMessage(flags=0, hdr=msghdr, data=rawmsg[coreapi.CoreMessage.header_len:]) self.session.broker.handle_message(msg) - def xmldoc(self, doctype): - """ - Returns an XML xml.minidom.Document with a DOCTYPE tag set to the - provided doctype string, and an initial element having the same - name. - """ - # we hack in the DOCTYPE using the parser - docstr = """ - - <%s/>""" % (doctype, doctype, doctype) - # normally this would be: doc = Document() - return parseString(docstr) - - def xmlparam(self, doc, name, value): - """ - Convenience function for building a parameter tag of the format: - - """ - p = doc.createElement("param") - p.setAttribute("name", name) - p.setAttribute("value", value) - return p - - def xmlshimdefinition(self, doc, name): - """ - Convenience function for building a definition tag of the format: - - """ - p = doc.createElement("shim") - p.setAttribute("definition", name) - return p - - def xmlwrite(self, doc, filename): - """ - Write the given XML document to the specified filename. - """ - pathname = os.path.join(self.session.session_dir, filename) - with open(pathname, "w") as xml_file: - doc.writexml(writer=xml_file, indent="", addindent=" ", newl="\n", encoding="UTF-8") - def check_node_models(self): """ Associate EmaneModel classes with EmaneNode nodes. The model @@ -633,74 +713,16 @@ class EmaneManager(ModelManager): count += len(emane_node.netifs()) return count - def newplatformxmldoc(self, otadev=None, eventdev=None): - """ - Start a new platform XML file. Use global EMANE config values - as keys. Override OTA manager and event service devices if - specified (in order to support Raw Transport). - """ - doc = self.xmldoc("platform") - plat = doc.getElementsByTagName("platform").pop() - - if otadev: - self.set_config("otamanagerdevice", otadev) - - if eventdev: - self.set_config("eventservicedevice", eventdev) - - # append all platform options (except starting id) to doc - for configuration in self.emane_config.emulator_config: - name = configuration.id - if name == "platform_id_start": - continue - - value = self.get_config(name) - param = self.xmlparam(doc, name, value) - plat.appendChild(param) - - return doc - def buildplatformxml(self, ctrlnet): """ Build a platform.xml file now that all nodes are configured. """ nemid = int(self.get_config("nem_id_start")) - platformxmls = {} # assume self._objslock is already held here for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] - nems = emane_node.buildplatformxmlentry(self.xmldoc("platform")) - - for netif in sorted(nems, key=lambda x: x.node.objid): - nementry = nems[netif] - nementry.setAttribute("id", "%d" % nemid) - key = netif.node.objid - if netif.transport_type == "raw": - key = "host" - otadev = ctrlnet.brname - eventdev = ctrlnet.brname - else: - otadev = None - eventdev = None - - if key not in platformxmls: - platformxmls[key] = self.newplatformxmldoc(otadev, eventdev) - - doc = platformxmls[key] - plat = doc.getElementsByTagName("platform").pop() - plat.appendChild(nementry) - emane_node.setnemid(netif, nemid) - macstr = self._hwaddr_prefix + ":00:00:" - macstr += "%02X:%02X" % ((nemid >> 8) & 0xFF, nemid & 0xFF) - netif.sethwaddr(MacAddress.from_string(macstr)) - nemid += 1 - - for key in sorted(platformxmls.keys()): - if key == "host": - self.xmlwrite(platformxmls["host"], "platform.xml") - continue - self.xmlwrite(platformxmls[key], "platform%d.xml" % key) + nemid = build_node_platform_xml(self, ctrlnet, emane_node, nemid) def buildnemxml(self): """ @@ -709,23 +731,7 @@ class EmaneManager(ModelManager): """ for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] - emane_node.build_xml_files(self) - - def appendtransporttonem(self, doc, nem, nodenum, ifc=None): - """ - Given a nem XML node and EMANE WLAN node number, append - a tag to the NEM definition, required for using - EMANE"s internal transport. - """ - emane_node = self._emane_nodes[nodenum] - transtag = doc.createElement("transport") - transtypestr = "virtual" - - if ifc and ifc.transport_type == "raw": - transtypestr = "raw" - - transtag.setAttribute("definition", emane_node.transportxmlname(transtypestr)) - nem.appendChild(transtag) + build_xml_files(self, emane_node) def buildtransportxml(self): """ @@ -758,14 +764,14 @@ class EmaneManager(ModelManager): return dev = self.get_config("eventservicedevice") - doc = self.xmldoc("emaneeventmsgsvc") - es = doc.getElementsByTagName("emaneeventmsgsvc").pop() - kvs = (("group", group), ("port", port), ("device", dev), ("mcloop", "1"), ("ttl", "32")) - xmlutils.add_text_elements_from_tuples(doc, es, kvs) - filename = "libemaneeventservice.xml" - self.xmlwrite(doc, filename) - pathname = os.path.join(self.session.session_dir, filename) - self.initeventservice(filename=pathname) + + event_element = etree.Element("emaneeventmsgsvc") + for name, value in (("group", group), ("port", port), ("device", dev), ("mcloop", "1"), ("ttl", "32")): + sub_element = etree.SubElement(event_element, name) + sub_element.text = value + file_name = "libemaneeventservice.xml" + file_path = os.path.join(self.session.session_dir, file_name) + emanexml.create_file(event_element, "emaneeventmsgsvc", file_path) def startdaemons(self): """ @@ -1062,8 +1068,5 @@ class EmaneGlobalModel(EmaneModel): def __init__(self, session, object_id=None): super(EmaneGlobalModel, self).__init__(session, object_id) - def build_xml_files(self, emane_manager, interface): - """ - Build the necessary nem, mac, and phy XMLs in the given path. - """ + def build_xml_files(self, config, interface=None): raise NotImplementedError diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index e69f377b..a4c35eb6 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -8,95 +8,8 @@ from lxml import etree from core import logger from core.conf import ConfigGroup from core.emane import emanemanifest -from core.misc import utils from core.mobility import WirelessModel -from core.xml import corexml - - -def emane_xml_doctype(name): - """ - Create emane xml doctype for xml files. - - :param str name: name for xml doctype - :return: emane xml doctype - :rtype: str - """ - return '' % (name, name) - - -def add_emane_configurations(xml_element, configurations, config, config_ignore): - """ - - - :param lxml.etree.Element xml_element: xml element to add emane configurations to - :param list[core.config.Configuration] configurations: configurations to add to xml - :param dict config: configuration values - :param set config_ignore: configuration options to ignore - :return: - """ - for configuration in configurations: - # ignore custom configurations - name = configuration.id - if name in config_ignore: - continue - - # check if value is a multi param - value = str(config[name]) - params = lxml_value_to_params(value) - if params: - params_element = etree.SubElement(xml_element, "paramlist", name=name) - for param in params: - etree.SubElement(params_element, "item", value=param) - else: - etree.SubElement(xml_element, "param", name=name, value=value) - - -def lxml_value_to_params(value): - """ - Helper to convert a parameter to a parameter tuple. - - :param str value: value string to convert to tuple - :return: parameter tuple, None otherwise - """ - try: - values = utils.make_tuple_fromstr(value, str) - - if not hasattr(values, "__iter__"): - return None - - if len(values) < 2: - return None - - return values - - except SyntaxError: - logger.exception("error in value string to param list") - return None - - -# def value_to_params(doc, name, value): -# """ -# Helper to convert a parameter to a paramlist. Returns an XML paramlist, or None if the value does not expand to -# multiple values. -# -# :param xml.dom.minidom.Document doc: xml document -# :param name: name of element for params -# :param str value: value string to convert to tuple -# :return: xml document with added params or None, when an invalid value has been provided -# """ -# try: -# values = utils.make_tuple_fromstr(value, str) -# except SyntaxError: -# logger.exception("error in value string to param list") -# return None -# -# if not hasattr(values, "__iter__"): -# return None -# -# if len(values) < 2: -# return None -# -# return xmlutils.add_param_list_to_parent(doc, parent=None, name=name, values=values) +from core.xml import emanexml class EmaneModel(WirelessModel): @@ -122,8 +35,6 @@ class EmaneModel(WirelessModel): phy_config = emanemanifest.parse(phy_xml, phy_defaults) config_ignore = set() - config_groups_override = None - configurations_override = None @classmethod def configurations(cls): @@ -138,29 +49,25 @@ class EmaneModel(WirelessModel): ConfigGroup("PHY Parameters", mac_len + 1, config_len) ] - def build_xml_files(self, emane_manager, interface): + def build_xml_files(self, config, interface=None): """ - Builds xml files for emane. Includes a nem.xml file that points to both mac.xml and phy.xml definitions. + Builds xml files for this emane model. Creates a nem.xml file that points to both mac.xml and phy.xml + definitions. - :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager + :param dict config: emane model configuration for the node and interface :param interface: interface for the emane node :return: nothing """ - # retrieve configuration values - config = emane_manager.getifcconfig(self.object_id, interface, self.name) - if not config: - return - # create document and write to disk - self.create_nem_doc(interface) + self.create_nem_xml(interface) # create mac document and write to disk - self.create_mac_doc(interface, config) + self.create_mac_xml(interface, config) # create phy document and write to disk - self.create_phy_doc(interface, config) + self.create_phy_xml(interface, config) - def create_nem_doc(self, interface): + def create_nem_xml(self, interface): """ Create the nem xml document. @@ -188,7 +95,7 @@ class EmaneModel(WirelessModel): nem_name = self.nem_name(interface) self.create_file(nem_element, nem_name, "nem") - def create_mac_doc(self, interface, config): + def create_mac_xml(self, interface, config): """ Create the mac xml document. @@ -200,13 +107,13 @@ class EmaneModel(WirelessModel): raise ValueError("must define emane model library") mac_element = etree.Element("mac", name="%s MAC" % self.name, library=self.mac_library) - add_emane_configurations(mac_element, self.mac_config, config, self.config_ignore) + emanexml.add_configurations(mac_element, self.mac_config, config, self.config_ignore) # write out xml mac_name = self.mac_name(interface) self.create_file(mac_element, mac_name, "mac") - def create_phy_doc(self, interface, config): + def create_phy_xml(self, interface, config): """ Create the phy xml document. @@ -218,7 +125,7 @@ class EmaneModel(WirelessModel): if self.phy_library: phy_element.set("library", self.phy_library) - add_emane_configurations(phy_element, self.phy_config, config, self.config_ignore) + emanexml.add_configurations(phy_element, self.phy_config, config, self.config_ignore) # write out xml phy_name = self.phy_name(interface) @@ -226,8 +133,7 @@ class EmaneModel(WirelessModel): def create_file(self, xml_element, file_name, doc_name): file_path = os.path.join(self.session.session_dir, file_name) - doctype = emane_xml_doctype(doc_name) - corexml.write_xml_file(xml_element, file_path, doctype=doctype) + emanexml.create_file(xml_element, doc_name, file_path) def post_startup(self): """ @@ -237,64 +143,6 @@ class EmaneModel(WirelessModel): """ logger.info("emane model(%s) has no post setup tasks", self.name) - def build_nem_xml(self, doc, emane_node, interface): - """ - Build the NEM definition that goes into the platform.xml file. - - This returns an XML element that will be added to the element. - - This default method supports per-interface config (e.g. - or per-EmaneNode config (e.g. . - - This can be overriden by a model for NEM flexibility; n is the EmaneNode. - - - - :param xml.dom.minidom.Document doc: xml document - :param core.emane.nodes.EmaneNode emane_node: emane node to get information from - :param interface: interface for the emane node - :return: created platform xml - """ - # if this netif contains a non-standard (per-interface) config, - # then we need to use a more specific xml file here - nem_name = self.nem_name(interface) - nem = doc.createElement("nem") - nem.setAttribute("name", interface.localname) - nem.setAttribute("definition", nem_name) - return nem - - def build_transport_xml(self, doc, emane_node, interface): - """ - Build the transport definition that goes into the platform.xml file. - This returns an XML element that will be added to the nem definition. - This default method supports raw and virtual transport types, but may be - overridden by a model to support the e.g. pluggable virtual transport. - - - - - - :param xml.dom.minidom.Document doc: xml document - :param core.emane.nodes.EmaneNode emane_node: emane node to get information from - :param interface: interface for the emane node - :return: created transport xml - """ - transport_type = interface.transport_type - if not transport_type: - logger.info("warning: %s interface type unsupported!", interface.name) - transport_type = "raw" - transport_name = emane_node.transportxmlname(transport_type) - - transport = doc.createElement("transport") - transport.setAttribute("definition", transport_name) - - param = doc.createElement("param") - param.setAttribute("name", "device") - param.setAttribute("value", interface.name) - - transport.appendChild(param) - return transport - def _basename(self, interface=None): """ Create name that is leveraged for configuration file creation. diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index be99f375..dd154b8b 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -4,8 +4,6 @@ control of an EMANE emulation. An EmaneNode has several attached NEMs that share the same MAC+PHY model. """ -import os - from core import logger from core.coreobj import PyCoreNet from core.enumerations import LinkTypes @@ -119,89 +117,6 @@ class EmaneNode(EmaneNet): """ return sorted(self._netif.values(), key=lambda ifc: ifc.node.objid) - def buildplatformxmlentry(self, doc): - """ - Return a dictionary of XML elements describing the NEMs - connected to this EmaneNode for inclusion in the platform.xml file. - """ - ret = {} - if self.model is None: - logger.info("warning: EmaneNode %s has no associated model", self.name) - return ret - - for netif in self.netifs(): - nementry = self.model.build_nem_xml(doc, self, netif) - trans = self.model.build_transport_xml(doc, self, netif) - nementry.appendChild(trans) - ret[netif] = nementry - - return ret - - def build_xml_files(self, emane_manager): - """ - Let the configured model build the necessary nem, mac, and phy XMLs. - - :param core.emane.emanemanager.EmaneManager emane_manager: core emane manager - :return: nothing - """ - if self.model is None: - return - - # build XML for overall network (EmaneNode) configs - self.model.build_xml_files(emane_manager, interface=None) - - # build XML for specific interface (NEM) configs - need_virtual = False - need_raw = False - vtype = "virtual" - rtype = "raw" - - for netif in self.netifs(): - self.model.build_xml_files(emane_manager, netif) - if "virtual" in netif.transport_type: - need_virtual = True - vtype = netif.transport_type - else: - need_raw = True - rtype = netif.transport_type - - # build transport XML files depending on type of interfaces involved - if need_virtual: - self.buildtransportxml(emane_manager, vtype) - - if need_raw: - self.buildtransportxml(emane_manager, rtype) - - def buildtransportxml(self, emane, transport_type): - """ - Write a transport XML file for the Virtual or Raw Transport. - """ - transdoc = emane.xmldoc("transport") - trans = transdoc.getElementsByTagName("transport").pop() - trans.setAttribute("name", "%s Transport" % transport_type.capitalize()) - trans.setAttribute("library", "trans%s" % transport_type.lower()) - trans.appendChild(emane.xmlparam(transdoc, "bitrate", "0")) - - config = emane.get_configs(node_id=self.objid, config_type=self.model.name) - logger.debug("transport xml config: %s", config) - flowcontrol = config.get("flowcontrolenable", "0") == "1" - - if "virtual" in transport_type.lower(): - if os.path.exists("/dev/net/tun_flowctl"): - trans.appendChild(emane.xmlparam(transdoc, "devicepath", "/dev/net/tun_flowctl")) - else: - trans.appendChild(emane.xmlparam(transdoc, "devicepath", "/dev/net/tun")) - if flowcontrol: - trans.appendChild(emane.xmlparam(transdoc, "flowcontrolenable", "on")) - - emane.xmlwrite(transdoc, self.transportxmlname(transport_type.lower())) - - def transportxmlname(self, _type): - """ - Return the string name for the Transport XML file, e.g. 'n3transvirtual.xml' - """ - return "n%strans%s.xml" % (self.objid, _type) - def installnetifs(self, do_netns=True): """ Install TAP devices into their namespaces. This is done after diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 551a75e0..757528a5 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -59,7 +59,7 @@ def create_emane_config(node_id, emane_config, config): emulator_element = etree.SubElement(emane_configuration, "emulator") for emulator_config in emane_config.emulator_config: value = config[emulator_config.id] - add_configuration(emulator_element, emane_config.id, value) + add_configuration(emulator_element, emulator_config.id, value) nem_element = etree.SubElement(emane_configuration, "nem") for nem_config in emane_config.nem_config: @@ -519,7 +519,7 @@ class CoreXmlWriter(object): for model_name, config in all_configs.iteritems(): logger.info("writing emane config: %s", config) if model_name == -1: - emane_configuration = create_emane_config(node_id, self.session.emane_config, config) + emane_configuration = create_emane_config(node_id, self.session.emane.emane_config, config) else: model = self.session.emane.models[model_name] emane_configuration = create_emane_model_config(node_id, model, config) diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py new file mode 100644 index 00000000..15d997ce --- /dev/null +++ b/daemon/core/xml/emanexml.py @@ -0,0 +1,64 @@ +from lxml import etree + +from core import logger +from core.misc import utils +from core.xml import corexml + + +def _value_to_params(value): + """ + Helper to convert a parameter to a parameter tuple. + + :param str value: value string to convert to tuple + :return: parameter tuple, None otherwise + """ + try: + values = utils.make_tuple_fromstr(value, str) + + if not hasattr(values, "__iter__"): + return None + + if len(values) < 2: + return None + + return values + + except SyntaxError: + logger.exception("error in value string to param list") + return None + + +def create_file(xml_element, doc_name, file_path): + doctype = '' % {"doc_name": doc_name} + corexml.write_xml_file(xml_element, file_path, doctype=doctype) + + +def add_param(xml_element, name, value): + etree.SubElement(xml_element, "param", name=name, value=value) + + +def add_configurations(xml_element, configurations, config, config_ignore): + """ + Add emane model configurations to xml element. + + :param lxml.etree.Element xml_element: xml element to add emane configurations to + :param list[core.config.Configuration] configurations: configurations to add to xml + :param dict config: configuration values + :param set config_ignore: configuration options to ignore + :return: + """ + for configuration in configurations: + # ignore custom configurations + name = configuration.id + if name in config_ignore: + continue + + # check if value is a multi param + value = str(config[name]) + params = _value_to_params(value) + if params: + params_element = etree.SubElement(xml_element, "paramlist", name=name) + for param in params: + etree.SubElement(params_element, "item", value=param) + else: + add_param(xml_element, name, value) diff --git a/daemon/examples/myemane/examplemodel.py b/daemon/examples/myemane/examplemodel.py index fe1af9f3..308b6f1d 100644 --- a/daemon/examples/myemane/examplemodel.py +++ b/daemon/examples/myemane/examplemodel.py @@ -1,8 +1,7 @@ - - from core.emane import emanemanifest from core.emane import emanemodel + ## Custom EMANE Model class ExampleModel(emanemodel.EmaneModel): ### MAC Definition @@ -47,10 +46,3 @@ class ExampleModel(emanemodel.EmaneModel): # Allows you to ignore options within phy/mac, used typically if you needed to add a custom option for display # within the gui. config_ignore = set() - # Allows you to override how options are displayed with the GUI, using the GUI format of - # "name:1-2|othername:3-4". This will be parsed into tabs, split by "|" and account for items based on the indexed - # numbers after ":" for including values in each tab. - config_groups_override = None - # Allows you to override the default config matrix list. This value by default is the mac_config + phy_config, in - # that order. - config_matrix_override = None From 875a8802509fdec2f39796fb016078db8322a40e Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 6 Jul 2018 11:41:32 -0700 Subject: [PATCH 66/83] refactored emane xml configurations into an isolated file, using lxml --- daemon/core/emane/commeffect.py | 16 +- daemon/core/emane/emanemanager.py | 169 +-------------- daemon/core/emane/emanemodel.py | 145 ++----------- daemon/core/xml/emanexml.py | 328 ++++++++++++++++++++++++++++++ 4 files changed, 354 insertions(+), 304 deletions(-) diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index e2011e15..efb279c7 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -2,6 +2,8 @@ commeffect.py: EMANE CommEffect model for CORE """ +import os + from lxml import etree from core import logger @@ -64,21 +66,22 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): :return: nothing """ # retrieve xml names - nem_name = self.nem_name(interface) - shim_name = self.shim_name(interface) + nem_name = emanexml.nem_file_name(self, interface) + shim_name = emanexml.shim_file_name(self, interface) # create and write nem document nem_element = etree.Element("nem", name="%s NEM" % self.name, type="unstructured") transport_type = "virtual" if interface and interface.transport_type == "raw": transport_type = "raw" - transport_name = "n%strans%s.xml" % (self.object_id, transport_type) - etree.SubElement(nem_element, "transport", definition=transport_name) + transport_file = emanexml.transport_file_name(self.object_id, transport_type) + etree.SubElement(nem_element, "transport", definition=transport_file) # set shim configuration etree.SubElement(nem_element, "shim", definition=shim_name) - self.create_file(nem_element, nem_name, "nem") + nem_file = os.path.join(self.session.session_dir, nem_name) + emanexml.create_file(nem_element, "nem", nem_file) # create and write shim document shim_element = etree.Element("shim", name="%s SHIM" % self.name, library=self.shim_library) @@ -96,7 +99,8 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): if ff.strip() != "": emanexml.add_param(shim_element, "filterfile", ff) - self.create_file(shim_element, shim_name, "shim") + shim_file = os.path.join(self.session.session_dir, shim_name) + emanexml.create_file(shim_element, "shim", shim_file) def linkconfig(self, netif, bw=None, delay=None, loss=None, duplicate=None, jitter=None, netif2=None): """ diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 11c58b73..e708c743 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -5,8 +5,6 @@ emane.py: definition of an Emane class for implementing configuration control of import os import threading -from lxml import etree - from core import CoreCommandError from core import constants from core import logger @@ -32,7 +30,6 @@ from core.enumerations import NodeTypes from core.enumerations import RegisterTlvs from core.misc import nodeutils from core.misc import utils -from core.misc.ipaddress import MacAddress from core.xml import emanexml try: @@ -56,159 +53,6 @@ EMANE_MODELS = [ ] -def build_node_platform_xml(emane_manager, control_net, node, nem_id): - nem_entries = {} - platform_xmls = {} - - if node.model is None: - logger.info("warning: EmaneNode %s has no associated model", node.name) - return nem_entries - - for netif in node.netifs(): - # build nem xml - nem_name = node.model.nem_name(netif) - nem_element = etree.Element("nem", id=str(nem_id), name=netif.localname, definition=nem_name) - - # build transport xml - transport_type = netif.transport_type - if not transport_type: - logger.info("warning: %s interface type unsupported!", netif.name) - transport_type = "raw" - transport_name = "n%strans%s.xml" % (node.objid, transport_type.lower()) - transport_element = etree.SubElement(nem_element, "transport", definition=transport_name) - - # add transport parameter - emanexml.add_param(transport_element, "device", netif.name) - - # add nem entry - nem_entries[netif] = nem_element - - # merging code - key = netif.node.objid - if netif.transport_type == "raw": - key = "host" - otadev = control_net.brname - eventdev = control_net.brname - else: - otadev = None - eventdev = None - - platform_element = platform_xmls.get(key) - if not platform_element: - platform_element = etree.Element("platform") - - if otadev: - emane_manager.set_config("otamanagerdevice", otadev) - - if eventdev: - emane_manager.set_config("eventservicedevice", eventdev) - - # append all platform options (except starting id) to doc - for configuration in emane_manager.emane_config.emulator_config: - name = configuration.id - if name == "platform_id_start": - continue - - value = emane_manager.get_config(name) - emanexml.add_param(platform_element, name, value) - - # add platform xml - platform_xmls[key] = platform_element - - platform_element.append(nem_element) - - node.setnemid(netif, nem_id) - macstr = emane_manager._hwaddr_prefix + ":00:00:" - macstr += "%02X:%02X" % ((nem_id >> 8) & 0xFF, nem_id & 0xFF) - netif.sethwaddr(MacAddress.from_string(macstr)) - - # increment nem id - nem_id += 1 - - for key in sorted(platform_xmls.keys()): - if key == "host": - file_name = "platform.xml" - else: - file_name = "platform%d.xml" % key - - platform_element = platform_xmls[key] - - doc_name = "platform" - file_path = os.path.join(emane_manager.session.session_dir, file_name) - emanexml.create_file(platform_element, doc_name, file_path) - - return nem_id - - -def build_transport_xml(emane_manager, node, transport_type): - transport_element = etree.Element( - "transport", - name="%s Transport" % transport_type.capitalize(), - library="trans%s" % transport_type.lower() - ) - - # add bitrate - emanexml.add_param(transport_element, "bitrate", "0") - - # get emane model cnfiguration - config = emane_manager.get_configs(node_id=node.objid, config_type=node.model.name) - flowcontrol = config.get("flowcontrolenable", "0") == "1" - - if "virtual" in transport_type.lower(): - device_path = "/dev/net/tun_flowctl" - if not os.path.exists(device_path): - device_path = "/dev/net/tun" - emanexml.add_param(transport_element, "devicepath", device_path) - - if flowcontrol: - emanexml.add_param(transport_element, "flowcontrolenable", "on") - - doc_name = "transport" - file_name = "n%strans%s.xml" % (node.objid, transport_type.lower()) - file_path = os.path.join(emane_manager.session.session_dir, file_name) - emanexml.create_file(transport_element, doc_name, file_path) - - -def build_xml_files(emane_manager, node): - if node.model is None: - return - - # get model configurations - config = emane_manager.getifcconfig(node.model.object_id, None, node.model.name) - if not config: - return - - # build XML for overall network (EmaneNode) configs - node.model.build_xml_files(config) - - # build XML for specific interface (NEM) configs - need_virtual = False - need_raw = False - vtype = "virtual" - rtype = "raw" - - for netif in node.netifs(): - # check for interface specific emane configuration and write xml files, if needed - config = emane_manager.getifcconfig(node.model.object_id, netif, node.model.name) - if config: - node.model.build_xml_files(config, netif) - - # check transport type needed for interface - if "virtual" in netif.transport_type: - need_virtual = True - vtype = netif.transport_type - else: - need_raw = True - rtype = netif.transport_type - - # build transport XML files depending on type of interfaces involved - if need_virtual: - build_transport_xml(emane_manager, node, vtype) - - if need_raw: - build_transport_xml(emane_manager, node, rtype) - - class EmaneManager(ModelManager): """ EMANE controller object. Lives in a Session instance and is used for @@ -217,7 +61,6 @@ class EmaneManager(ModelManager): """ name = "emane" config_type = RegisterTlvs.EMULATION_SERVER.value - _hwaddr_prefix = "02:02" SUCCESS, NOT_NEEDED, NOT_READY = (0, 1, 2) EVENTCFGVAR = "LIBEMANEEVENTSERVICECONFIG" DEFAULT_LOG_LEVEL = 3 @@ -722,7 +565,7 @@ class EmaneManager(ModelManager): # assume self._objslock is already held here for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] - nemid = build_node_platform_xml(self, ctrlnet, emane_node, nemid) + nemid = emanexml.build_node_platform_xml(self, ctrlnet, emane_node, nemid) def buildnemxml(self): """ @@ -731,7 +574,7 @@ class EmaneManager(ModelManager): """ for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] - build_xml_files(self, emane_node) + emanexml.build_xml_files(self, emane_node) def buildtransportxml(self): """ @@ -765,13 +608,7 @@ class EmaneManager(ModelManager): dev = self.get_config("eventservicedevice") - event_element = etree.Element("emaneeventmsgsvc") - for name, value in (("group", group), ("port", port), ("device", dev), ("mcloop", "1"), ("ttl", "32")): - sub_element = etree.SubElement(event_element, name) - sub_element.text = value - file_name = "libemaneeventservice.xml" - file_path = os.path.join(self.session.session_dir, file_name) - emanexml.create_file(event_element, "emaneeventmsgsvc", file_path) + emanexml.create_event_service_xml(group, port, dev, self.session.session_dir) def startdaemons(self): """ diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index a4c35eb6..55092741 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -3,8 +3,6 @@ Defines Emane Models used within CORE. """ import os -from lxml import etree - from core import logger from core.conf import ConfigGroup from core.emane import emanemanifest @@ -58,82 +56,26 @@ class EmaneModel(WirelessModel): :param interface: interface for the emane node :return: nothing """ - # create document and write to disk - self.create_nem_xml(interface) + nem_name = emanexml.nem_file_name(self, interface) + mac_name = emanexml.mac_file_name(self, interface) + phy_name = emanexml.phy_file_name(self, interface) - # create mac document and write to disk - self.create_mac_xml(interface, config) - - # create phy document and write to disk - self.create_phy_xml(interface, config) - - def create_nem_xml(self, interface): - """ - Create the nem xml document. - - :param interface: interface for the emane node - :return: nothing - """ - nem_element = etree.Element("nem", name="%s NEM" % self.name) - - # add transport transport_type = "virtual" if interface and interface.transport_type == "raw": transport_type = "raw" - transport_type = "n%strans%s.xml" % (self.object_id, transport_type) - etree.SubElement(nem_element, "transport", definition=transport_type) + transport_name = emanexml.transport_file_name(self.object_id, transport_type) - # create mac - mac_name = self.mac_name(interface) - etree.SubElement(nem_element, "mac", definition=mac_name) + # create nem xml file + nem_file = os.path.join(self.session.session_dir, nem_name) + emanexml.create_nem_xml(self, nem_file, transport_name, mac_name, phy_name) - # create phy - phy_name = self.phy_name(interface) - etree.SubElement(nem_element, "phy", definition=phy_name) + # create mac xml file + mac_file = os.path.join(self.session.session_dir, mac_name) + emanexml.create_mac_xml(self, config, mac_file) - # write out xml - nem_name = self.nem_name(interface) - self.create_file(nem_element, nem_name, "nem") - - def create_mac_xml(self, interface, config): - """ - Create the mac xml document. - - :param interface: interface for the emane node - :param dict config: all current configuration values, mac + phy - :return: nothing - """ - if not self.mac_library: - raise ValueError("must define emane model library") - - mac_element = etree.Element("mac", name="%s MAC" % self.name, library=self.mac_library) - emanexml.add_configurations(mac_element, self.mac_config, config, self.config_ignore) - - # write out xml - mac_name = self.mac_name(interface) - self.create_file(mac_element, mac_name, "mac") - - def create_phy_xml(self, interface, config): - """ - Create the phy xml document. - - :param interface: interface for the emane node - :param dict config: all current configuration values, mac + phy - :return: nothing - """ - phy_element = etree.Element("phy", name="%s PHY" % self.name) - if self.phy_library: - phy_element.set("library", self.phy_library) - - emanexml.add_configurations(phy_element, self.phy_config, config, self.config_ignore) - - # write out xml - phy_name = self.phy_name(interface) - self.create_file(phy_element, phy_name, "phy") - - def create_file(self, xml_element, file_name, doc_name): - file_path = os.path.join(self.session.session_dir, file_name) - emanexml.create_file(xml_element, doc_name, file_path) + # create phy xml file + phy_file = os.path.join(self.session.session_dir, phy_name) + emanexml.create_phy_xml(self, config, phy_file) def post_startup(self): """ @@ -143,67 +85,6 @@ class EmaneModel(WirelessModel): """ logger.info("emane model(%s) has no post setup tasks", self.name) - def _basename(self, interface=None): - """ - Create name that is leveraged for configuration file creation. - - :param interface: interface for this model - :return: basename used for file creation - :rtype: str - """ - name = "n%s" % self.object_id - - if interface: - node_id = interface.node.objid - if self.session.emane.getifcconfig(node_id, interface, self.name): - name = interface.localname.replace(".", "_") - - return "%s%s" % (name, self.name) - - def nem_name(self, interface=None): - """ - Return the string name for the NEM XML file, e.g. "n3rfpipenem.xml" - - :param interface: interface for this model - :return: nem xml filename - :rtype: str - """ - basename = self._basename(interface) - append = "" - if interface and interface.transport_type == "raw": - append = "_raw" - return "%snem%s.xml" % (basename, append) - - def shim_name(self, interface=None): - """ - Return the string name for the SHIM XML file, e.g. "commeffectshim.xml" - - :param interface: interface for this model - :return: shim xml filename - :rtype: str - """ - return "%sshim.xml" % self._basename(interface) - - def mac_name(self, interface=None): - """ - Return the string name for the MAC XML file, e.g. "n3rfpipemac.xml" - - :param interface: interface for this model - :return: mac xml filename - :rtype: str - """ - return "%smac.xml" % self._basename(interface) - - def phy_name(self, interface=None): - """ - Return the string name for the PHY XML file, e.g. "n3rfpipephy.xml" - - :param interface: interface for this model - :return: phy xml filename - :rtype: str - """ - return "%sphy.xml" % self._basename(interface) - def update(self, moved, moved_netifs): """ Invoked from MobilityModel when nodes are moved; this causes diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 15d997ce..220a39c3 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -1,9 +1,14 @@ +import os + from lxml import etree from core import logger from core.misc import utils +from core.misc.ipaddress import MacAddress from core.xml import corexml +_hwaddr_prefix = "02:02" + def _value_to_params(value): """ @@ -62,3 +67,326 @@ def add_configurations(xml_element, configurations, config, config_ignore): etree.SubElement(params_element, "item", value=param) else: add_param(xml_element, name, value) + + +def build_node_platform_xml(emane_manager, control_net, node, nem_id): + """ + Create platform xml for a specific node. + + :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane configurations + :param core.netns.nodes.CtrlNet control_net: control net node for this emane network + :param core.emane.nodes.EmaneNode node: node to write platform xml for + :param int nem_id: nem id to use for interfaces for this node + :return: the next nem id that can be used for creating platform xml files + :rtype: int + """ + logger.debug("building emane platform xml for node(%s): %s", node, node.name) + nem_entries = {} + platform_xmls = {} + + if node.model is None: + logger.info("warning: EmaneNode %s has no associated model", node.name) + return nem_entries + + for netif in node.netifs(): + # build nem xml + nem_definition = nem_file_name(node.model, netif) + nem_element = etree.Element("nem", id=str(nem_id), name=netif.localname, definition=nem_definition) + + # build transport xml + transport_type = netif.transport_type + if not transport_type: + logger.info("warning: %s interface type unsupported!", netif.name) + transport_type = "raw" + transport_file = transport_file_name(node.objid, transport_type) + transport_element = etree.SubElement(nem_element, "transport", definition=transport_file) + + # add transport parameter + add_param(transport_element, "device", netif.name) + + # add nem entry + nem_entries[netif] = nem_element + + # merging code + key = netif.node.objid + if netif.transport_type == "raw": + key = "host" + otadev = control_net.brname + eventdev = control_net.brname + else: + otadev = None + eventdev = None + + platform_element = platform_xmls.get(key) + if not platform_element: + platform_element = etree.Element("platform") + + if otadev: + emane_manager.set_config("otamanagerdevice", otadev) + + if eventdev: + emane_manager.set_config("eventservicedevice", eventdev) + + # append all platform options (except starting id) to doc + for configuration in emane_manager.emane_config.emulator_config: + name = configuration.id + if name == "platform_id_start": + continue + + value = emane_manager.get_config(name) + add_param(platform_element, name, value) + + # add platform xml + platform_xmls[key] = platform_element + + platform_element.append(nem_element) + + node.setnemid(netif, nem_id) + macstr = _hwaddr_prefix + ":00:00:" + macstr += "%02X:%02X" % ((nem_id >> 8) & 0xFF, nem_id & 0xFF) + netif.sethwaddr(MacAddress.from_string(macstr)) + + # increment nem id + nem_id += 1 + + for key in sorted(platform_xmls.keys()): + if key == "host": + file_name = "platform.xml" + else: + file_name = "platform%d.xml" % key + + platform_element = platform_xmls[key] + + doc_name = "platform" + file_path = os.path.join(emane_manager.session.session_dir, file_name) + create_file(platform_element, doc_name, file_path) + + return nem_id + + +def build_xml_files(emane_manager, node): + """ + Generate emane xml files required for node. + + :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane configurations + :param core.emane.nodes.EmaneNode node: node to write platform xml for + :return: nothing + """ + logger.debug("building all emane xml for node(%s): %s", node, node.name) + if node.model is None: + return + + # get model configurations + config = emane_manager.get_configs(node.model.object_id, node.model.name) + if not config: + return + + # build XML for overall network (EmaneNode) configs + node.model.build_xml_files(config) + + # build XML for specific interface (NEM) configs + need_virtual = False + need_raw = False + vtype = "virtual" + rtype = "raw" + + for netif in node.netifs(): + # check for interface specific emane configuration and write xml files, if needed + config = emane_manager.getifcconfig(node.model.object_id, netif, node.model.name) + if config: + node.model.build_xml_files(config, netif) + + # check transport type needed for interface + if "virtual" in netif.transport_type: + need_virtual = True + vtype = netif.transport_type + else: + need_raw = True + rtype = netif.transport_type + + # build transport XML files depending on type of interfaces involved + if need_virtual: + build_transport_xml(emane_manager, node, vtype) + + if need_raw: + build_transport_xml(emane_manager, node, rtype) + + +def build_transport_xml(emane_manager, node, transport_type): + """ + Build transport xml file for node and transport type. + + :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane configurations + :param core.emane.nodes.EmaneNode node: node to write platform xml for + :param str transport_type: transport type to build xml for + :return: nothing + """ + transport_element = etree.Element( + "transport", + name="%s Transport" % transport_type.capitalize(), + library="trans%s" % transport_type.lower() + ) + + # add bitrate + add_param(transport_element, "bitrate", "0") + + # get emane model cnfiguration + config = emane_manager.get_configs(node.objid, node.model.name) + flowcontrol = config.get("flowcontrolenable", "0") == "1" + + if "virtual" in transport_type.lower(): + device_path = "/dev/net/tun_flowctl" + if not os.path.exists(device_path): + device_path = "/dev/net/tun" + add_param(transport_element, "devicepath", device_path) + + if flowcontrol: + add_param(transport_element, "flowcontrolenable", "on") + + doc_name = "transport" + file_name = transport_file_name(node.objid, transport_type) + file_path = os.path.join(emane_manager.session.session_dir, file_name) + create_file(transport_element, doc_name, file_path) + + +def create_phy_xml(emane_model, config, file_path): + """ + Create the phy xml document. + + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param dict config: all current configuration values, mac + phy + :param str file_path: path to write file to + :return: nothing + """ + phy_element = etree.Element("phy", name="%s PHY" % emane_model.name) + if emane_model.phy_library: + phy_element.set("library", emane_model.phy_library) + + add_configurations(phy_element, emane_model.phy_config, config, emane_model.config_ignore) + create_file(phy_element, "phy", file_path) + + +def create_mac_xml(emane_model, config, file_path): + """ + Create the mac xml document. + + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param dict config: all current configuration values, mac + phy + :param str file_path: path to write file to + :return: nothing + """ + if not emane_model.mac_library: + raise ValueError("must define emane model library") + + mac_element = etree.Element("mac", name="%s MAC" % emane_model.name, library=emane_model.mac_library) + add_configurations(mac_element, emane_model.mac_config, config, emane_model.config_ignore) + create_file(mac_element, "mac", file_path) + + +def create_nem_xml(emane_model, nem_file, transport_definition, mac_definition, phy_definition): + """ + Create the nem xml document. + + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param str nem_file: nem file path to write + :param str transport_definition: transport file definition path + :param str mac_definition: mac file definition path + :param str phy_definition: phy file definition path + :return: nothing + """ + nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) + etree.SubElement(nem_element, "transport", definition=transport_definition) + etree.SubElement(nem_element, "mac", definition=mac_definition) + etree.SubElement(nem_element, "phy", definition=phy_definition) + create_file(nem_element, "nem", nem_file) + + +def create_event_service_xml(group, port, device, file_directory): + event_element = etree.Element("emaneeventmsgsvc") + for name, value in (("group", group), ("port", port), ("device", device), ("mcloop", "1"), ("ttl", "32")): + sub_element = etree.SubElement(event_element, name) + sub_element.text = value + file_name = "libemaneeventservice.xml" + file_path = os.path.join(file_directory, file_name) + create_file(event_element, "emaneeventmsgsvc", file_path) + + +def transport_file_name(node_id, transport_type): + """ + Create name for a transport xml file. + + :param int node_id: node id to generate transport file name for + :param str transport_type: transport type to generate transport file + :return: + """ + return "n%strans%s.xml" % (node_id, transport_type) + + +def _basename(emane_model, interface=None): + """ + Create name that is leveraged for configuration file creation. + + :param interface: interface for this model + :return: basename used for file creation + :rtype: str + """ + name = "n%s" % emane_model.object_id + + if interface: + node_id = interface.node.objid + if emane_model.session.emane.getifcconfig(node_id, interface, emane_model.name): + name = interface.localname.replace(".", "_") + + return "%s%s" % (name, emane_model.name) + + +def nem_file_name(emane_model, interface=None): + """ + Return the string name for the NEM XML file, e.g. "n3rfpipenem.xml" + + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param interface: interface for this model + :return: nem xml filename + :rtype: str + """ + basename = _basename(emane_model, interface) + append = "" + if interface and interface.transport_type == "raw": + append = "_raw" + return "%snem%s.xml" % (basename, append) + + +def shim_file_name(emane_model, interface=None): + """ + Return the string name for the SHIM XML file, e.g. "commeffectshim.xml" + + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param interface: interface for this model + :return: shim xml filename + :rtype: str + """ + return "%sshim.xml" % _basename(emane_model, interface) + + +def mac_file_name(emane_model, interface=None): + """ + Return the string name for the MAC XML file, e.g. "n3rfpipemac.xml" + + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param interface: interface for this model + :return: mac xml filename + :rtype: str + """ + return "%smac.xml" % _basename(emane_model, interface) + + +def phy_file_name(emane_model, interface=None): + """ + Return the string name for the PHY XML file, e.g. "n3rfpipephy.xml" + + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param interface: interface for this model + :return: phy xml filename + :rtype: str + """ + return "%sphy.xml" % _basename(emane_model, interface) From d988ff86d170ac8d6597867579bf21430ae206cc Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 6 Jul 2018 11:56:09 -0700 Subject: [PATCH 67/83] added documentation to emanexml --- daemon/core/emane/emanemanager.py | 2 +- daemon/core/xml/emanexml.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index e708c743..d3fec411 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -209,7 +209,7 @@ class EmaneManager(ModelManager): def load_models(self, emane_models): """ - load EMANE models and make them available. + Load EMANE models and make them available. """ for emane_model in emane_models: logger.info("loading emane model: %s", emane_model.__name__) diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 220a39c3..db811f76 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -34,11 +34,27 @@ def _value_to_params(value): def create_file(xml_element, doc_name, file_path): + """ + Create xml file. + + :param lxml.etree.Element xml_element: root element to write to file + :param str doc_name: name to use in the emane doctype + :param str file_path: file path to write xml file to + :return: nothing + """ doctype = '' % {"doc_name": doc_name} corexml.write_xml_file(xml_element, file_path, doctype=doctype) def add_param(xml_element, name, value): + """ + Add emane configuration parameter to xml element. + + :param lxml.etree.Element xml_element: element to append parameter to + :param str name: name of parameter + :param str value: value for parameter + :return: nothing + """ etree.SubElement(xml_element, "param", name=name, value=value) @@ -302,6 +318,15 @@ def create_nem_xml(emane_model, nem_file, transport_definition, mac_definition, def create_event_service_xml(group, port, device, file_directory): + """ + Create a emane event service xml file. + + :param str group: event group + :param str port: event port + :param str device: event device + :param str file_directory: directory to create file in + :return: nothing + """ event_element = etree.Element("emaneeventmsgsvc") for name, value in (("group", group), ("port", port), ("device", device), ("mcloop", "1"), ("ttl", "32")): sub_element = etree.SubElement(event_element, name) From 90f95c633f1eeaddb75be78b213fcb7828dbd310 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 6 Jul 2018 14:09:03 -0700 Subject: [PATCH 68/83] removed old script file from 5.1 merge --- daemon/sbin/core-daemon | 357 ---------------------------------------- 1 file changed, 357 deletions(-) delete mode 100755 daemon/sbin/core-daemon diff --git a/daemon/sbin/core-daemon b/daemon/sbin/core-daemon deleted file mode 100755 index 0e615147..00000000 --- a/daemon/sbin/core-daemon +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env python -# -# CORE -# Copyright (c)2010-2013 the Boeing Company. -# See the LICENSE file included in this distribution. -# -# authors: Tom Goff -# Jeff Ahrenholz -# - -""" -core-daemon: the CORE daemon is a server process that receives CORE API -messages and instantiates emulated nodes and networks within the kernel. Various -message handlers are defined and some support for sending messages. -""" - -import ConfigParser -import atexit -import importlib -import optparse -import os -import signal -import socket -import sys -import threading -import time - -from pkg_resources import require -require("core_python", "corens3_python", "core_python_netns") - -from core import constants -from core import corehandlers -from core import coreserver -from core import enumerations -from core import logger -from core import services -from core.api import coreapi -from core.corehandlers import CoreDatagramRequestHandler -from core.enumerations import MessageFlags -from core.enumerations import RegisterTlvs -from core.misc import nodemaps -from core.misc import nodeutils -from core.misc.utils import closeonexec -from core.misc.utils import daemonize - -DEFAULT_MAXFD = 1024 - - -def startudp(core_server, server_address): - """ - Start a thread running a UDP server on the same host,port for connectionless requests. - - :param core.coreserver.CoreServer core_server: core server instance - :param tuple[str, int] server_address: server address - :return: created core udp server - :rtype: core.coreserver.CoreUdpServer - """ - core_server.udpserver = coreserver.CoreUdpServer(server_address, CoreDatagramRequestHandler, core_server) - core_server.udpthread = threading.Thread(target=core_server.udpserver.start) - core_server.udpthread.daemon = True - core_server.udpthread.start() - return core_server.udpserver - - -def startaux(core_server, aux_address, aux_handler): - """ - Start a thread running an auxiliary TCP server on the given address. - This server will communicate with client requests using a handler - using the aux_handler class. The aux_handler can provide an alternative - API to CORE. - - :param core.coreserver.CoreServer core_server: core server instance - :param tuple[str, int] aux_address: auxiliary server address - :param str aux_handler: auxiliary handler string to import - :return: auxiliary server - """ - handlermodname, dot, handlerclassname = aux_handler.rpartition(".") - handlermod = importlib.import_module(handlermodname) - handlerclass = getattr(handlermod, handlerclassname) - core_server.auxserver = coreserver.CoreAuxServer(aux_address, handlerclass, core_server) - core_server.auxthread = threading.Thread(target=core_server.auxserver.start) - core_server.auxthread.daemon = True - core_server.auxthread.start() - return core_server.auxserver - - -def banner(): - """ - Output the program banner printed to the terminal or log file. - - :return: nothing - """ - logger.info("CORE daemon v.%s started %s\n" % (constants.COREDPY_VERSION, time.ctime())) - - -def cored(cfg=None): - """ - Start the CoreServer object and enter the server loop. - - :param dict cfg: core configuration - :return: nothing - """ - host = cfg["listenaddr"] - port = int(cfg["port"]) - if host == "" or host is None: - host = "localhost" - try: - server = coreserver.CoreServer((host, port), corehandlers.CoreRequestHandler, cfg) - except: - logger.exception("error starting main server on: %s:%s", host, port) - sys.exit(1) - - closeonexec(server.fileno()) - logger.info("main server started, listening on: %s:%s\n" % (host, port)) - - udpserver = startudp(server, (host, port)) - closeonexec(udpserver.fileno()) - - auxreqhandler = cfg["aux_request_handler"] - if auxreqhandler: - handler, auxport = auxreqhandler.rsplit(":") - auxserver = startaux(server, (host, int(auxport)), handler) - closeonexec(auxserver.fileno()) - - server.serve_forever() - - -# TODO: should sessions and the main core daemon both catch at exist to shutdown independently? -def cleanup(): - """ - Runs server shutdown and cleanup when catching an exit signal. - - :return: nothing - """ - while coreserver.CoreServer.servers: - server = coreserver.CoreServer.servers.pop() - server.shutdown() - - -atexit.register(cleanup) - - -def sighandler(signum, stackframe): - """ - Signal handler when different signals are sent. - - :param int signum: singal number sent - :param stackframe: stack frame sent - :return: nothing - """ - logger.error("terminated by signal: %s", signum) - sys.exit(signum) - - -signal.signal(signal.SIGHUP, sighandler) -signal.signal(signal.SIGINT, sighandler) -signal.signal(signal.SIGTERM, sighandler) -signal.signal(signal.SIGUSR1, sighandler) -signal.signal(signal.SIGUSR2, sighandler) - - -def logrotate(stdout, stderr, stdoutmode=0644, stderrmode=0644): - """ - Log rotation method. - - :param stdout: stdout - :param stderr: stderr - :param int stdoutmode: stdout mode - :param int stderrmode: stderr mode - :return: - """ - - def reopen(fileno, filename, mode): - err = 0 - fd = -1 - try: - fd = os.open(filename, - os.O_WRONLY | os.O_CREAT | os.O_APPEND, mode) - os.dup2(fd, fileno) - except OSError as e: - err = e.errno - finally: - if fd >= 0: - os.close(fd) - return err - - if stdout: - err = reopen(1, stdout, stdoutmode) - if stderr: - if stderr == stdout and not err: - try: - os.dup2(1, 2) - except OSError as e: - pass - else: - reopen(2, stderr, stderrmode) - - -def get_merged_config(filename): - """ - Return a configuration after merging config file and command-line arguments. - - :param str filename: file name to merge configuration settings with - :return: merged configuration - :rtype: dict - """ - # these are the defaults used in the config file - defaults = {"port": "%d" % enumerations.CORE_API_PORT, - "listenaddr": "localhost", - "pidfile": "%s/run/core-daemon.pid" % constants.CORE_STATE_DIR, - "logfile": "%s/log/core-daemon.log" % constants.CORE_STATE_DIR, - "xmlfilever": "1.0", - "numthreads": "1", - "verbose": "False", - "daemonize": "False", - "debug": "False", - "execfile": None, - "aux_request_handler": None, - } - - usagestr = "usage: %prog [-h] [options] [args]\n\n" + \ - "CORE daemon v.%s instantiates Linux network namespace " \ - "nodes." % constants.COREDPY_VERSION - parser = optparse.OptionParser(usage=usagestr) - parser.add_option("-f", "--configfile", dest="configfile", - type="string", - help="read config from specified file; default = %s" % - filename) - parser.add_option("-d", "--daemonize", dest="daemonize", - action="store_true", - help="run in background as daemon; default=%s" % \ - defaults["daemonize"]) - parser.add_option("-e", "--execute", dest="execfile", type="string", - help="execute a Python/XML-based session") - parser.add_option("-l", "--logfile", dest="logfile", type="string", - help="log output to specified file; default = %s" % - defaults["logfile"]) - parser.add_option("-p", "--port", dest="port", type=int, - help="port number to listen on; default = %s" % \ - defaults["port"]) - parser.add_option("-i", "--pidfile", dest="pidfile", - help="filename to write pid to; default = %s" % \ - defaults["pidfile"]) - parser.add_option("-t", "--numthreads", dest="numthreads", type=int, - help="number of server threads; default = %s" % \ - defaults["numthreads"]) - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="enable verbose logging; default = %s" % \ - defaults["verbose"]) - parser.add_option("-g", "--debug", dest="debug", action="store_true", - help="enable debug logging; default = %s" % \ - defaults["debug"]) - - # parse command line options - options, args = parser.parse_args() - - # read the config file - if options.configfile is not None: - filename = options.configfile - del options.configfile - cfg = ConfigParser.SafeConfigParser(defaults) - cfg.read(filename) - - section = "core-daemon" - if not cfg.has_section(section): - cfg.add_section(section) - # gracefully support legacy configs (cored.py/cored now core-daemon) - if cfg.has_section("cored.py"): - for name, val in cfg.items("cored.py"): - if name == "pidfile" or name == "logfile": - bn = os.path.basename(val).replace("coredpy", "core-daemon") - val = os.path.join(os.path.dirname(val), bn) - cfg.set(section, name, val) - if cfg.has_section("cored"): - for name, val in cfg.items("cored"): - if name == "pidfile" or name == "logfile": - bn = os.path.basename(val).replace("cored", "core-daemon") - val = os.path.join(os.path.dirname(val), bn) - cfg.set(section, name, val) - - # merge command line with config file - for opt in options.__dict__: - val = options.__dict__[opt] - if val is not None: - cfg.set(section, opt, val.__str__()) - - return dict(cfg.items(section)), args - - -def exec_file(cfg): - """ - Send a Register Message to execute a new session based on XML or Python script file. - - :param dict cfg: configuration settings - :return: 0 - """ - filename = cfg["execfile"] - logger.info("Telling daemon to execute file: %s...", filename) - tlvdata = coreapi.CoreRegisterTlv.pack(RegisterTlvs.EXECUTE_SERVER.value, filename) - msg = coreapi.CoreRegMessage.pack(MessageFlags.ADD.value, tlvdata) - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - # TODO: connect address option - sock.connect(("localhost", int(cfg["port"]))) - sock.sendall(msg) - return 0 - - -def main(): - """ - Main program startup. - - :return: nothing - """ - # get a configuration merged from config file and command-line arguments - cfg, args = get_merged_config("%s/core.conf" % constants.CORE_CONF_DIR) - for a in args: - logger.error("ignoring command line argument: %s", a) - - if cfg["daemonize"] == "True": - daemonize(rootdir=None, umask=0, close_fds=False, - stdin=os.devnull, - stdout=cfg["logfile"], stderr=cfg["logfile"], - pidfilename=cfg["pidfile"], - defaultmaxfd=DEFAULT_MAXFD) - signal.signal(signal.SIGUSR1, lambda signum, stackframe: - logrotate(stdout=cfg["logfile"], stderr=cfg["logfile"])) - - banner() - if cfg["execfile"]: - cfg["execfile"] = os.path.abspath(cfg["execfile"]) - sys.exit(exec_file(cfg)) - try: - cored(cfg) - except KeyboardInterrupt: - logger.info("keyboard interrupt, stopping core daemon") - - sys.exit(0) - - -if __name__ == "__main__": - # configure nodes to use - node_map = nodemaps.NODES - if len(sys.argv) == 2 and sys.argv[1] == "ovs": - from core.netns.openvswitch import OVS_NODES - node_map.update(OVS_NODES) - - # update with BSD based nodes - if os.uname()[0] == "FreeBSD": - from core.bsd.nodes import BSD_NODES - node_map.update(BSD_NODES) - - nodeutils.set_node_map(node_map) - - # load default services - services.load() - - main() From 3fb8ae0439f9575ec8a8cd50976978f968f2f52d Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 6 Jul 2018 14:26:33 -0700 Subject: [PATCH 69/83] updated versioning to 5.2 --- configure.ac | 4 ++-- daemon/data/logging.conf | 2 +- daemon/setup.py | 2 +- netns/setup.py | 2 +- ns3/setup.py | 2 +- sonar-project.properties | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 8c522601..80cceaaa 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. # this defines the CORE version number, must be static for AC_INIT -AC_INIT(core, 5.1, core-dev@nrl.navy.mil) +AC_INIT(core, 5.2, core-dev@nrl.navy.mil) # autoconf and automake initialization AC_CONFIG_SRCDIR([netns/version.h.in]) @@ -66,7 +66,7 @@ AC_PROG_SED want_python=no want_linux_netns=no -if test "x$enable_daemon" = "xyes"; then +if test "x$enable_daemon" = "xyes"; then want_python=yes want_linux_netns=yes diff --git a/daemon/data/logging.conf b/daemon/data/logging.conf index 46de6e92..7f3d496f 100644 --- a/daemon/data/logging.conf +++ b/daemon/data/logging.conf @@ -14,7 +14,7 @@ } }, "root": { - "level": "DEBUG", + "level": "INFO", "handlers": ["console"] } } diff --git a/daemon/setup.py b/daemon/setup.py index 1355ad8b..7ad302c9 100644 --- a/daemon/setup.py +++ b/daemon/setup.py @@ -39,7 +39,7 @@ data_files.extend(recursive_files(_EXAMPLES_DIR, "examples")) setup( name="core", - version="5.1", + version="5.2", packages=find_packages(), install_requires=[ "enum34", diff --git a/netns/setup.py b/netns/setup.py index 839faeae..9afa3694 100644 --- a/netns/setup.py +++ b/netns/setup.py @@ -29,7 +29,7 @@ vcmd = Extension( setup( name="core-netns", - version="5.1", + version="5.2", description="Extension modules to support virtual nodes using Linux network namespaces", scripts=["vcmd", "vnoded", "netns"], ext_modules=[ diff --git a/ns3/setup.py b/ns3/setup.py index c49d225e..08641acb 100644 --- a/ns3/setup.py +++ b/ns3/setup.py @@ -6,7 +6,7 @@ _EXAMPLES_DIR = "share/corens3/examples" setup( name="core-ns3", - version="5.1", + version="5.2", packages=[ "corens3", ], diff --git a/sonar-project.properties b/sonar-project.properties index 13f1d64e..23786e96 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,11 +1,11 @@ # branch name for the sonar analysis -sonar.branch.name=5.1_shell_cleanup +sonar.branch.name=rel/5.2 sonar.branch.target=master # required metadata sonar.projectKey=CORE sonar.projectName=CORE -sonar.projectVersion=5.1 +sonar.projectVersion=5.2 # define modules sonar.modules=daemon,ns3,netns From 5b0ed13e78c92ae0ea7d3d82f36796be1657c28e Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 6 Jul 2018 14:40:51 -0700 Subject: [PATCH 70/83] fixed linkconfig parameter name to match other linkconfig signatures --- daemon/core/netns/openvswitch.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/daemon/core/netns/openvswitch.py b/daemon/core/netns/openvswitch.py index ba47131f..04c6d875 100644 --- a/daemon/core/netns/openvswitch.py +++ b/daemon/core/netns/openvswitch.py @@ -185,55 +185,55 @@ class OvsNet(PyCoreNet): ebtables_queue.ebchange(self) - def linkconfig(self, interface, bw=None, delay=None, loss=None, duplicate=None, + def linkconfig(self, netif, bw=None, delay=None, loss=None, duplicate=None, jitter=None, netif2=None, devname=None): """ Configure link parameters by applying tc queuing disciplines on the interface. """ if not devname: - devname = interface.localname + devname = netif.localname tc = [constants.TC_BIN, "qdisc", "replace", "dev", devname] parent = ["root"] # attempt to set bandwidth and update as needed if value changed - bandwidth_changed = interface.setparam("bw", bw) + bandwidth_changed = netif.setparam("bw", bw) if bandwidth_changed: # from tc-tbf(8): minimum value for burst is rate / kernel_hz if bw > 0: if self.up: - burst = max(2 * interface.mtu, bw / 1000) + burst = max(2 * netif.mtu, bw / 1000) limit = 0xffff # max IP payload tbf = ["tbf", "rate", str(bw), "burst", str(burst), "limit", str(limit)] logger.info("linkconfig: %s" % [tc + parent + ["handle", "1:"] + tbf]) utils.check_cmd(tc + parent + ["handle", "1:"] + tbf) - interface.setparam("has_tbf", True) - elif interface.getparam("has_tbf") and bw <= 0: + netif.setparam("has_tbf", True) + elif netif.getparam("has_tbf") and bw <= 0: tcd = [] + tc tcd[2] = "delete" if self.up: utils.check_cmd(tcd + parent) - interface.setparam("has_tbf", False) + netif.setparam("has_tbf", False) # removing the parent removes the child - interface.setparam("has_netem", False) + netif.setparam("has_netem", False) - if interface.getparam("has_tbf"): + if netif.getparam("has_tbf"): parent = ["parent", "1:1"] netem = ["netem"] - delay_changed = interface.setparam("delay", delay) + delay_changed = netif.setparam("delay", delay) if loss is not None: loss = float(loss) - loss_changed = interface.setparam("loss", loss) + loss_changed = netif.setparam("loss", loss) if duplicate is not None: duplicate = float(duplicate) - duplicate_changed = interface.setparam("duplicate", duplicate) - jitter_changed = interface.setparam("jitter", jitter) + duplicate_changed = netif.setparam("duplicate", duplicate) + jitter_changed = netif.setparam("jitter", jitter) # if nothing changed return if not any([bandwidth_changed, delay_changed, loss_changed, duplicate_changed, jitter_changed]): @@ -256,7 +256,7 @@ class OvsNet(PyCoreNet): if delay <= 0 and jitter <= 0 and loss <= 0 and duplicate <= 0: # possibly remove netem if it exists and parent queue wasn"t removed - if not interface.getparam("has_netem"): + if not netif.getparam("has_netem"): return tc[2] = "delete" @@ -264,12 +264,12 @@ class OvsNet(PyCoreNet): if self.up: logger.info("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) utils.check_cmd(tc + parent + ["handle", "10:"]) - interface.setparam("has_netem", False) + netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: logger.info("linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],)) utils.check_cmd(tc + parent + ["handle", "10:"] + netem) - interface.setparam("has_netem", True) + netif.setparam("has_netem", True) def linknet(self, network): """ From aed3126a6a6f0f27bc0e56687e0e256dd4c27c19 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 6 Jul 2018 14:51:28 -0700 Subject: [PATCH 71/83] changed corexml log statement --- daemon/core/xml/corexml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 757528a5..9d022625 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -517,7 +517,7 @@ class CoreXmlWriter(object): for node_id in self.session.emane.nodes(): all_configs = self.session.emane.get_all_configs(node_id) for model_name, config in all_configs.iteritems(): - logger.info("writing emane config: %s", config) + logger.info("writing emane config node(%s) model(%s)", node_id, model_name) if model_name == -1: emane_configuration = create_emane_config(node_id, self.session.emane.emane_config, config) else: @@ -533,6 +533,7 @@ class CoreXmlWriter(object): for node_id in self.session.mobility.nodes(): all_configs = self.session.mobility.get_all_configs(node_id) for model_name, config in all_configs.iteritems(): + logger.info("writing mobility config node(%s) model(%s)", node_id, model_name) mobility_configuration = etree.SubElement(mobility_configurations, "mobility_configuration") add_attribute(mobility_configuration, "node", node_id) add_attribute(mobility_configuration, "model", model_name) From bf8b7c6d5f3b675035f4127c6cc99e39042e6a51 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Fri, 6 Jul 2018 14:58:59 -0700 Subject: [PATCH 72/83] small updates to conf.py documentation --- daemon/core/conf.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index ffe71541..b232b101 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -304,7 +304,14 @@ class ConfigurableOptions(object): class ModelManager(ConfigurableManager): + """ + Helps handle setting models for nodes and managing their model configurations. + """ + def __init__(self): + """ + Creates a ModelManager object. + """ super(ModelManager, self).__init__() self.models = {} self.node_models = {} @@ -359,6 +366,14 @@ class ModelManager(ConfigurableManager): return config def set_model(self, node, model_class, config=None): + """ + Set model and model configuration for node. + + :param node: node to set model for + :param model_class: model class to set for node + :param dict config: model configuration, None for default configuration + :return: nothing + """ logger.info("setting mobility model(%s) for node(%s): %s", model_class.name, node.objid, config) self.set_model_config(node.objid, model_class.name, config) config = self.get_model_config(node.objid, model_class.name) From bf222cd5b4655246764079415f06387c40e2b171 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Mon, 9 Jul 2018 08:37:45 -0700 Subject: [PATCH 73/83] fixed issue with OvsCtrlNet startup ordering and variable definition --- daemon/core/netns/openvswitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/netns/openvswitch.py b/daemon/core/netns/openvswitch.py index 04c6d875..9fa6138a 100644 --- a/daemon/core/netns/openvswitch.py +++ b/daemon/core/netns/openvswitch.py @@ -351,12 +351,12 @@ class OvsCtrlNet(OvsNet): def __init__(self, session, objid="ctrlnet", name=None, prefix=None, hostid=None, start=True, assign_address=True, updown_script=None, serverintf=None): - OvsNet.__init__(self, session, objid=objid, name=name, start=start) self.prefix = ipaddress.Ipv4Prefix(prefix) self.hostid = hostid self.assign_address = assign_address self.updown_script = updown_script self.serverintf = serverintf + OvsNet.__init__(self, session, objid=objid, name=name, start=start) def startup(self): if self.detectoldbridge(): From bfbee35a5314b5842292fb839bbe9d2936e4c206 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 11 Jul 2018 09:19:06 -0700 Subject: [PATCH 74/83] updates to support external transport configuration and an emane transport service to generate and run emanetransport for a configured external transport model --- daemon/core/conf.py | 2 +- daemon/core/emane/commeffect.py | 3 +- daemon/core/emane/emanemanager.py | 7 +++-- daemon/core/emane/emanemodel.py | 18 ++++++++++-- daemon/core/emane/nodes.py | 2 +- daemon/core/netns/vif.py | 2 ++ daemon/core/services/emaneservices.py | 40 +++++++++++++++++++++++++++ daemon/core/xml/corexml.py | 11 ++++++++ daemon/core/xml/emanexml.py | 37 ++++++++++++++++++++----- daemon/data/logging.conf | 2 +- 10 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 daemon/core/services/emaneservices.py diff --git a/daemon/core/conf.py b/daemon/core/conf.py index b232b101..f9e6e29d 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -105,7 +105,7 @@ class Configuration(object): Represents a configuration options. """ - def __init__(self, _id, _type, label, default="", options=None): + def __init__(self, _id, _type, label=None, default="", options=None): """ Creates a Configuration object. diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index efb279c7..bf0eee79 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -41,8 +41,9 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): shim_defaults = {} config_shim = emanemanifest.parse(shim_xml, shim_defaults) - # comm effect does not need the default phy configuration + # comm effect does not need the default phy and external configurations phy_config = () + external_config = () @classmethod def configurations(cls): diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index d3fec411..e7b831e8 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -128,7 +128,9 @@ class EmaneManager(ModelManager): if not config and self.has_configs(interface.node.objid): config = self.get_configs(node_id=interface.node.objid, config_type=model_name) - if not config and interface.transport_type == "raw": + # get non interface config, when none found + # if not config and interface.transport_type == "raw": + if not config and self.has_configs(node_id): # with EMANE 0.9.2+, we need an extra NEM XML from # model.buildnemxmlfiles(), so defaults are returned here config = self.get_configs(node_id=node_id, config_type=model_name) @@ -561,11 +563,12 @@ class EmaneManager(ModelManager): Build a platform.xml file now that all nodes are configured. """ nemid = int(self.get_config("nem_id_start")) + platform_xmls = {} # assume self._objslock is already held here for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] - nemid = emanexml.build_node_platform_xml(self, ctrlnet, emane_node, nemid) + nemid = emanexml.build_node_platform_xml(self, ctrlnet, emane_node, nemid, platform_xmls) def buildnemxml(self): """ diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 55092741..d1e5df22 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -5,7 +5,9 @@ import os from core import logger from core.conf import ConfigGroup +from core.conf import Configuration from core.emane import emanemanifest +from core.enumerations import ConfigDataTypes from core.mobility import WirelessModel from core.xml import emanexml @@ -32,19 +34,28 @@ class EmaneModel(WirelessModel): } phy_config = emanemanifest.parse(phy_xml, phy_defaults) + # support for external configurations + external_config = [ + Configuration("external", ConfigDataTypes.BOOL, default="0"), + Configuration("platformendpoint", ConfigDataTypes.STRING, default="127.0.0.1:40001"), + Configuration("transportendpoint", ConfigDataTypes.STRING, default="127.0.0.1:50002") + ] + config_ignore = set() @classmethod def configurations(cls): - return cls.mac_config + cls.phy_config + return cls.mac_config + cls.phy_config + cls.external_config @classmethod def config_groups(cls): mac_len = len(cls.mac_config) + phy_len = len(cls.phy_config) + mac_len config_len = len(cls.configurations()) return [ ConfigGroup("MAC Parameters", 1, mac_len), - ConfigGroup("PHY Parameters", mac_len + 1, config_len) + ConfigGroup("PHY Parameters", mac_len + 1, phy_len), + ConfigGroup("External Parameters", phy_len + 1, config_len) ] def build_xml_files(self, config, interface=None): @@ -60,6 +71,7 @@ class EmaneModel(WirelessModel): mac_name = emanexml.mac_file_name(self, interface) phy_name = emanexml.phy_file_name(self, interface) + # check if this is external transport_type = "virtual" if interface and interface.transport_type == "raw": transport_type = "raw" @@ -67,7 +79,7 @@ class EmaneModel(WirelessModel): # create nem xml file nem_file = os.path.join(self.session.session_dir, nem_name) - emanexml.create_nem_xml(self, nem_file, transport_name, mac_name, phy_name) + emanexml.create_nem_xml(self, config, nem_file, transport_name, mac_name, phy_name) # create mac xml file mac_file = os.path.join(self.session.session_dir, mac_name) diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index dd154b8b..6eb788d3 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -131,7 +131,7 @@ class EmaneNode(EmaneNet): for netif in self.netifs(): if do_netns and "virtual" in netif.transport_type.lower(): netif.install() - netif.setaddrs() + # netif.setaddrs() if not self.session.emane.genlocationevents(): netif.poshook = None continue diff --git a/daemon/core/netns/vif.py b/daemon/core/netns/vif.py index b7d77d23..2e5b66fc 100644 --- a/daemon/core/netns/vif.py +++ b/daemon/core/netns/vif.py @@ -169,6 +169,7 @@ class TunTap(PyCoreNetIf): :return: wait for device local response :rtype: int """ + logger.debug("waiting for device local: %s", self.localname) def localdevexists(): args = [constants.IP_BIN, "link", "show", self.localname] @@ -182,6 +183,7 @@ class TunTap(PyCoreNetIf): :return: nothing """ + logger.debug("waiting for device node: %s", self.name) def nodedevexists(): args = [constants.IP_BIN, "link", "show", self.name] diff --git a/daemon/core/services/emaneservices.py b/daemon/core/services/emaneservices.py new file mode 100644 index 00000000..26c59282 --- /dev/null +++ b/daemon/core/services/emaneservices.py @@ -0,0 +1,40 @@ +from core.enumerations import NodeTypes +from core.misc import nodeutils +from core.service import CoreService +from core.xml import emanexml + + +class EmaneTransportService(CoreService): + name = "transportd" + executables = ("emanetransportd", "emanegentransportxml") + group = "EMANE" + dependencies = () + dirs = () + configs = ("emanetransport.sh",) + startup = ("sh %s" % configs[0],) + validate = ("pidof %s" % executables[0],) + validation_timer = 0.5 + shutdown = ("killall %s" % executables[0],) + + @classmethod + def generate_config(cls, node, filename): + if filename == cls.configs[0]: + transport_commands = [] + for interface in node.netifs(sort=True): + network_node = node.session.get_object(interface.net.objid) + if nodeutils.is_node(network_node, NodeTypes.EMANE): + if not node.session.emane.has_configs(network_node.objid): + continue + all_configs = node.session.emane.get_all_configs(network_node.objid) + config = all_configs.get(network_node.model.name) + if emanexml.is_external(config): + nem_id = network_node.getnemid(interface) + command = "emanetransportd -r -l 0 -d ../transportdaemon%s.xml" % nem_id + transport_commands.append(command) + transport_commands = "\n".join(transport_commands) + return """ +emanegentransportxml -o ../ ../platform%s.xml +%s +""" % (node.objid, transport_commands) + else: + raise ValueError diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 9d022625..df01a178 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -84,6 +84,11 @@ def create_emane_model_config(node_id, model, config): value = config[phy_config.id] add_configuration(phy_element, phy_config.id, value) + external_element = etree.SubElement(emane_element, "external") + for external_config in model.external_config: + value = config[external_config.id] + add_configuration(external_element, external_config.id, value) + return emane_element @@ -774,6 +779,12 @@ class CoreXmlReader(object): value = config.get("value") configs[name] = value + external_configuration = emane_configuration.find("external") + for config in external_configuration.iterchildren(): + name = config.get("name") + value = config.get("value") + configs[name] = value + logger.info("reading emane configuration node(%s) model(%s)", node_id, model_name) self.session.emane.set_model_config(node_id, model_name, configs) diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index db811f76..b235872c 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -10,6 +10,17 @@ from core.xml import corexml _hwaddr_prefix = "02:02" +def is_external(config): + """ + Checks if the configuration is for an external transport. + + :param dict config: configuration to check + :return: True if external, False otherwise + :rtype: bool + """ + return config.get("external") == "1" + + def _value_to_params(value): """ Helper to convert a parameter to a parameter tuple. @@ -85,7 +96,7 @@ def add_configurations(xml_element, configurations, config, config_ignore): add_param(xml_element, name, value) -def build_node_platform_xml(emane_manager, control_net, node, nem_id): +def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_xmls): """ Create platform xml for a specific node. @@ -93,15 +104,15 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id): :param core.netns.nodes.CtrlNet control_net: control net node for this emane network :param core.emane.nodes.EmaneNode node: node to write platform xml for :param int nem_id: nem id to use for interfaces for this node + :param dict platform_xmls: stores platform xml elements to append nem entries to :return: the next nem id that can be used for creating platform xml files :rtype: int """ logger.debug("building emane platform xml for node(%s): %s", node, node.name) nem_entries = {} - platform_xmls = {} if node.model is None: - logger.info("warning: EmaneNode %s has no associated model", node.name) + logger.warn("warning: EmaneNode %s has no associated model", node.name) return nem_entries for netif in node.netifs(): @@ -109,6 +120,16 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id): nem_definition = nem_file_name(node.model, netif) nem_element = etree.Element("nem", id=str(nem_id), name=netif.localname, definition=nem_definition) + # check if this is an external transport, get default config if an interface specific one does not exist + config = emane_manager.getifcconfig(node.model.object_id, netif, node.model.name) + + if is_external(config): + nem_element.set("transport", "external") + platform_endpoint = "platformendpoint" + add_param(nem_element, platform_endpoint, config[platform_endpoint]) + transport_endpoint = "transportendpoint" + add_param(nem_element, transport_endpoint, config[transport_endpoint]) + # build transport xml transport_type = netif.transport_type if not transport_type: @@ -220,7 +241,6 @@ def build_xml_files(emane_manager, node): need_raw = True rtype = netif.transport_type - # build transport XML files depending on type of interfaces involved if need_virtual: build_transport_xml(emane_manager, node, vtype) @@ -270,7 +290,7 @@ def create_phy_xml(emane_model, config, file_path): Create the phy xml document. :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for - :param dict config: all current configuration values, mac + phy + :param dict config: all current configuration values :param str file_path: path to write file to :return: nothing """ @@ -287,7 +307,7 @@ def create_mac_xml(emane_model, config, file_path): Create the mac xml document. :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for - :param dict config: all current configuration values, mac + phy + :param dict config: all current configuration values :param str file_path: path to write file to :return: nothing """ @@ -299,11 +319,12 @@ def create_mac_xml(emane_model, config, file_path): create_file(mac_element, "mac", file_path) -def create_nem_xml(emane_model, nem_file, transport_definition, mac_definition, phy_definition): +def create_nem_xml(emane_model, config, nem_file, transport_definition, mac_definition, phy_definition): """ Create the nem xml document. :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param dict config: all current configuration values :param str nem_file: nem file path to write :param str transport_definition: transport file definition path :param str mac_definition: mac file definition path @@ -311,6 +332,8 @@ def create_nem_xml(emane_model, nem_file, transport_definition, mac_definition, :return: nothing """ nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) + if is_external(config): + nem_element.set("type", "unstructured") etree.SubElement(nem_element, "transport", definition=transport_definition) etree.SubElement(nem_element, "mac", definition=mac_definition) etree.SubElement(nem_element, "phy", definition=phy_definition) diff --git a/daemon/data/logging.conf b/daemon/data/logging.conf index 7f3d496f..46de6e92 100644 --- a/daemon/data/logging.conf +++ b/daemon/data/logging.conf @@ -14,7 +14,7 @@ } }, "root": { - "level": "INFO", + "level": "DEBUG", "handlers": ["console"] } } From b03662dbeb64421ad2982300cb24721e91ef3663 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 11 Jul 2018 09:24:44 -0700 Subject: [PATCH 75/83] removed emane nodes netif install and setaddrs, install was always skipped and setaddrs is done by zebra/quagga --- daemon/core/emane/emanemanager.py | 6 +++--- daemon/core/emane/nodes.py | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index e7b831e8..fe8e5d83 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -318,7 +318,7 @@ class EmaneManager(ModelManager): if self.numnems() > 0: self.startdaemons() - self.installnetifs(do_netns=False) + self.installnetifs() for emane_node in self._emane_nodes.itervalues(): for netif in emane_node.netifs(): @@ -704,7 +704,7 @@ class EmaneManager(ModelManager): except CoreCommandError: logger.exception("error shutting down emane daemons") - def installnetifs(self, do_netns=True): + def installnetifs(self): """ Install TUN/TAP virtual interfaces into their proper namespaces now that the EMANE daemons are running. @@ -712,7 +712,7 @@ class EmaneManager(ModelManager): for key in sorted(self._emane_nodes.keys()): emane_node = self._emane_nodes[key] logger.info("emane install netifs for node: %d", key) - emane_node.installnetifs(do_netns) + emane_node.installnetifs() def deinstallnetifs(self): """ diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 6eb788d3..ad96105e 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -117,7 +117,7 @@ class EmaneNode(EmaneNet): """ return sorted(self._netif.values(), key=lambda ifc: ifc.node.objid) - def installnetifs(self, do_netns=True): + def installnetifs(self): """ Install TAP devices into their namespaces. This is done after EMANE daemons have been started, because that is their only chance @@ -129,9 +129,6 @@ class EmaneNode(EmaneNet): logger.error(warntxt) for netif in self.netifs(): - if do_netns and "virtual" in netif.transport_type.lower(): - netif.install() - # netif.setaddrs() if not self.session.emane.genlocationevents(): netif.poshook = None continue From 3d59a6829912298379ee75f3acf117b3eb06437d Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 11 Jul 2018 21:34:21 -0700 Subject: [PATCH 76/83] refactored how getters for configurations worked, to avoid setting defaults and causing undesireable states --- daemon/core/conf.py | 54 ++++++++++++--------------- daemon/core/emane/emanemanager.py | 9 ++--- daemon/core/mobility.py | 3 +- daemon/core/services/emaneservices.py | 7 +--- daemon/core/xml/corexml.py | 18 +++++++-- daemon/tests/test_conf.py | 2 +- 6 files changed, 46 insertions(+), 47 deletions(-) diff --git a/daemon/core/conf.py b/daemon/core/conf.py index f9e6e29d..f42107ac 100644 --- a/daemon/core/conf.py +++ b/daemon/core/conf.py @@ -152,16 +152,6 @@ class ConfigurableManager(object): """ return [node_id for node_id in self.node_configurations.iterkeys() if node_id != self._default_node] - def has_configs(self, node_id): - """ - Checks if this manager contains a configuration for the node id. - - :param int node_id: node id to check for a configuration - :return: True if a node configuration exists, False otherwise - :rtype: bool - """ - return node_id in self.node_configurations - def config_reset(self, node_id=None): """ Clears all configurations or configuration for a specific node. @@ -186,8 +176,9 @@ class ConfigurableManager(object): :return: nothing """ logger.debug("setting config for node(%s) type(%s): %s=%s", node_id, config_type, _id, value) - node_type_map = self.get_configs(node_id, config_type) - node_type_map[_id] = value + node_configs = self.node_configurations.setdefault(node_id, OrderedDict()) + node_type_configs = node_configs.setdefault(config_type, OrderedDict()) + node_type_configs[_id] = value def set_configs(self, config, node_id=_default_node, config_type=_default_type): """ @@ -199,9 +190,7 @@ class ConfigurableManager(object): :return: nothing """ logger.debug("setting config for node(%s) type(%s): %s", node_id, config_type, config) - node_configs = self.get_all_configs(node_id) - if config_type in node_configs: - node_configs.pop(config_type) + node_configs = self.node_configurations.setdefault(node_id, OrderedDict()) node_configs[config_type] = config def get_config(self, _id, node_id=_default_node, config_type=_default_type, default=None): @@ -211,13 +200,16 @@ class ConfigurableManager(object): :param str _id: specific configuration to retrieve :param int node_id: node id to store configuration for :param str config_type: configuration type to store configuration for - :param default: + :param default: default value to return when value is not found :return: configuration value :rtype str """ logger.debug("getting config for node(%s) type(%s): %s", node_id, config_type, _id) - node_type_map = self.get_configs(node_id, config_type) - return node_type_map.get(_id, default) + result = default + node_type_configs = self.get_configs(node_id, config_type) + if node_type_configs: + result = node_type_configs.get(_id, default) + return result def get_configs(self, node_id=_default_node, config_type=_default_type): """ @@ -229,8 +221,11 @@ class ConfigurableManager(object): :rtype: dict """ logger.debug("getting configs for node(%s) type(%s)", node_id, config_type) - node_map = self.get_all_configs(node_id) - return node_map.setdefault(config_type, {}) + result = None + node_configs = self.node_configurations.get(node_id) + if node_configs: + result = node_configs.get(config_type) + return result def get_all_configs(self, node_id=_default_node): """ @@ -241,7 +236,7 @@ class ConfigurableManager(object): :rtype: dict """ logger.debug("getting all configs for node(%s)", node_id) - return self.node_configurations.setdefault(node_id, OrderedDict()) + return self.node_configurations.get(node_id) class ConfigGroup(object): @@ -331,17 +326,17 @@ class ModelManager(ConfigurableManager): raise ValueError("%s is an invalid model" % model_name) # retrieve default values - node_config = self.get_model_config(node_id, model_name) + model_config = self.get_model_config(node_id, model_name) if not config: config = {} for key, value in config.iteritems(): - node_config[key] = value + model_config[key] = value # set as node model for startup self.node_models[node_id] = model_name # set configuration - self.set_configs(node_config, node_id=node_id, config_type=model_name) + self.set_configs(model_config, node_id=node_id, config_type=model_name) def get_model_config(self, node_id, model_name): """ @@ -388,16 +383,15 @@ class ModelManager(ConfigurableManager): :return: list of model and values tuples for the network node :rtype: list """ - models = [] - all_configs = {} - if self.has_configs(node_id=node.objid): - all_configs = self.get_all_configs(node_id=node.objid) + all_configs = self.get_all_configs(node.objid) + if not all_configs: + all_configs = {} - for model_name in all_configs.iterkeys(): + models = [] + for model_name, config in all_configs.iteritems(): if model_name == ModelManager._default_node: continue model_class = self.models[model_name] - config = self.get_configs(node_id=node.objid, config_type=model_name) models.append((model_class, config)) logger.debug("models for node(%s): %s", node.objid, models) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index fe8e5d83..966a3ac2 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -120,17 +120,14 @@ class EmaneManager(ModelManager): key += interface.netindex # try retrieve interface specific configuration, avoid getting defaults - config = {} - if self.has_configs(key): - config = self.get_configs(key) + config = self.get_configs(node_id=key, config_type=model_name) # otherwise retrieve the interfaces node configuration, avoid using defaults - if not config and self.has_configs(interface.node.objid): + if not config: config = self.get_configs(node_id=interface.node.objid, config_type=model_name) # get non interface config, when none found - # if not config and interface.transport_type == "raw": - if not config and self.has_configs(node_id): + if not config: # with EMANE 0.9.2+, we need an extra NEM XML from # model.buildnemxmlfiles(), so defaults are returned here config = self.get_configs(node_id=node_id, config_type=model_name) diff --git a/daemon/core/mobility.py b/daemon/core/mobility.py index 30436eec..3645bede 100644 --- a/daemon/core/mobility.py +++ b/daemon/core/mobility.py @@ -73,8 +73,7 @@ class MobilityManager(ModelManager): continue for model_name in self.models.iterkeys(): - all_configs = self.get_all_configs(node_id) - config = all_configs.get(model_name) + config = self.get_configs(node_id, model_name) if not config: continue model_class = self.models[model_name] diff --git a/daemon/core/services/emaneservices.py b/daemon/core/services/emaneservices.py index 26c59282..6c9ea0a7 100644 --- a/daemon/core/services/emaneservices.py +++ b/daemon/core/services/emaneservices.py @@ -23,11 +23,8 @@ class EmaneTransportService(CoreService): for interface in node.netifs(sort=True): network_node = node.session.get_object(interface.net.objid) if nodeutils.is_node(network_node, NodeTypes.EMANE): - if not node.session.emane.has_configs(network_node.objid): - continue - all_configs = node.session.emane.get_all_configs(network_node.objid) - config = all_configs.get(network_node.model.name) - if emanexml.is_external(config): + config = node.session.emane.get_configs(network_node.objid, network_node.model.name) + if config and emanexml.is_external(config): nem_id = network_node.getnemid(interface) command = "emanetransportd -r -l 0 -d ../transportdaemon%s.xml" % nem_id transport_commands.append(command) diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index df01a178..394649be 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -496,11 +496,13 @@ class CoreXmlWriter(object): self.scenario.append(hooks) def write_session_options(self): - # options option_elements = etree.Element("session_options") - # TODO: should we just save the current config regardless, since it may change? options_config = self.session.options.get_configs() + if not options_config: + return + for _id, default_value in self.session.options.default_values().iteritems(): + # TODO: should we just save the current config regardless, since it may change? value = options_config[_id] if value != default_value: add_configuration(option_elements, _id, value) @@ -511,7 +513,11 @@ class CoreXmlWriter(object): def write_session_metadata(self): # metadata metadata_elements = etree.Element("session_metadata") - for _id, value in self.session.metadata.get_configs().iteritems(): + config = self.session.metadata.get_configs() + if not config: + return + + for _id, value in config.iteritems(): add_configuration(metadata_elements, _id, value) if metadata_elements.getchildren(): @@ -521,6 +527,9 @@ class CoreXmlWriter(object): emane_configurations = etree.Element("emane_configurations") for node_id in self.session.emane.nodes(): all_configs = self.session.emane.get_all_configs(node_id) + if not all_configs: + continue + for model_name, config in all_configs.iteritems(): logger.info("writing emane config node(%s) model(%s)", node_id, model_name) if model_name == -1: @@ -537,6 +546,9 @@ class CoreXmlWriter(object): mobility_configurations = etree.Element("mobility_configurations") for node_id in self.session.mobility.nodes(): all_configs = self.session.mobility.get_all_configs(node_id) + if not all_configs: + continue + for model_name, config in all_configs.iteritems(): logger.info("writing mobility config node(%s) model(%s)", node_id, model_name) mobility_configuration = etree.SubElement(mobility_configurations, "mobility_configuration") diff --git a/daemon/tests/test_conf.py b/daemon/tests/test_conf.py index 96e3f6d9..6516de04 100644 --- a/daemon/tests/test_conf.py +++ b/daemon/tests/test_conf.py @@ -85,7 +85,7 @@ class TestConf: config_manager.config_reset(node_id) # then - assert not config_manager.has_configs(node_id) + assert not config_manager.get_configs(node_id=node_id) assert config_manager.get_configs() def test_configs_setget(self): From a6f2b9d7c1c24935b77f999562f3995baf0a6b2e Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 12 Jul 2018 15:28:26 -0700 Subject: [PATCH 77/83] removed emane transport definitions from external configurations --- daemon/core/xml/emanexml.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index b235872c..2d44ae50 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -129,17 +129,17 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x add_param(nem_element, platform_endpoint, config[platform_endpoint]) transport_endpoint = "transportendpoint" add_param(nem_element, transport_endpoint, config[transport_endpoint]) + else: + # build transport xml + transport_type = netif.transport_type + if not transport_type: + logger.info("warning: %s interface type unsupported!", netif.name) + transport_type = "raw" + transport_file = transport_file_name(node.objid, transport_type) + transport_element = etree.SubElement(nem_element, "transport", definition=transport_file) - # build transport xml - transport_type = netif.transport_type - if not transport_type: - logger.info("warning: %s interface type unsupported!", netif.name) - transport_type = "raw" - transport_file = transport_file_name(node.objid, transport_type) - transport_element = etree.SubElement(nem_element, "transport", definition=transport_file) - - # add transport parameter - add_param(transport_element, "device", netif.name) + # add transport parameter + add_param(transport_element, "device", netif.name) # add nem entry nem_entries[netif] = nem_element @@ -334,7 +334,8 @@ def create_nem_xml(emane_model, config, nem_file, transport_definition, mac_defi nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) if is_external(config): nem_element.set("type", "unstructured") - etree.SubElement(nem_element, "transport", definition=transport_definition) + else: + etree.SubElement(nem_element, "transport", definition=transport_definition) etree.SubElement(nem_element, "mac", definition=mac_definition) etree.SubElement(nem_element, "phy", definition=phy_definition) create_file(nem_element, "nem", nem_file) From 5bb23c1d93fff65d9ea7bc87cee74a28033bc38e Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 12 Jul 2018 20:04:04 -0700 Subject: [PATCH 78/83] fixed issue and determined better way to generate boot dependency paths for services --- daemon/core/service.py | 26 ++++++++++++++++---------- daemon/data/logging.conf | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/daemon/core/service.py b/daemon/core/service.py index 7e9cade6..500e9eda 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -228,16 +228,19 @@ class CoreServices(object): """ # generate service map and find starting points node_services = {service.name: service for service in services} - is_dependency = set() all_services = set() + has_dependency = set() + dependency_map = {} for service in services: all_services.add(service.name) - for service_name in service.dependencies: - # check service needed is valid - if service_name not in node_services: - raise ValueError("service(%s) dependency does not exist: %s" % (service.name, service_name)) - is_dependency.add(service_name) - starting_points = all_services - is_dependency + if service.dependencies: + has_dependency.add(service.name) + + for dependency in service.dependencies: + dependents = dependency_map.setdefault(dependency, set()) + dependents.add(service.name) + + starting_points = all_services - has_dependency # cycles means no starting points if not starting_points: @@ -270,8 +273,9 @@ class CoreServices(object): path.append(service.name) path_set.add(service.name) - # retrieve and set dependencies to the stack - stack.append(iter(service.dependencies)) + # retrieve and dependent services and add to stack + dependents = iter(dependency_map.get(service.name, [])) + stack.append(dependents) startup.append(service) break # for loop completed without a break @@ -281,8 +285,10 @@ class CoreServices(object): path_set.remove(path.pop()) if not path and startup: - startup.reverse() + # finalize startup path startups.append(startup) + + # reset new startup path startup = [] stack.pop() diff --git a/daemon/data/logging.conf b/daemon/data/logging.conf index 46de6e92..7f3d496f 100644 --- a/daemon/data/logging.conf +++ b/daemon/data/logging.conf @@ -14,7 +14,7 @@ } }, "root": { - "level": "DEBUG", + "level": "INFO", "handlers": ["console"] } } From e3e25463ebf9efc2347c0bcbc5fd93f0ff17cbf9 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Thu, 19 Jul 2018 16:45:05 -0700 Subject: [PATCH 79/83] removed line sending service load errors to the gui and it interferes with current coresendmsg --- daemon/core/corehandlers.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/daemon/core/corehandlers.py b/daemon/core/corehandlers.py index 0ba3a61b..90113693 100644 --- a/daemon/core/corehandlers.py +++ b/daemon/core/corehandlers.py @@ -550,14 +550,6 @@ class CoreHandler(SocketServer.BaseRequestHandler): # set initial session state self.session.set_state(EventTypes.DEFINITION_STATE) - # send service errors, if present - if self.coreemu.service_errors: - self.send_exception( - ExceptionLevels.ERROR, - "ServiceManager.load()", - "Failed to load services: %s" % " ".join(self.coreemu.service_errors) - ) - while True: try: message = self.receive_message() From 087a0f011bb7c161e74e106720171f880e21539b Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Tue, 24 Jul 2018 13:34:33 -0700 Subject: [PATCH 80/83] added back in core emane address creation, only when not external --- daemon/core/emane/nodes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index ad96105e..896cd7ff 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -129,9 +129,17 @@ class EmaneNode(EmaneNet): logger.error(warntxt) for netif in self.netifs(): + external = self.session.emane.get_config("external", self.objid, self.model.name) + if external == "0": + logger.info("I AM NOT SKIPPING EMANE ADDRESSES") + netif.setaddrs() + else: + logger.info("I AM SKIPPING EMANE ADDRESSES") + if not self.session.emane.genlocationevents(): netif.poshook = None continue + # at this point we register location handlers for generating # EMANE location events netif.poshook = self.setnemposition From 9ce28da658ef35c9ae553761c7652ecaab8f57f3 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 25 Jul 2018 09:37:59 -0700 Subject: [PATCH 81/83] fixed bug with custom service files using the same dict across instances of the class --- daemon/core/service.py | 8 +------ daemon/tests/test_services.py | 44 +++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/daemon/core/service.py b/daemon/core/service.py index 500e9eda..e741da7a 100644 --- a/daemon/core/service.py +++ b/daemon/core/service.py @@ -761,13 +761,7 @@ class CoreService(object): configuration is used to override their default parameters. """ self.custom = True - self.dirs = self.__class__.dirs - self.configs = self.__class__.configs - self.startup = self.__class__.startup - self.shutdown = self.__class__.shutdown - self.validate = self.__class__.validate - self.meta = self.__class__.meta - self.config_data = self.__class__.config_data + self.config_data = self.__class__.config_data.copy() @classmethod def on_load(cls): diff --git a/daemon/tests/test_services.py b/daemon/tests/test_services.py index 4c20d031..217378e0 100644 --- a/daemon/tests/test_services.py +++ b/daemon/tests/test_services.py @@ -177,24 +177,48 @@ class TestServices: # then assert status - def test_service_set_file(self, session): + def test_service_custom_startup(self, session): # given ServiceManager.add_services(_SERVICES_PATH) my_service = ServiceManager.get(SERVICE_ONE) node = session.add_node() - file_name = my_service.configs[0] - file_path = node.hostfilename(file_name) - file_data = "# custom file" - session.services.set_service_file(node.objid, my_service.name, file_name, file_data) - custom_service = session.services.get_service(node.objid, my_service.name) # when - session.services.create_service_files(node, custom_service) + session.services.set_service(node.objid, my_service.name) + custom_my_service = session.services.get_service(node.objid, my_service.name) + custom_my_service.startup = ("sh custom.sh",) # then - assert os.path.exists(file_path) - with open(file_path, "r") as custom_file: - assert custom_file.read() == file_data + assert my_service.startup != custom_my_service.startup + + def test_service_set_file(self, session): + # given + ServiceManager.add_services(_SERVICES_PATH) + my_service = ServiceManager.get(SERVICE_ONE) + node_one = session.add_node() + node_two = session.add_node() + file_name = my_service.configs[0] + file_data_one = "# custom file one" + file_data_two = "# custom file two" + session.services.set_service_file(node_one.objid, my_service.name, file_name, file_data_one) + session.services.set_service_file(node_two.objid, my_service.name, file_name, file_data_two) + + # when + custom_service_one = session.services.get_service(node_one.objid, my_service.name) + session.services.create_service_files(node_one, custom_service_one) + custom_service_two = session.services.get_service(node_two.objid, my_service.name) + session.services.create_service_files(node_two, custom_service_two) + + # then + file_path_one = node_one.hostfilename(file_name) + assert os.path.exists(file_path_one) + with open(file_path_one, "r") as custom_file: + assert custom_file.read() == file_data_one + + file_path_two = node_two.hostfilename(file_name) + assert os.path.exists(file_path_two) + with open(file_path_two, "r") as custom_file: + assert custom_file.read() == file_data_two def test_service_import(self): """ From 055f732cc7e8530a49767176780b4869d744ee77 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 25 Jul 2018 12:50:19 -0700 Subject: [PATCH 82/83] updated gui help menu to point to github home page and github documentation --- gui/initgui.tcl | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/gui/initgui.tcl b/gui/initgui.tcl index 042120a0..4d64cdb2 100644 --- a/gui/initgui.tcl +++ b/gui/initgui.tcl @@ -43,46 +43,46 @@ # global variables # FUNCTION # GUI-related global varibles -# +# # * newlink -- helps when creating a new link. If there is no # link currently created, this value is set to an empty string. # * selectbox -- the value of the box representing all the selected items # * selected -- containes the list of node_id's of all selected nodes. -# * newCanvas -- +# * newCanvas -- # -# * animatephase -- starting dashoffset. With this value the effect of -# rotating line around selected itme is achived. +# * animatephase -- starting dashoffset. With this value the effect of +# rotating line around selected itme is achived. # * undolevel -- control variable for undo. # * redolevel -- control variable for redo. # * undolog -- control variable for saving all the past configurations. -# * changed -- control variable for indicating that there something changed +# * changed -- control variable for indicating that there something changed # in active configuration. # * badentry -- control variable indicating that there has been a bad entry # in the text box. # * cursorstate -- control variable for animating cursor. # * clock_seconds -- control variable for animating cursor. -# * oper_mode -- control variable reresenting operating mode, possible +# * oper_mode -- control variable reresenting operating mode, possible # values are edit and exec. -# * grid -- control variable representing grid distance. All new -# elements on the +# * grid -- control variable representing grid distance. All new +# elements on the # canvas are snaped to grid. Default value is 24. # * sizex -- X size of the canvas. # * sizey -- Y size of the canvas. # * curcanvas -- the value of the current canvas. -# * autorearrange_enabled -- control variable indicating is +# * autorearrange_enabled -- control variable indicating is # autorearrange enabled. # # * defLinkColor -- defines the default link color, default link color is set # to red. # * defLinkWidth -- defines the width of the link, default is 2. -# * defEthBandwidth -- defines the ethernet bandwidth, default is set to +# * defEthBandwidth -- defines the ethernet bandwidth, default is set to # 100000000. # * defSerBandwidth -- defines the serail link bandwidth, default is 2048000. # * defSerDelay -- defines the serail link delay, default is 2500. # * showIfNames -- control variable for showing interface names, default is 1 -# * showIfIPaddrs -- control variable for showing interface IPv4 addresses, +# * showIfIPaddrs -- control variable for showing interface IPv4 addresses, # default is 1 (addresses are visible). -# * showIfIPv6addrs -- control variable for showing interface IPv4 +# * showIfIPv6addrs -- control variable for showing interface IPv4 # addresses, default is 1 (addresses are visible). # * showNodeLabels -- control variable for showing node labels, default is 1. # * showLinkLabels -- control variable for showing link labels, default is 1. @@ -118,19 +118,19 @@ set resizemode false set thruplotResize false # dictionary that maps cursor style to resize mode -set cursorToResizemode [dict create top_left_corner lu] +set cursorToResizemode [dict create top_left_corner lu] dict set cursorToResizemode bottom_left_corner ld dict set cursorToResizemode left_side l dict set cursorToResizemode top_right_corner ru dict set cursorToResizemode bottom_right_corner rd dict set cursorToResizemode right_side r dict set cursorToResizemode top_side u - dict set cursorToResizemode bottom_side d + dict set cursorToResizemode bottom_side d -# dictionary that maps thruplot to color +# dictionary that maps thruplot to color set thruPlotColor [dict create default blue] set thruPlotDragStart false -set thruPlotCur null +set thruPlotCur null set curPlotLineColor blue set curPlotFillColor "#7f9eee" @@ -143,7 +143,7 @@ set defThruPlotMaxKBPS 10 # # Initialize a few variables to default values # -set defLinkColor Red +set defLinkColor Red set defFillColor Gray set defLinkWidth 2 set defEthBandwidth 0 @@ -224,7 +224,7 @@ bind . "fileNewDialogBox" bind . "fileOpenDialogBox" .menubar.file add command -label "Reload" -underline 0 \ - -command { openFile $currentFile } + -command { openFile $currentFile } .menubar.file add command -label Save -underline 0 \ -accelerator "Ctrl+S" -command { fileSaveDialogBox "" } @@ -245,7 +245,7 @@ bind . "fileSaveDialogBox {}" .menubar.file add separator .menubar.file add command -label "Open current file in editor..." \ - -underline 21 -command { + -underline 21 -command { global currentFile set ed [get_text_editor false] set t [get_term_prog false] @@ -428,10 +428,10 @@ menu .menubar.tools.experimental toplevel .ns2im-dialog wm transient .ns2im-dialog . wm title .ns2im-dialog "ns2imunes converter" - + set f1 [frame .ns2im-dialog.entry1] set f2 [frame .ns2im-dialog.buttons] - + label $f1.l -text "ns2 file:" entry $f1.e -width 25 -textvariable ns2srcfile button $f1.b -text "Browse" -width 8 \ @@ -440,13 +440,13 @@ menu .menubar.tools.experimental -initialfile $ns2srcfile] $f1.e delete 0 end $f1.e insert 0 "$srcfile" - } + } button $f2.b1 -text "OK" -command { ns2im $srcfile destroy .ns2im-dialog } button $f2.b2 -text "Cancel" -command { destroy .ns2im-dialog} - + pack $f1.b $f1.e -side right pack $f1.l -side right -fill x -expand true pack $f2.b1 -side left -expand true -anchor e @@ -594,10 +594,10 @@ menu .menubar.session -tearoff 1 # Help # menu .menubar.help -tearoff 0 -.menubar.help add command -label "Online manual (www)" -command \ - "_launchBrowser https://downloads.pf.itd.nrl.navy.mil/docs/core/core-html/" -.menubar.help add command -label "CORE website (www)" -command \ - "_launchBrowser https://www.nrl.navy.mil/itd/ncs/products/core" +.menubar.help add command -label "CORE GitHub (www)" -command \ + "_launchBrowser https://github.com/coreemu/core" +.menubar.help add command -label "CORE Documentation (www)" -command \ + "_launchBrowser https://coreemu.github.io/core/" .menubar.help add command -label "Mailing list (www)" -command \ "_launchBrowser https://publists.nrl.navy.mil/mailman/listinfo/core-users" .menubar.help add command -label "About" -command popupAbout @@ -686,7 +686,7 @@ pack .bottom -side bottom -fill x label .bottom.textbox -relief sunken -bd 1 -anchor w -width 999 label .bottom.zoom -relief sunken -bd 1 -anchor w -width 10 label .bottom.cpu_load -relief sunken -bd 1 -anchor w -width 9 -label .bottom.mbuf -relief sunken -bd 1 -anchor w -width 9 +label .bottom.mbuf -relief sunken -bd 1 -anchor w -width 9 label .bottom.indicators -relief sunken -bd 1 -anchor w -width 5 pack .bottom.indicators .bottom.mbuf .bottom.cpu_load \ .bottom.zoom .bottom.textbox -side right -padx 0 -fill both @@ -808,7 +808,7 @@ set ::tk::dialog::file::showHiddenBtn 1 # switchCanvas first -focus -force . +focus -force . # # Fire up the animation loop - used basically for selectbox From f1863a874d497243ae3e579ef32a4b98729d01c0 Mon Sep 17 00:00:00 2001 From: "Blake J. Harnden" Date: Wed, 25 Jul 2018 15:24:54 -0700 Subject: [PATCH 83/83] removed unwanted logging --- daemon/core/emane/nodes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 896cd7ff..5711a4f6 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -131,10 +131,7 @@ class EmaneNode(EmaneNet): for netif in self.netifs(): external = self.session.emane.get_config("external", self.objid, self.model.name) if external == "0": - logger.info("I AM NOT SKIPPING EMANE ADDRESSES") netif.setaddrs() - else: - logger.info("I AM SKIPPING EMANE ADDRESSES") if not self.session.emane.genlocationevents(): netif.poshook = None