updated emane config files to be generated for remote servers, fixed services not using node remote server compatible commands

daemon/core/emane/emanemanager.py

@@ -18,6 +18,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel
 from core.emane.nodes import EmaneNet
 from core.emane.rfpipe import EmaneRfPipeModel
 from core.emane.tdma import EmaneTdmaModel
+from core.emulator import distributed
 from core.emulator.enumerations import (
     ConfigDataTypes,
     ConfigFlags,
@@ -679,8 +680,12 @@ class EmaneManager(ModelManager):
             return
 
         dev = self.get_config("eventservicedevice")
-
         emanexml.create_event_service_xml(group, port, dev, self.session.session_dir)
+        for server in self.session.servers:
+            conn = self.session.servers[server]
+            emanexml.create_event_service_xml(
+                group, port, dev, self.session.session_dir, conn
+            )
 
     def startdaemons(self):
         """
@@ -745,7 +750,7 @@ class EmaneManager(ModelManager):
                 os.path.join(path, "emane%d.log" % n),
                 os.path.join(path, "platform%d.xml" % n),
             ]
-            output = node.check_cmd(args)
+            output = node.node_net_cmd(args)
             logging.info("node(%s) emane daemon running: %s", node.name, args)
             logging.info("node(%s) emane daemon output: %s", node.name, output)
 
@@ -756,6 +761,10 @@ class EmaneManager(ModelManager):
         emanecmd += ["-f", os.path.join(path, "emane.log")]
         args = emanecmd + [os.path.join(path, "platform.xml")]
         utils.check_cmd(args, cwd=path)
+        args = " ".join(args)
+        for server in self.session.servers:
+            conn = self.session.servers[server]
+            distributed.remote_cmd(conn, args, cwd=path)
         logging.info("host emane daemon running: %s", args)
 
     def stopdaemons(self):

daemon/core/emane/emanemodel.py

@@ -102,6 +102,11 @@ class EmaneModel(WirelessModel):
         mac_name = emanexml.mac_file_name(self, interface)
         phy_name = emanexml.phy_file_name(self, interface)
 
+        # remote server for file
+        server = None
+        if interface is not None:
+            server = interface.node.server
+
         # check if this is external
         transport_type = "virtual"
         if interface and interface.transport_type == "raw":
@@ -111,16 +116,16 @@ class EmaneModel(WirelessModel):
         # create nem xml file
         nem_file = os.path.join(self.session.session_dir, nem_name)
         emanexml.create_nem_xml(
-            self, config, nem_file, transport_name, mac_name, phy_name
+            self, config, nem_file, transport_name, mac_name, phy_name, server
         )
 
         # create mac xml file
         mac_file = os.path.join(self.session.session_dir, mac_name)
-        emanexml.create_mac_xml(self, config, mac_file)
+        emanexml.create_mac_xml(self, config, mac_file, server)
 
         # create phy xml file
         phy_file = os.path.join(self.session.session_dir, phy_name)
-        emanexml.create_phy_xml(self, config, phy_file)
+        emanexml.create_phy_xml(self, config, phy_file, server)
 
     def post_startup(self):
         """

daemon/core/emulator/distributed.py

@@ -1,12 +1,16 @@
 import logging
+import os
 import threading
+from tempfile import NamedTemporaryFile
+
+from invoke import UnexpectedExit
 
 from core.errors import CoreCommandError
 
 LOCK = threading.Lock()
 
 
-def remote_cmd(server, cmd, env=None):
+def remote_cmd(server, cmd, env=None, cwd=None):
     """
     Run command remotely using server connection.
 
@@ -14,23 +18,38 @@ def remote_cmd(server, cmd, env=None):
         default is None for localhost
     :param str cmd: command to run
     :param dict env: environment for remote command, default is None
+    :param str cwd: directory to run command in, defaults to None, which is the user's
+        home directory
     :return: stdout when success
     :rtype: str
     :raises CoreCommandError: when a non-zero exit status occurs
     """
     logging.info("remote cmd server(%s): %s", server, cmd)
-    with LOCK:
+    replace_env = env is not None
-        if env is None:
+    try:
-            result = server.run(cmd, hide=False)
+        with LOCK:
-        else:
+            if cwd is None:
-            result = server.run(cmd, hide=False, env=env, replace_env=True)
+                result = server.run(cmd, hide=False, env=env, replace_env=replace_env)
-    if result.exited:
+            else:
-        raise CoreCommandError(
+                with server.cd(cwd):
-            result.exited, result.command, result.stdout, result.stderr
+                    result = server.run(
-        )
+                        cmd, hide=False, env=env, replace_env=replace_env
-    return result.stdout.strip()
+                    )
+        return result.stdout.strip()
+    except UnexpectedExit as e:
+        stdout, stderr = e.streams_for_display()
+        raise CoreCommandError(e.result.exited, cmd, stdout, stderr)
 
 
 def remote_put(server, source, destination):
     with LOCK:
         server.put(source, destination)
+
+
+def remote_put_temp(server, destination, data):
+    with LOCK:
+        temp = NamedTemporaryFile(delete=False)
+        temp.write(data.encode("utf-8"))
+        temp.close()
+        server.put(temp.name, destination)
+        os.unlink(temp.name)

daemon/core/nodes/base.py

@@ -11,7 +11,6 @@ import string
 import threading
 from builtins import range
 from socket import AF_INET, AF_INET6
-from tempfile import NamedTemporaryFile
 
 from core import constants, utils
 from core.emulator import distributed
@@ -997,11 +996,8 @@ class CoreNode(CoreNodeBase):
                 open_file.write(contents)
                 os.chmod(open_file.name, mode)
         else:
-            temp = NamedTemporaryFile(delete=False)
-            temp.write(contents.encode("utf-8"))
-            temp.close()
             self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname])
-            distributed.remote_put(self.server, temp.name, hostfilename)
+            distributed.remote_put_temp(self.server, hostfilename, contents)
             self.net_cmd(["chmod", "%o" % mode, hostfilename])
         logging.debug(
             "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode

daemon/core/services/coreservices.py

@@ -9,6 +9,7 @@ services.
 
 import enum
 import logging
+import shlex
 import time
 from multiprocessing.pool import ThreadPool
 
@@ -597,8 +598,9 @@ class CoreServices(object):
         status = 0
         for cmd in cmds:
             logging.debug("validating service(%s) using: %s", service.name, cmd)
+            cmd = shlex.split(cmd)
             try:
-                node.check_cmd(cmd)
+                node.node_net_cmd(cmd)
             except CoreCommandError as e:
                 logging.debug(
                     "node(%s) service(%s) validate failed", node.name, service.name
@@ -728,11 +730,11 @@ class CoreServices(object):
 
         status = 0
         for cmd in cmds:
+            cmd = shlex.split(cmd)
             try:
                 if wait:
-                    node.check_cmd(cmd)
+                    cmd.append("&")
-                else:
+                node.node_net_cmd(cmd)
-                    node.cmd(cmd, wait=False)
             except CoreCommandError:
                 logging.exception("error starting command")
                 status = -1

daemon/core/xml/emanexml.py

@@ -1,9 +1,11 @@
 import logging
 import os
+from tempfile import NamedTemporaryFile
 
 from lxml import etree
 
 from core import utils
+from core.emulator import distributed
 from core.nodes.ipaddress import MacAddress
 from core.xml import corexml
 
@@ -44,20 +46,29 @@ def _value_to_params(value):
     return None
 
 
-def create_file(xml_element, doc_name, file_path):
+def create_file(xml_element, doc_name, file_path, server=None):
     """
     Create xml file.
 
     :param lxml.etree.Element xml_element: root element to write to file
     :param str doc_name: name to use in the emane doctype
     :param str file_path: file path to write xml file to
+    :param fabric.connection.Connection server: remote server node will run on,
+            default is None for localhost
     :return: nothing
     """
     doctype = (
         '<!DOCTYPE %(doc_name)s SYSTEM "file:///usr/share/emane/dtd/%(doc_name)s.dtd">'
         % {"doc_name": doc_name}
     )
-    corexml.write_xml_file(xml_element, file_path, doctype=doctype)
+    if server is not None:
+        temp = NamedTemporaryFile(delete=False)
+        create_file(xml_element, doc_name, temp.name)
+        temp.close()
+        distributed.remote_put(server, temp.name, file_path)
+        os.unlink(temp.name)
+    else:
+        corexml.write_xml_file(xml_element, file_path, doctype=doctype)
 
 
 def add_param(xml_element, name, value):
@@ -204,17 +215,18 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x
         # increment nem id
         nem_id += 1
 
+    doc_name = "platform"
     for key in sorted(platform_xmls.keys()):
+        platform_element = platform_xmls[key]
         if key == "host":
             file_name = "platform.xml"
+            file_path = os.path.join(emane_manager.session.session_dir, file_name)
+            create_file(platform_element, doc_name, file_path)
         else:
             file_name = "platform%d.xml" % key
-
+            file_path = os.path.join(emane_manager.session.session_dir, file_name)
-        platform_element = platform_xmls[key]
+            linked_node = emane_manager.session.nodes[key]
-
+            create_file(platform_element, doc_name, file_path, linked_node.server)
-        doc_name = "platform"
-        file_path = os.path.join(emane_manager.session.session_dir, file_name)
-        create_file(platform_element, doc_name, file_path)
 
     return nem_id
 
@@ -303,15 +315,20 @@ def build_transport_xml(emane_manager, node, transport_type):
     file_name = transport_file_name(node.id, transport_type)
     file_path = os.path.join(emane_manager.session.session_dir, file_name)
     create_file(transport_element, doc_name, file_path)
+    for server in emane_manager.session.servers:
+        conn = emane_manager.session.servers[server]
+        create_file(transport_element, doc_name, file_path, conn)
 
 
-def create_phy_xml(emane_model, config, file_path):
+def create_phy_xml(emane_model, config, file_path, server):
     """
     Create the phy xml document.
 
     :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml
     :param dict config: all current configuration values
     :param str file_path: path to write file to
+    :param fabric.connection.Connection server: remote server node will run on,
+            default is None for localhost
     :return: nothing
     """
     phy_element = etree.Element("phy", name="%s PHY" % emane_model.name)
@@ -322,15 +339,24 @@ def create_phy_xml(emane_model, config, file_path):
         phy_element, emane_model.phy_config, config, emane_model.config_ignore
     )
     create_file(phy_element, "phy", file_path)
+    if server is not None:
+        create_file(phy_element, "phy", file_path, server)
+    else:
+        create_file(phy_element, "phy", file_path)
+        for server in emane_model.session.servers:
+            conn = emane_model.session.servers[server]
+            create_file(phy_element, "phy", file_path, conn)
 
 
-def create_mac_xml(emane_model, config, file_path):
+def create_mac_xml(emane_model, config, file_path, server):
     """
     Create the mac xml document.
 
     :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml
     :param dict config: all current configuration values
     :param str file_path: path to write file to
+    :param fabric.connection.Connection server: remote server node will run on,
+            default is None for localhost
     :return: nothing
     """
     if not emane_model.mac_library:
@@ -343,10 +369,23 @@ def create_mac_xml(emane_model, config, file_path):
         mac_element, emane_model.mac_config, config, emane_model.config_ignore
     )
     create_file(mac_element, "mac", file_path)
+    if server is not None:
+        create_file(mac_element, "mac", file_path, server)
+    else:
+        create_file(mac_element, "mac", file_path)
+        for server in emane_model.session.servers:
+            conn = emane_model.session.servers[server]
+            create_file(mac_element, "mac", file_path, conn)
 
 
 def create_nem_xml(
-    emane_model, config, nem_file, transport_definition, mac_definition, phy_definition
+    emane_model,
+    config,
+    nem_file,
+    transport_definition,
+    mac_definition,
+    phy_definition,
+    server,
 ):
     """
     Create the nem xml document.
@@ -357,6 +396,8 @@ def create_nem_xml(
     :param str transport_definition: transport file definition path
     :param str mac_definition: mac file definition path
     :param str phy_definition: phy file definition path
+    :param fabric.connection.Connection server: remote server node will run on,
+            default is None for localhost
     :return: nothing
     """
     nem_element = etree.Element("nem", name="%s NEM" % emane_model.name)
@@ -366,10 +407,16 @@ def create_nem_xml(
         etree.SubElement(nem_element, "transport", definition=transport_definition)
     etree.SubElement(nem_element, "mac", definition=mac_definition)
     etree.SubElement(nem_element, "phy", definition=phy_definition)
-    create_file(nem_element, "nem", nem_file)
+    if server is not None:
+        create_file(nem_element, "nem", nem_file, server)
+    else:
+        create_file(nem_element, "nem", nem_file)
+        for server in emane_model.session.servers:
+            conn = emane_model.session.servers[server]
+            create_file(nem_element, "nem", nem_file, conn)
 
 
-def create_event_service_xml(group, port, device, file_directory):
+def create_event_service_xml(group, port, device, file_directory, server=None):
     """
     Create a emane event service xml file.
 
@@ -377,6 +424,8 @@ def create_event_service_xml(group, port, device, file_directory):
     :param str port: event port
     :param str device: event device
     :param str file_directory: directory to create  file in
+    :param fabric.connection.Connection server: remote server node will run on,
+            default is None for localhost
     :return: nothing
     """
     event_element = etree.Element("emaneeventmsgsvc")
@@ -391,7 +440,7 @@ def create_event_service_xml(group, port, device, file_directory):
         sub_element.text = value
     file_name = "libemaneeventservice.xml"
     file_path = os.path.join(file_directory, file_name)
-    create_file(event_element, "emaneeventmsgsvc", file_path)
+    create_file(event_element, "emaneeventmsgsvc", file_path, server)
 
 
 def transport_file_name(node_id, transport_type):

daemon/examples/python/distributed_emane.py

@@ -0,0 +1,65 @@
+import logging
+import pdb
+import sys
+
+from core.emane.ieee80211abg import EmaneIeee80211abgModel
+from core.emulator.coreemu import CoreEmu
+from core.emulator.emudata import IpPrefixes, NodeOptions
+from core.emulator.enumerations import EventTypes, NodeTypes
+
+
+def main():
+    # ip generator for example
+    prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16")
+
+    # create emulator instance for creating sessions and utility methods
+    coreemu = CoreEmu()
+    session = coreemu.create_session()
+
+    # set controlnet
+    session.options.set_config(
+        "controlnet",
+        "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 "
+        "core4:172.16.4.0/24 core5:172.16.5.0/24",
+    )
+
+    # initialize distributed
+    address = sys.argv[1]
+    remote = sys.argv[2]
+    session.address = address
+    session.add_distributed(remote)
+
+    # must be in configuration state for nodes to start, when using "node_add" below
+    session.set_state(EventTypes.CONFIGURATION_STATE)
+
+    # create local node, switch, and remote nodes
+    options = NodeOptions(model="mdr")
+    options.set_position(0, 0)
+    node_one = session.add_node(node_options=options)
+    emane_net = session.add_node(_type=NodeTypes.EMANE)
+    session.emane.set_model(emane_net, EmaneIeee80211abgModel)
+    options.emulation_server = remote
+    node_two = session.add_node(node_options=options)
+
+    # create node interfaces and link
+    interface_one = prefixes.create_interface(node_one)
+    interface_two = prefixes.create_interface(node_two)
+    session.add_link(node_one.id, emane_net.id, interface_one=interface_one)
+    session.add_link(node_two.id, emane_net.id, interface_one=interface_two)
+
+    # instantiate session
+    try:
+        session.instantiate()
+    except Exception:
+        logging.exception("error during instantiate")
+
+    # pause script for verification
+    pdb.set_trace()
+
+    # shutdown session
+    coreemu.shutdown()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()