From 7aa013d351980918ab4908dcdff524285ae7ab66 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 20:38:32 -0700 Subject: [PATCH 01/38] start to wrapping commands to support remote ssh --- daemon/core/nodes/base.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index ff34984d..b655c70a 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,6 +14,8 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 +from fabric import Connection + from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes @@ -82,17 +84,23 @@ class NodeBase(object): """ raise NotImplementedError - def net_cmd(self, args): + def net_cmd(self, args, env=None): """ Runs a command that is used to configure and setup the network on the host system. :param list[str]|str args: command to run + :param dict env: environment to run command with :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - return utils.check_cmd(args) + if self.server is None: + return utils.check_cmd(args, env=env) + else: + args = " ".join(args) + result = Connection(self.server, user="root").run(args, hide=True) + return result.stderr def setposition(self, x=None, y=None, z=None): """ @@ -515,7 +523,7 @@ class CoreNode(CoreNodeBase): env["NODE_NUMBER"] = str(self.id) env["NODE_NAME"] = str(self.name) - output = utils.check_cmd(vnoded, env=env) + output = self.net_cmd(vnoded, env=env) self.pid = int(output) # create vnode client @@ -660,8 +668,8 @@ class CoreNode(CoreNodeBase): """ source = os.path.abspath(source) logging.debug("node(%s) mounting: %s at %s", self.name, source, target) - self.client.check_cmd(["mkdir", "-p", target]) - self.client.check_cmd([constants.MOUNT_BIN, "-n", "--bind", source, target]) + self.node_net_cmd(["mkdir", "-p", target]) + self.node_net_cmd([constants.MOUNT_BIN, "-n", "--bind", source, target]) self._mounts.append((source, target)) def newifindex(self): From 031517ba56c9e69cf31c1d3616d78a93c7aaaedc Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 4 Oct 2019 09:29:10 -0700 Subject: [PATCH 02/38] fixed base.py imports with isort --- daemon/core/nodes/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index b655c70a..7efad49e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,14 +14,13 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 -from fabric import Connection - from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, OvsNetClient +from fabric import Connection _DEFAULT_MTU = 1500 From f83f98262f3523a76b1fa76c6b1a415adfb65900 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 4 Oct 2019 17:33:44 -0700 Subject: [PATCH 03/38] some initial remote node commands using fabric --- daemon/core/emulator/session.py | 18 ++++++- daemon/core/nodes/base.py | 68 ++++++++++++++++++++++----- daemon/examples/python/distributed.py | 55 ++++++++++++++++++++++ 3 files changed, 127 insertions(+), 14 deletions(-) create mode 100644 daemon/examples/python/distributed.py diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 7b2a03b9..f5625232 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -14,6 +14,8 @@ import threading import time from multiprocessing.pool import ThreadPool +from fabric import Connection + from core import constants, utils from core.api.tlv import coreapi from core.api.tlv.broker import CoreBroker @@ -144,6 +146,9 @@ class Session(object): self.emane = EmaneManager(session=self) self.sdt = Sdt(session=self) + # distributed servers + self.servers = set() + # initialize default node services self.services.default_services = { "mdr": ("zebra", "OSPFv3MDR", "IPForward"), @@ -153,6 +158,11 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } + def init_distributed(self): + for server in self.servers: + cmd = "mkdir -p %s" % self.session_dir + Connection(server, user="root").run(cmd, hide=False) + @classmethod def get_node_class(cls, _type): """ @@ -683,7 +693,13 @@ class Session(object): image=node_options.image, ) else: - node = self.create_node(cls=node_class, _id=_id, name=name, start=start) + node = self.create_node( + cls=node_class, + _id=_id, + name=name, + start=start, + server=node_options.emulation_server, + ) # set node attributes node.icon = node_options.icon diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 7efad49e..b08c7cd4 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,13 +14,15 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 +from fabric import Connection + from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes +from core.errors import CoreCommandError from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, OvsNetClient -from fabric import Connection _DEFAULT_MTU = 1500 @@ -33,7 +35,7 @@ class NodeBase(object): apitype = None # TODO: appears start has no usage, verify and remove - def __init__(self, session, _id=None, name=None, start=True): + def __init__(self, session, _id=None, name=None, start=True, server=None): """ Creates a PyCoreObj instance. @@ -41,7 +43,7 @@ class NodeBase(object): :param int _id: id :param str name: object name :param bool start: start value - :return: + :param str server: remote server node will run on, default is None for localhost """ self.session = session @@ -51,8 +53,11 @@ class NodeBase(object): if name is None: name = "o%s" % self.id self.name = name + self.server = server + if self.server is not None: + self.server_conn = Connection(self.server, user="root") + self.type = None - self.server = None self.services = None # ifindex is key, CoreInterface instance is value self._netif = {} @@ -94,12 +99,23 @@ class NodeBase(object): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ + logging.info("net cmd server(%s): %s", self.server, args) if self.server is None: return utils.check_cmd(args, env=env) else: args = " ".join(args) - result = Connection(self.server, user="root").run(args, hide=True) - return result.stderr + result = self.server_conn.run(args, hide=False) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr + ) + + logging.info( + "fabric result:\n\tstdout: %s\n\tstderr: %s", + result.stdout.strip(), + result.stderr.strip(), + ) + return result.stdout.strip() def setposition(self, x=None, y=None, z=None): """ @@ -243,7 +259,7 @@ class CoreNodeBase(NodeBase): Base class for CORE nodes. """ - def __init__(self, session, _id=None, name=None, start=True): + def __init__(self, session, _id=None, name=None, start=True, server=None): """ Create a CoreNodeBase instance. @@ -251,8 +267,9 @@ class CoreNodeBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: boolean for starting + :param str server: remote server node will run on, default is None for localhost """ - super(CoreNodeBase, self).__init__(session, _id, name, start=start) + super(CoreNodeBase, self).__init__(session, _id, name, start, server) self.services = [] self.nodedir = None self.tmpnodedir = False @@ -265,7 +282,7 @@ class CoreNodeBase(NodeBase): """ if self.nodedir is None: self.nodedir = os.path.join(self.session.session_dir, self.name + ".conf") - os.makedirs(self.nodedir) + self.net_cmd(["mkdir", "-p", self.nodedir]) self.tmpnodedir = True else: self.tmpnodedir = False @@ -446,7 +463,14 @@ class CoreNode(CoreNodeBase): valid_address_types = {"inet", "inet6", "inet6link"} def __init__( - self, session, _id=None, name=None, nodedir=None, bootsh="boot.sh", start=True + self, + session, + _id=None, + name=None, + nodedir=None, + bootsh="boot.sh", + start=True, + server=None, ): """ Create a CoreNode instance. @@ -457,8 +481,9 @@ class CoreNode(CoreNodeBase): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost """ - super(CoreNode, self).__init__(session, _id, name, start) + super(CoreNode, self).__init__(session, _id, name, start, server) self.nodedir = nodedir self.ctrlchnlname = os.path.abspath( os.path.join(self.session.session_dir, self.name) @@ -619,7 +644,24 @@ class CoreNode(CoreNodeBase): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - return self.check_cmd(args) + logging.info("net cmd server(%s): %s", self.server, args) + if self.server is None: + return self.check_cmd(args) + else: + args = self.client._cmd_args() + args + args = " ".join(args) + result = self.server_conn.run(args, hide=False) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr + ) + + logging.info( + "fabric result:\n\tstdout: %s\n\tstderr: %s", + result.stdout.strip(), + result.stderr.strip(), + ) + return result.stdout.strip() def check_cmd(self, args): """ @@ -653,7 +695,7 @@ class CoreNode(CoreNodeBase): hostpath = os.path.join( self.nodedir, os.path.normpath(path).strip("/").replace("/", ".") ) - os.mkdir(hostpath) + self.net_cmd(["mkdir", "-p", hostpath]) self.mount(hostpath, path) def mount(self, source, target): diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py new file mode 100644 index 00000000..bed75a47 --- /dev/null +++ b/daemon/examples/python/distributed.py @@ -0,0 +1,55 @@ +import logging + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import NodeOptions +from core.emulator.enumerations import EventTypes + + +def main(): + # ip generator for example + # prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # initialize distributed + session.servers.add("core2") + session.init_distributed() + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create switch network node + # switch = session.add_node(_type=NodeTypes.SWITCH) + + # create nodes + options = NodeOptions() + options.emulation_server = "10.10.4.38" + options.emulation_server = "core2" + session.add_node(node_options=options) + # interface = prefixes.create_interface(node_one) + # session.add_link(node_one.id, switch.id, interface_one=interface) + + # node_two = session.add_node() + # interface = prefixes.create_interface(node_two) + # session.add_link(node_two.id, switch.id, interface_one=interface) + + # instantiate session + session.instantiate() + + # print("starting iperf server on node: %s" % node_one.name) + # node_one.cmd(["iperf", "-s", "-D"]) + # node_one_address = prefixes.ip4_address(node_one) + # + # print("node %s connecting to %s" % (node_two.name, node_one_address)) + # node_two.client.icmd(["iperf", "-t", "10", "-c", node_one_address]) + # node_one.cmd(["killall", "-9", "iperf"]) + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From 931ee65235cea91923c16d00aaa554876f22a165 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Oct 2019 09:48:30 -0700 Subject: [PATCH 04/38] added remote_cmd func for nodes to avoid duplication --- daemon/core/nodes/base.py | 44 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index b08c7cd4..8e4d33d2 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -104,18 +104,29 @@ class NodeBase(object): return utils.check_cmd(args, env=env) else: args = " ".join(args) - result = self.server_conn.run(args, hide=False) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) + return self.remote_cmd(args) - logging.info( - "fabric result:\n\tstdout: %s\n\tstderr: %s", - result.stdout.strip(), - result.stderr.strip(), + def remote_cmd(self, cmd): + """ + Run command remotely using server connection. + + :param str cmd: command to run + :return: stdout when success + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + result = self.server_conn.run(cmd, hide=False) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr ) - return result.stdout.strip() + + logging.info( + "fabric result:\n\tstdout: %s\n\tstderr: %s", + result.stdout.strip(), + result.stderr.strip(), + ) + return result.stdout.strip() def setposition(self, x=None, y=None, z=None): """ @@ -650,18 +661,7 @@ class CoreNode(CoreNodeBase): else: args = self.client._cmd_args() + args args = " ".join(args) - result = self.server_conn.run(args, hide=False) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) - - logging.info( - "fabric result:\n\tstdout: %s\n\tstderr: %s", - result.stdout.strip(), - result.stderr.strip(), - ) - return result.stdout.strip() + return self.remote_cmd(args) def check_cmd(self, args): """ From 95296988c5e92f52ed66290b1b5d2893d464cc03 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Oct 2019 11:16:57 -0700 Subject: [PATCH 05/38] updates to Pipefile.lock and for nodes to add server to constructor --- daemon/Pipfile.lock | 414 +++++++++++++++++++++++----------- daemon/core/emane/nodes.py | 4 +- daemon/core/nodes/base.py | 5 +- daemon/core/nodes/network.py | 28 ++- daemon/core/nodes/physical.py | 6 +- ns3/corens3/obj.py | 6 +- 6 files changed, 310 insertions(+), 153 deletions(-) diff --git a/daemon/Pipfile.lock b/daemon/Pipfile.lock index 1e1cf60c..73400b8b 100644 --- a/daemon/Pipfile.lock +++ b/daemon/Pipfile.lock @@ -14,6 +14,67 @@ ] }, "default": { + "asn1crypto": { + "hashes": [ + "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", + "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" + ], + "version": "==1.0.1" + }, + "bcrypt": { + "hashes": [ + "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", + "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", + "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", + "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", + "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", + "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", + "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", + "sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0", + "sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de", + "sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e", + "sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052", + "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", + "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", + "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", + "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", + "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" + ], + "version": "==3.1.7" + }, + "cffi": { + "hashes": [ + "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", + "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", + "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", + "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", + "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", + "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", + "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", + "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", + "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", + "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", + "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", + "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", + "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", + "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", + "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", + "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", + "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", + "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", + "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", + "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", + "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", + "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", + "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", + "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", + "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", + "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", + "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", + "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" + ], + "version": "==1.12.3" + }, "configparser": { "hashes": [ "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", @@ -25,14 +86,33 @@ "editable": true, "path": "." }, - "enum34": { + "cryptography": { "hashes": [ - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", + "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", + "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", + "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", + "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", + "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", + "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", + "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", + "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", + "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", + "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", + "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", + "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", + "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", + "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", + "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" ], - "version": "==1.1.6" + "version": "==2.7" + }, + "fabric": { + "hashes": [ + "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389", + "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6" + ], + "version": "==2.5.0" }, "future": { "hashes": [ @@ -42,40 +122,48 @@ }, "grpcio": { "hashes": [ - "sha256:1303578092f1f6e4bfbc354c04ac422856c393723d3ffa032fff0f7cb5cfd693", - "sha256:229c6b313cd82bec8f979b059d87f03cc1a48939b543fe170b5a9c5cf6a6bc69", - "sha256:3cd3d99a8b5568d0d186f9520c16121a0f2a4bcad8e2b9884b76fb88a85a7774", - "sha256:41cfb222db358227521f9638a6fbc397f310042a4db5539a19dea01547c621cd", - "sha256:43330501660f636fd6547d1e196e395cd1e2c2ae57d62219d6184a668ffebda0", - "sha256:45d7a2bd8b4f25a013296683f4140d636cdbb507d94a382ea5029a21e76b1648", - "sha256:47dc935658a13b25108823dabd010194ddea9610357c5c1ef1ad7b3f5157ebee", - "sha256:480aa7e2b56238badce0b9413a96d5b4c90c3bfbd79eba5a0501e92328d9669e", - "sha256:4a0934c8b0f97e1d8c18e76c45afc0d02d33ab03125258179f2ac6c7a13f3626", - "sha256:5624dab19e950f99e560400c59d87b685809e4cfcb2c724103f1ab14c06071f7", - "sha256:60515b1405bb3dadc55e6ca99429072dad3e736afcf5048db5452df5572231ff", - "sha256:610f97ebae742a57d336a69b09a9c7d7de1f62aa54aaa8adc635b38f55ba4382", - "sha256:64ea189b2b0859d1f7b411a09185028744d494ef09029630200cc892e366f169", - "sha256:686090c6c1e09e4f49585b8508d0a31d58bc3895e4049ea55b197d1381e9f70f", - "sha256:7745c365195bb0605e3d47b480a2a4d1baa8a41a5fd0a20de5fa48900e2c886a", - "sha256:79491e0d2b77a1c438116bf9e5f9e2e04e78b78524615e2ce453eff62db59a09", - "sha256:825177dd4c601c487836b7d6b4ba268db59787157911c623ba59a7c03c8d3adc", - "sha256:8a060e1f72fb94eee8a035ed29f1201ce903ad14cbe27bda56b4a22a8abda045", - "sha256:90168cc6353e2766e47b650c963f21cfff294654b10b3a14c67e26a4e3683634", - "sha256:94b7742734bceeff6d8db5edb31ac844cb68fc7f13617eca859ff1b78bb20ba1", - "sha256:962aebf2dd01bbb2cdb64580e61760f1afc470781f9ecd5fe8f3d8dcd8cf4556", - "sha256:9c8d9eacdce840b72eee7924c752c31b675f8aec74790e08cff184a4ea8aa9c1", - "sha256:af5b929debc336f6bab9b0da6915f9ee5e41444012aed6a79a3c7e80d7662fdf", - "sha256:b9cdb87fc77e9a3eabdc42a512368538d648fa0760ad30cf97788076985c790a", - "sha256:c5e6380b90b389454669dc67d0a39fb4dc166416e01308fcddd694236b8329ef", - "sha256:d60c90fe2bfbee735397bf75a2f2c4e70c5deab51cd40c6e4fa98fae018c8db6", - "sha256:d8582c8b1b1063249da1588854251d8a91df1e210a328aeb0ece39da2b2b763b", - "sha256:ddbf86ba3aa0ad8fed2867910d2913ee237d55920b55f1d619049b3399f04efc", - "sha256:e46bc0664c5c8a0545857aa7a096289f8db148e7f9cca2d0b760113e8994bddc", - "sha256:f6437f70ec7fed0ca3a0eef1146591bb754b418bb6c6b21db74f0333d624e135", - "sha256:f71693c3396530c6b00773b029ea85e59272557e9bd6077195a6593e4229892a", - "sha256:f79f7455f8fbd43e8e9d61914ecf7f48ba1c8e271801996fef8d6a8f3cc9f39f" + "sha256:0302331e014fc4bac028b6ad480b33f7abfe20b9bdcca7be417124dda8f22115", + "sha256:0aa0cce9c5eb1261b32173a20ed42b51308d55ce28ecc2021e868b3cb90d9503", + "sha256:0c83947575300499adbc308e986d754e7f629be0bdd9bea1ffdd5cf76e1f1eff", + "sha256:0ca26ff968d45efd4ef73447c4d4b34322ea8c7d06fbb6907ce9e5db78f1bbcb", + "sha256:0cf80a7955760c2498f8821880242bb657d70998065ff0d2a082de5ffce230a7", + "sha256:0d40706e57d9833fe0e023a08b468f33940e8909affa12547874216d36bba208", + "sha256:11872069156de34c6f3f9a1deb46cc88bc35dfde88262c4c73eb22b39b16fc55", + "sha256:16065227faae0ab0abf1789bfb92a2cd2ab5da87630663f93f8178026da40e0d", + "sha256:1e33778277685f6fabb22539136269c87c029e39b6321ef1a639b756a1c0a408", + "sha256:2b16be15b1ae656bc7a36642b8c7045be2dde2048bb4b67478003e9d9db8022a", + "sha256:3701dfca3ada27ceef0d17f728ce9dfef155ed20c57979c2b05083082258c6c1", + "sha256:41912ecaf482abf2de74c69f509878f99223f5dd6b2de1a09c955afd4de3cf9b", + "sha256:4332cbd20544fe7406910137590f38b5b3a1f6170258e038652cf478c639430f", + "sha256:44068ecbdc6467c2bff4d8198816c8a2701b6dd1ec16078fceb6adc7c1f577d6", + "sha256:53115960e37059420e2d16a4b04b00dd2ab3b6c3c67babd01ffbfdcd7881a69b", + "sha256:6e7027bcd4070414751e2a5e60706facb98a1fc636497c9bac5442fe37b8ae6b", + "sha256:6ff57fb2f07b7226b5bec89e8e921ea9bd220f35f11e094f2ba38f09eecd49c6", + "sha256:73240e244d7644654bbda1f309f4911748b6a1804b7a8897ddbe8a04c90f7407", + "sha256:785234bbc469bc75e26c868789a2080ffb30bd6e93930167797729889ad06b0b", + "sha256:82f9d3c7f91d2d1885631335c003c5d45ae1cd69cc0bc4893f21fef50b8151bc", + "sha256:86bdc2a965510658407a1372eb61f0c92f763fdfb2795e4d038944da4320c950", + "sha256:95e925b56676a55e6282b3de80a1cbad5774072159779c61eac02791dface049", + "sha256:96673bb4f14bd3263613526d1e7e33fdb38a9130e3ce87bf52314965706e1900", + "sha256:970014205e76920484679035b6fb4b16e02fc977e5aac4d22025da849c79dab9", + "sha256:ace5e8bf11a1571f855f5dab38a9bd34109b6c9bc2864abf24a597598c7e3695", + "sha256:ad375f03eb3b9cb75a24d91eab8609e134d34605f199efc41e20dd642bdac855", + "sha256:b819c4c7dcf0de76788ce5f95daad6d4e753d6da2b6a5f84e5bb5b5ce95fddc4", + "sha256:c17943fd340cbd906db49f3f03c7545e5a66b617e8348b2c7a0d2c759d216af1", + "sha256:d21247150dea86dabd3b628d8bc4b563036db3d332b3f4db3c5b1b0b122cb4f6", + "sha256:d4d500a7221116de9767229ff5dd10db91f789448d85befb0adf5a37b0cd83b5", + "sha256:e2a942a3cfccbbca21a90c144867112698ef36486345c285da9e98c466f22b22", + "sha256:e983273dca91cb8a5043bc88322eb48e2b8d4e4998ff441a1ee79ced89db3909" ], - "version": "==1.23.0" + "version": "==1.24.1" + }, + "invoke": { + "hashes": [ + "sha256:c52274d2e8a6d64ef0d61093e1983268ea1fc0cd13facb9448c4ef0c9a7ac7da", + "sha256:f4ec8a134c0122ea042c8912529f87652445d9f4de590b353d23f95bfa1f0efd", + "sha256:fc803a5c9052f15e63310aa81a43498d7c55542beb18564db88a9d75a176fa44" + ], + "version": "==1.3.0" }, "lxml": { "hashes": [ @@ -104,6 +192,64 @@ ], "version": "==4.4.1" }, + "paramiko": { + "hashes": [ + "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", + "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" + ], + "version": "==2.6.0" + }, + "protobuf": { + "hashes": [ + "sha256:125713564d8cfed7610e52444c9769b8dcb0b55e25cc7841f2290ee7bc86636f", + "sha256:1accdb7a47e51503be64d9a57543964ba674edac103215576399d2d0e34eac77", + "sha256:27003d12d4f68e3cbea9eb67427cab3bfddd47ff90670cb367fcd7a3a89b9657", + "sha256:3264f3c431a631b0b31e9db2ae8c927b79fc1a7b1b06b31e8e5bcf2af91fe896", + "sha256:3c5ab0f5c71ca5af27143e60613729e3488bb45f6d3f143dc918a20af8bab0bf", + "sha256:45dcf8758873e3f69feab075e5f3177270739f146255225474ee0b90429adef6", + "sha256:56a77d61a91186cc5676d8e11b36a5feb513873e4ae88d2ee5cf530d52bbcd3b", + "sha256:5984e4947bbcef5bd849d6244aec507d31786f2dd3344139adc1489fb403b300", + "sha256:6b0441da73796dd00821763bb4119674eaf252776beb50ae3883bed179a60b2a", + "sha256:6f6677c5ade94d4fe75a912926d6796d5c71a2a90c2aeefe0d6f211d75c74789", + "sha256:84a825a9418d7196e2acc48f8746cf1ee75877ed2f30433ab92a133f3eaf8fbe", + "sha256:b842c34fe043ccf78b4a6cf1019d7b80113707d68c88842d061fa2b8fb6ddedc", + "sha256:ca33d2f09dae149a1dcf942d2d825ebb06343b77b437198c9e2ef115cf5d5bc1", + "sha256:db83b5c12c0cd30150bb568e6feb2435c49ce4e68fe2d7b903113f0e221e58fe", + "sha256:f50f3b1c5c1c1334ca7ce9cad5992f098f460ffd6388a3cabad10b66c2006b09", + "sha256:f99f127909731cafb841c52f9216e447d3e4afb99b17bebfad327a75aee206de" + ], + "version": "==3.10.0" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "pynacl": { + "hashes": [ + "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", + "sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c", + "sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e", + "sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae", + "sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621", + "sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56", + "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", + "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", + "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", + "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", + "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", + "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", + "sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b", + "sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f", + "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", + "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", + "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", + "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", + "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" + ], + "version": "==1.3.0" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -136,10 +282,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", + "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" ], - "version": "==19.1.0" + "version": "==19.2.0" }, "black": { "hashes": [ @@ -180,78 +326,78 @@ }, "grpcio": { "hashes": [ - "sha256:1303578092f1f6e4bfbc354c04ac422856c393723d3ffa032fff0f7cb5cfd693", - "sha256:229c6b313cd82bec8f979b059d87f03cc1a48939b543fe170b5a9c5cf6a6bc69", - "sha256:3cd3d99a8b5568d0d186f9520c16121a0f2a4bcad8e2b9884b76fb88a85a7774", - "sha256:41cfb222db358227521f9638a6fbc397f310042a4db5539a19dea01547c621cd", - "sha256:43330501660f636fd6547d1e196e395cd1e2c2ae57d62219d6184a668ffebda0", - "sha256:45d7a2bd8b4f25a013296683f4140d636cdbb507d94a382ea5029a21e76b1648", - "sha256:47dc935658a13b25108823dabd010194ddea9610357c5c1ef1ad7b3f5157ebee", - "sha256:480aa7e2b56238badce0b9413a96d5b4c90c3bfbd79eba5a0501e92328d9669e", - "sha256:4a0934c8b0f97e1d8c18e76c45afc0d02d33ab03125258179f2ac6c7a13f3626", - "sha256:5624dab19e950f99e560400c59d87b685809e4cfcb2c724103f1ab14c06071f7", - "sha256:60515b1405bb3dadc55e6ca99429072dad3e736afcf5048db5452df5572231ff", - "sha256:610f97ebae742a57d336a69b09a9c7d7de1f62aa54aaa8adc635b38f55ba4382", - "sha256:64ea189b2b0859d1f7b411a09185028744d494ef09029630200cc892e366f169", - "sha256:686090c6c1e09e4f49585b8508d0a31d58bc3895e4049ea55b197d1381e9f70f", - "sha256:7745c365195bb0605e3d47b480a2a4d1baa8a41a5fd0a20de5fa48900e2c886a", - "sha256:79491e0d2b77a1c438116bf9e5f9e2e04e78b78524615e2ce453eff62db59a09", - "sha256:825177dd4c601c487836b7d6b4ba268db59787157911c623ba59a7c03c8d3adc", - "sha256:8a060e1f72fb94eee8a035ed29f1201ce903ad14cbe27bda56b4a22a8abda045", - "sha256:90168cc6353e2766e47b650c963f21cfff294654b10b3a14c67e26a4e3683634", - "sha256:94b7742734bceeff6d8db5edb31ac844cb68fc7f13617eca859ff1b78bb20ba1", - "sha256:962aebf2dd01bbb2cdb64580e61760f1afc470781f9ecd5fe8f3d8dcd8cf4556", - "sha256:9c8d9eacdce840b72eee7924c752c31b675f8aec74790e08cff184a4ea8aa9c1", - "sha256:af5b929debc336f6bab9b0da6915f9ee5e41444012aed6a79a3c7e80d7662fdf", - "sha256:b9cdb87fc77e9a3eabdc42a512368538d648fa0760ad30cf97788076985c790a", - "sha256:c5e6380b90b389454669dc67d0a39fb4dc166416e01308fcddd694236b8329ef", - "sha256:d60c90fe2bfbee735397bf75a2f2c4e70c5deab51cd40c6e4fa98fae018c8db6", - "sha256:d8582c8b1b1063249da1588854251d8a91df1e210a328aeb0ece39da2b2b763b", - "sha256:ddbf86ba3aa0ad8fed2867910d2913ee237d55920b55f1d619049b3399f04efc", - "sha256:e46bc0664c5c8a0545857aa7a096289f8db148e7f9cca2d0b760113e8994bddc", - "sha256:f6437f70ec7fed0ca3a0eef1146591bb754b418bb6c6b21db74f0333d624e135", - "sha256:f71693c3396530c6b00773b029ea85e59272557e9bd6077195a6593e4229892a", - "sha256:f79f7455f8fbd43e8e9d61914ecf7f48ba1c8e271801996fef8d6a8f3cc9f39f" + "sha256:0302331e014fc4bac028b6ad480b33f7abfe20b9bdcca7be417124dda8f22115", + "sha256:0aa0cce9c5eb1261b32173a20ed42b51308d55ce28ecc2021e868b3cb90d9503", + "sha256:0c83947575300499adbc308e986d754e7f629be0bdd9bea1ffdd5cf76e1f1eff", + "sha256:0ca26ff968d45efd4ef73447c4d4b34322ea8c7d06fbb6907ce9e5db78f1bbcb", + "sha256:0cf80a7955760c2498f8821880242bb657d70998065ff0d2a082de5ffce230a7", + "sha256:0d40706e57d9833fe0e023a08b468f33940e8909affa12547874216d36bba208", + "sha256:11872069156de34c6f3f9a1deb46cc88bc35dfde88262c4c73eb22b39b16fc55", + "sha256:16065227faae0ab0abf1789bfb92a2cd2ab5da87630663f93f8178026da40e0d", + "sha256:1e33778277685f6fabb22539136269c87c029e39b6321ef1a639b756a1c0a408", + "sha256:2b16be15b1ae656bc7a36642b8c7045be2dde2048bb4b67478003e9d9db8022a", + "sha256:3701dfca3ada27ceef0d17f728ce9dfef155ed20c57979c2b05083082258c6c1", + "sha256:41912ecaf482abf2de74c69f509878f99223f5dd6b2de1a09c955afd4de3cf9b", + "sha256:4332cbd20544fe7406910137590f38b5b3a1f6170258e038652cf478c639430f", + "sha256:44068ecbdc6467c2bff4d8198816c8a2701b6dd1ec16078fceb6adc7c1f577d6", + "sha256:53115960e37059420e2d16a4b04b00dd2ab3b6c3c67babd01ffbfdcd7881a69b", + "sha256:6e7027bcd4070414751e2a5e60706facb98a1fc636497c9bac5442fe37b8ae6b", + "sha256:6ff57fb2f07b7226b5bec89e8e921ea9bd220f35f11e094f2ba38f09eecd49c6", + "sha256:73240e244d7644654bbda1f309f4911748b6a1804b7a8897ddbe8a04c90f7407", + "sha256:785234bbc469bc75e26c868789a2080ffb30bd6e93930167797729889ad06b0b", + "sha256:82f9d3c7f91d2d1885631335c003c5d45ae1cd69cc0bc4893f21fef50b8151bc", + "sha256:86bdc2a965510658407a1372eb61f0c92f763fdfb2795e4d038944da4320c950", + "sha256:95e925b56676a55e6282b3de80a1cbad5774072159779c61eac02791dface049", + "sha256:96673bb4f14bd3263613526d1e7e33fdb38a9130e3ce87bf52314965706e1900", + "sha256:970014205e76920484679035b6fb4b16e02fc977e5aac4d22025da849c79dab9", + "sha256:ace5e8bf11a1571f855f5dab38a9bd34109b6c9bc2864abf24a597598c7e3695", + "sha256:ad375f03eb3b9cb75a24d91eab8609e134d34605f199efc41e20dd642bdac855", + "sha256:b819c4c7dcf0de76788ce5f95daad6d4e753d6da2b6a5f84e5bb5b5ce95fddc4", + "sha256:c17943fd340cbd906db49f3f03c7545e5a66b617e8348b2c7a0d2c759d216af1", + "sha256:d21247150dea86dabd3b628d8bc4b563036db3d332b3f4db3c5b1b0b122cb4f6", + "sha256:d4d500a7221116de9767229ff5dd10db91f789448d85befb0adf5a37b0cd83b5", + "sha256:e2a942a3cfccbbca21a90c144867112698ef36486345c285da9e98c466f22b22", + "sha256:e983273dca91cb8a5043bc88322eb48e2b8d4e4998ff441a1ee79ced89db3909" ], - "version": "==1.23.0" + "version": "==1.24.1" }, "grpcio-tools": { "hashes": [ - "sha256:056f2a274edda4315e825ac2e3a9536f5415b43aa51669196860c8de6e76d847", - "sha256:0c953251585fdcd422072e4b7f4243fce215f22e21db94ec83c5970e41db6e18", - "sha256:142a73f5769f37bf2e4a8e4a77ef60f7af5f55635f60428322b49c87bd8f9cc0", - "sha256:1b333e2a068d8ef89a01eb23a098d2a789659c3178de79da9bd3d0ffb944cc6d", - "sha256:2124f19cc51d63405a0204ae38ef355732ab0a235975ab41ff6f6f9701905432", - "sha256:24c3a04adfb6c6f1bc4a2f8498d7661ca296ae352b498e538832c22ddde7bf81", - "sha256:3a2054e9640cbdd0ce8a345afb86be52875c5a8f9f5973a5c64791a8002da2dd", - "sha256:3fd15a09eecef83440ac849dcda2ff522f8ee1603ebfcdbb0e9b320ef2012e41", - "sha256:457e7a7dfa0b6bb608a766edba6f20c9d626a790df802016b930ad242fec4470", - "sha256:49ad5661d54ff0d164e4b441ee5e05191187d497380afa16d36d72eb8ef048de", - "sha256:561078e425d21a6720c3c3828385d949e24c0765e2852a46ecc3ad3fca2706e5", - "sha256:5a4f65ab06b32dc34112ed114dee3b698c8463670474334ece5b0b531073804c", - "sha256:8883e0e34676356d219a4cd37d239c3ead655cc550836236b52068e091416fff", - "sha256:8d2b45b1faf81098780e07d6a1c207b328b07e913160b8baa7e2e8d89723e06c", - "sha256:b0ebddb6ecc4c161369f93bb3a74c6120a498d3ddc759b64679709a885dd6d4f", - "sha256:b786ba4842c50de865dd3885b5570690a743e84a327b7213dd440eb0e6b996f8", - "sha256:be8efa010f5a80f1862ead80c3b19b5eb97dc954a0f59a1e2487078576105e03", - "sha256:c29106eaff0e2e708a9a89628dc0134ef145d0d3631f0ef421c09f380c30e354", - "sha256:c3c71236a056ec961b2b8b3b7c0b3b5a826283bc69c4a1c6415d23b70fea8243", - "sha256:cbc35031ec2b29af36947d085a7fbbcd8b79b84d563adf6156103d82565f78db", - "sha256:d47307c22744918e803c1eec7263a14f36aaf34fe496bff9ccbcae67c02b40ae", - "sha256:db088c98e563c1bb070da5984c8df08b45b61e4d9c6d2a8a1ffeed2af89fd1f3", - "sha256:df4dd1cb670062abdacc1fbce41cae4e08a4a212d28dd94fdbbf90615d027f73", - "sha256:e3adcf1499ca08d1e036ff44aedf55ed78653d946f4c4426b6e72ab757cc4dec", - "sha256:e3b3e32e0cda4dc382ec5bed8599dab644e4b3fc66a9ab54eb58248e207880b9", - "sha256:ed524195b35304c670755efa1eca579e5c290a66981f97004a5b2c0d12d6897d", - "sha256:edb42432790b1f8ec9f08faf9326d7e5dfe6e1d8c8fe4db39abc0a49c1c76537", - "sha256:eff1f995e5aa4cc941b6bbc45b5b57842f8f62bbe1a99427281c2c70cf42312c", - "sha256:f2fcdc2669662d77b400f80e20315a3661466e3cb3df1730f8083f9e49465cbc", - "sha256:f52ec9926daf48f41389d39d01570967b99c7dbc12bffc134cc3a3c5b5540ba2", - "sha256:fd007d67fdfbd2a13bf8a8c8ced8353b42a92ca72dbee54e951d8ddbc6ca12bc", - "sha256:ff9045e928dbb7943ea8559bfabebee95a43a830e00bf52c16202d2d805780fb" + "sha256:0a849994d7d6411ca6147bb1db042b61ba6232eb5c90c69de5380a441bf80a75", + "sha256:0db96ed52816471ceec8807aedf5cb4fd133ca201f614464cb46ca58584edf84", + "sha256:1b98720459204e9afa33928e4fd53aeec6598afb7f704ed497f6926c67f12b9b", + "sha256:200479310cc083c41a5020f6e5e916a99ee0f7c588b6affe317b96a839120bf4", + "sha256:25543b8f2e59ddcc9929d6f6111faa5c474b21580d2996f93347bb55f2ecba84", + "sha256:2d4609996616114c155c1e697a9faf604d81f2508cd9a4168a0bafd53c799e24", + "sha256:2fdb2a1ed2b3e43514d9c29c9de415c953a46caabbc8a9b7de1439a0c1bd3b89", + "sha256:3886a7983d8ae19df0c11a54114d6546fcdf76cf18cdccf25c3b14200fd5478a", + "sha256:408d111b9341f107bdafc523e2345471547ffe8a4104e6f2ce690b7a25c4bae5", + "sha256:60b3dd5e76c1389fc836bf83675985b92d158ff9a8d3d6d3f0a670f0c227ef13", + "sha256:629be7ce8504530b4adbf0425a44dd53007ccb6212344804294888c9662cc38f", + "sha256:6af3dde07b1051e954230e650a6ef74073cf993cf473c2078580f8a73c4fe46a", + "sha256:7a1e77539d28e90517c55561f40f7872f1348d0e23f25a38d68abbfb5b0eff88", + "sha256:87917a18b3b5951b6c9badd7b5ef09f63f61611966b58427b856bdf5c1d68e91", + "sha256:8823d0ebd185a77edb506e286c88d06847f75620a033ad96ef9c0fd7efc1d859", + "sha256:8bd3e12e1969beb813b861a2a65d4f2d4faaa87de0b60bf7f848da2d8ffc4eb2", + "sha256:8f37e9acc46e75ed9786ece89afeacd86182893eacc3f0642d81531b90fbe25f", + "sha256:9b358dd2f4142e89d760a52a7a8f4ec5dbaf955e7ada09f703f3a5d05dddd12e", + "sha256:9cb43007c4a8aa7adaacf896f5109b578028f23d259615e3fa5866e38855b311", + "sha256:9cf594bfbfbf84dcd462b20a4a753362be7ed376d2b5020a083dac24400b7b6c", + "sha256:ab79940e5c5ed949e1f95e7f417dd916b0992d29f45d073dd64501a76d128e2c", + "sha256:ba8aab6c78a82755477bb8c79f3be0824b297422d1edb21b94ae5a45407bf3ba", + "sha256:bcc00b83bf39f6e60a13f0b24ec3951f4d2ae810b01e6e125b7ff238a85da1ac", + "sha256:c1fcf5cbe6a2ecdc587b469156520b9128ccdb7c5908060c7d9712cd97e76db5", + "sha256:c6e640d39b9615388b59036b29970292b15f4519043e43833e28c674f740d1f7", + "sha256:c6ea2c385da620049b17f0135cf9307a4750e9d9c9988e15bfeeaf1f209c4ada", + "sha256:cec4f37120f93fe2ab4ab9a7eab9a877163d74c232c93a275a624971f8557b81", + "sha256:d2dbb42d237bcdecb7284535ec074c85bbf880124c1cbbff362ed3bd81ed7d41", + "sha256:d5c98a41abd4f7de43b256c21bbba2a97c57e25bf6a170927a90638b18f7509c", + "sha256:dcf5965a24179aa7dcfa00b5ff70f4f2f202e663657e0c74a642307beecda053", + "sha256:e11e3aacf0200d6e00a9b74534e0174738768fe1c41e5aa2f4aab881d6b43afd", + "sha256:e550816bdb2e49bba94bcd7f342004a8adbc46e9a25c8c4ed3fd58f2435c655f" ], "index": "pypi", - "version": "==1.23.0" + "version": "==1.24.1" }, "identify": { "hashes": [ @@ -262,11 +408,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:652234b6ab8f2506ae58e528b6fbcc668831d3cc758e1bc01ef438d328b68cdb", - "sha256:6f264986fb88042bc1f0535fa9a557e6a376cfe5679dc77caac7fe8b5d43d05f" + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" ], "markers": "python_version < '3.8'", - "version": "==0.22" + "version": "==0.23" }, "importlib-resources": { "hashes": [ @@ -314,10 +460,10 @@ }, "packaging": { "hashes": [ - "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", - "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "version": "==19.1" + "version": "==19.2" }, "pluggy": { "hashes": [ @@ -336,24 +482,24 @@ }, "protobuf": { "hashes": [ - "sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295", - "sha256:01acbca2d2c8c3f7f235f1842440adbe01bbc379fa1cbdd80753801432b3fae9", - "sha256:0a795bca65987b62d6b8a2d934aa317fd1a4d06a6dd4df36312f5b0ade44a8d9", - "sha256:0ec035114213b6d6e7713987a759d762dd94e9f82284515b3b7331f34bfaec7f", - "sha256:31b18e1434b4907cb0113e7a372cd4d92c047ce7ba0fa7ea66a404d6388ed2c1", - "sha256:32a3abf79b0bef073c70656e86d5bd68a28a1fbb138429912c4fc07b9d426b07", - "sha256:55f85b7808766e5e3f526818f5e2aeb5ba2edcc45bcccede46a3ccc19b569cb0", - "sha256:64ab9bc971989cbdd648c102a96253fdf0202b0c38f15bd34759a8707bdd5f64", - "sha256:64cf847e843a465b6c1ba90fb6c7f7844d54dbe9eb731e86a60981d03f5b2e6e", - "sha256:917c8662b585470e8fd42f052661fc66d59fccaae450a60044307dcbf82a3335", - "sha256:afed9003d7f2be2c3df20f64220c30faec441073731511728a2cb4cab4cd46a6", - "sha256:bf8e05d638b585d1752c5a84247134a0350d3a8b73d3632489a014a9f6f1e758", - "sha256:d831b047bd69becaf64019a47179eb22118a50dd008340655266a906c69c6417", - "sha256:de2760583ed28749ff885789c1cbc6c9c06d6de92fc825740ab99deb2f25ea4d", - "sha256:eabc4cf1bc19689af8022ba52fd668564a8d96e0d08f3b4732d26a64255216a4", - "sha256:fcff6086c86fb1628d94ea455c7b9de898afc50378042927a59df8065a79a549" + "sha256:125713564d8cfed7610e52444c9769b8dcb0b55e25cc7841f2290ee7bc86636f", + "sha256:1accdb7a47e51503be64d9a57543964ba674edac103215576399d2d0e34eac77", + "sha256:27003d12d4f68e3cbea9eb67427cab3bfddd47ff90670cb367fcd7a3a89b9657", + "sha256:3264f3c431a631b0b31e9db2ae8c927b79fc1a7b1b06b31e8e5bcf2af91fe896", + "sha256:3c5ab0f5c71ca5af27143e60613729e3488bb45f6d3f143dc918a20af8bab0bf", + "sha256:45dcf8758873e3f69feab075e5f3177270739f146255225474ee0b90429adef6", + "sha256:56a77d61a91186cc5676d8e11b36a5feb513873e4ae88d2ee5cf530d52bbcd3b", + "sha256:5984e4947bbcef5bd849d6244aec507d31786f2dd3344139adc1489fb403b300", + "sha256:6b0441da73796dd00821763bb4119674eaf252776beb50ae3883bed179a60b2a", + "sha256:6f6677c5ade94d4fe75a912926d6796d5c71a2a90c2aeefe0d6f211d75c74789", + "sha256:84a825a9418d7196e2acc48f8746cf1ee75877ed2f30433ab92a133f3eaf8fbe", + "sha256:b842c34fe043ccf78b4a6cf1019d7b80113707d68c88842d061fa2b8fb6ddedc", + "sha256:ca33d2f09dae149a1dcf942d2d825ebb06343b77b437198c9e2ef115cf5d5bc1", + "sha256:db83b5c12c0cd30150bb568e6feb2435c49ce4e68fe2d7b903113f0e221e58fe", + "sha256:f50f3b1c5c1c1334ca7ce9cad5992f098f460ffd6388a3cabad10b66c2006b09", + "sha256:f99f127909731cafb841c52f9216e447d3e4afb99b17bebfad327a75aee206de" ], - "version": "==3.9.1" + "version": "==3.10.0" }, "py": { "hashes": [ @@ -385,11 +531,11 @@ }, "pytest": { "hashes": [ - "sha256:95d13143cc14174ca1a01ec68e84d76ba5d9d493ac02716fd9706c949a505210", - "sha256:b78fe2881323bd44fd9bd76e5317173d4316577e7b1cddebae9136a4495ec865" + "sha256:13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5", + "sha256:d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.2.0" }, "pyyaml": { "hashes": [ diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 89c97b6b..5451506f 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -29,8 +29,8 @@ class EmaneNet(CoreNetworkBase): type = "wlan" is_emane = True - def __init__(self, session, _id=None, name=None, start=True): - super(EmaneNet, self).__init__(session, _id, name, start) + def __init__(self, session, _id=None, name=None, start=True, server=None): + super(EmaneNet, self).__init__(session, _id, name, start, server) self.conf = "" self.up = False self.nemidmap = {} diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 8e4d33d2..d0f03e6e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -1045,7 +1045,7 @@ class CoreNetworkBase(NodeBase): linktype = LinkTypes.WIRED.value is_emane = False - def __init__(self, session, _id, name, start=True): + def __init__(self, session, _id, name, start=True, server=None): """ Create a CoreNetworkBase instance. @@ -1053,8 +1053,9 @@ class CoreNetworkBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: should object start + :param str server: remote server node will run on, default is None for localhost """ - super(CoreNetworkBase, self).__init__(session, _id, name, start=start) + super(CoreNetworkBase, self).__init__(session, _id, name, start, server) self._linked = {} self._linked_lock = threading.Lock() diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 6af1ed9e..444ded56 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -281,7 +281,9 @@ class CoreNetwork(CoreNetworkBase): policy = "DROP" - def __init__(self, session, _id=None, name=None, start=True, policy=None): + def __init__( + self, session, _id=None, name=None, start=True, server=None, policy=None + ): """ Creates a LxBrNet instance. @@ -289,9 +291,10 @@ class CoreNetwork(CoreNetworkBase): :param int _id: object id :param str name: object name :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :param policy: network policy """ - CoreNetworkBase.__init__(self, session, _id, name, start) + CoreNetworkBase.__init__(self, session, _id, name, start, server) if name is None: name = str(self.id) if policy is not None: @@ -649,6 +652,7 @@ class GreTapBridge(CoreNetwork): ttl=255, key=None, start=True, + server=None, ): """ Create a GreTapBridge instance. @@ -663,9 +667,7 @@ class GreTapBridge(CoreNetwork): :param key: gre tap key :param bool start: start flag """ - CoreNetwork.__init__( - self, session=session, _id=_id, name=name, policy=policy, start=False - ) + CoreNetwork.__init__(self, session, _id, name, False, server, policy) self.grekey = key if self.grekey is None: self.grekey = self.session.id ^ self.id @@ -769,6 +771,7 @@ class CtrlNet(CoreNetwork): prefix=None, hostid=None, start=True, + server=None, assign_address=True, updown_script=None, serverintf=None, @@ -782,6 +785,7 @@ class CtrlNet(CoreNetwork): :param prefix: control network ipv4 prefix :param hostid: host id :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :param str assign_address: assigned address :param str updown_script: updown script :param serverintf: server interface @@ -792,7 +796,7 @@ class CtrlNet(CoreNetwork): self.assign_address = assign_address self.updown_script = updown_script self.serverintf = serverintf - CoreNetwork.__init__(self, session, _id=_id, name=name, start=start) + CoreNetwork.__init__(self, session, _id, name, start, server) def startup(self): """ @@ -1028,7 +1032,7 @@ class HubNode(CoreNetwork): policy = "ACCEPT" type = "hub" - def __init__(self, session, _id=None, name=None, start=True): + def __init__(self, session, _id=None, name=None, start=True, server=None): """ Creates a HubNode instance. @@ -1036,9 +1040,10 @@ class HubNode(CoreNetwork): :param int _id: node id :param str name: node namee :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ - CoreNetwork.__init__(self, session, _id, name, start) + CoreNetwork.__init__(self, session, _id, name, start, server) # TODO: move to startup method if start: @@ -1055,7 +1060,9 @@ class WlanNode(CoreNetwork): policy = "DROP" type = "wlan" - def __init__(self, session, _id=None, name=None, start=True, policy=None): + def __init__( + self, session, _id=None, name=None, start=True, server=None, policy=None + ): """ Create a WlanNode instance. @@ -1063,9 +1070,10 @@ class WlanNode(CoreNetwork): :param int _id: node id :param str name: node name :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :param policy: wlan policy """ - CoreNetwork.__init__(self, session, _id, name, start, policy) + CoreNetwork.__init__(self, session, _id, name, start, server, policy) # wireless model such as basic range self.model = None # mobility model such as scripted diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 0035f97a..ba8cdc25 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -277,7 +277,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): apitype = NodeTypes.RJ45.value type = "rj45" - def __init__(self, session, _id=None, name=None, mtu=1500, start=True): + def __init__(self, session, _id=None, name=None, mtu=1500, start=True, server=None): """ Create an RJ45Node instance. @@ -286,9 +286,9 @@ class Rj45Node(CoreNodeBase, CoreInterface): :param str name: node name :param mtu: rj45 mtu :param bool start: start flag - :return: + :param str server: remote server node will run on, default is None for localhost """ - CoreNodeBase.__init__(self, session, _id, name, start=start) + CoreNodeBase.__init__(self, session, _id, name, start, server) CoreInterface.__init__(self, node=self, name=name, mtu=mtu) self.up = False self.lock = threading.RLock() diff --git a/ns3/corens3/obj.py b/ns3/corens3/obj.py index 70291d3b..c1907f03 100644 --- a/ns3/corens3/obj.py +++ b/ns3/corens3/obj.py @@ -117,8 +117,10 @@ class CoreNs3Net(CoreNetworkBase): # icon used type = "wlan" - def __init__(self, session, _id=None, name=None, start=True, policy=None): - CoreNetworkBase.__init__(self, session, _id, name) + def __init__( + self, session, _id=None, name=None, start=True, server=None, policy=None + ): + CoreNetworkBase.__init__(self, session, _id, name, start, server) self.tapbridge = ns.tap_bridge.TapBridgeHelper() self._ns3devs = {} self._tapdevs = {} From cca57bba47fc8312f85afe2774e0500aaa809f9b Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Oct 2019 16:10:01 -0700 Subject: [PATCH 06/38] updated other node system commands to be ran in such a way that should work if local or remote using shell commands --- daemon/core/nodes/base.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d0f03e6e..848fae34 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -2,12 +2,9 @@ Defines the base logic for nodes used within core. """ -import errno import logging import os import random -import shutil -import signal import socket import string import threading @@ -309,7 +306,7 @@ class CoreNodeBase(NodeBase): return if self.tmpnodedir: - shutil.rmtree(self.nodedir, ignore_errors=True) + self.net_cmd(["rm", "-rf", self.nodedir]) def addnetif(self, netif, ifindex): """ @@ -522,8 +519,8 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - os.kill(self.pid, 0) - except OSError: + self.net_cmd(["kill", "-9", str(self.pid)]) + except CoreCommandError: return False return True @@ -560,6 +557,7 @@ class CoreNode(CoreNodeBase): output = self.net_cmd(vnoded, env=env) self.pid = int(output) + logging.debug("node(%s) pid: %s", self.name, self.pid) # create vnode client self.client = client.VnodeClient(self.name, self.ctrlchnlname) @@ -599,21 +597,17 @@ class CoreNode(CoreNodeBase): for netif in self.netifs(): netif.shutdown() - # attempt to kill node process and wait for termination of children + # kill node process if present try: - os.kill(self.pid, signal.SIGTERM) - os.waitpid(self.pid, 0) - except OSError as e: - if e.errno != 10: - logging.exception("error killing process") + self.net_cmd(["kill", "-9", str(self.pid)]) + except CoreCommandError: + logging.exception("error killing process") # remove node directory if present try: - os.unlink(self.ctrlchnlname) - except OSError as e: - # no such file or directory - if e.errno != errno.ENOENT: - logging.exception("error removing node directory") + self.net_cmd(["rm", "-rf", self.ctrlchnlname]) + except CoreCommandError: + logging.exception("error removing node directory") # clear interface data, close client, and mark self and not up self._netif.clear() @@ -1029,9 +1023,9 @@ class CoreNode(CoreNodeBase): :return: nothing """ hostfilename = self.hostfilename(filename) - shutil.copy2(srcfilename, hostfilename) + self.net_cmd(["cp", "-a", srcfilename, hostfilename]) if mode is not None: - os.chmod(hostfilename, mode) + self.net_cmd(["chmod", oct(mode), hostfilename]) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) From 4eacd815d13b9bf4a93054e744d13f2026bd66fc Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sun, 6 Oct 2019 00:06:29 -0700 Subject: [PATCH 07/38] updated to use fabric scp for copying files to remote nodes --- daemon/core/nodes/base.py | 64 ++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 848fae34..901d08a6 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -5,11 +5,13 @@ Defines the base logic for nodes used within core. import logging import os import random +import shutil import socket import string import threading from builtins import range from socket import AF_INET, AF_INET6 +from tempfile import NamedTemporaryFile from fabric import Connection @@ -519,7 +521,7 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - self.net_cmd(["kill", "-9", str(self.pid)]) + self.net_cmd(["kill", "-0", str(self.pid)]) except CoreCommandError: return False @@ -961,9 +963,13 @@ class CoreNode(CoreNodeBase): """ logging.info("adding file from %s to %s", srcname, filename) directory = os.path.dirname(filename) - self.client.check_cmd(["mkdir", "-p", directory]) - self.client.check_cmd(["mv", srcname, filename]) - self.client.check_cmd(["sync"]) + if self.server is None: + self.client.check_cmd(["mkdir", "-p", directory]) + self.client.check_cmd(["mv", srcname, filename]) + self.client.check_cmd(["sync"]) + else: + self.net_cmd(["mkdir", "-p", directory]) + self.server_conn.put(srcname, filename) def hostfilename(self, filename): """ @@ -981,21 +987,6 @@ class CoreNode(CoreNodeBase): dirname = os.path.join(self.nodedir, dirname) return os.path.join(dirname, basename) - def opennodefile(self, filename, mode="w"): - """ - Open a node file, within it"s directory. - - :param str filename: file name to open - :param str mode: mode to open file in - :return: open file - :rtype: file - """ - hostfilename = self.hostfilename(filename) - dirname, _basename = os.path.split(hostfilename) - if not os.path.isdir(dirname): - os.makedirs(dirname, mode=0o755) - return open(hostfilename, mode) - def nodefile(self, filename, contents, mode=0o644): """ Create a node file with a given mode. @@ -1005,12 +996,24 @@ class CoreNode(CoreNodeBase): :param int mode: mode for file :return: nothing """ - with self.opennodefile(filename, "w") as open_file: - open_file.write(contents) - os.chmod(open_file.name, mode) - logging.debug( - "node(%s) added file: %s; mode: 0%o", self.name, open_file.name, mode - ) + hostfilename = self.hostfilename(filename) + dirname, _basename = os.path.split(hostfilename) + if self.server is None: + if not os.path.isdir(dirname): + os.makedirs(dirname, mode=0o755) + with open(hostfilename, "w") as open_file: + open_file.write(contents) + os.chmod(open_file.name, mode) + else: + temp = NamedTemporaryFile() + temp.write(contents) + temp.close() + self.net_cmd(["mkdir", "-m", oct(0o755), "-p", dirname]) + self.server_conn.put(temp.name, hostfilename) + self.net_cmd(["chmod", oct(mode), hostfilename]) + logging.debug( + "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode + ) def nodefilecopy(self, filename, srcfilename, mode=None): """ @@ -1023,9 +1026,14 @@ class CoreNode(CoreNodeBase): :return: nothing """ hostfilename = self.hostfilename(filename) - self.net_cmd(["cp", "-a", srcfilename, hostfilename]) - if mode is not None: - self.net_cmd(["chmod", oct(mode), hostfilename]) + if self.server is None: + shutil.copy2(srcfilename, hostfilename) + if mode is not None: + os.chmod(hostfilename, mode) + else: + self.server_conn.put(srcfilename, hostfilename) + if mode is not None: + self.net_cmd(["chmod", oct(mode), hostfilename]) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) From 212fec916b77a2d9e594026a5e0caec973a5c25e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 7 Oct 2019 11:58:27 -0700 Subject: [PATCH 08/38] updated how distributed servers are added and connections are created to reduce duplicate connections --- daemon/core/emulator/session.py | 29 +++++++++++++++------ daemon/core/nodes/base.py | 36 ++++++++++++++------------- daemon/examples/python/distributed.py | 8 +++--- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index f5625232..e0afc53c 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -147,7 +147,7 @@ class Session(object): self.sdt = Sdt(session=self) # distributed servers - self.servers = set() + self.servers = {} # initialize default node services self.services.default_services = { @@ -158,10 +158,21 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } + def add_distributed(self, server): + conn = Connection(server, user="root") + self.servers[server] = conn + def init_distributed(self): for server in self.servers: + conn = self.servers[server] cmd = "mkdir -p %s" % self.session_dir - Connection(server, user="root").run(cmd, hide=False) + conn.run(cmd, hide=False) + + def shutdown_distributed(self): + for server in self.servers: + conn = self.servers[server] + cmd = "rm -rf %s" % self.session_dir + conn.run(cmd, hide=False) @classmethod def get_node_class(cls, _type): @@ -676,6 +687,13 @@ class Session(object): if not name: name = "%s%s" % (node_class.__name__, _id) + # verify distributed server + server = self.servers.get(node_options.emulation_server) + if node_options.emulation_server is not None and server is None: + raise CoreError( + "invalid distributed server: %s" % node_options.emulation_server + ) + # create node logging.info( "creating node(%s) id(%s) name(%s) start(%s)", @@ -694,11 +712,7 @@ class Session(object): ) else: node = self.create_node( - cls=node_class, - _id=_id, - name=name, - start=start, - server=node_options.emulation_server, + cls=node_class, _id=_id, name=name, start=start, server=server ) # set node attributes @@ -972,6 +986,7 @@ class Session(object): preserve = self.options.get_config("preservedir") == "1" if not preserve: shutil.rmtree(self.session_dir, ignore_errors=True) + self.shutdown_distributed() # call session shutdown handlers for handler in self.shutdown_handlers: diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 901d08a6..21324c59 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -13,8 +13,6 @@ from builtins import range from socket import AF_INET, AF_INET6 from tempfile import NamedTemporaryFile -from fabric import Connection - from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes @@ -42,7 +40,8 @@ class NodeBase(object): :param int _id: id :param str name: object name :param bool start: start value - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ self.session = session @@ -53,8 +52,6 @@ class NodeBase(object): name = "o%s" % self.id self.name = name self.server = server - if self.server is not None: - self.server_conn = Connection(self.server, user="root") self.type = None self.services = None @@ -103,18 +100,23 @@ class NodeBase(object): return utils.check_cmd(args, env=env) else: args = " ".join(args) - return self.remote_cmd(args) + return self.remote_cmd(args, env=env) - def remote_cmd(self, cmd): + def remote_cmd(self, cmd, env=None): """ Run command remotely using server connection. :param str cmd: command to run + :param dict env: environment for remote command, default is None :return: stdout when success :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - result = self.server_conn.run(cmd, hide=False) + if env is None: + result = self.server.run(cmd, hide=False) + else: + logging.info("command env: %s", env) + result = self.server.run(cmd, hide=False, env=env, replace_env=True) if result.exited: raise CoreCommandError( result.exited, result.command, result.stdout, result.stderr @@ -969,7 +971,7 @@ class CoreNode(CoreNodeBase): self.client.check_cmd(["sync"]) else: self.net_cmd(["mkdir", "-p", directory]) - self.server_conn.put(srcname, filename) + self.server.put(srcname, filename) def hostfilename(self, filename): """ @@ -992,7 +994,7 @@ class CoreNode(CoreNodeBase): Create a node file with a given mode. :param str filename: name of file to create - :param contents: contents of file + :param str contents: contents of file :param int mode: mode for file :return: nothing """ @@ -1005,12 +1007,12 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - temp = NamedTemporaryFile() - temp.write(contents) + temp = NamedTemporaryFile(delete=False) + temp.write(contents.encode("utf-8")) temp.close() - self.net_cmd(["mkdir", "-m", oct(0o755), "-p", dirname]) - self.server_conn.put(temp.name, hostfilename) - self.net_cmd(["chmod", oct(mode), hostfilename]) + self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) + self.server.put(temp.name, hostfilename) + self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode ) @@ -1031,9 +1033,9 @@ class CoreNode(CoreNodeBase): if mode is not None: os.chmod(hostfilename, mode) else: - self.server_conn.put(srcfilename, hostfilename) + self.server.put(srcfilename, hostfilename) if mode is not None: - self.net_cmd(["chmod", oct(mode), hostfilename]) + self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index bed75a47..5b5174f6 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -1,4 +1,5 @@ import logging +import pdb from core.emulator.coreemu import CoreEmu from core.emulator.emudata import NodeOptions @@ -14,7 +15,7 @@ def main(): session = coreemu.create_session() # initialize distributed - session.servers.add("core2") + session.add_distributed("core2") session.init_distributed() # must be in configuration state for nodes to start, when using "node_add" below @@ -25,13 +26,12 @@ def main(): # create nodes options = NodeOptions() - options.emulation_server = "10.10.4.38" options.emulation_server = "core2" session.add_node(node_options=options) # interface = prefixes.create_interface(node_one) # session.add_link(node_one.id, switch.id, interface_one=interface) - # node_two = session.add_node() + session.add_node() # interface = prefixes.create_interface(node_two) # session.add_link(node_two.id, switch.id, interface_one=interface) @@ -46,6 +46,8 @@ def main(): # node_two.client.icmd(["iperf", "-t", "10", "-c", node_one_address]) # node_one.cmd(["killall", "-9", "iperf"]) + pdb.set_trace() + # shutdown session coreemu.shutdown() From b7b0e4222c5ee8312133a3d332528a9fed0dea2d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 8 Oct 2019 15:09:26 -0700 Subject: [PATCH 09/38] updates for basic working distrbuted network using fabric --- daemon/core/emulator/distributed.py | 27 ++++++ daemon/core/emulator/session.py | 83 +++++++++++++++++-- daemon/core/nodes/base.py | 75 ++++++++--------- daemon/core/nodes/interface.py | 46 ++++++---- daemon/core/nodes/network.py | 39 +++++++-- daemon/core/utils.py | 2 +- daemon/examples/python/distributed.py | 43 +++++----- .../examples/python/distributed_switches.py | 42 ++++++++++ 8 files changed, 261 insertions(+), 96 deletions(-) create mode 100644 daemon/core/emulator/distributed.py create mode 100644 daemon/examples/python/distributed_switches.py diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py new file mode 100644 index 00000000..104d939d --- /dev/null +++ b/daemon/core/emulator/distributed.py @@ -0,0 +1,27 @@ +import logging + +from core.errors import CoreCommandError + + +def remote_cmd(server, cmd, env=None): + """ + Run command remotely using server connection. + + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost + :param str cmd: command to run + :param dict env: environment for remote command, default is None + :return: stdout when success + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + logging.info("remote cmd server(%s): %s", server, cmd) + if env is None: + result = server.run(cmd, hide=False) + else: + result = server.run(cmd, hide=False, env=env, replace_env=True) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr + ) + return result.stdout.strip() diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index e0afc53c..9eb02a07 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -37,9 +37,11 @@ from core.location.event import EventLoop from core.location.mobility import MobilityManager from core.nodes.base import CoreNetworkBase, CoreNode, CoreNodeBase from core.nodes.docker import DockerNode -from core.nodes.ipaddress import MacAddress +from core.nodes.interface import GreTap +from core.nodes.ipaddress import IpAddress, MacAddress from core.nodes.lxd import LxcNode from core.nodes.network import ( + CoreNetwork, CtrlNet, GreTapBridge, HubNode, @@ -148,6 +150,8 @@ class Session(object): # distributed servers self.servers = {} + self.tunnels = {} + self.address = None # initialize default node services self.services.default_services = { @@ -161,19 +165,81 @@ class Session(object): def add_distributed(self, server): conn = Connection(server, user="root") self.servers[server] = conn - - def init_distributed(self): - for server in self.servers: - conn = self.servers[server] - cmd = "mkdir -p %s" % self.session_dir - conn.run(cmd, hide=False) + cmd = "mkdir -p %s" % self.session_dir + conn.run(cmd, hide=False) def shutdown_distributed(self): + # shutdown all tunnels + for key in self.tunnels: + tunnels = self.tunnels[key] + for tunnel in tunnels: + tunnel.shutdown() + + # remove all remote session directories for server in self.servers: conn = self.servers[server] cmd = "rm -rf %s" % self.session_dir conn.run(cmd, hide=False) + # clear tunnels + self.tunnels.clear() + + def initialize_distributed(self): + for node_id in self.nodes: + node = self.nodes[node_id] + + if not isinstance(node, CoreNetwork): + continue + + if isinstance(node, CtrlNet) and node.serverintf is not None: + continue + + for server in self.servers: + conn = self.servers[server] + key = self.tunnelkey(node_id, IpAddress.to_int(server)) + + # local to server + logging.info( + "local tunnel node(%s) to remote(%s) key(%s)", + node.name, + server, + key, + ) + local_tap = GreTap(session=self, remoteip=server, key=key) + local_tap.net_client.create_interface(node.brname, local_tap.localname) + + # server to local + logging.info( + "remote tunnel node(%s) to local(%s) key(%s)", + node.name, + self.address, + key, + ) + remote_tap = GreTap( + session=self, remoteip=self.address, key=key, server=conn + ) + remote_tap.net_client.create_interface( + node.brname, remote_tap.localname + ) + + # save tunnels for shutdown + self.tunnels[key] = [local_tap, remote_tap] + + def tunnelkey(self, n1num, n2num): + """ + Compute a 32-bit key used to uniquely identify a GRE tunnel. + The hash(n1num), hash(n2num) values are used, so node numbers may be + None or string values (used for e.g. "ctrlnet"). + + :param int n1num: node one id + :param int n2num: node two id + :return: tunnel key for the node pair + :rtype: int + """ + logging.debug("creating tunnel key for: %s, %s", n1num, n2num) + key = (self.id << 16) ^ utils.hashkey(n1num) ^ (utils.hashkey(n2num) << 8) + return key & 0xFFFFFFFF + @classmethod def get_node_class(cls, _type): """ @@ -1493,6 +1559,9 @@ class Session(object): self.add_remove_control_interface(node=None, remove=False) self.broker.startup() + # initialize distributed tunnels + self.initialize_distributed() + # instantiate will be invoked again upon Emane configure if self.emane.startup() == self.emane.NOT_READY: return diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 21324c59..82915b38 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,6 +14,7 @@ from socket import AF_INET, AF_INET6 from tempfile import NamedTemporaryFile from core import constants, utils +from core.emulator import distributed from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.errors import CoreCommandError @@ -95,39 +96,7 @@ class NodeBase(object): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - logging.info("net cmd server(%s): %s", self.server, args) - if self.server is None: - return utils.check_cmd(args, env=env) - else: - args = " ".join(args) - return self.remote_cmd(args, env=env) - - def remote_cmd(self, cmd, env=None): - """ - Run command remotely using server connection. - - :param str cmd: command to run - :param dict env: environment for remote command, default is None - :return: stdout when success - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - if env is None: - result = self.server.run(cmd, hide=False) - else: - logging.info("command env: %s", env) - result = self.server.run(cmd, hide=False, env=env, replace_env=True) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) - - logging.info( - "fabric result:\n\tstdout: %s\n\tstderr: %s", - result.stdout.strip(), - result.stderr.strip(), - ) - return result.stdout.strip() + raise NotImplementedError def setposition(self, x=None, y=None, z=None): """ @@ -279,7 +248,8 @@ class CoreNodeBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: boolean for starting - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ super(CoreNodeBase, self).__init__(session, _id, name, start, server) self.services = [] @@ -412,6 +382,23 @@ class CoreNodeBase(NodeBase): return common + def net_cmd(self, args, env=None): + """ + Runs a command that is used to configure and setup the network on the host + system. + + :param list[str]|str args: command to run + :param dict env: environment to run command with + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + if self.server is None: + return utils.check_cmd(args, env=env) + else: + args = " ".join(args) + return distributed.remote_cmd(self.server, args, env=env) + def node_net_cmd(self, args): """ Runs a command that is used to configure and setup the network within a @@ -493,7 +480,8 @@ class CoreNode(CoreNodeBase): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ super(CoreNode, self).__init__(session, _id, name, start, server) self.nodedir = nodedir @@ -653,13 +641,13 @@ class CoreNode(CoreNodeBase): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - logging.info("net cmd server(%s): %s", self.server, args) if self.server is None: + logging.info("node(%s) cmd: %s", self.name, args) return self.check_cmd(args) else: args = self.client._cmd_args() + args args = " ".join(args) - return self.remote_cmd(args) + return distributed.remote_cmd(self.server, args) def check_cmd(self, args): """ @@ -753,7 +741,11 @@ class CoreNode(CoreNodeBase): raise ValueError("interface name (%s) too long" % name) veth = Veth( - node=self, name=name, localname=localname, net=net, start=self.up + node=self, + name=name, + localname=localname, + start=self.up, + server=self.server, ) if self.up: @@ -806,9 +798,7 @@ class CoreNode(CoreNodeBase): sessionid = self.session.short_session_id() localname = "tap%s.%s.%s" % (self.id, ifindex, sessionid) name = ifname - tuntap = TunTap( - node=self, name=name, localname=localname, net=net, start=self.up - ) + tuntap = TunTap(node=self, name=name, localname=localname, start=self.up) try: self.addnetif(tuntap, ifindex) @@ -1057,7 +1047,8 @@ class CoreNetworkBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: should object start - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ super(CoreNetworkBase, self).__init__(session, _id, name, start, server) self._linked = {} diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 51859e3a..8b73b1b7 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -7,6 +7,7 @@ import time from builtins import int, range from core import utils +from core.emulator import distributed from core.errors import CoreCommandError from core.nodes.netclient import LinuxNetClient @@ -16,13 +17,15 @@ class CoreInterface(object): Base class for network interfaces. """ - def __init__(self, node, name, mtu): + def __init__(self, node, name, mtu, server=None): """ Creates a PyCoreNetIf instance. :param core.nodes.base.CoreNode node: node for interface :param str name: interface name :param mtu: mtu value + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ self.node = node @@ -42,7 +45,15 @@ class CoreInterface(object): self.netindex = None # index used to find flow data self.flow_id = None - self.net_client = LinuxNetClient(utils.check_cmd) + self.server = server + self.net_client = LinuxNetClient(self.net_cmd) + + def net_cmd(self, args): + if self.server is None: + return utils.check_cmd(args) + else: + args = " ".join(args) + return distributed.remote_cmd(self.server, args) def startup(self): """ @@ -191,8 +202,7 @@ class Veth(CoreInterface): Provides virtual ethernet functionality for core nodes. """ - # TODO: network is not used, why was it needed? - def __init__(self, node, name, localname, mtu=1500, net=None, start=True): + def __init__(self, node, name, localname, mtu=1500, server=None, start=True): """ Creates a VEth instance. @@ -200,12 +210,13 @@ class Veth(CoreInterface): :param str name: interface name :param str localname: interface local name :param mtu: interface mtu - :param net: network + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param bool start: start flag :raises CoreCommandError: when there is a command exception """ # note that net arg is ignored - CoreInterface.__init__(self, node=node, name=name, mtu=mtu) + CoreInterface.__init__(self, node, name, mtu, server) self.localname = localname self.up = False if start: @@ -251,8 +262,7 @@ class TunTap(CoreInterface): TUN/TAP virtual device in TAP mode """ - # TODO: network is not used, why was it needed? - def __init__(self, node, name, localname, mtu=1500, net=None, start=True): + def __init__(self, node, name, localname, mtu=1500, server=None, start=True): """ Create a TunTap instance. @@ -260,10 +270,11 @@ class TunTap(CoreInterface): :param str name: interface name :param str localname: local interface name :param mtu: interface mtu - :param core.nodes.base.CoreNetworkBase net: related network + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param bool start: start flag """ - CoreInterface.__init__(self, node=node, name=name, mtu=mtu) + CoreInterface.__init__(self, node, name, mtu, server) self.localname = localname self.up = False self.transport_type = "virtual" @@ -427,6 +438,7 @@ class GreTap(CoreInterface): ttl=255, key=None, start=True, + server=None, ): """ Creates a GreTap instance. @@ -441,9 +453,11 @@ class GreTap(CoreInterface): :param ttl: ttl value :param key: gre tap key :param bool start: start flag + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :raises CoreCommandError: when there is a command exception """ - CoreInterface.__init__(self, node=node, name=name, mtu=mtu) + CoreInterface.__init__(self, node, name, mtu, server) self.session = session if _id is None: # from PyCoreObj @@ -460,9 +474,13 @@ class GreTap(CoreInterface): if remoteip is None: raise ValueError("missing remote IP required for GRE TAP device") - self.net_client.create_gretap( - self.localname, str(remoteip), str(localip), str(ttl), str(key) - ) + if localip is not None: + localip = str(localip) + if ttl is not None: + ttl = str(ttl) + if key is not None: + key = str(key) + self.net_client.create_gretap(self.localname, str(remoteip), localip, ttl, key) self.net_client.device_up(self.localname) self.up = True diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 444ded56..3ec84282 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -10,6 +10,7 @@ import time from socket import AF_INET, AF_INET6 from core import constants, utils +from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs from core.errors import CoreCommandError, CoreError @@ -291,7 +292,8 @@ class CoreNetwork(CoreNetworkBase): :param int _id: object id :param str name: object name :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param policy: network policy """ CoreNetworkBase.__init__(self, session, _id, name, start, server) @@ -307,6 +309,27 @@ class CoreNetwork(CoreNetworkBase): self.startup() ebq.startupdateloop(self) + def net_cmd(self, args, env=None): + """ + Runs a command that is used to configure and setup the network on the host + system. + + :param list[str]|str args: command to run + :param dict env: environment to run command with + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + logging.info("network node(%s) cmd", self.name) + output = utils.check_cmd(args, env=env) + + args = " ".join(args) + for server in self.session.servers: + conn = self.session.servers[server] + distributed.remote_cmd(conn, args, env=env) + + return output + def startup(self): """ Linux bridge starup logic. @@ -381,11 +404,11 @@ class CoreNetwork(CoreNetworkBase): """ Attach a network interface. - :param core.netns.vnode.VEth netif: network interface to attach + :param core.nodes.interface.Veth netif: network interface to attach :return: nothing """ if self.up: - self.net_client.create_interface(self.brname, netif.localname) + netif.net_client.create_interface(self.brname, netif.localname) CoreNetworkBase.attach(self, netif) @@ -397,7 +420,7 @@ class CoreNetwork(CoreNetworkBase): :return: nothing """ if self.up: - self.net_client.delete_interface(self.brname, netif.localname) + netif.net_client.delete_interface(self.brname, netif.localname) CoreNetworkBase.detach(self, netif) @@ -591,13 +614,11 @@ class CoreNetwork(CoreNetworkBase): if len(name) >= 16: raise ValueError("interface name %s too long" % name) - netif = Veth( - node=None, name=name, localname=localname, mtu=1500, net=self, start=self.up - ) + netif = Veth(node=None, name=name, localname=localname, mtu=1500, start=self.up) self.attach(netif) if net.up: # this is similar to net.attach() but uses netif.name instead of localname - self.net_client.create_interface(net.brname, netif.name) + netif.net_client.create_interface(net.brname, netif.name) i = net.newifindex() net._netif[i] = netif with net._linked_lock: @@ -666,6 +687,8 @@ class GreTapBridge(CoreNetwork): :param ttl: ttl value :param key: gre tap key :param bool start: start flag + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ CoreNetwork.__init__(self, session, _id, name, False, server, policy) self.grekey = key diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 20d2384e..8e59a050 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -263,7 +263,7 @@ def check_cmd(args, **kwargs): kwargs["stdout"] = subprocess.PIPE kwargs["stderr"] = subprocess.STDOUT args = split_args(args) - logging.debug("command: %s", args) + logging.info("command: %s", args) try: p = subprocess.Popen(args, **kwargs) stdout, _ = p.communicate() diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index 5b5174f6..feb5e8bb 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -1,51 +1,46 @@ import logging import pdb +import sys from core.emulator.coreemu import CoreEmu -from core.emulator.emudata import NodeOptions -from core.emulator.enumerations import EventTypes +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes def main(): # ip generator for example - # prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods coreemu = CoreEmu() session = coreemu.create_session() # initialize distributed - session.add_distributed("core2") - session.init_distributed() + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) - # create switch network node - # switch = session.add_node(_type=NodeTypes.SWITCH) - - # create nodes + # create local node, switch, and remote nodes + node_one = session.add_node() + switch = session.add_node(_type=NodeTypes.SWITCH) options = NodeOptions() - options.emulation_server = "core2" - session.add_node(node_options=options) - # interface = prefixes.create_interface(node_one) - # session.add_link(node_one.id, switch.id, interface_one=interface) + options.emulation_server = remote + node_two = session.add_node(node_options=options) - session.add_node() - # interface = prefixes.create_interface(node_two) - # session.add_link(node_two.id, switch.id, interface_one=interface) + # create not interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, switch.id, interface_one=interface_one) + session.add_link(node_two.id, switch.id, interface_one=interface_two) # instantiate session session.instantiate() - # print("starting iperf server on node: %s" % node_one.name) - # node_one.cmd(["iperf", "-s", "-D"]) - # node_one_address = prefixes.ip4_address(node_one) - # - # print("node %s connecting to %s" % (node_two.name, node_one_address)) - # node_two.client.icmd(["iperf", "-t", "10", "-c", node_one_address]) - # node_one.cmd(["killall", "-9", "iperf"]) - + # pause script for verification pdb.set_trace() # shutdown session diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py new file mode 100644 index 00000000..c6366d5d --- /dev/null +++ b/daemon/examples/python/distributed_switches.py @@ -0,0 +1,42 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.enumerations import EventTypes, NodeTypes + + +def main(): + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + switch_one = session.add_node(_type=NodeTypes.SWITCH) + switch_two = session.add_node(_type=NodeTypes.SWITCH) + + # create not interfaces and link + session.add_link(switch_one.id, switch_two.id) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From c8d68c332a65704097ae0fd251eb910402b9c808 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 8 Oct 2019 21:06:22 -0700 Subject: [PATCH 10/38] updates for testing using examples --- daemon/core/emulator/session.py | 9 ++++++++- daemon/core/nodes/network.py | 3 ++- daemon/examples/python/distributed.py | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 9eb02a07..31e75de2 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -1869,7 +1869,14 @@ class Session(object): assign_address = self.master prefix = prefixes[0] - logging.info("controlnet prefix: %s - %s", type(prefix), prefix) + logging.info( + "controlnet(%s) prefix(%s) assign(%s) updown(%s) serverintf(%s)", + _id, + prefix, + assign_address, + updown_script, + server_interface, + ) control_net = self.create_node( cls=CtrlNet, _id=_id, diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 3ec84282..f7d6af69 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -808,7 +808,8 @@ class CtrlNet(CoreNetwork): :param prefix: control network ipv4 prefix :param hostid: host id :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param str assign_address: assigned address :param str updown_script: updown script :param serverintf: server interface diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index feb5e8bb..3cb3debd 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -15,6 +15,9 @@ def main(): coreemu = CoreEmu() session = coreemu.create_session() + # set controlnet + session.options.set_config("controlnet", "172.16.0.0/24") + # initialize distributed address = sys.argv[1] remote = sys.argv[2] From 7e45168e777a01835589f249b7ca2647ae29b244 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 8 Oct 2019 21:17:15 -0700 Subject: [PATCH 11/38] distributed example for ptp --- daemon/core/nodes/network.py | 4 +- daemon/examples/python/distributed.py | 2 +- daemon/examples/python/distributed_ptp.py | 50 +++++++++++++++++++ .../examples/python/distributed_switches.py | 2 +- 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 daemon/examples/python/distributed_ptp.py diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index f7d6af69..c404db5a 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -852,7 +852,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - utils.check_cmd([self.updown_script, self.brname, "startup"]) + self.net_cmd([self.updown_script, self.brname, "startup"]) if self.serverintf: self.net_client.create_interface(self.brname, self.serverintf) @@ -880,7 +880,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - utils.check_cmd([self.updown_script, self.brname, "shutdown"]) + self.net_cmd([self.updown_script, self.brname, "shutdown"]) except CoreCommandError: logging.exception("error issuing shutdown script shutdown") diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index 3cb3debd..ca9ca928 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -34,7 +34,7 @@ def main(): options.emulation_server = remote node_two = session.add_node(node_options=options) - # create not interfaces and link + # create node interfaces and link interface_one = prefixes.create_interface(node_one) interface_two = prefixes.create_interface(node_two) session.add_link(node_one.id, switch.id, interface_one=interface_one) diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py new file mode 100644 index 00000000..6ab5b9dc --- /dev/null +++ b/daemon/examples/python/distributed_ptp.py @@ -0,0 +1,50 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes + + +def main(): + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + node_one = session.add_node() + options = NodeOptions() + options.emulation_server = remote + node_two = session.add_node(node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, node_two.id, interface_one, interface_two) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py index c6366d5d..b7ed166b 100644 --- a/daemon/examples/python/distributed_switches.py +++ b/daemon/examples/python/distributed_switches.py @@ -24,7 +24,7 @@ def main(): switch_one = session.add_node(_type=NodeTypes.SWITCH) switch_two = session.add_node(_type=NodeTypes.SWITCH) - # create not interfaces and link + # create node interfaces and link session.add_link(switch_one.id, switch_two.id) # instantiate session From 859f473ba9f50b5207a09494f7c7c7fd1f8ad422 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Oct 2019 12:13:26 -0700 Subject: [PATCH 12/38] updated ebtables to use net_cmd --- daemon/core/nodes/network.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index c404db5a..2fab566c 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -3,7 +3,6 @@ Defines network nodes used within core. """ import logging -import os import socket import threading import time @@ -165,20 +164,20 @@ class EbtablesQueue(object): """ # save kernel ebtables snapshot to a file args = self.ebatomiccmd(["--atomic-save"]) - utils.check_cmd(args) + wlan.net_cmd(args) # modify the table file using queued ebtables commands for c in self.cmds: args = self.ebatomiccmd(c) - utils.check_cmd(args) + wlan.net_cmd(args) self.cmds = [] # commit the table file to the kernel args = self.ebatomiccmd(["--atomic-commit"]) - utils.check_cmd(args) + wlan.net_cmd(args) try: - os.unlink(self.atomic_file) + wlan.net_cmd(["rm", "-f", self.atomic_file]) except OSError: logging.exception("error removing atomic file: %s", self.atomic_file) @@ -312,7 +311,7 @@ class CoreNetwork(CoreNetworkBase): def net_cmd(self, args, env=None): """ Runs a command that is used to configure and setup the network on the host - system. + system and all configured distributed servers. :param list[str]|str args: command to run :param dict env: environment to run command with @@ -341,7 +340,7 @@ class CoreNetwork(CoreNetworkBase): # create a new ebtables chain for this bridge ebtablescmds( - utils.check_cmd, + self.net_cmd, [ [constants.EBTABLES_BIN, "-N", self.brname, "-P", self.policy], [ @@ -372,7 +371,7 @@ class CoreNetwork(CoreNetworkBase): try: self.net_client.delete_bridge(self.brname) ebtablescmds( - utils.check_cmd, + self.net_cmd, [ [ constants.EBTABLES_BIN, @@ -844,7 +843,6 @@ class CtrlNet(CoreNetwork): if self.assign_address: addrlist = ["%s/%s" % (addr, self.prefix.prefixlen)] self.addrconfig(addrlist=addrlist) - logging.info("address %s", addr) if self.updown_script: logging.info( From a4b6b8be510e2a0ea04e365abbaf5c52bbe446ed Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Oct 2019 15:44:45 -0700 Subject: [PATCH 13/38] updated link config to work distributed, added crude locking for fabric --- daemon/core/emulator/distributed.py | 17 +++++-- daemon/core/nodes/base.py | 6 +-- daemon/core/nodes/network.py | 8 +-- daemon/examples/python/distributed_ptp.py | 2 +- daemon/examples/python/distributed_wlan.py | 58 ++++++++++++++++++++++ 5 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 daemon/examples/python/distributed_wlan.py diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 104d939d..2c7d7bbb 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -1,7 +1,10 @@ import logging +import threading from core.errors import CoreCommandError +LOCK = threading.Lock() + def remote_cmd(server, cmd, env=None): """ @@ -16,12 +19,18 @@ def remote_cmd(server, cmd, env=None): :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("remote cmd server(%s): %s", server, cmd) - if env is None: - result = server.run(cmd, hide=False) - else: - result = server.run(cmd, hide=False, env=env, replace_env=True) + with LOCK: + if env is None: + result = server.run(cmd, hide=False) + else: + result = server.run(cmd, hide=False, env=env, replace_env=True) if result.exited: raise CoreCommandError( result.exited, result.command, result.stdout, result.stderr ) return result.stdout.strip() + + +def remote_put(server, source, destination): + with LOCK: + server.put(source, destination) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 82915b38..4f95c56e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -961,7 +961,7 @@ class CoreNode(CoreNodeBase): self.client.check_cmd(["sync"]) else: self.net_cmd(["mkdir", "-p", directory]) - self.server.put(srcname, filename) + distributed.remote_put(self.server, srcname, filename) def hostfilename(self, filename): """ @@ -1001,7 +1001,7 @@ class CoreNode(CoreNodeBase): temp.write(contents.encode("utf-8")) temp.close() self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) - self.server.put(temp.name, hostfilename) + distributed.remote_put(self.server, temp.name, hostfilename) self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode @@ -1023,7 +1023,7 @@ class CoreNode(CoreNodeBase): if mode is not None: os.chmod(hostfilename, mode) else: - self.server.put(srcfilename, hostfilename) + distributed.remote_put(self.server, srcfilename, hostfilename) if mode is not None: self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.info( diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 2fab566c..81dfc34b 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -525,14 +525,14 @@ class CoreNetwork(CoreNetworkBase): logging.debug( "linkconfig: %s" % ([tc + parent + ["handle", "1:"] + tbf],) ) - utils.check_cmd(tc + parent + ["handle", "1:"] + tbf) + netif.net_cmd(tc + parent + ["handle", "1:"] + tbf) netif.setparam("has_tbf", True) changed = True elif netif.getparam("has_tbf") and bw <= 0: tcd = [] + tc tcd[2] = "delete" if self.up: - utils.check_cmd(tcd + parent) + netif.net_cmd(tcd + parent) netif.setparam("has_tbf", False) # removing the parent removes the child netif.setparam("has_netem", False) @@ -575,14 +575,14 @@ class CoreNetwork(CoreNetworkBase): tc[2] = "delete" if self.up: logging.debug("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) - utils.check_cmd(tc + parent + ["handle", "10:"]) + netif.net_cmd(tc + parent + ["handle", "10:"]) netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: logging.debug( "linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],) ) - utils.check_cmd(tc + parent + ["handle", "10:"] + netem) + netif.net_cmd(tc + parent + ["handle", "10:"] + netem) netif.setparam("has_netem", True) def linknet(self, net): diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index 6ab5b9dc..2b611816 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -25,8 +25,8 @@ def main(): session.set_state(EventTypes.CONFIGURATION_STATE) # create local node, switch, and remote nodes - node_one = session.add_node() options = NodeOptions() + node_one = session.add_node(node_options=options) options.emulation_server = remote node_two = session.add_node(node_options=options) diff --git a/daemon/examples/python/distributed_wlan.py b/daemon/examples/python/distributed_wlan.py new file mode 100644 index 00000000..ca64ee01 --- /dev/null +++ b/daemon/examples/python/distributed_wlan.py @@ -0,0 +1,58 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes +from core.location.mobility import BasicRangeModel + + +def main(): + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # set controlnet + # session.options.set_config("controlnet", "172.16.0.0/24") + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + options = NodeOptions() + options.set_position(0, 0) + options.emulation_server = remote + node_one = session.add_node(node_options=options) + wlan = session.add_node(_type=NodeTypes.WIRELESS_LAN) + session.mobility.set_model(wlan, BasicRangeModel) + node_two = session.add_node(node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, wlan.id, interface_one=interface_one) + session.add_link(node_two.id, wlan.id, interface_one=interface_two) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From bc586933399fe7b227222a26f87cfa496d760fe7 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 10 Oct 2019 11:53:52 -0700 Subject: [PATCH 14/38] updated emane config files to be generated for remote servers, fixed services not using node remote server compatible commands --- daemon/core/emane/emanemanager.py | 13 +++- daemon/core/emane/emanemodel.py | 11 ++- daemon/core/emulator/distributed.py | 41 ++++++++--- daemon/core/nodes/base.py | 6 +- daemon/core/services/coreservices.py | 10 +-- daemon/core/xml/emanexml.py | 77 +++++++++++++++++---- daemon/examples/python/distributed_emane.py | 65 +++++++++++++++++ 7 files changed, 184 insertions(+), 39 deletions(-) create mode 100644 daemon/examples/python/distributed_emane.py diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 746016f9..2902c47c 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -18,6 +18,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel +from core.emulator import distributed from core.emulator.enumerations import ( ConfigDataTypes, ConfigFlags, @@ -679,8 +680,12 @@ class EmaneManager(ModelManager): return dev = self.get_config("eventservicedevice") - emanexml.create_event_service_xml(group, port, dev, self.session.session_dir) + for server in self.session.servers: + conn = self.session.servers[server] + emanexml.create_event_service_xml( + group, port, dev, self.session.session_dir, conn + ) def startdaemons(self): """ @@ -745,7 +750,7 @@ class EmaneManager(ModelManager): os.path.join(path, "emane%d.log" % n), os.path.join(path, "platform%d.xml" % n), ] - output = node.check_cmd(args) + output = node.node_net_cmd(args) logging.info("node(%s) emane daemon running: %s", node.name, args) logging.info("node(%s) emane daemon output: %s", node.name, output) @@ -756,6 +761,10 @@ class EmaneManager(ModelManager): emanecmd += ["-f", os.path.join(path, "emane.log")] args = emanecmd + [os.path.join(path, "platform.xml")] utils.check_cmd(args, cwd=path) + args = " ".join(args) + for server in self.session.servers: + conn = self.session.servers[server] + distributed.remote_cmd(conn, args, cwd=path) logging.info("host emane daemon running: %s", args) def stopdaemons(self): diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 56eee289..3ca2a18f 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -102,6 +102,11 @@ class EmaneModel(WirelessModel): mac_name = emanexml.mac_file_name(self, interface) phy_name = emanexml.phy_file_name(self, interface) + # remote server for file + server = None + if interface is not None: + server = interface.node.server + # check if this is external transport_type = "virtual" if interface and interface.transport_type == "raw": @@ -111,16 +116,16 @@ class EmaneModel(WirelessModel): # create nem xml file nem_file = os.path.join(self.session.session_dir, nem_name) emanexml.create_nem_xml( - self, config, nem_file, transport_name, mac_name, phy_name + self, config, nem_file, transport_name, mac_name, phy_name, server ) # create mac xml file mac_file = os.path.join(self.session.session_dir, mac_name) - emanexml.create_mac_xml(self, config, mac_file) + emanexml.create_mac_xml(self, config, mac_file, server) # create phy xml file phy_file = os.path.join(self.session.session_dir, phy_name) - emanexml.create_phy_xml(self, config, phy_file) + emanexml.create_phy_xml(self, config, phy_file, server) def post_startup(self): """ diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 2c7d7bbb..35cbf208 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -1,12 +1,16 @@ import logging +import os import threading +from tempfile import NamedTemporaryFile + +from invoke import UnexpectedExit from core.errors import CoreCommandError LOCK = threading.Lock() -def remote_cmd(server, cmd, env=None): +def remote_cmd(server, cmd, env=None, cwd=None): """ Run command remotely using server connection. @@ -14,23 +18,38 @@ def remote_cmd(server, cmd, env=None): default is None for localhost :param str cmd: command to run :param dict env: environment for remote command, default is None + :param str cwd: directory to run command in, defaults to None, which is the user's + home directory :return: stdout when success :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("remote cmd server(%s): %s", server, cmd) - with LOCK: - if env is None: - result = server.run(cmd, hide=False) - else: - result = server.run(cmd, hide=False, env=env, replace_env=True) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) - return result.stdout.strip() + replace_env = env is not None + try: + with LOCK: + if cwd is None: + result = server.run(cmd, hide=False, env=env, replace_env=replace_env) + else: + with server.cd(cwd): + result = server.run( + cmd, hide=False, env=env, replace_env=replace_env + ) + return result.stdout.strip() + except UnexpectedExit as e: + stdout, stderr = e.streams_for_display() + raise CoreCommandError(e.result.exited, cmd, stdout, stderr) def remote_put(server, source, destination): with LOCK: server.put(source, destination) + + +def remote_put_temp(server, destination, data): + with LOCK: + temp = NamedTemporaryFile(delete=False) + temp.write(data.encode("utf-8")) + temp.close() + server.put(temp.name, destination) + os.unlink(temp.name) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 4f95c56e..d8cac8c5 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -11,7 +11,6 @@ import string import threading from builtins import range from socket import AF_INET, AF_INET6 -from tempfile import NamedTemporaryFile from core import constants, utils from core.emulator import distributed @@ -997,11 +996,8 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - temp = NamedTemporaryFile(delete=False) - temp.write(contents.encode("utf-8")) - temp.close() self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) - distributed.remote_put(self.server, temp.name, hostfilename) + distributed.remote_put_temp(self.server, hostfilename, contents) self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index 4563d4b7..b34daa73 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -9,6 +9,7 @@ services. import enum import logging +import shlex import time from multiprocessing.pool import ThreadPool @@ -597,8 +598,9 @@ class CoreServices(object): status = 0 for cmd in cmds: logging.debug("validating service(%s) using: %s", service.name, cmd) + cmd = shlex.split(cmd) try: - node.check_cmd(cmd) + node.node_net_cmd(cmd) except CoreCommandError as e: logging.debug( "node(%s) service(%s) validate failed", node.name, service.name @@ -728,11 +730,11 @@ class CoreServices(object): status = 0 for cmd in cmds: + cmd = shlex.split(cmd) try: if wait: - node.check_cmd(cmd) - else: - node.cmd(cmd, wait=False) + cmd.append("&") + node.node_net_cmd(cmd) except CoreCommandError: logging.exception("error starting command") status = -1 diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index d73f3d5b..3b4fafef 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -1,9 +1,11 @@ import logging import os +from tempfile import NamedTemporaryFile from lxml import etree from core import utils +from core.emulator import distributed from core.nodes.ipaddress import MacAddress from core.xml import corexml @@ -44,20 +46,29 @@ def _value_to_params(value): return None -def create_file(xml_element, doc_name, file_path): +def create_file(xml_element, doc_name, file_path, server=None): """ Create xml file. :param lxml.etree.Element xml_element: root element to write to file :param str doc_name: name to use in the emane doctype :param str file_path: file path to write xml file to + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ doctype = ( '' % {"doc_name": doc_name} ) - corexml.write_xml_file(xml_element, file_path, doctype=doctype) + if server is not None: + temp = NamedTemporaryFile(delete=False) + create_file(xml_element, doc_name, temp.name) + temp.close() + distributed.remote_put(server, temp.name, file_path) + os.unlink(temp.name) + else: + corexml.write_xml_file(xml_element, file_path, doctype=doctype) def add_param(xml_element, name, value): @@ -204,17 +215,18 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x # increment nem id nem_id += 1 + doc_name = "platform" for key in sorted(platform_xmls.keys()): + platform_element = platform_xmls[key] if key == "host": file_name = "platform.xml" + file_path = os.path.join(emane_manager.session.session_dir, file_name) + create_file(platform_element, doc_name, file_path) else: file_name = "platform%d.xml" % key - - platform_element = platform_xmls[key] - - doc_name = "platform" - file_path = os.path.join(emane_manager.session.session_dir, file_name) - create_file(platform_element, doc_name, file_path) + file_path = os.path.join(emane_manager.session.session_dir, file_name) + linked_node = emane_manager.session.nodes[key] + create_file(platform_element, doc_name, file_path, linked_node.server) return nem_id @@ -303,15 +315,20 @@ def build_transport_xml(emane_manager, node, transport_type): file_name = transport_file_name(node.id, transport_type) file_path = os.path.join(emane_manager.session.session_dir, file_name) create_file(transport_element, doc_name, file_path) + for server in emane_manager.session.servers: + conn = emane_manager.session.servers[server] + create_file(transport_element, doc_name, file_path, conn) -def create_phy_xml(emane_model, config, file_path): +def create_phy_xml(emane_model, config, file_path, server): """ Create the phy xml document. :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ phy_element = etree.Element("phy", name="%s PHY" % emane_model.name) @@ -322,15 +339,24 @@ def create_phy_xml(emane_model, config, file_path): phy_element, emane_model.phy_config, config, emane_model.config_ignore ) create_file(phy_element, "phy", file_path) + if server is not None: + create_file(phy_element, "phy", file_path, server) + else: + create_file(phy_element, "phy", file_path) + for server in emane_model.session.servers: + conn = emane_model.session.servers[server] + create_file(phy_element, "phy", file_path, conn) -def create_mac_xml(emane_model, config, file_path): +def create_mac_xml(emane_model, config, file_path, server): """ Create the mac xml document. :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ if not emane_model.mac_library: @@ -343,10 +369,23 @@ def create_mac_xml(emane_model, config, file_path): mac_element, emane_model.mac_config, config, emane_model.config_ignore ) create_file(mac_element, "mac", file_path) + if server is not None: + create_file(mac_element, "mac", file_path, server) + else: + create_file(mac_element, "mac", file_path) + for server in emane_model.session.servers: + conn = emane_model.session.servers[server] + create_file(mac_element, "mac", file_path, conn) def create_nem_xml( - emane_model, config, nem_file, transport_definition, mac_definition, phy_definition + emane_model, + config, + nem_file, + transport_definition, + mac_definition, + phy_definition, + server, ): """ Create the nem xml document. @@ -357,6 +396,8 @@ def create_nem_xml( :param str transport_definition: transport file definition path :param str mac_definition: mac file definition path :param str phy_definition: phy file definition path + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) @@ -366,10 +407,16 @@ def create_nem_xml( etree.SubElement(nem_element, "transport", definition=transport_definition) etree.SubElement(nem_element, "mac", definition=mac_definition) etree.SubElement(nem_element, "phy", definition=phy_definition) - create_file(nem_element, "nem", nem_file) + if server is not None: + create_file(nem_element, "nem", nem_file, server) + else: + create_file(nem_element, "nem", nem_file) + for server in emane_model.session.servers: + conn = emane_model.session.servers[server] + create_file(nem_element, "nem", nem_file, conn) -def create_event_service_xml(group, port, device, file_directory): +def create_event_service_xml(group, port, device, file_directory, server=None): """ Create a emane event service xml file. @@ -377,6 +424,8 @@ def create_event_service_xml(group, port, device, file_directory): :param str port: event port :param str device: event device :param str file_directory: directory to create file in + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ event_element = etree.Element("emaneeventmsgsvc") @@ -391,7 +440,7 @@ def create_event_service_xml(group, port, device, file_directory): sub_element.text = value file_name = "libemaneeventservice.xml" file_path = os.path.join(file_directory, file_name) - create_file(event_element, "emaneeventmsgsvc", file_path) + create_file(event_element, "emaneeventmsgsvc", file_path, server) def transport_file_name(node_id, transport_type): diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py new file mode 100644 index 00000000..1ffe5795 --- /dev/null +++ b/daemon/examples/python/distributed_emane.py @@ -0,0 +1,65 @@ +import logging +import pdb +import sys + +from core.emane.ieee80211abg import EmaneIeee80211abgModel +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes + + +def main(): + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # set controlnet + session.options.set_config( + "controlnet", + "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 " + "core4:172.16.4.0/24 core5:172.16.5.0/24", + ) + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + options = NodeOptions(model="mdr") + options.set_position(0, 0) + node_one = session.add_node(node_options=options) + emane_net = session.add_node(_type=NodeTypes.EMANE) + session.emane.set_model(emane_net, EmaneIeee80211abgModel) + options.emulation_server = remote + node_two = session.add_node(node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, emane_net.id, interface_one=interface_one) + session.add_link(node_two.id, emane_net.id, interface_one=interface_two) + + # instantiate session + try: + session.instantiate() + except Exception: + logging.exception("error during instantiate") + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From f6cdeb23de01aba78089c7c8192883ab9ece365b Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 10 Oct 2019 15:25:12 -0700 Subject: [PATCH 15/38] changes to update commands to leverage either node_net_cmd/net_cmd --- daemon/core/api/tlv/corehandlers.py | 13 +++++++++---- daemon/core/emane/emanemanager.py | 24 ++++++++++++++++++------ daemon/core/emulator/distributed.py | 10 ++++++++-- daemon/core/emulator/session.py | 3 ++- daemon/core/errors.py | 7 ++++++- daemon/core/nodes/base.py | 10 ++++++---- daemon/core/nodes/client.py | 5 +++-- daemon/core/nodes/physical.py | 2 +- daemon/core/services/coreservices.py | 9 +++------ daemon/examples/python/switch.py | 9 ++++++--- daemon/examples/python/wlan.py | 6 +++--- daemon/tests/emane/test_emane.py | 2 +- daemon/tests/test_core.py | 4 +--- 13 files changed, 67 insertions(+), 37 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index e520ce95..a531efe2 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -37,7 +37,7 @@ from core.emulator.enumerations import ( RegisterTlvs, SessionTlvs, ) -from core.errors import CoreError +from core.errors import CoreCommandError, CoreError from core.location.mobility import BasicRangeModel from core.nodes.network import WlanNode from core.services.coreservices import ServiceManager, ServiceShim @@ -882,16 +882,21 @@ class CoreHandler(socketserver.BaseRequestHandler): return (reply,) else: logging.info("execute message with cmd=%s", command) + command = utils.split_args(command) # execute command and send a response if ( message.flags & MessageFlags.STRING.value or message.flags & MessageFlags.TEXT.value ): - # shlex.split() handles quotes within the string if message.flags & MessageFlags.LOCAL.value: status, res = utils.cmd_output(command) else: - status, res = node.cmd_output(command) + try: + res = node.node_net_cmd(command) + status = 0 + except CoreCommandError as e: + res = e.stderr + status = e.returncode logging.info( "done exec cmd=%s with status=%d res=(%d bytes)", command, @@ -913,7 +918,7 @@ class CoreHandler(socketserver.BaseRequestHandler): if message.flags & MessageFlags.LOCAL.value: utils.mute_detach(command) else: - node.cmd(command, wait=False) + node.node_net_cmd(command, wait=False) except CoreError: logging.exception("error getting object: %s", node_num) # XXX wait and queue this message to try again later diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 2902c47c..49cf4f24 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -772,7 +772,8 @@ class EmaneManager(ModelManager): Kill the appropriate EMANE daemons. """ # TODO: we may want to improve this if we had the PIDs from the specific EMANE daemons that we"ve started - args = ["killall", "-q", "emane"] + kill_emaned = ["killall", "-q", "emane"] + kill_transortd = ["killall", "-q", "emanetransportd"] stop_emane_on_host = False for node in self.getnodes(): if hasattr(node, "transport_type") and node.transport_type == "raw": @@ -780,13 +781,19 @@ class EmaneManager(ModelManager): continue if node.up: - node.cmd(args, wait=False) + node.node_net_cmd(kill_emaned, wait=False) # TODO: RJ45 node if stop_emane_on_host: try: - utils.check_cmd(args) - utils.check_cmd(["killall", "-q", "emanetransportd"]) + utils.check_cmd(kill_emaned) + utils.check_cmd(kill_transortd) + kill_emaned = " ".join(kill_emaned) + kill_transortd = " ".join(kill_transortd) + for server in self.session.servers: + conn = self.session[server] + distributed.remote_cmd(conn, kill_emaned) + distributed.remote_cmd(conn, kill_transortd) except CoreCommandError: logging.exception("error shutting down emane daemons") @@ -976,8 +983,13 @@ class EmaneManager(ModelManager): Return True if an EMANE process associated with the given node is running, False otherwise. """ args = ["pkill", "-0", "-x", "emane"] - status = node.cmd(args) - return status == 0 + try: + node.node_net_cmd(args) + result = True + except CoreCommandError: + result = False + + return result class EmaneGlobalModel(EmaneModel): diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 35cbf208..abec0a57 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -10,7 +10,7 @@ from core.errors import CoreCommandError LOCK = threading.Lock() -def remote_cmd(server, cmd, env=None, cwd=None): +def remote_cmd(server, cmd, env=None, cwd=None, wait=True): """ Run command remotely using server connection. @@ -20,12 +20,18 @@ def remote_cmd(server, cmd, env=None, cwd=None): :param dict env: environment for remote command, default is None :param str cwd: directory to run command in, defaults to None, which is the user's home directory + :param bool wait: True to wait for status, False to background process :return: stdout when success :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - logging.info("remote cmd server(%s): %s", server, cmd) + replace_env = env is not None + if not wait: + cmd += " &" + logging.info( + "remote cmd server(%s) cwd(%s) wait(%s): %s", server.host, cwd, wait, cmd + ) try: with LOCK: if cwd is None: diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 31e75de2..9bc2ab80 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -2044,4 +2044,5 @@ class Session(object): utils.mute_detach(data) else: node = self.get_node(node_id) - node.cmd(data, wait=False) + data = utils.split_args(data) + node.node_net_cmd(data, wait=False) diff --git a/daemon/core/errors.py b/daemon/core/errors.py index bb124434..5b76abb3 100644 --- a/daemon/core/errors.py +++ b/daemon/core/errors.py @@ -10,7 +10,12 @@ class CoreCommandError(subprocess.CalledProcessError): """ def __str__(self): - return "Command(%s), Status(%s):\n%s" % (self.cmd, self.returncode, self.output) + return "Command(%s), Status(%s):\nstdout: %s\nstderr: %s" % ( + self.cmd, + self.returncode, + self.output, + self.stderr, + ) class CoreError(Exception): diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d8cac8c5..147989d7 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -630,29 +630,31 @@ class CoreNode(CoreNodeBase): """ return self.client.cmd_output(args) - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): """ Runs a command that is used to configure and setup the network within a node. - :param list[str]|str args: command to run + :param list[str] args: command to run + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: logging.info("node(%s) cmd: %s", self.name, args) - return self.check_cmd(args) + return self.client.check_cmd(args, wait=wait) else: args = self.client._cmd_args() + args args = " ".join(args) - return distributed.remote_cmd(self.server, args) + return distributed.remote_cmd(self.server, args, wait=wait) def check_cmd(self, args): """ Runs shell command on node. :param list[str]|str args: command to run + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 6b7fa44c..6c72547b 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -97,17 +97,18 @@ class VnodeClient(object): status = p.wait() return status, output.decode("utf-8").strip() - def check_cmd(self, args): + def check_cmd(self, args, wait=True): """ Run command and return exit status and combined stdout and stderr. :param list[str]|str args: command to run + :param bool wait: True to wait for command status, False otherwise :return: combined stdout and stderr :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ status, output = self.cmd_output(args) - if status != 0: + if wait and status != 0: raise CoreCommandError(status, args, output) return output.strip() diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index ba8cdc25..48bd5a5a 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -94,7 +94,7 @@ class PhysicalNode(CoreNodeBase): return output.strip() def shcmd(self, cmdstr, sh="/bin/sh"): - return self.cmd([sh, "-c", cmdstr]) + return self.node_net_cmd([sh, "-c", cmdstr]) def sethwaddr(self, ifindex, addr): """ diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index b34daa73..d6eeb1b5 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -9,7 +9,6 @@ services. import enum import logging -import shlex import time from multiprocessing.pool import ThreadPool @@ -598,7 +597,7 @@ class CoreServices(object): status = 0 for cmd in cmds: logging.debug("validating service(%s) using: %s", service.name, cmd) - cmd = shlex.split(cmd) + cmd = utils.split_args(cmd) try: node.node_net_cmd(cmd) except CoreCommandError as e: @@ -730,11 +729,9 @@ class CoreServices(object): status = 0 for cmd in cmds: - cmd = shlex.split(cmd) + cmd = utils.split_args(cmd) try: - if wait: - cmd.append("&") - node.node_net_cmd(cmd) + node.node_net_cmd(cmd, wait) except CoreCommandError: logging.exception("error starting command") status = -1 diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index d669478d..3c6ec383 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -43,11 +43,14 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd(["iperf", "-s", "-D"]) first_node_address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, first_node_address)) - last_node.client.icmd(["iperf", "-t", str(options.time), "-c", first_node_address]) - first_node.cmd(["killall", "-9", "iperf"]) + output = last_node.node_net_cmd( + ["iperf", "-t", str(options.time), "-c", first_node_address] + ) + print(output) + first_node.node_net_cmd(["killall", "-9", "iperf"]) # shutdown session coreemu.shutdown() diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index 376f34d0..3d5171c2 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -47,11 +47,11 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd(["iperf", "-s", "-D"]) address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, address)) - last_node.client.icmd(["iperf", "-t", str(options.time), "-c", address]) - first_node.cmd(["killall", "-9", "iperf"]) + last_node.node_net_cmd(["iperf", "-t", str(options.time), "-c", address]) + first_node.node_net_cmd(["killall", "-9", "iperf"]) # shutdown session coreemu.shutdown() diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 85836605..d9001065 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -26,7 +26,7 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) - return from_node.cmd(["ping", "-c", str(count), address]) + return from_node.node_net_cmd(["ping", "-c", str(count), address]) class TestEmane: diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index e120dcc8..7d64ae69 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -38,7 +38,7 @@ def createclients(sessiondir, clientcls=VnodeClient, cmdchnlfilterfunc=None): def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) - return from_node.cmd(["ping", "-c", "3", address]) + return from_node.node_net_cmd(["ping", "-c", "3", address]) class TestCore: @@ -102,7 +102,6 @@ class TestCore: # check various command using vcmd module command = ["ls"] - assert not client.cmd(command) status, output = client.cmd_output(command) assert not status p, stdin, stdout, stderr = client.popen(command) @@ -110,7 +109,6 @@ class TestCore: assert not client.icmd(command) # check various command using command line - assert not client.cmd(command) status, output = client.cmd_output(command) assert not status p, stdin, stdout, stderr = client.popen(command) From c3d27eb8a53b380d19e7656e6b865aef4cb8adde Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 10 Oct 2019 23:01:16 -0700 Subject: [PATCH 16/38] removed utils.cmd and related node functions --- daemon/core/nodes/base.py | 22 ---------------------- daemon/core/nodes/docker.py | 20 -------------------- daemon/core/nodes/lxd.py | 25 ------------------------- daemon/core/nodes/physical.py | 13 ------------- daemon/core/utils.py | 20 -------------------- daemon/tests/emane/test_emane.py | 9 +++++++-- daemon/tests/test_core.py | 8 +++++++- 7 files changed, 14 insertions(+), 103 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 147989d7..d2d49fb4 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -421,17 +421,6 @@ class CoreNodeBase(NodeBase): """ raise NotImplementedError - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - raise NotImplementedError - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. @@ -609,17 +598,6 @@ class CoreNode(CoreNodeBase): finally: self.rmnodedir() - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - return self.client.cmd(args, wait) - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index ad7deff2..69bf376f 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -47,15 +47,6 @@ class DockerClient(object): name=self.name )) - def cmd(self, cmd, wait=True): - if isinstance(cmd, list): - cmd = " ".join(cmd) - logging.info("docker cmd wait(%s): %s", wait, cmd) - return utils.cmd("docker exec {name} {cmd}".format( - name=self.name, - cmd=cmd - ), wait) - def cmd_output(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) @@ -155,17 +146,6 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - return self.client.cmd(args, wait) - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 86ca192e..54a66182 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -55,13 +55,6 @@ class LxdClient(object): logging.info("lxc cmd output: %s", args) return utils.cmd_output(args) - def cmd(self, cmd, wait=True): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._cmd_args(cmd) - logging.info("lxc cmd: %s", args) - return utils.cmd(args, wait) - def _ns_args(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) @@ -72,13 +65,6 @@ class LxdClient(object): logging.info("ns cmd: %s", args) return utils.cmd_output(args) - def ns_cmd(self, cmd, wait=True): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._ns_args(cmd) - logging.info("ns cmd: %s", args) - return utils.cmd(args, wait) - def copy_file(self, source, destination): if destination[0] != "/": destination = os.path.join("/root/", destination) @@ -158,17 +144,6 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - return self.client.cmd(args, wait) - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 48bd5a5a..159c746d 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -52,19 +52,6 @@ class PhysicalNode(CoreNodeBase): """ return sh - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - os.chdir(self.nodedir) - status = utils.cmd(args, wait) - return status - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 8e59a050..83a18c6a 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -207,26 +207,6 @@ def mute_detach(args, **kwargs): return subprocess.Popen(args, **kwargs).pid -def cmd(args, wait=True): - """ - Runs a command on and returns the exit status. - - :param list[str]|str args: command arguments - :param bool wait: wait for command to end or not - :return: command status - :rtype: int - """ - args = split_args(args) - logging.debug("command: %s", args) - try: - p = subprocess.Popen(args) - if not wait: - return 0 - return p.wait() - except OSError: - raise CoreCommandError(-1, args) - - def cmd_output(args): """ Execute a command on the host and return a tuple containing the exit status and diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index d9001065..3d9b9eb2 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -12,7 +12,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel from core.emulator.emudata import NodeOptions -from core.errors import CoreError +from core.errors import CoreCommandError, CoreError _EMANE_MODELS = [ EmaneIeee80211abgModel, @@ -26,7 +26,12 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) - return from_node.node_net_cmd(["ping", "-c", str(count), address]) + try: + from_node.node_net_cmd(["ping", "-c", str(count), address]) + status = 0 + except CoreCommandError as e: + status = e.returncode + return status class TestEmane: diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 7d64ae69..9a59b4ce 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -10,6 +10,7 @@ import pytest from core.emulator.emudata import NodeOptions from core.emulator.enumerations import MessageFlags, NodeTypes +from core.errors import CoreCommandError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility from core.nodes.client import VnodeClient @@ -38,7 +39,12 @@ def createclients(sessiondir, clientcls=VnodeClient, cmdchnlfilterfunc=None): def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) - return from_node.node_net_cmd(["ping", "-c", "3", address]) + try: + from_node.node_net_cmd(["ping", "-c", "3", address]) + status = 0 + except CoreCommandError as e: + status = e.returncode + return status class TestCore: From 4a6d69bb09eb27724e7b10342a666f1fd4725487 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 09:34:49 -0700 Subject: [PATCH 17/38] removing cmd_output function from utils and nodes --- daemon/core/api/grpc/server.py | 9 ++++-- daemon/core/api/tlv/corehandlers.py | 7 +++- daemon/core/nodes/base.py | 20 ------------ daemon/core/nodes/client.py | 50 +++++------------------------ daemon/core/nodes/docker.py | 46 ++++++++------------------ daemon/core/nodes/lxd.py | 37 ++++++--------------- daemon/core/nodes/physical.py | 42 ++---------------------- daemon/core/services/utility.py | 8 +++-- daemon/core/utils.py | 21 ------------ daemon/tests/test_core.py | 4 --- 10 files changed, 51 insertions(+), 193 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 7fb6ed31..3d589fab 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,6 +10,7 @@ from queue import Empty, Queue import grpc +from core import utils from core.api.grpc import core_pb2, core_pb2_grpc from core.emane.nodes import EmaneNet from core.emulator.data import ( @@ -22,7 +23,7 @@ from core.emulator.data import ( ) from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions from core.emulator.enumerations import EventTypes, LinkTypes, NodeTypes -from core.errors import CoreError +from core.errors import CoreCommandError, CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility from core.nodes.base import CoreNetworkBase from core.nodes.docker import DockerNode @@ -882,7 +883,11 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): logging.debug("sending node command: %s", request) session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) - _, output = node.cmd_output(request.command) + try: + args = utils.split_args(request.command) + output = node.node_net_cmd(args) + except CoreCommandError as e: + output = e.stderr return core_pb2.NodeCommandResponse(output=output) def GetNodeTerminal(self, request, context): diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index a531efe2..e1a32e5f 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -889,7 +889,12 @@ class CoreHandler(socketserver.BaseRequestHandler): or message.flags & MessageFlags.TEXT.value ): if message.flags & MessageFlags.LOCAL.value: - status, res = utils.cmd_output(command) + try: + res = utils.check_cmd(command) + status = 0 + except CoreCommandError as e: + res = e.stderr + status = e.returncode else: try: res = node.node_net_cmd(command) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d2d49fb4..78279b5e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -421,16 +421,6 @@ class CoreNodeBase(NodeBase): """ raise NotImplementedError - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - raise NotImplementedError - def termcmdstring(self, sh): """ Create a terminal command string. @@ -598,16 +588,6 @@ class CoreNode(CoreNodeBase): finally: self.rmnodedir() - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - return self.client.cmd_output(args) - def node_net_cmd(self, args, wait=True): """ Runs a command that is used to configure and setup the network within a diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 6c72547b..3b0b2843 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -57,46 +57,6 @@ class VnodeClient(object): def _cmd_args(self): return [constants.VCMD_BIN, "-c", self.ctrlchnlname, "--"] - def cmd(self, args, wait=True): - """ - Execute a command on a node and return the status (return code). - - :param list[str]|str args: command arguments - :param bool wait: wait for command to end or not - :return: command status - :rtype: int - """ - self._verify_connection() - args = utils.split_args(args) - - # run command, return process when not waiting - cmd = self._cmd_args() + args - logging.debug("cmd wait(%s): %s", wait, cmd) - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - if not wait: - return 0 - - # wait for and return exit status - return p.wait() - - def cmd_output(self, args): - """ - Execute a command on a node and return a tuple containing the - exit status and result string. stderr output - is folded into the stdout result string. - - :param list[str]|str args: command to run - :return: command status and combined stdout and stderr output - :rtype: tuple[int, str] - """ - p, stdin, stdout, stderr = self.popen(args) - stdin.close() - output = stdout.read() + stderr.read() - stdout.close() - stderr.close() - status = p.wait() - return status, output.decode("utf-8").strip() - def check_cmd(self, args, wait=True): """ Run command and return exit status and combined stdout and stderr. @@ -107,10 +67,16 @@ class VnodeClient(object): :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ - status, output = self.cmd_output(args) + p, stdin, stdout, stderr = self.popen(args) + stdin.close() + output = stdout.read() + stderr.read() + output = output.decode("utf-8").strip() + stdout.close() + stderr.close() + status = p.wait() if wait and status != 0: raise CoreCommandError(status, args, output) - return output.strip() + return output def popen(self, args): """ diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 69bf376f..028b3e0b 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -27,12 +27,12 @@ class DockerClient(object): def get_info(self): args = "docker inspect {name}".format(name=self.name) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + output = utils.check_cmd(args) data = json.loads(output) if not data: - raise CoreCommandError(status, args, "docker({name}) not present".format(name=self.name)) + raise CoreCommandError( + -1, args, "docker({name}) not present".format(name=self.name) + ) return data[0] def is_alive(self): @@ -47,11 +47,11 @@ class DockerClient(object): name=self.name )) - def cmd_output(self, cmd): + def check_cmd(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) logging.info("docker cmd output: %s", cmd) - return utils.cmd_output("docker exec {name} {cmd}".format( + return utils.check_cmd("docker exec {name} {cmd}".format( name=self.name, cmd=cmd )) @@ -64,13 +64,11 @@ class DockerClient(object): cmd=cmd ) logging.info("ns cmd: %s", args) - return utils.cmd_output(args) + return utils.check_cmd(args) def get_pid(self): args = "docker inspect -f '{{{{.State.Pid}}}}' {name}".format(name=self.name) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + output = utils.check_cmd(args) self.pid = output logging.debug("node(%s) pid: %s", self.name, self.pid) return output @@ -81,9 +79,7 @@ class DockerClient(object): name=self.name, destination=destination ) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + return utils.check_cmd(args) class DockerNode(CoreNode): @@ -146,16 +142,6 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - return self.client.cmd_output(args) - def check_cmd(self, args): """ Runs shell command on node. @@ -165,20 +151,14 @@ class DockerNode(CoreNode): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - status, output = self.client.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) - return output + return self.client.check_cmd(args) - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) - return 0 + return "" - status, output = self.client.ns_cmd(args) - if status: - raise CoreCommandError(status, args, output) - return output + return self.client.ns_cmd(args) def termcmdstring(self, sh="/bin/sh"): """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 54a66182..c28306b6 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -25,13 +25,11 @@ class LxdClient(object): def get_info(self): args = "lxc list {name} --format json".format(name=self.name) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + output = utils.check_cmd(args) data = json.loads(output) if not data: raise CoreCommandError( - status, args, "LXC({name}) not present".format(name=self.name) + -1, args, "LXC({name}) not present".format(name=self.name) ) return data[0] @@ -48,22 +46,22 @@ class LxdClient(object): def _cmd_args(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def cmd_output(self, cmd): + def check_cmd(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) args = self._cmd_args(cmd) logging.info("lxc cmd output: %s", args) - return utils.cmd_output(args) + return utils.check_cmd(args) def _ns_args(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) - def ns_cmd_output(self, cmd): + def ns_check_cmd(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) args = self._ns_args(cmd) logging.info("ns cmd: %s", args) - return utils.cmd_output(args) + return utils.check_cmd(args) def copy_file(self, source, destination): if destination[0] != "/": @@ -72,9 +70,7 @@ class LxdClient(object): args = "lxc file push {source} {name}/{destination}".format( source=source, name=self.name, destination=destination ) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + utils.check_cmd(args) class LxcNode(CoreNode): @@ -144,16 +140,6 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - return self.client.cmd_output(args) - def check_cmd(self, args): """ Runs shell command on node. @@ -163,15 +149,12 @@ class LxcNode(CoreNode): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - status, output = self.client.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) - return output + return self.client.check_cmd(args) - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) - return 0 + return "" return self.check_cmd(args) def termcmdstring(self, sh="/bin/sh"): diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 159c746d..60e445ea 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -4,7 +4,6 @@ PhysicalNode class for including real systems in the emulated network. import logging import os -import subprocess import threading from core import constants, utils @@ -52,20 +51,6 @@ class PhysicalNode(CoreNodeBase): """ return sh - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - os.chdir(self.nodedir) - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout, _ = p.communicate() - status = p.wait() - return status, stdout.strip() - def check_cmd(self, args): """ Runs shell command on node. @@ -75,10 +60,8 @@ class PhysicalNode(CoreNodeBase): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - status, output = self.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) - return output.strip() + os.chdir(self.nodedir) + return utils.check_cmd(args) def shcmd(self, cmdstr, sh="/bin/sh"): return self.node_net_cmd([sh, "-c", cmdstr]) @@ -526,27 +509,6 @@ class Rj45Node(CoreNodeBase, CoreInterface): """ raise NotImplementedError - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - raise NotImplementedError - - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - raise NotImplementedError - def termcmdstring(self, sh): """ Create a terminal command string. diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 8088b149..66a84dd6 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -415,9 +415,11 @@ class HttpService(UtilService): Detect the apache2 version using the 'a2query' command. """ try: - status, result = utils.cmd_output(["a2query", "-v"]) - except CoreCommandError: - status = -1 + result = utils.check_cmd(["a2query", "-v"]) + status = 0 + except CoreCommandError as e: + status = e.returncode + result = e.stderr if status == 0 and result[:3] == "2.4": return cls.APACHEVER24 diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 83a18c6a..ead65eeb 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -207,27 +207,6 @@ def mute_detach(args, **kwargs): return subprocess.Popen(args, **kwargs).pid -def cmd_output(args): - """ - Execute a command on the host and return a tuple containing the exit status and - result string. stderr output is folded into the stdout result string. - - :param list[str]|str args: command arguments - :return: command status and stdout - :rtype: tuple[int, str] - :raises CoreCommandError: when the file to execute is not found - """ - args = split_args(args) - logging.debug("command: %s", args) - try: - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout, _ = p.communicate() - status = p.wait() - return status, stdout.decode("utf-8").strip() - except OSError: - raise CoreCommandError(-1, args) - - def check_cmd(args, **kwargs): """ Execute a command on the host and return a tuple containing the exit status and diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 9a59b4ce..7432ee58 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -108,15 +108,11 @@ class TestCore: # check various command using vcmd module command = ["ls"] - status, output = client.cmd_output(command) - assert not status p, stdin, stdout, stderr = client.popen(command) assert not p.wait() assert not client.icmd(command) # check various command using command line - status, output = client.cmd_output(command) - assert not status p, stdin, stdout, stderr = client.popen(command) assert not p.wait() assert not client.icmd(command) From d326f246a7adbc2cb49a5ded4f568f5e8f326902 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 12:57:37 -0700 Subject: [PATCH 18/38] removed node based check_cmd, updated to use appropriate function --- daemon/core/emane/emanemanager.py | 7 +++- daemon/core/emulator/session.py | 10 +++--- daemon/core/nodes/base.py | 53 ++++++---------------------- daemon/core/nodes/docker.py | 13 +------ daemon/core/nodes/interface.py | 17 +++++++-- daemon/core/nodes/lxd.py | 15 ++------ daemon/core/nodes/netclient.py | 3 +- daemon/core/nodes/network.py | 8 +++-- daemon/core/nodes/physical.py | 33 +++-------------- daemon/core/services/coreservices.py | 3 +- daemon/tests/test_nodes.py | 2 +- 11 files changed, 51 insertions(+), 113 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 49cf4f24..90ca9dfe 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -152,8 +152,13 @@ class EmaneManager(ModelManager): """ try: # check for emane - emane_version = utils.check_cmd(["emane", "--version"]) + args = ["emane", "--version"] + emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) + args = " ".join(args) + for server in self.session.servers: + conn = self.session.servers[server] + distributed.remote_cmd(conn, args) # load default emane models self.load_models(EMANE_MODELS) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 9bc2ab80..e3a2389b 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -140,6 +140,11 @@ class Session(object): self.options.set_config(key, value) self.metadata = SessionMetaData() + # distributed servers + self.servers = {} + self.tunnels = {} + self.address = None + # initialize session feature helpers self.broker = CoreBroker(session=self) self.location = CoreLocation() @@ -148,11 +153,6 @@ class Session(object): self.emane = EmaneManager(session=self) self.sdt = Sdt(session=self) - # distributed servers - self.servers = {} - self.tunnels = {} - self.address = None - # initialize default node services self.services.default_services = { "mdr": ("zebra", "OSPFv3MDR", "IPForward"), diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 78279b5e..2b9e08eb 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -84,18 +84,24 @@ class NodeBase(object): """ raise NotImplementedError - def net_cmd(self, args, env=None): + def net_cmd(self, args, env=None, cwd=None, wait=True): """ Runs a command that is used to configure and setup the network on the host system. :param list[str]|str args: command to run :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - raise NotImplementedError + if self.server is None: + return utils.check_cmd(args, env=env, cwd=cwd) + else: + args = " ".join(args) + return distributed.remote_cmd(self.server, args, env, cwd, wait) def setposition(self, x=None, y=None, z=None): """ @@ -381,40 +387,13 @@ class CoreNodeBase(NodeBase): return common - def net_cmd(self, args, env=None): - """ - Runs a command that is used to configure and setup the network on the host - system. - - :param list[str]|str args: command to run - :param dict env: environment to run command with - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - if self.server is None: - return utils.check_cmd(args, env=env) - else: - args = " ".join(args) - return distributed.remote_cmd(self.server, args, env=env) - - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): """ Runs a command that is used to configure and setup the network within a node. :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - raise NotImplementedError - - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs @@ -607,18 +586,6 @@ class CoreNode(CoreNodeBase): args = " ".join(args) return distributed.remote_cmd(self.server, args, wait=wait) - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :param bool wait: True to wait for status, False otherwise - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - return self.client.check_cmd(args) - def termcmdstring(self, sh="/bin/sh"): """ Create a terminal command string. diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 028b3e0b..28a92607 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -142,17 +142,6 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - return self.client.check_cmd(args) - def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) @@ -178,7 +167,7 @@ class DockerNode(CoreNode): """ logging.debug("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - self.check_cmd(args) + self.client.check_cmd(args) def mount(self, source, target): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 8b73b1b7..08f1d662 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -48,12 +48,23 @@ class CoreInterface(object): self.server = server self.net_client = LinuxNetClient(self.net_cmd) - def net_cmd(self, args): + def net_cmd(self, args, env=None, cwd=None, wait=True): + """ + Runs a command on the host system or distributed servers. + + :param list[str]|str args: command to run + :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ if self.server is None: - return utils.check_cmd(args) + return utils.check_cmd(args, env=env, cwd=cwd) else: args = " ".join(args) - return distributed.remote_cmd(self.server, args) + return distributed.remote_cmd(self.server, args, env, cwd, wait) def startup(self): """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index c28306b6..f16df715 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -140,22 +140,11 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - return self.client.check_cmd(args) - def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) return "" - return self.check_cmd(args) + return self.client.check_cmd(args) def termcmdstring(self, sh="/bin/sh"): """ @@ -175,7 +164,7 @@ class LxcNode(CoreNode): """ logging.info("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - self.check_cmd(args) + return self.client.check_cmd(args) def mount(self, source, target): """ diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 34fee343..a689dde9 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -5,7 +5,6 @@ Clients for dealing with bridge/interface commands. import os from core.constants import BRCTL_BIN, ETHTOOL_BIN, IP_BIN, OVS_BIN, TC_BIN -from core.utils import check_cmd class LinuxNetClient(object): @@ -275,7 +274,7 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - check_cmd([BRCTL_BIN, "setageing", name, "0"]) + self.run([BRCTL_BIN, "setageing", name, "0"]) class OvsNetClient(LinuxNetClient): diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 81dfc34b..b236b96b 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -308,24 +308,26 @@ class CoreNetwork(CoreNetworkBase): self.startup() ebq.startupdateloop(self) - def net_cmd(self, args, env=None): + def net_cmd(self, args, env=None, cwd=None, wait=True): """ Runs a command that is used to configure and setup the network on the host system and all configured distributed servers. :param list[str]|str args: command to run :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("network node(%s) cmd", self.name) - output = utils.check_cmd(args, env=env) + output = utils.check_cmd(args, env=env, cwd=cwd) args = " ".join(args) for server in self.session.servers: conn = self.session.servers[server] - distributed.remote_cmd(conn, args, env=env) + distributed.remote_cmd(conn, args, env, cwd, wait) return output diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 60e445ea..4c219258 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -51,21 +51,6 @@ class PhysicalNode(CoreNodeBase): """ return sh - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - os.chdir(self.nodedir) - return utils.check_cmd(args) - - def shcmd(self, cmdstr, sh="/bin/sh"): - return self.node_net_cmd([sh, "-c", cmdstr]) - def sethwaddr(self, ifindex, addr): """ Set hardware address for an interface. @@ -205,13 +190,13 @@ class PhysicalNode(CoreNodeBase): source = os.path.abspath(source) logging.info("mounting %s at %s", source, target) os.makedirs(target) - self.check_cmd([constants.MOUNT_BIN, "--bind", source, target]) + self.net_cmd([constants.MOUNT_BIN, "--bind", source, target], cwd=self.nodedir) self._mounts.append((source, target)) def umount(self, target): logging.info("unmounting '%s'" % target) try: - self.check_cmd([constants.UMOUNT_BIN, "-l", target]) + self.net_cmd([constants.UMOUNT_BIN, "-l", target], cwd=self.nodedir) except CoreCommandError: logging.exception("unmounting failed for %s", target) @@ -256,7 +241,8 @@ class Rj45Node(CoreNodeBase, CoreInterface): :param str name: node name :param mtu: rj45 mtu :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ CoreNodeBase.__init__(self, session, _id, name, start, server) CoreInterface.__init__(self, node=self, name=name, mtu=mtu) @@ -498,17 +484,6 @@ class Rj45Node(CoreNodeBase, CoreInterface): CoreInterface.setposition(self, x, y, z) return result - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: exist status and combined stdout and stderr - :rtype: tuple[int, str] - :raises CoreCommandError: when a non-zero exit status occurs - """ - raise NotImplementedError - def termcmdstring(self, sh): """ Create a terminal command string. diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index d6eeb1b5..dc45fa33 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -631,8 +631,9 @@ class CoreServices(object): """ status = 0 for args in service.shutdown: + args = utils.split_args(args) try: - node.check_cmd(args) + node.node_net_cmd(args) except CoreCommandError: logging.exception("error running stop command %s", args) status = -1 diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index baf0c20c..ad5476d4 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -30,7 +30,7 @@ class TestNodes: assert os.path.exists(node.nodedir) assert node.alive() assert node.up - assert node.check_cmd(["ip", "addr", "show", "lo"]) + assert node.node_net_cmd(["ip", "addr", "show", "lo"]) def test_node_update(self, session): # given From fc7a161221b44fffcebba3bca2a6189efa7661b8 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 13:15:57 -0700 Subject: [PATCH 19/38] updated utils.check_cmd to accept the same parameters as other commands and be leveraged for node cmds --- daemon/core/nodes/base.py | 2 +- daemon/core/nodes/docker.py | 6 +++--- daemon/core/nodes/interface.py | 2 +- daemon/core/nodes/lxd.py | 6 +++--- daemon/core/nodes/network.py | 2 +- daemon/core/utils.py | 29 ++++++++++++++++------------- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 2b9e08eb..6a3bebf0 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -98,7 +98,7 @@ class NodeBase(object): :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - return utils.check_cmd(args, env=env, cwd=cwd) + return utils.check_cmd(args, env, cwd, wait) else: args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 28a92607..1b8322ae 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -56,7 +56,7 @@ class DockerClient(object): cmd=cmd )) - def ns_cmd(self, cmd): + def ns_cmd(self, cmd, wait): if isinstance(cmd, list): cmd = " ".join(cmd) args = "nsenter -t {pid} -u -i -p -n {cmd}".format( @@ -64,7 +64,7 @@ class DockerClient(object): cmd=cmd ) logging.info("ns cmd: %s", args) - return utils.check_cmd(args) + return utils.check_cmd(args, wait=wait) def get_pid(self): args = "docker inspect -f '{{{{.State.Pid}}}}' {name}".format(name=self.name) @@ -147,7 +147,7 @@ class DockerNode(CoreNode): logging.debug("node down, not running network command: %s", args) return "" - return self.client.ns_cmd(args) + return self.client.ns_cmd(args, wait) def termcmdstring(self, sh="/bin/sh"): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 08f1d662..d749bbde 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -61,7 +61,7 @@ class CoreInterface(object): :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - return utils.check_cmd(args, env=env, cwd=cwd) + return utils.check_cmd(args, env, cwd, wait) else: args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index f16df715..df01f4ad 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -46,12 +46,12 @@ class LxdClient(object): def _cmd_args(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def check_cmd(self, cmd): + def check_cmd(self, cmd, wait): if isinstance(cmd, list): cmd = " ".join(cmd) args = self._cmd_args(cmd) logging.info("lxc cmd output: %s", args) - return utils.check_cmd(args) + return utils.check_cmd(args, wait=wait) def _ns_args(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) @@ -144,7 +144,7 @@ class LxcNode(CoreNode): if not self.up: logging.debug("node down, not running network command: %s", args) return "" - return self.client.check_cmd(args) + return self.client.check_cmd(args, wait) def termcmdstring(self, sh="/bin/sh"): """ diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index b236b96b..7197c77f 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -322,7 +322,7 @@ class CoreNetwork(CoreNetworkBase): :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("network node(%s) cmd", self.name) - output = utils.check_cmd(args, env=env, cwd=cwd) + output = utils.check_cmd(args, env, cwd, wait) args = " ".join(args) for server in self.session.servers: diff --git a/daemon/core/utils.py b/daemon/core/utils.py index ead65eeb..df59f9ea 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -11,8 +11,8 @@ import logging import logging.config import os import shlex -import subprocess import sys +from subprocess import PIPE, STDOUT, Popen from past.builtins import basestring @@ -203,33 +203,36 @@ def mute_detach(args, **kwargs): args = split_args(args) kwargs["preexec_fn"] = _detach_init kwargs["stdout"] = DEVNULL - kwargs["stderr"] = subprocess.STDOUT - return subprocess.Popen(args, **kwargs).pid + kwargs["stderr"] = STDOUT + return Popen(args, **kwargs).pid -def check_cmd(args, **kwargs): +def check_cmd(args, env=None, cwd=None, wait=True): """ Execute a command on the host and return a tuple containing the exit status and result string. stderr output is folded into the stdout result string. :param list[str]|str args: command arguments - :param dict kwargs: keyword arguments to pass to subprocess.Popen + :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when there is a non-zero exit status or the file to execute is not found """ - kwargs["stdout"] = subprocess.PIPE - kwargs["stderr"] = subprocess.STDOUT args = split_args(args) logging.info("command: %s", args) try: - p = subprocess.Popen(args, **kwargs) - stdout, _ = p.communicate() - status = p.wait() - if status != 0: - raise CoreCommandError(status, args, stdout) - return stdout.decode("utf-8").strip() + p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd) + if wait: + stdout, stderr = p.communicate() + status = p.wait() + if status != 0: + raise CoreCommandError(status, args, stdout, stderr) + return stdout.decode("utf-8").strip() + else: + return "" except OSError: raise CoreCommandError(-1, args) From b5d71bab8243ad8700b397530cf2d889cf0158ca Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 13:36:00 -0700 Subject: [PATCH 20/38] removed VnodeClient.popen --- daemon/core/nodes/base.py | 1 - daemon/core/nodes/client.py | 28 ++-------------------------- daemon/core/utils.py | 2 +- daemon/tests/test_core.py | 4 ---- 4 files changed, 3 insertions(+), 32 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 6a3bebf0..120c53ca 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -579,7 +579,6 @@ class CoreNode(CoreNodeBase): :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - logging.info("node(%s) cmd: %s", self.name, args) return self.client.check_cmd(args, wait=wait) else: args = self.client._cmd_args() + args diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 3b0b2843..13072c05 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -4,12 +4,9 @@ over a control channel to the vnoded process running in a network namespace. The control channel can be accessed via calls using the vcmd shell. """ -import logging import os -from subprocess import PIPE, Popen from core import constants, utils -from core.errors import CoreCommandError class VnodeClient(object): @@ -67,31 +64,10 @@ class VnodeClient(object): :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ - p, stdin, stdout, stderr = self.popen(args) - stdin.close() - output = stdout.read() + stderr.read() - output = output.decode("utf-8").strip() - stdout.close() - stderr.close() - status = p.wait() - if wait and status != 0: - raise CoreCommandError(status, args, output) - return output - - def popen(self, args): - """ - Execute a popen command against the node. - - :param list[str]|str args: command arguments - :return: popen object, stdin, stdout, and stderr - :rtype: tuple - """ self._verify_connection() args = utils.split_args(args) - cmd = self._cmd_args() + args - logging.debug("popen: %s", cmd) - p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) - return p, p.stdin, p.stdout, p.stderr + args = self._cmd_args() + args + return utils.check_cmd(args, wait=wait) def icmd(self, args): """ diff --git a/daemon/core/utils.py b/daemon/core/utils.py index df59f9ea..003c7134 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -222,7 +222,7 @@ def check_cmd(args, env=None, cwd=None, wait=True): execute is not found """ args = split_args(args) - logging.info("command: %s", args) + logging.info("command cwd(%s) wait(%s): %s", cwd, wait, args) try: p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd) if wait: diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 7432ee58..4360ba06 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -108,13 +108,9 @@ class TestCore: # check various command using vcmd module command = ["ls"] - p, stdin, stdout, stderr = client.popen(command) - assert not p.wait() assert not client.icmd(command) # check various command using command line - p, stdin, stdout, stderr = client.popen(command) - assert not p.wait() assert not client.icmd(command) # check module methods From 69772f993c47b90357f93c7c8a1a5157ac864600 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 13:55:06 -0700 Subject: [PATCH 21/38] removed VnodeClient.icmd and VnodeClient.term --- daemon/core/nodes/client.py | 52 ---------------------------- daemon/examples/python/emane80211.py | 4 --- daemon/tests/test_core.py | 31 ++--------------- 3 files changed, 2 insertions(+), 85 deletions(-) diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 13072c05..81297cf5 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -4,8 +4,6 @@ over a control channel to the vnoded process running in a network namespace. The control channel can be accessed via calls using the vcmd shell. """ -import os - from core import constants, utils @@ -69,56 +67,6 @@ class VnodeClient(object): args = self._cmd_args() + args return utils.check_cmd(args, wait=wait) - def icmd(self, args): - """ - Execute an icmd against a node. - - :param list[str]|str args: command arguments - :return: command result - :rtype: int - """ - args = utils.split_args(args) - return os.spawnlp( - os.P_WAIT, - constants.VCMD_BIN, - constants.VCMD_BIN, - "-c", - self.ctrlchnlname, - "--", - *args - ) - - def term(self, sh="/bin/sh"): - """ - Open a terminal on a node. - - :param str sh: shell to open terminal with - :return: terminal command result - :rtype: int - """ - args = ( - "xterm", - "-ut", - "-title", - self.name, - "-e", - constants.VCMD_BIN, - "-c", - self.ctrlchnlname, - "--", - sh, - ) - if "SUDO_USER" in os.environ: - args = ( - "su", - "-s", - "/bin/sh", - "-c", - "exec " + " ".join(map(lambda x: "'%s'" % x, args)), - os.environ["SUDO_USER"], - ) - return os.spawnvp(os.P_NOWAIT, args[0], args) - def termcmdstring(self, sh="/bin/sh"): """ Create a terminal command string. diff --git a/daemon/examples/python/emane80211.py b/daemon/examples/python/emane80211.py index 0e42be95..adf6959b 100644 --- a/daemon/examples/python/emane80211.py +++ b/daemon/examples/python/emane80211.py @@ -40,10 +40,6 @@ def example(options): # instantiate session session.instantiate() - # start a shell on the first node - node = session.get_node(2) - node.client.term("bash") - # shutdown session input("press enter to exit...") coreemu.shutdown() diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 4360ba06..392794d0 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -3,7 +3,6 @@ Unit tests for testing basic CORE networks. """ import os -import stat import threading import pytest @@ -12,31 +11,12 @@ from core.emulator.emudata import NodeOptions from core.emulator.enumerations import MessageFlags, NodeTypes from core.errors import CoreCommandError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility -from core.nodes.client import VnodeClient _PATH = os.path.abspath(os.path.dirname(__file__)) _MOBILITY_FILE = os.path.join(_PATH, "mobility.scen") _WIRED = [NodeTypes.PEER_TO_PEER, NodeTypes.HUB, NodeTypes.SWITCH] -def createclients(sessiondir, clientcls=VnodeClient, cmdchnlfilterfunc=None): - """ - Create clients - - :param str sessiondir: session directory to create clients - :param class clientcls: class to create clients from - :param func cmdchnlfilterfunc: command channel filter function - :return: list of created clients - :rtype: list - """ - direntries = map(lambda x: os.path.join(sessiondir, x), os.listdir(sessiondir)) - cmdchnls = list(filter(lambda x: stat.S_ISSOCK(os.stat(x).st_mode), direntries)) - if cmdchnlfilterfunc: - cmdchnls = list(filter(cmdchnlfilterfunc, cmdchnls)) - cmdchnls.sort() - return map(lambda x: clientcls(os.path.basename(x), x), cmdchnls) - - def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) try: @@ -106,15 +86,8 @@ class TestCore: # check we are connected assert client.connected() - # check various command using vcmd module - command = ["ls"] - assert not client.icmd(command) - - # check various command using command line - assert not client.icmd(command) - - # check module methods - assert createclients(session.session_dir) + # validate command + assert client.check_cmd("echo hello") == "hello" def test_netif(self, session, ip_prefixes): """ From 02ef91242eb3c9b33354021bfed01983854e4889 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 16:36:57 -0700 Subject: [PATCH 22/38] initial changes to convert all commands to be string based for consistency --- daemon/core/api/grpc/server.py | 4 +- daemon/core/api/tlv/corehandlers.py | 1 - daemon/core/emane/emanemanager.py | 41 ++++----- daemon/core/emane/tdma.py | 3 +- daemon/core/emulator/session.py | 1 - daemon/core/location/mobility.py | 2 +- daemon/core/nodes/base.py | 60 ++++++------- daemon/core/nodes/client.py | 10 +-- daemon/core/nodes/docker.py | 1 - daemon/core/nodes/interface.py | 3 +- daemon/core/nodes/ipaddress.py | 6 +- daemon/core/nodes/lxd.py | 20 ++--- daemon/core/nodes/netclient.py | 86 +++++++++---------- daemon/core/nodes/network.py | 123 ++++++++------------------- daemon/core/services/coreservices.py | 3 - daemon/core/services/utility.py | 2 +- daemon/core/utils.py | 22 +---- daemon/core/xml/corexmldeployment.py | 5 +- daemon/tests/emane/test_emane.py | 2 +- daemon/tests/test_core.py | 2 +- daemon/tests/test_nodes.py | 4 +- 21 files changed, 145 insertions(+), 256 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 3d589fab..84d019e3 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,7 +10,6 @@ from queue import Empty, Queue import grpc -from core import utils from core.api.grpc import core_pb2, core_pb2_grpc from core.emane.nodes import EmaneNet from core.emulator.data import ( @@ -884,8 +883,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) try: - args = utils.split_args(request.command) - output = node.node_net_cmd(args) + output = node.node_net_cmd(request.command) except CoreCommandError as e: output = e.stderr return core_pb2.NodeCommandResponse(output=output) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index e1a32e5f..444ae5a5 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -882,7 +882,6 @@ class CoreHandler(socketserver.BaseRequestHandler): return (reply,) else: logging.info("execute message with cmd=%s", command) - command = utils.split_args(command) # execute command and send a response if ( message.flags & MessageFlags.STRING.value diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 90ca9dfe..65fed8bd 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -152,10 +152,9 @@ class EmaneManager(ModelManager): """ try: # check for emane - args = ["emane", "--version"] + args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - args = " ".join(args) for server in self.session.servers: conn = self.session.servers[server] distributed.remote_cmd(conn, args) @@ -652,14 +651,6 @@ class EmaneManager(ModelManager): emane_net = self._emane_nets[key] emanexml.build_xml_files(self, emane_net) - def buildtransportxml(self): - """ - Calls emanegentransportxml using a platform.xml file to build the transportdaemon*.xml. - """ - utils.check_cmd( - ["emanegentransportxml", "platform.xml"], cwd=self.session.session_dir - ) - def buildeventservicexml(self): """ Build the libemaneeventservice.xml file if event service options @@ -705,9 +696,9 @@ class EmaneManager(ModelManager): logging.info("setting user-defined EMANE log level: %d", cfgloglevel) loglevel = str(cfgloglevel) - emanecmd = ["emane", "-d", "-l", loglevel] + emanecmd = "emane -d -l %s" % loglevel if realtime: - emanecmd += ("-r",) + emanecmd += " -r" otagroup, _otaport = self.get_config("otamanagergroup").split(":") otadev = self.get_config("otamanagerdevice") @@ -750,11 +741,11 @@ class EmaneManager(ModelManager): node.node_net_client.create_route(eventgroup, eventdev) # start emane - args = emanecmd + [ - "-f", + args = "%s -f %s %s" % ( + emanecmd, os.path.join(path, "emane%d.log" % n), os.path.join(path, "platform%d.xml" % n), - ] + ) output = node.node_net_cmd(args) logging.info("node(%s) emane daemon running: %s", node.name, args) logging.info("node(%s) emane daemon output: %s", node.name, output) @@ -763,22 +754,22 @@ class EmaneManager(ModelManager): return path = self.session.session_dir - emanecmd += ["-f", os.path.join(path, "emane.log")] - args = emanecmd + [os.path.join(path, "platform.xml")] - utils.check_cmd(args, cwd=path) - args = " ".join(args) + emanecmd += " -f %s" % os.path.join(path, "emane.log") + emanecmd += " %s" % os.path.join(path, "platform.xml") + utils.check_cmd(emanecmd, cwd=path) for server in self.session.servers: conn = self.session.servers[server] - distributed.remote_cmd(conn, args, cwd=path) - logging.info("host emane daemon running: %s", args) + distributed.remote_cmd(conn, emanecmd, cwd=path) + logging.info("host emane daemon running: %s", emanecmd) def stopdaemons(self): """ Kill the appropriate EMANE daemons. """ - # TODO: we may want to improve this if we had the PIDs from the specific EMANE daemons that we"ve started - kill_emaned = ["killall", "-q", "emane"] - kill_transortd = ["killall", "-q", "emanetransportd"] + # TODO: we may want to improve this if we had the PIDs from the specific EMANE + # daemons that we"ve started + kill_emaned = "killall -q emane" + kill_transortd = "killall -q emanetransportd" stop_emane_on_host = False for node in self.getnodes(): if hasattr(node, "transport_type") and node.transport_type == "raw": @@ -793,8 +784,6 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - kill_emaned = " ".join(kill_emaned) - kill_transortd = " ".join(kill_transortd) for server in self.session.servers: conn = self.session[server] distributed.remote_cmd(conn, kill_emaned) diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index dfd36b5d..249e81b6 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -62,4 +62,5 @@ class EmaneTdmaModel(emanemodel.EmaneModel): logging.info( "setting up tdma schedule: schedule(%s) device(%s)", schedule, event_device ) - utils.check_cmd(["emaneevent-tdmaschedule", "-i", event_device, schedule]) + args = "emaneevent-tdmaschedule -i %s %s" % (event_device, schedule) + utils.check_cmd(args) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index e3a2389b..7c4cd651 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -2044,5 +2044,4 @@ class Session(object): utils.mute_detach(data) else: node = self.get_node(node_id) - data = utils.split_args(data) node.node_net_cmd(data, wait=False) diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index f6ce60ca..3522a3f7 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -1281,7 +1281,7 @@ class Ns2ScriptedMobility(WayPointMobility): if filename is None or filename == "": return filename = self.findfile(filename) - args = ["/bin/sh", filename, typestr] + args = "/bin/sh %s %s" % (filename, typestr) utils.check_cmd( args, cwd=self.session.session_dir, env=self.session.get_environment() ) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 120c53ca..85e25074 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -12,7 +12,8 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 -from core import constants, utils +from core import utils +from core.constants import MOUNT_BIN, VNODED_BIN from core.emulator import distributed from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes @@ -89,7 +90,7 @@ class NodeBase(object): Runs a command that is used to configure and setup the network on the host system. - :param list[str]|str args: command to run + :param str args: command to run :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -100,7 +101,6 @@ class NodeBase(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) def setposition(self, x=None, y=None, z=None): @@ -269,7 +269,7 @@ class CoreNodeBase(NodeBase): """ if self.nodedir is None: self.nodedir = os.path.join(self.session.session_dir, self.name + ".conf") - self.net_cmd(["mkdir", "-p", self.nodedir]) + self.net_cmd("mkdir -p %s" % self.nodedir) self.tmpnodedir = True else: self.tmpnodedir = False @@ -285,7 +285,7 @@ class CoreNodeBase(NodeBase): return if self.tmpnodedir: - self.net_cmd(["rm", "-rf", self.nodedir]) + self.net_cmd("rm -rf %s" % self.nodedir) def addnetif(self, netif, ifindex): """ @@ -334,7 +334,7 @@ class CoreNodeBase(NodeBase): Attach a network. :param int ifindex: interface of index to attach - :param core.nodes.interface.CoreInterface net: network to attach + :param core.nodes.base.CoreNetworkBase net: network to attach :return: nothing """ if ifindex not in self._netif: @@ -392,7 +392,7 @@ class CoreNodeBase(NodeBase): Runs a command that is used to configure and setup the network within a node. - :param list[str]|str args: command to run + :param str args: command to run :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str @@ -468,7 +468,7 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - self.net_cmd(["kill", "-0", str(self.pid)]) + self.net_cmd("kill -0 %s" % self.pid) except CoreCommandError: return False @@ -488,18 +488,11 @@ class CoreNode(CoreNodeBase): raise ValueError("starting a node that is already up") # create a new namespace for this node using vnoded - vnoded = [ - constants.VNODED_BIN, - "-v", - "-c", - self.ctrlchnlname, - "-l", - self.ctrlchnlname + ".log", - "-p", - self.ctrlchnlname + ".pid", - ] + vnoded = "{cmd} -v -c {name} -l {name}.log -p {name}.pid".format( + cmd=VNODED_BIN, name=self.ctrlchnlname + ) if self.nodedir: - vnoded += ["-C", self.nodedir] + vnoded += " -C %s" % self.nodedir env = self.session.get_environment(state=False) env["NODE_NUMBER"] = str(self.id) env["NODE_NAME"] = str(self.name) @@ -548,13 +541,13 @@ class CoreNode(CoreNodeBase): # kill node process if present try: - self.net_cmd(["kill", "-9", str(self.pid)]) + self.net_cmd("kill -9 %s" % self.pid) except CoreCommandError: logging.exception("error killing process") # remove node directory if present try: - self.net_cmd(["rm", "-rf", self.ctrlchnlname]) + self.net_cmd("rm -rf %s" % self.ctrlchnlname) except CoreCommandError: logging.exception("error removing node directory") @@ -572,7 +565,7 @@ class CoreNode(CoreNodeBase): Runs a command that is used to configure and setup the network within a node. - :param list[str] args: command to run + :param str args: command to run :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str @@ -581,8 +574,7 @@ class CoreNode(CoreNodeBase): if self.server is None: return self.client.check_cmd(args, wait=wait) else: - args = self.client._cmd_args() + args - args = " ".join(args) + args = self.client.create_cmd(args) return distributed.remote_cmd(self.server, args, wait=wait) def termcmdstring(self, sh="/bin/sh"): @@ -606,7 +598,7 @@ class CoreNode(CoreNodeBase): hostpath = os.path.join( self.nodedir, os.path.normpath(path).strip("/").replace("/", ".") ) - self.net_cmd(["mkdir", "-p", hostpath]) + self.net_cmd("mkdir -p %s" % hostpath) self.mount(hostpath, path) def mount(self, source, target): @@ -620,8 +612,8 @@ class CoreNode(CoreNodeBase): """ source = os.path.abspath(source) logging.debug("node(%s) mounting: %s at %s", self.name, source, target) - self.node_net_cmd(["mkdir", "-p", target]) - self.node_net_cmd([constants.MOUNT_BIN, "-n", "--bind", source, target]) + self.node_net_cmd("mkdir -p %s" % target) + self.node_net_cmd("%s -n --bind %s %s" % (MOUNT_BIN, source, target)) self._mounts.append((source, target)) def newifindex(self): @@ -881,11 +873,11 @@ class CoreNode(CoreNodeBase): logging.info("adding file from %s to %s", srcname, filename) directory = os.path.dirname(filename) if self.server is None: - self.client.check_cmd(["mkdir", "-p", directory]) - self.client.check_cmd(["mv", srcname, filename]) - self.client.check_cmd(["sync"]) + self.client.check_cmd("mkdir -p %s" % directory) + self.client.check_cmd("mv %s %s" % (srcname, filename)) + self.client.check_cmd("sync") else: - self.net_cmd(["mkdir", "-p", directory]) + self.net_cmd("mkdir -p %s" % directory) distributed.remote_put(self.server, srcname, filename) def hostfilename(self, filename): @@ -922,9 +914,9 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) + self.net_cmd("mkdir -m %o -p %s" % (0o755, dirname)) distributed.remote_put_temp(self.server, hostfilename, contents) - self.net_cmd(["chmod", "%o" % mode, hostfilename]) + self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode ) @@ -947,7 +939,7 @@ class CoreNode(CoreNodeBase): else: distributed.remote_put(self.server, srcfilename, hostfilename) if mode is not None: - self.net_cmd(["chmod", "%o" % mode, hostfilename]) + self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 81297cf5..e09c72fa 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -5,6 +5,7 @@ The control channel can be accessed via calls using the vcmd shell. """ from core import constants, utils +from core.constants import VCMD_BIN class VnodeClient(object): @@ -49,22 +50,21 @@ class VnodeClient(object): """ pass - def _cmd_args(self): - return [constants.VCMD_BIN, "-c", self.ctrlchnlname, "--"] + def create_cmd(self, args): + return "%s -c %s -- %s" % (VCMD_BIN, self.ctrlchnlname, args) def check_cmd(self, args, wait=True): """ Run command and return exit status and combined stdout and stderr. - :param list[str]|str args: command to run + :param str args: command to run :param bool wait: True to wait for command status, False otherwise :return: combined stdout and stderr :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ self._verify_connection() - args = utils.split_args(args) - args = self._cmd_args() + args + args = self.create_cmd(args) return utils.check_cmd(args, wait=wait) def termcmdstring(self, sh="/bin/sh"): diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 1b8322ae..416b31d1 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -63,7 +63,6 @@ class DockerClient(object): pid=self.pid, cmd=cmd ) - logging.info("ns cmd: %s", args) return utils.check_cmd(args, wait=wait) def get_pid(self): diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index d749bbde..4e834fd3 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -52,7 +52,7 @@ class CoreInterface(object): """ Runs a command on the host system or distributed servers. - :param list[str]|str args: command to run + :param str args: command to run :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -63,7 +63,6 @@ class CoreInterface(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) def startup(self): diff --git a/daemon/core/nodes/ipaddress.py b/daemon/core/nodes/ipaddress.py index 00aed74c..c7860dbc 100644 --- a/daemon/core/nodes/ipaddress.py +++ b/daemon/core/nodes/ipaddress.py @@ -19,7 +19,7 @@ class MacAddress(object): """ Creates a MacAddress instance. - :param str address: mac address + :param bytes address: mac address """ self.addr = address @@ -42,7 +42,7 @@ class MacAddress(object): """ if not self.addr: return IpAddress.from_string("::") - tmp = struct.unpack("!Q", "\x00\x00" + self.addr)[0] + tmp = struct.unpack("!Q", b"\x00\x00" + self.addr)[0] nic = int(tmp) & 0x000000FFFFFF oui = int(tmp) & 0xFFFFFF000000 # toggle U/L bit @@ -88,7 +88,7 @@ class IpAddress(object): Create a IpAddress instance. :param int af: address family - :param str address: ip address + :param bytes address: ip address :return: """ # check if (af, addr) is valid diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index df01f4ad..96b107f8 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -43,25 +43,19 @@ class LxdClient(object): def stop_container(self): utils.check_cmd("lxc delete --force {name}".format(name=self.name)) - def _cmd_args(self, cmd): + def create_cmd(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def check_cmd(self, cmd, wait): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._cmd_args(cmd) - logging.info("lxc cmd output: %s", args) + def check_cmd(self, cmd, wait=True): + args = self.create_cmd(cmd) return utils.check_cmd(args, wait=wait) - def _ns_args(self, cmd): + def create_ns_cmd(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) - def ns_check_cmd(self, cmd): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._ns_args(cmd) - logging.info("ns cmd: %s", args) - return utils.check_cmd(args) + def ns_check_cmd(self, cmd, wait=True): + args = self.create_ns_cmd(cmd) + return utils.check_cmd(args, wait=wait) def copy_file(self, source, destination): if destination[0] != "/": diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index a689dde9..930e13b4 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -27,7 +27,7 @@ class LinuxNetClient(object): :param str name: name for hostname :return: nothing """ - self.run(["hostname", name]) + self.run("hostname %s" % name) def create_route(self, route, device): """ @@ -37,7 +37,7 @@ class LinuxNetClient(object): :param str device: device to add route to :return: nothing """ - self.run([IP_BIN, "route", "add", route, "dev", device]) + self.run("%s route add %s dev %s" % (IP_BIN, route, device)) def device_up(self, device): """ @@ -46,7 +46,7 @@ class LinuxNetClient(object): :param str device: device to bring up :return: nothing """ - self.run([IP_BIN, "link", "set", device, "up"]) + self.run("%s link set %s up" % (IP_BIN, device)) def device_down(self, device): """ @@ -55,7 +55,7 @@ class LinuxNetClient(object): :param str device: device to bring down :return: nothing """ - self.run([IP_BIN, "link", "set", device, "down"]) + self.run("%s link set %s down" % (IP_BIN, device)) def device_name(self, device, name): """ @@ -65,7 +65,7 @@ class LinuxNetClient(object): :param str name: name to set :return: nothing """ - self.run([IP_BIN, "link", "set", device, "name", name]) + self.run("%s link set %s name %s" % (IP_BIN, device, name)) def device_show(self, device): """ @@ -75,7 +75,7 @@ class LinuxNetClient(object): :return: device information :rtype: str """ - return self.run([IP_BIN, "link", "show", device]) + return self.run("%s link show %s" % (IP_BIN, device)) def device_ns(self, device, namespace): """ @@ -85,7 +85,7 @@ class LinuxNetClient(object): :param str namespace: namespace to set device to :return: nothing """ - self.run([IP_BIN, "link", "set", device, "netns", namespace]) + self.run("%s link set %s netns %s" % (IP_BIN, device, namespace)) def device_flush(self, device): """ @@ -94,7 +94,7 @@ class LinuxNetClient(object): :param str device: device to flush :return: nothing """ - self.run([IP_BIN, "-6", "address", "flush", "dev", device]) + self.run("%s -6 address flush dev %s" % (IP_BIN, device)) def device_mac(self, device, mac): """ @@ -104,7 +104,7 @@ class LinuxNetClient(object): :param str mac: mac to set :return: nothing """ - self.run([IP_BIN, "link", "set", "dev", device, "address", mac]) + self.run("%s link set dev %s address %s" % (IP_BIN, device, mac)) def delete_device(self, device): """ @@ -113,7 +113,7 @@ class LinuxNetClient(object): :param str device: device to delete :return: nothing """ - self.run([IP_BIN, "link", "delete", device]) + self.run("%s link delete %s" % (IP_BIN, device)) def delete_tc(self, device): """ @@ -122,7 +122,7 @@ class LinuxNetClient(object): :param str device: device to remove tc :return: nothing """ - self.run([TC_BIN, "qdisc", "del", "dev", device, "root"]) + self.run("%s qdisc del dev %s root" % (TC_BIN, device)) def checksums_off(self, interface_name): """ @@ -131,7 +131,7 @@ class LinuxNetClient(object): :param str interface_name: interface to update :return: nothing """ - self.run([ETHTOOL_BIN, "-K", interface_name, "rx", "off", "tx", "off"]) + self.run("%s -K %s rx off tx off" % (ETHTOOL_BIN, interface_name)) def create_address(self, device, address, broadcast=None): """ @@ -144,19 +144,11 @@ class LinuxNetClient(object): """ if broadcast is not None: self.run( - [ - IP_BIN, - "address", - "add", - address, - "broadcast", - broadcast, - "dev", - device, - ] + "%s address add %s broadcast %s dev %s" + % (IP_BIN, address, broadcast, device) ) else: - self.run([IP_BIN, "address", "add", address, "dev", device]) + self.run("%s address add %s dev %s" % (IP_BIN, address, device)) def delete_address(self, device, address): """ @@ -166,7 +158,7 @@ class LinuxNetClient(object): :param str address: address to remove :return: nothing """ - self.run([IP_BIN, "address", "delete", address, "dev", device]) + self.run("%s address delete %s dev %s" % (IP_BIN, address, device)) def create_veth(self, name, peer): """ @@ -176,9 +168,7 @@ class LinuxNetClient(object): :param str peer: peer name :return: nothing """ - self.run( - [IP_BIN, "link", "add", "name", name, "type", "veth", "peer", "name", peer] - ) + self.run("%s link add name %s type veth peer name %s" % (IP_BIN, name, peer)) def create_gretap(self, device, address, local, ttl, key): """ @@ -191,13 +181,13 @@ class LinuxNetClient(object): :param str key: key for tap :return: nothing """ - cmd = [IP_BIN, "link", "add", device, "type", "gretap", "remote", address] + cmd = "%s link add %s type gretap remote %s" % (IP_BIN, device, address) if local is not None: - cmd.extend(["local", local]) + cmd += " local %s" % local if ttl is not None: - cmd.extend(["ttl", ttl]) + cmd += " ttl %s" % ttl if key is not None: - cmd.extend(["key", key]) + cmd += " key %s" % key self.run(cmd) def create_bridge(self, name): @@ -207,9 +197,9 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - self.run([BRCTL_BIN, "addbr", name]) - self.run([BRCTL_BIN, "stp", name, "off"]) - self.run([BRCTL_BIN, "setfd", name, "0"]) + self.run("%s addbr %s" % (BRCTL_BIN, name)) + self.run("%s stp %s off" % (BRCTL_BIN, name)) + self.run("%s setfd %s 0" % (BRCTL_BIN, name)) self.device_up(name) # turn off multicast snooping so forwarding occurs w/o IGMP joins @@ -226,7 +216,7 @@ class LinuxNetClient(object): :return: nothing """ self.device_down(name) - self.run([BRCTL_BIN, "delbr", name]) + self.run("%s delbr %s" % (BRCTL_BIN, name)) def create_interface(self, bridge_name, interface_name): """ @@ -236,7 +226,7 @@ class LinuxNetClient(object): :param str interface_name: interface name :return: nothing """ - self.run([BRCTL_BIN, "addif", bridge_name, interface_name]) + self.run("%s addif %s %s" % (BRCTL_BIN, bridge_name, interface_name)) self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): @@ -247,7 +237,7 @@ class LinuxNetClient(object): :param str interface_name: interface name :return: nothing """ - self.run([BRCTL_BIN, "delif", bridge_name, interface_name]) + self.run("%s delif %s %s" % (BRCTL_BIN, bridge_name, interface_name)) def existing_bridges(self, _id): """ @@ -255,7 +245,7 @@ class LinuxNetClient(object): :param _id: node id to check bridges for """ - output = self.run([BRCTL_BIN, "show"]) + output = self.run("%s show" % BRCTL_BIN) lines = output.split("\n") for line in lines[1:]: columns = line.split() @@ -274,7 +264,7 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - self.run([BRCTL_BIN, "setageing", name, "0"]) + self.run("%s setageing %s 0" % (BRCTL_BIN, name)) class OvsNetClient(LinuxNetClient): @@ -289,10 +279,10 @@ class OvsNetClient(LinuxNetClient): :param str name: bridge name :return: nothing """ - self.run([OVS_BIN, "add-br", name]) - self.run([OVS_BIN, "set", "bridge", name, "stp_enable=false"]) - self.run([OVS_BIN, "set", "bridge", name, "other_config:stp-max-age=6"]) - self.run([OVS_BIN, "set", "bridge", name, "other_config:stp-forward-delay=4"]) + self.run("%s add-br %s" % (OVS_BIN, name)) + self.run("%s set bridge %s stp_enable=false" % (OVS_BIN, name)) + self.run("%s set bridge %s other_config:stp-max-age=6" % (OVS_BIN, name)) + self.run("%s set bridge %s other_config:stp-forward-delay=4" % (OVS_BIN, name)) self.device_up(name) def delete_bridge(self, name): @@ -303,7 +293,7 @@ class OvsNetClient(LinuxNetClient): :return: nothing """ self.device_down(name) - self.run([OVS_BIN, "del-br", name]) + self.run("%s del-br %s" % (OVS_BIN, name)) def create_interface(self, bridge_name, interface_name): """ @@ -313,7 +303,7 @@ class OvsNetClient(LinuxNetClient): :param str interface_name: interface name :return: nothing """ - self.run([OVS_BIN, "add-port", bridge_name, interface_name]) + self.run("%s add-port %s %s" % (OVS_BIN, bridge_name, interface_name)) self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): @@ -324,7 +314,7 @@ class OvsNetClient(LinuxNetClient): :param str interface_name: interface name :return: nothing """ - self.run([OVS_BIN, "del-port", bridge_name, interface_name]) + self.run("%s del-port %s %s" % (OVS_BIN, bridge_name, interface_name)) def existing_bridges(self, _id): """ @@ -332,7 +322,7 @@ class OvsNetClient(LinuxNetClient): :param _id: node id to check bridges for """ - output = self.run([OVS_BIN, "list-br"]) + output = self.run("%s list-br" % OVS_BIN) if output: for line in output.split("\n"): fields = line.split(".") @@ -347,4 +337,4 @@ class OvsNetClient(LinuxNetClient): :param str name: bridge name :return: nothing """ - self.run([OVS_BIN, "set", "bridge", name, "other_config:mac-aging-time=0"]) + self.run("%s set bridge %s other_config:mac-aging-time=0" % (OVS_BIN, name)) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 7197c77f..e0fcd839 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -9,6 +9,7 @@ import time from socket import AF_INET, AF_INET6 from core import constants, utils +from core.constants import EBTABLES_BIN from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs @@ -92,14 +93,11 @@ class EbtablesQueue(object): """ Helper for building ebtables atomic file command list. - :param list[str] cmd: ebtable command + :param str cmd: ebtable command :return: ebtable atomic command :rtype: list[str] """ - r = [constants.EBTABLES_BIN, "--atomic-file", self.atomic_file] - if cmd: - r.extend(cmd) - return r + return "%s --atomic-file %s %s" % (EBTABLES_BIN, self.atomic_file, cmd) def lastupdate(self, wlan): """ @@ -163,7 +161,7 @@ class EbtablesQueue(object): :return: nothing """ # save kernel ebtables snapshot to a file - args = self.ebatomiccmd(["--atomic-save"]) + args = self.ebatomiccmd("--atomic-save") wlan.net_cmd(args) # modify the table file using queued ebtables commands @@ -173,12 +171,12 @@ class EbtablesQueue(object): self.cmds = [] # commit the table file to the kernel - args = self.ebatomiccmd(["--atomic-commit"]) + args = self.ebatomiccmd("--atomic-commit") wlan.net_cmd(args) try: - wlan.net_cmd(["rm", "-f", self.atomic_file]) - except OSError: + wlan.net_cmd("rm -f %s" % self.atomic_file) + except CoreCommandError: logging.exception("error removing atomic file: %s", self.atomic_file) def ebchange(self, wlan): @@ -200,58 +198,26 @@ class EbtablesQueue(object): """ with wlan._linked_lock: # flush the chain - self.cmds.extend([["-F", wlan.brname]]) + self.cmds.append("-F %s" % wlan.brname) # rebuild the chain for netif1, v in wlan._linked.items(): for netif2, linked in v.items(): if wlan.policy == "DROP" and linked: self.cmds.extend( [ - [ - "-A", - wlan.brname, - "-i", - netif1.localname, - "-o", - netif2.localname, - "-j", - "ACCEPT", - ], - [ - "-A", - wlan.brname, - "-o", - netif1.localname, - "-i", - netif2.localname, - "-j", - "ACCEPT", - ], + "-A %s -i %s -o %s -j ACCEPT" + % (wlan.brname, netif1.localname, netif2.localname), + "-A %s -o %s -i %s -j ACCEPT" + % (wlan.brname, netif1.localname, netif2.localname), ] ) elif wlan.policy == "ACCEPT" and not linked: self.cmds.extend( [ - [ - "-A", - wlan.brname, - "-i", - netif1.localname, - "-o", - netif2.localname, - "-j", - "DROP", - ], - [ - "-A", - wlan.brname, - "-o", - netif1.localname, - "-i", - netif2.localname, - "-j", - "DROP", - ], + "-A %s -i %s -o %s -j DROP" + % (wlan.brname, netif1.localname, netif2.localname), + "-A %s -o %s -i %s -j DROP" + % (wlan.brname, netif1.localname, netif2.localname), ] ) @@ -313,7 +279,7 @@ class CoreNetwork(CoreNetworkBase): Runs a command that is used to configure and setup the network on the host system and all configured distributed servers. - :param list[str]|str args: command to run + :param str args: command to run :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -323,12 +289,9 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - - args = " ".join(args) for server in self.session.servers: conn = self.session.servers[server] distributed.remote_cmd(conn, args, env, cwd, wait) - return output def startup(self): @@ -341,21 +304,12 @@ class CoreNetwork(CoreNetworkBase): self.net_client.create_bridge(self.brname) # create a new ebtables chain for this bridge - ebtablescmds( - self.net_cmd, - [ - [constants.EBTABLES_BIN, "-N", self.brname, "-P", self.policy], - [ - constants.EBTABLES_BIN, - "-A", - "FORWARD", - "--logical-in", - self.brname, - "-j", - self.brname, - ], - ], - ) + cmds = [ + "%s -N %s -P %s" % (EBTABLES_BIN, self.brname, self.policy), + "%s -A FORWARD --logical-in %s -j %s" + % (EBTABLES_BIN, self.brname, self.brname), + ] + ebtablescmds(self.net_cmd, cmds) self.up = True @@ -372,21 +326,12 @@ class CoreNetwork(CoreNetworkBase): try: self.net_client.delete_bridge(self.brname) - ebtablescmds( - self.net_cmd, - [ - [ - constants.EBTABLES_BIN, - "-D", - "FORWARD", - "--logical-in", - self.brname, - "-j", - self.brname, - ], - [constants.EBTABLES_BIN, "-X", self.brname], - ], - ) + cmds = [ + "%s -D FORWARD --logical-in %s -j %s" + % (EBTABLES_BIN, self.brname, self.brname), + "%s -X %s" % (EBTABLES_BIN, self.brname), + ] + ebtablescmds(self.net_cmd, cmds) except CoreCommandError: logging.exception("error during shutdown") @@ -852,7 +797,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd([self.updown_script, self.brname, "startup"]) + self.net_cmd("%s %s startup" % (self.updown_script, self.brname)) if self.serverintf: self.net_client.create_interface(self.brname, self.serverintf) @@ -880,7 +825,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd([self.updown_script, self.brname, "shutdown"]) + self.net_cmd("%s %s shutdown" % (self.updown_script, self.brname)) except CoreCommandError: logging.exception("error issuing shutdown script shutdown") @@ -1064,7 +1009,8 @@ class HubNode(CoreNetwork): :param int _id: node id :param str name: node namee :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :raises CoreCommandError: when there is a command exception """ CoreNetwork.__init__(self, session, _id, name, start, server) @@ -1094,7 +1040,8 @@ class WlanNode(CoreNetwork): :param int _id: node id :param str name: node name :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param policy: wlan policy """ CoreNetwork.__init__(self, session, _id, name, start, server, policy) diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index dc45fa33..6db2d8ed 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -597,7 +597,6 @@ class CoreServices(object): status = 0 for cmd in cmds: logging.debug("validating service(%s) using: %s", service.name, cmd) - cmd = utils.split_args(cmd) try: node.node_net_cmd(cmd) except CoreCommandError as e: @@ -631,7 +630,6 @@ class CoreServices(object): """ status = 0 for args in service.shutdown: - args = utils.split_args(args) try: node.node_net_cmd(args) except CoreCommandError: @@ -730,7 +728,6 @@ class CoreServices(object): status = 0 for cmd in cmds: - cmd = utils.split_args(cmd) try: node.node_net_cmd(cmd, wait) except CoreCommandError: diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 66a84dd6..14bd5a90 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -415,7 +415,7 @@ class HttpService(UtilService): Detect the apache2 version using the 'a2query' command. """ try: - result = utils.check_cmd(["a2query", "-v"]) + result = utils.check_cmd("a2query -v") status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 003c7134..8f8da19c 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -14,8 +14,6 @@ import shlex import sys from subprocess import PIPE, STDOUT, Popen -from past.builtins import basestring - from core.errors import CoreCommandError DEVNULL = open(os.devnull, "wb") @@ -177,20 +175,6 @@ def make_tuple_fromstr(s, value_type): return tuple(value_type(i) for i in values) -def split_args(args): - """ - Convenience method for splitting potential string commands into a shell-like - syntax list. - - :param list/str args: command list or string - :return: shell-like syntax list - :rtype: list - """ - if isinstance(args, basestring): - args = shlex.split(args) - return args - - def mute_detach(args, **kwargs): """ Run a muted detached process by forking it. @@ -200,7 +184,7 @@ def mute_detach(args, **kwargs): :return: process id of the command :rtype: int """ - args = split_args(args) + args = shlex.split(args) kwargs["preexec_fn"] = _detach_init kwargs["stdout"] = DEVNULL kwargs["stderr"] = STDOUT @@ -212,7 +196,7 @@ def check_cmd(args, env=None, cwd=None, wait=True): Execute a command on the host and return a tuple containing the exit status and result string. stderr output is folded into the stdout result string. - :param list[str]|str args: command arguments + :param str args: command arguments :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -221,8 +205,8 @@ def check_cmd(args, env=None, cwd=None, wait=True): :raises CoreCommandError: when there is a non-zero exit status or the file to execute is not found """ - args = split_args(args) logging.info("command cwd(%s) wait(%s): %s", cwd, wait, args) + args = shlex.split(args) try: p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd) if wait: diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index c410ef5f..ee316ffc 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -3,7 +3,8 @@ import socket from lxml import etree -from core import constants, utils +from core import utils +from core.constants import IP_BIN from core.emane.nodes import EmaneNet from core.nodes import ipaddress from core.nodes.base import CoreNodeBase @@ -67,7 +68,7 @@ def get_address_type(address): def get_ipv4_addresses(hostname): if hostname == "localhost": addresses = [] - args = [constants.IP_BIN, "-o", "-f", "inet", "addr", "show"] + args = "%s -o -f inet address show" % IP_BIN output = utils.check_cmd(args) for line in output.split(os.linesep): split = line.split() diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 3d9b9eb2..65065665 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -27,7 +27,7 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd(["ping", "-c", str(count), address]) + from_node.node_net_cmd("ping -c %s %s" % (count, address)) status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 392794d0..3fc90da8 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -20,7 +20,7 @@ _WIRED = [NodeTypes.PEER_TO_PEER, NodeTypes.HUB, NodeTypes.SWITCH] def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd(["ping", "-c", "3", address]) + from_node.node_net_cmd("ping -c 3 %s" % address) status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index ad5476d4..1f18c87e 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -30,7 +30,7 @@ class TestNodes: assert os.path.exists(node.nodedir) assert node.alive() assert node.up - assert node.node_net_cmd(["ip", "addr", "show", "lo"]) + assert node.node_net_cmd("ip address show lo") def test_node_update(self, session): # given @@ -67,4 +67,4 @@ class TestNodes: # then assert node assert node.up - assert utils.check_cmd(["brctl", "show", node.brname]) + assert utils.check_cmd("brctl show %s" % node.brname) From 5b3308a23159553621b51e5b79c48277afe5d44d Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 22:27:04 -0700 Subject: [PATCH 23/38] updated linkconfig to use string commands, fixed issues for wlan configuration --- daemon/core/api/tlv/corehandlers.py | 2 +- daemon/core/location/mobility.py | 2 +- daemon/core/nodes/netclient.py | 2 +- daemon/core/nodes/network.py | 49 ++++++++++++++--------------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 444ae5a5..6ff7f55b 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1425,7 +1425,7 @@ class CoreHandler(socketserver.BaseRequestHandler): parsed_config = ConfigShim.str_to_dict(values_str) self.session.mobility.set_model_config(node_id, object_name, parsed_config) - if self.session.state == EventTypes.RUNTIME_STATE.value: + if self.session.state == EventTypes.RUNTIME_STATE.value and parsed_config: try: node = self.session.get_node(node_id) if object_name == BasicRangeModel.name: diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index 3522a3f7..2f323783 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -426,7 +426,7 @@ class BasicRangeModel(WirelessModel): self.delay = int(config["delay"]) if self.delay == 0: self.delay = None - self.loss = int(config["error"]) + self.loss = int(float(config["error"])) if self.loss == 0: self.loss = None self.jitter = int(config["jitter"]) diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 930e13b4..43d21ead 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -122,7 +122,7 @@ class LinuxNetClient(object): :param str device: device to remove tc :return: nothing """ - self.run("%s qdisc del dev %s root" % (TC_BIN, device)) + self.run("%s qdisc delete dev %s root" % (TC_BIN, device)) def checksums_off(self, interface_name): """ diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index e0fcd839..4df7b28a 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -8,8 +8,8 @@ import threading import time from socket import AF_INET, AF_INET6 -from core import constants, utils -from core.constants import EBTABLES_BIN +from core import utils +from core.constants import EBTABLES_BIN, TC_BIN from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs @@ -457,8 +457,8 @@ class CoreNetwork(CoreNetworkBase): """ if devname is None: devname = netif.localname - tc = [constants.TC_BIN, "qdisc", "replace", "dev", devname] - parent = ["root"] + tc = "%s qdisc replace dev %s" % (TC_BIN, devname) + parent = "root" changed = False if netif.setparam("bw", bw): # from tc-tbf(8): minimum value for burst is rate / kernel_hz @@ -466,27 +466,24 @@ class CoreNetwork(CoreNetworkBase): burst = max(2 * netif.mtu, bw / 1000) # max IP payload limit = 0xFFFF - tbf = ["tbf", "rate", str(bw), "burst", str(burst), "limit", str(limit)] + tbf = "tbf rate %s burst %s limit %s" % (bw, burst, limit) if bw > 0: if self.up: - logging.debug( - "linkconfig: %s" % ([tc + parent + ["handle", "1:"] + tbf],) - ) - netif.net_cmd(tc + parent + ["handle", "1:"] + tbf) + cmd = "%s %s handle 1: %s" % (tc, parent, tbf) + netif.net_cmd(cmd) netif.setparam("has_tbf", True) changed = True elif netif.getparam("has_tbf") and bw <= 0: - tcd = [] + tc - tcd[2] = "delete" if self.up: - netif.net_cmd(tcd + parent) + cmd = "%s qdisc delete dev %s %s" % (TC_BIN, devname, parent) + netif.net_cmd(cmd) netif.setparam("has_tbf", False) # removing the parent removes the child netif.setparam("has_netem", False) changed = True if netif.getparam("has_tbf"): - parent = ["parent", "1:1"] - netem = ["netem"] + parent = "parent 1:1" + netem = "netem" changed = max(changed, netif.setparam("delay", delay)) if loss is not None: loss = float(loss) @@ -499,17 +496,17 @@ class CoreNetwork(CoreNetworkBase): return # jitter and delay use the same delay statement if delay is not None: - netem += ["delay", "%sus" % delay] + netem += " delay %sus" % delay if jitter is not None: if delay is None: - netem += ["delay", "0us", "%sus" % jitter, "25%"] + netem += " delay 0us %sus 25%%" % jitter else: - netem += ["%sus" % jitter, "25%"] + netem += " %sus 25%%" % jitter if loss is not None and loss > 0: - netem += ["loss", "%s%%" % min(loss, 100)] + netem += " loss %s%%" % min(loss, 100) if duplicate is not None and duplicate > 0: - netem += ["duplicate", "%s%%" % min(duplicate, 100)] + netem += " duplicate %s%%" % min(duplicate, 100) delay_check = delay is None or delay <= 0 jitter_check = jitter is None or jitter <= 0 @@ -519,17 +516,19 @@ class CoreNetwork(CoreNetworkBase): # possibly remove netem if it exists and parent queue wasn't removed if not netif.getparam("has_netem"): return - tc[2] = "delete" if self.up: - logging.debug("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) - netif.net_cmd(tc + parent + ["handle", "10:"]) + cmd = "%s qdisc delete dev %s %s handle 10:" % (TC_BIN, devname, parent) + netif.net_cmd(cmd) netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: - logging.debug( - "linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],) + cmd = "%s qdisc replace dev %s %s handle 10: %s" % ( + TC_BIN, + devname, + parent, + netem, ) - netif.net_cmd(tc + parent + ["handle", "10:"] + netem) + netif.net_cmd(cmd) netif.setparam("has_netem", True) def linknet(self, net): From 2bfd05088094ed8d93d6f9737843793e4f66e0a4 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 22:37:33 -0700 Subject: [PATCH 24/38] updated missed commands to be string based --- daemon/core/emane/emanemanager.py | 5 +++-- daemon/core/emulator/session.py | 3 ++- daemon/core/nodes/physical.py | 7 ++++--- daemon/examples/python/switch.py | 6 +++--- daemon/examples/python/wlan.py | 6 +++--- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 65fed8bd..35cc7ca9 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -974,9 +974,10 @@ class EmaneManager(ModelManager): def emanerunning(self, node): """ - Return True if an EMANE process associated with the given node is running, False otherwise. + Return True if an EMANE process associated with the given node is running, + False otherwise. """ - args = ["pkill", "-0", "-x", "emane"] + args = "pkill -0 -x emane" try: node.node_net_cmd(args) result = True diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 7c4cd651..7d3928b2 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -2025,7 +2025,8 @@ class Session(object): data, ) - # TODO: if data is None, this blows up, but this ties into how event functions are ran, need to clean that up + # TODO: if data is None, this blows up, but this ties into how event functions + # are ran, need to clean that up def run_event(self, node_id=None, name=None, data=None): """ Run a scheduled event, executing commands in the data string. diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 4c219258..9bd04c53 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -6,7 +6,8 @@ import logging import os import threading -from core import constants, utils +from core import utils +from core.constants import MOUNT_BIN, UMOUNT_BIN from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNodeBase @@ -190,13 +191,13 @@ class PhysicalNode(CoreNodeBase): source = os.path.abspath(source) logging.info("mounting %s at %s", source, target) os.makedirs(target) - self.net_cmd([constants.MOUNT_BIN, "--bind", source, target], cwd=self.nodedir) + self.net_cmd("%s --bind %s %s" % (MOUNT_BIN, source, target), cwd=self.nodedir) self._mounts.append((source, target)) def umount(self, target): logging.info("unmounting '%s'" % target) try: - self.net_cmd([constants.UMOUNT_BIN, "-l", target], cwd=self.nodedir) + self.net_cmd("%s -l %s" % (UMOUNT_BIN, target), cwd=self.nodedir) except CoreCommandError: logging.exception("unmounting failed for %s", target) diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index 3c6ec383..e4d0fd02 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -43,14 +43,14 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.node_net_cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd("iperf -s -D") first_node_address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, first_node_address)) output = last_node.node_net_cmd( - ["iperf", "-t", str(options.time), "-c", first_node_address] + "iperf -t %s -c %s" % (options.time, first_node_address) ) print(output) - first_node.node_net_cmd(["killall", "-9", "iperf"]) + first_node.node_net_cmd("killall -9 iperf") # shutdown session coreemu.shutdown() diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index 3d5171c2..b16af7cd 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -47,11 +47,11 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.node_net_cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd("iperf -s -D") address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, address)) - last_node.node_net_cmd(["iperf", "-t", str(options.time), "-c", address]) - first_node.node_net_cmd(["killall", "-9", "iperf"]) + last_node.node_net_cmd("iperf -t %s -c %s" % (options.time, address)) + first_node.node_net_cmd("killall -9 iperf") # shutdown session coreemu.shutdown() From 82bdbd776b3d36b88f45c9397097735cf5ad1f93 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 14 Oct 2019 12:31:41 -0700 Subject: [PATCH 25/38] removed parameter conversion for creating GreTap commands --- daemon/core/nodes/interface.py | 14 ++++---------- daemon/core/nodes/netclient.py | 4 ++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 4e834fd3..2f42fdfe 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -456,12 +456,12 @@ class GreTap(CoreInterface): :param core.nodes.base.CoreNode node: related core node :param str name: interface name :param core.emulator.session.Session session: core session instance - :param mtu: interface mtu + :param int mtu: interface mtu :param str remoteip: remote address :param int _id: object id :param str localip: local address - :param ttl: ttl value - :param key: gre tap key + :param int ttl: ttl value + :param int key: gre tap key :param bool start: start flag :param fabric.connection.Connection server: remote server node will run on, default is None for localhost @@ -484,13 +484,7 @@ class GreTap(CoreInterface): if remoteip is None: raise ValueError("missing remote IP required for GRE TAP device") - if localip is not None: - localip = str(localip) - if ttl is not None: - ttl = str(ttl) - if key is not None: - key = str(key) - self.net_client.create_gretap(self.localname, str(remoteip), localip, ttl, key) + self.net_client.create_gretap(self.localname, remoteip, localip, ttl, key) self.net_client.device_up(self.localname) self.up = True diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 43d21ead..6de5d698 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -177,8 +177,8 @@ class LinuxNetClient(object): :param str device: device to add tap to :param str address: address to add tap for :param str local: local address to tie to - :param str ttl: time to live value - :param str key: key for tap + :param int ttl: time to live value + :param int key: key for tap :return: nothing """ cmd = "%s link add %s type gretap remote %s" % (IP_BIN, device, address) From 5f282bb6950e575c8d6c9f1c920c0072dbf376a1 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 14 Oct 2019 14:28:18 -0700 Subject: [PATCH 26/38] updates to lxd/docker to work with net_cmd/node_net_cmd --- daemon/core/nodes/base.py | 20 ++++++-- daemon/core/nodes/client.py | 11 +---- daemon/core/nodes/docker.py | 89 ++++++++++++++++++++++++---------- daemon/core/nodes/lxd.py | 70 +++++++++++++++----------- daemon/examples/lxd/lxd2lxd.py | 4 +- 5 files changed, 122 insertions(+), 72 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 85e25074..f263d0f3 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -452,14 +452,24 @@ class CoreNode(CoreNodeBase): self._mounts = [] self.bootsh = bootsh - if session.options.get_config("ovs") == "True": - self.node_net_client = OvsNetClient(self.node_net_cmd) - else: - self.node_net_client = LinuxNetClient(self.node_net_cmd) + use_ovs = session.options.get_config("ovs") == "True" + self.node_net_client = self.create_node_net_client(use_ovs) if start: self.startup() + def create_node_net_client(self, use_ovs): + """ + Create a client for running network orchestration commands. + + :param bool use_ovs: True to use OVS bridges, False for Linux bridge + :return: network client + """ + if use_ovs: + return OvsNetClient(self.node_net_cmd) + else: + return LinuxNetClient(self.node_net_cmd) + def alive(self): """ Check if the node is alive. @@ -584,7 +594,7 @@ class CoreNode(CoreNodeBase): :param str sh: shell to execute command in :return: str """ - return self.client.termcmdstring(sh) + return self.client.create_cmd(sh) def privatedir(self, path): """ diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index e09c72fa..632e12bc 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -4,7 +4,7 @@ over a control channel to the vnoded process running in a network namespace. The control channel can be accessed via calls using the vcmd shell. """ -from core import constants, utils +from core import utils from core.constants import VCMD_BIN @@ -66,12 +66,3 @@ class VnodeClient(object): self._verify_connection() args = self.create_cmd(args) return utils.check_cmd(args, wait=wait) - - def termcmdstring(self, sh="/bin/sh"): - """ - Create a terminal command string. - - :param str sh: shell to execute command in - :return: str - """ - return "%s -c %s -- %s" % (constants.VCMD_BIN, self.ctrlchnlname, sh) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 416b31d1..e465a768 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -1,21 +1,25 @@ import json import logging import os +from tempfile import NamedTemporaryFile from core import utils +from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode +from core.nodes.netclient import LinuxNetClient, OvsNetClient class DockerClient(object): - def __init__(self, name, image): + def __init__(self, name, image, run): self.name = name self.image = image + self.run = run self.pid = None def create_container(self): - utils.check_cmd( + self.run( "docker run -td --init --net=none --hostname {name} --name {name} " "--sysctl net.ipv6.conf.all.disable_ipv6=0 " "{image} /bin/bash".format( @@ -27,7 +31,7 @@ class DockerClient(object): def get_info(self): args = "docker inspect {name}".format(name=self.name) - output = utils.check_cmd(args) + output = self.run(args) data = json.loads(output) if not data: raise CoreCommandError( @@ -43,22 +47,24 @@ class DockerClient(object): return False def stop_container(self): - utils.check_cmd("docker rm -f {name}".format( + self.run("docker rm -f {name}".format( name=self.name )) def check_cmd(self, cmd): - if isinstance(cmd, list): - cmd = " ".join(cmd) logging.info("docker cmd output: %s", cmd) return utils.check_cmd("docker exec {name} {cmd}".format( name=self.name, cmd=cmd )) + def create_ns_cmd(self, cmd): + return "nsenter -t {pid} -u -i -p -n {cmd}".format( + pid=self.pid, + cmd=cmd + ) + def ns_cmd(self, cmd, wait): - if isinstance(cmd, list): - cmd = " ".join(cmd) args = "nsenter -t {pid} -u -i -p -n {cmd}".format( pid=self.pid, cmd=cmd @@ -67,7 +73,7 @@ class DockerClient(object): def get_pid(self): args = "docker inspect -f '{{{{.State.Pid}}}}' {name}".format(name=self.name) - output = utils.check_cmd(args) + output = self.run(args) self.pid = output logging.debug("node(%s) pid: %s", self.name, self.pid) return output @@ -78,7 +84,7 @@ class DockerClient(object): name=self.name, destination=destination ) - return utils.check_cmd(args) + return self.run(args) class DockerNode(CoreNode): @@ -101,6 +107,12 @@ class DockerNode(CoreNode): self.image = image super(DockerNode, self).__init__(session, _id, name, nodedir, bootsh, start) + def create_node_net_client(self, use_ovs): + if use_ovs: + return OvsNetClient(self.nsenter_cmd) + else: + return LinuxNetClient(self.nsenter_cmd) + def alive(self): """ Check if the node is alive. @@ -122,7 +134,7 @@ class DockerNode(CoreNode): if self.up: raise ValueError("starting a node that is already up") self.makenodedir() - self.client = DockerClient(self.name, self.image) + self.client = DockerClient(self.name, self.image, self.net_cmd) self.pid = self.client.create_container() self.up = True @@ -141,12 +153,13 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def node_net_cmd(self, args, wait=True): - if not self.up: - logging.debug("node down, not running network command: %s", args) - return "" - - return self.client.ns_cmd(args, wait) + def nsenter_cmd(self, args, wait=True): + if self.server is None: + args = self.client.create_ns_cmd(args) + return utils.check_cmd(args, wait=wait) + else: + args = self.client.create_ns_cmd(args) + return distributed.remote_cmd(self.server, args, wait=wait) def termcmdstring(self, sh="/bin/sh"): """ @@ -166,7 +179,7 @@ class DockerNode(CoreNode): """ logging.debug("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - self.client.check_cmd(args) + self.node_net_cmd(args) def mount(self, source, target): """ @@ -189,13 +202,24 @@ class DockerNode(CoreNode): :param int mode: mode for file :return: nothing """ - logging.debug("node dir(%s) ctrlchannel(%s)", self.nodedir, self.ctrlchnlname) logging.debug("nodefile filename(%s) mode(%s)", filename, mode) - file_path = os.path.join(self.nodedir, filename) - with open(file_path, "w") as f: - os.chmod(f.name, mode) - f.write(contents) - self.client.copy_file(file_path, filename) + directory = os.path.dirname(filename) + temp = NamedTemporaryFile(delete=False) + temp.write(contents.encode("utf-8")) + temp.close() + + if directory: + self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) + if self.server is not None: + distributed.remote_put(self.server, temp.name, temp.name) + self.client.copy_file(temp.name, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) + if self.server is not None: + self.net_cmd("rm -f %s" % temp.name) + os.unlink(temp.name) + logging.debug( + "node(%s) added file: %s; mode: 0%o", self.name, filename, mode + ) def nodefilecopy(self, filename, srcfilename, mode=None): """ @@ -207,5 +231,18 @@ class DockerNode(CoreNode): :param int mode: mode to copy to :return: nothing """ - logging.info("node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode) - raise Exception("not supported") + logging.info( + "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode + ) + directory = os.path.dirname(filename) + self.node_net_cmd("mkdir -p %s" % directory) + + if self.server is None: + source = srcfilename + else: + temp = NamedTemporaryFile(delete=False) + source = temp.name + distributed.remote_put(self.server, source, temp.name) + + self.client.copy_file(source, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 96b107f8..afd36db2 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -2,30 +2,31 @@ import json import logging import os import time +from tempfile import NamedTemporaryFile from core import utils +from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode class LxdClient(object): - def __init__(self, name, image): + def __init__(self, name, image, run): self.name = name self.image = image + self.run = run self.pid = None def create_container(self): - utils.check_cmd( - "lxc launch {image} {name}".format(name=self.name, image=self.image) - ) + self.run("lxc launch {image} {name}".format(name=self.name, image=self.image)) data = self.get_info() self.pid = data["state"]["pid"] return self.pid def get_info(self): args = "lxc list {name} --format json".format(name=self.name) - output = utils.check_cmd(args) + output = self.run(args) data = json.loads(output) if not data: raise CoreCommandError( @@ -41,20 +42,16 @@ class LxdClient(object): return False def stop_container(self): - utils.check_cmd("lxc delete --force {name}".format(name=self.name)) + self.run("lxc delete --force {name}".format(name=self.name)) def create_cmd(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def check_cmd(self, cmd, wait=True): - args = self.create_cmd(cmd) - return utils.check_cmd(args, wait=wait) - def create_ns_cmd(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) - def ns_check_cmd(self, cmd, wait=True): - args = self.create_ns_cmd(cmd) + def check_cmd(self, cmd, wait=True): + args = self.create_cmd(cmd) return utils.check_cmd(args, wait=wait) def copy_file(self, source, destination): @@ -64,7 +61,7 @@ class LxdClient(object): args = "lxc file push {source} {name}/{destination}".format( source=source, name=self.name, destination=destination ) - utils.check_cmd(args) + self.run(args) class LxcNode(CoreNode): @@ -115,7 +112,7 @@ class LxcNode(CoreNode): if self.up: raise ValueError("starting a node that is already up") self.makenodedir() - self.client = LxdClient(self.name, self.image) + self.client = LxdClient(self.name, self.image, self.net_cmd) self.pid = self.client.create_container() self.up = True @@ -134,12 +131,6 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def node_net_cmd(self, args, wait=True): - if not self.up: - logging.debug("node down, not running network command: %s", args) - return "" - return self.client.check_cmd(args, wait) - def termcmdstring(self, sh="/bin/sh"): """ Create a terminal command string. @@ -147,7 +138,7 @@ class LxcNode(CoreNode): :param str sh: shell to execute command in :return: str """ - return "lxc exec {name} -- bash".format(name=self.name) + return "lxc exec {name} -- {sh}".format(name=self.name, sh=sh) def privatedir(self, path): """ @@ -158,7 +149,7 @@ class LxcNode(CoreNode): """ logging.info("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - return self.client.check_cmd(args) + return self.node_net_cmd(args) def mount(self, source, target): """ @@ -181,13 +172,23 @@ class LxcNode(CoreNode): :param int mode: mode for file :return: nothing """ - logging.debug("node dir(%s) ctrlchannel(%s)", self.nodedir, self.ctrlchnlname) logging.debug("nodefile filename(%s) mode(%s)", filename, mode) - file_path = os.path.join(self.nodedir, filename) - with open(file_path, "w") as f: - os.chmod(f.name, mode) - f.write(contents) - self.client.copy_file(file_path, filename) + + directory = os.path.dirname(filename) + temp = NamedTemporaryFile(delete=False) + temp.write(contents.encode("utf-8")) + temp.close() + + if directory: + self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) + if self.server is not None: + distributed.remote_put(self.server, temp.name, temp.name) + self.client.copy_file(temp.name, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) + if self.server is not None: + self.net_cmd("rm -f %s" % temp.name) + os.unlink(temp.name) + logging.debug("node(%s) added file: %s; mode: 0%o", self.name, filename, mode) def nodefilecopy(self, filename, srcfilename, mode=None): """ @@ -202,7 +203,18 @@ class LxcNode(CoreNode): logging.info( "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode ) - raise Exception("not supported") + directory = os.path.dirname(filename) + self.node_net_cmd("mkdir -p %s" % directory) + + if self.server is None: + source = srcfilename + else: + temp = NamedTemporaryFile(delete=False) + source = temp.name + distributed.remote_put(self.server, source, temp.name) + + self.client.copy_file(source, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) def addnetif(self, netif, ifindex): super(LxcNode, self).addnetif(netif, ifindex) diff --git a/daemon/examples/lxd/lxd2lxd.py b/daemon/examples/lxd/lxd2lxd.py index e0ff13a3..4f27de95 100644 --- a/daemon/examples/lxd/lxd2lxd.py +++ b/daemon/examples/lxd/lxd2lxd.py @@ -5,7 +5,7 @@ from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes if __name__ == "__main__": - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.INFO) coreemu = CoreEmu() session = coreemu.create_session() @@ -14,7 +14,7 @@ if __name__ == "__main__": # create nodes and interfaces try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(image="ubuntu") + options = NodeOptions(image="ubuntu:18.04") # create node one node_one = session.add_node(_type=NodeTypes.LXC, node_options=options) From 6570f22ccf1151df3120f87428805f0edd3d060a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 14 Oct 2019 15:43:57 -0700 Subject: [PATCH 27/38] refactor fabric distributed to use a class and update sessions to create and provide these to nodes --- daemon/core/emane/emanemanager.py | 21 +++--- daemon/core/emulator/distributed.py | 113 ++++++++++++++++++---------- daemon/core/emulator/session.py | 32 ++++---- daemon/core/nodes/base.py | 35 ++++----- daemon/core/nodes/docker.py | 7 +- daemon/core/nodes/interface.py | 19 +++-- daemon/core/nodes/lxd.py | 5 +- daemon/core/nodes/network.py | 27 ++++--- daemon/core/nodes/physical.py | 4 +- daemon/core/xml/emanexml.py | 23 +++--- 10 files changed, 153 insertions(+), 133 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 35cc7ca9..f48d2e2e 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -18,7 +18,6 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel -from core.emulator import distributed from core.emulator.enumerations import ( ConfigDataTypes, ConfigFlags, @@ -155,9 +154,9 @@ class EmaneManager(ModelManager): args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - for server in self.session.servers: - conn = self.session.servers[server] - distributed.remote_cmd(conn, args) + for host in self.session.servers: + server = self.session.servers[host] + server.remote_cmd(args) # load default emane models self.load_models(EMANE_MODELS) @@ -757,9 +756,9 @@ class EmaneManager(ModelManager): emanecmd += " -f %s" % os.path.join(path, "emane.log") emanecmd += " %s" % os.path.join(path, "platform.xml") utils.check_cmd(emanecmd, cwd=path) - for server in self.session.servers: - conn = self.session.servers[server] - distributed.remote_cmd(conn, emanecmd, cwd=path) + for host in self.session.servers: + server = self.session.servers[host] + server.remote_cmd(emanecmd, cwd=path) logging.info("host emane daemon running: %s", emanecmd) def stopdaemons(self): @@ -784,10 +783,10 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - for server in self.session.servers: - conn = self.session[server] - distributed.remote_cmd(conn, kill_emaned) - distributed.remote_cmd(conn, kill_transortd) + for host in self.session.servers: + server = self.session[host] + server.remote_cmd(kill_emaned) + server.remote_cmd(kill_transortd) except CoreCommandError: logging.exception("error shutting down emane daemons") diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index abec0a57..19594ae1 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -1,8 +1,13 @@ +""" +Defines distributed server functionality. +""" + import logging import os import threading from tempfile import NamedTemporaryFile +from fabric import Connection from invoke import UnexpectedExit from core.errors import CoreCommandError @@ -10,52 +15,80 @@ from core.errors import CoreCommandError LOCK = threading.Lock() -def remote_cmd(server, cmd, env=None, cwd=None, wait=True): +class DistributedServer(object): """ - Run command remotely using server connection. - - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost - :param str cmd: command to run - :param dict env: environment for remote command, default is None - :param str cwd: directory to run command in, defaults to None, which is the user's - home directory - :param bool wait: True to wait for status, False to background process - :return: stdout when success - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs + Provides distributed server interactions. """ - replace_env = env is not None - if not wait: - cmd += " &" - logging.info( - "remote cmd server(%s) cwd(%s) wait(%s): %s", server.host, cwd, wait, cmd - ) - try: - with LOCK: - if cwd is None: - result = server.run(cmd, hide=False, env=env, replace_env=replace_env) - else: - with server.cd(cwd): - result = server.run( + def __init__(self, host): + """ + Create a DistributedServer instance. + + :param str host: host to connect to + """ + self.host = host + self.conn = Connection(host, user="root") + self.lock = threading.Lock() + + def remote_cmd(self, cmd, env=None, cwd=None, wait=True): + """ + Run command remotely using server connection. + + :param str cmd: command to run + :param dict env: environment for remote command, default is None + :param str cwd: directory to run command in, defaults to None, which is the user's + home directory + :param bool wait: True to wait for status, False to background process + :return: stdout when success + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + + replace_env = env is not None + if not wait: + cmd += " &" + logging.info( + "remote cmd server(%s) cwd(%s) wait(%s): %s", self.host, cwd, wait, cmd + ) + try: + with self.lock: + if cwd is None: + result = self.conn.run( cmd, hide=False, env=env, replace_env=replace_env ) - return result.stdout.strip() - except UnexpectedExit as e: - stdout, stderr = e.streams_for_display() - raise CoreCommandError(e.result.exited, cmd, stdout, stderr) + else: + with self.conn.cd(cwd): + result = self.conn.run( + cmd, hide=False, env=env, replace_env=replace_env + ) + return result.stdout.strip() + except UnexpectedExit as e: + stdout, stderr = e.streams_for_display() + raise CoreCommandError(e.result.exited, cmd, stdout, stderr) + def remote_put(self, source, destination): + """ + Push file to remote server. -def remote_put(server, source, destination): - with LOCK: - server.put(source, destination) + :param str source: source file to push + :param str destination: destination file location + :return: nothing + """ + with self.lock: + self.conn.put(source, destination) + def remote_put_temp(self, destination, data): + """ + Remote push file contents to a remote server, using a temp file as an + intermediate step. -def remote_put_temp(server, destination, data): - with LOCK: - temp = NamedTemporaryFile(delete=False) - temp.write(data.encode("utf-8")) - temp.close() - server.put(temp.name, destination) - os.unlink(temp.name) + :param str destination: file destination for data + :param str data: data to store in remote file + :return: nothing + """ + with self.lock: + temp = NamedTemporaryFile(delete=False) + temp.write(data.encode("utf-8")) + temp.close() + self.conn.put(temp.name, destination) + os.unlink(temp.name) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 7d3928b2..a0b4ec38 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -14,14 +14,13 @@ import threading import time from multiprocessing.pool import ThreadPool -from fabric import Connection - from core import constants, utils from core.api.tlv import coreapi from core.api.tlv.broker import CoreBroker from core.emane.emanemanager import EmaneManager from core.emane.nodes import EmaneNet from core.emulator.data import EventData, ExceptionData, NodeData +from core.emulator.distributed import DistributedServer from core.emulator.emudata import ( IdGen, LinkOptions, @@ -162,11 +161,11 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } - def add_distributed(self, server): - conn = Connection(server, user="root") - self.servers[server] = conn + def add_distributed(self, host): + server = DistributedServer(host) + self.servers[host] = server cmd = "mkdir -p %s" % self.session_dir - conn.run(cmd, hide=False) + server.remote_cmd(cmd) def shutdown_distributed(self): # shutdown all tunnels @@ -176,10 +175,10 @@ class Session(object): tunnel.shutdown() # remove all remote session directories - for server in self.servers: - conn = self.servers[server] + for host in self.servers: + server = self.servers[host] cmd = "rm -rf %s" % self.session_dir - conn.run(cmd, hide=False) + server.remote_cmd(cmd) # clear tunnels self.tunnels.clear() @@ -194,18 +193,15 @@ class Session(object): if isinstance(node, CtrlNet) and node.serverintf is not None: continue - for server in self.servers: - conn = self.servers[server] - key = self.tunnelkey(node_id, IpAddress.to_int(server)) + for host in self.servers: + server = self.servers[host] + key = self.tunnelkey(node_id, IpAddress.to_int(host)) # local to server logging.info( - "local tunnel node(%s) to remote(%s) key(%s)", - node.name, - server, - key, + "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key ) - local_tap = GreTap(session=self, remoteip=server, key=key) + local_tap = GreTap(session=self, remoteip=host, key=key) local_tap.net_client.create_interface(node.brname, local_tap.localname) # server to local @@ -216,7 +212,7 @@ class Session(object): key, ) remote_tap = GreTap( - session=self, remoteip=self.address, key=key, server=conn + session=self, remoteip=self.address, key=key, server=server ) remote_tap.net_client.create_interface( node.brname, remote_tap.localname diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index f263d0f3..7758e4af 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,7 +14,6 @@ from socket import AF_INET, AF_INET6 from core import utils from core.constants import MOUNT_BIN, VNODED_BIN -from core.emulator import distributed from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.errors import CoreCommandError @@ -41,8 +40,8 @@ class NodeBase(object): :param int _id: id :param str name: object name :param bool start: start value - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ self.session = session @@ -101,7 +100,7 @@ class NodeBase(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - return distributed.remote_cmd(self.server, args, env, cwd, wait) + return self.server.remote_cmd(args, env, cwd, wait) def setposition(self, x=None, y=None, z=None): """ @@ -200,7 +199,7 @@ class NodeBase(object): x, y, _ = self.getposition() model = self.type - emulation_server = self.server + emulation_server = self.server.host services = self.services if services is not None: @@ -253,8 +252,8 @@ class CoreNodeBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: boolean for starting - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ super(CoreNodeBase, self).__init__(session, _id, name, start, server) self.services = [] @@ -437,8 +436,8 @@ class CoreNode(CoreNodeBase): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ super(CoreNode, self).__init__(session, _id, name, start, server) self.nodedir = nodedir @@ -585,7 +584,7 @@ class CoreNode(CoreNodeBase): return self.client.check_cmd(args, wait=wait) else: args = self.client.create_cmd(args) - return distributed.remote_cmd(self.server, args, wait=wait) + return self.server.remote_cmd(args, wait=wait) def termcmdstring(self, sh="/bin/sh"): """ @@ -888,7 +887,7 @@ class CoreNode(CoreNodeBase): self.client.check_cmd("sync") else: self.net_cmd("mkdir -p %s" % directory) - distributed.remote_put(self.server, srcname, filename) + self.server.remote_put(srcname, filename) def hostfilename(self, filename): """ @@ -925,7 +924,7 @@ class CoreNode(CoreNodeBase): os.chmod(open_file.name, mode) else: self.net_cmd("mkdir -m %o -p %s" % (0o755, dirname)) - distributed.remote_put_temp(self.server, hostfilename, contents) + self.server.remote_put_temp(hostfilename, contents) self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode @@ -944,12 +943,10 @@ class CoreNode(CoreNodeBase): hostfilename = self.hostfilename(filename) if self.server is None: shutil.copy2(srcfilename, hostfilename) - if mode is not None: - os.chmod(hostfilename, mode) else: - distributed.remote_put(self.server, srcfilename, hostfilename) - if mode is not None: - self.net_cmd("chmod %o %s" % (mode, hostfilename)) + self.server.remote_put(srcfilename, hostfilename) + if mode is not None: + self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) @@ -971,8 +968,8 @@ class CoreNetworkBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: should object start - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ super(CoreNetworkBase, self).__init__(session, _id, name, start, server) self._linked = {} diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index e465a768..2679704f 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -4,7 +4,6 @@ import os from tempfile import NamedTemporaryFile from core import utils -from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode @@ -159,7 +158,7 @@ class DockerNode(CoreNode): return utils.check_cmd(args, wait=wait) else: args = self.client.create_ns_cmd(args) - return distributed.remote_cmd(self.server, args, wait=wait) + return self.server.remote_cmd(args, wait=wait) def termcmdstring(self, sh="/bin/sh"): """ @@ -211,7 +210,7 @@ class DockerNode(CoreNode): if directory: self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) if self.server is not None: - distributed.remote_put(self.server, temp.name, temp.name) + self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) if self.server is not None: @@ -242,7 +241,7 @@ class DockerNode(CoreNode): else: temp = NamedTemporaryFile(delete=False) source = temp.name - distributed.remote_put(self.server, source, temp.name) + self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 2f42fdfe..bfcb8583 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -7,7 +7,6 @@ import time from builtins import int, range from core import utils -from core.emulator import distributed from core.errors import CoreCommandError from core.nodes.netclient import LinuxNetClient @@ -24,8 +23,8 @@ class CoreInterface(object): :param core.nodes.base.CoreNode node: node for interface :param str name: interface name :param mtu: mtu value - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ self.node = node @@ -63,7 +62,7 @@ class CoreInterface(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - return distributed.remote_cmd(self.server, args, env, cwd, wait) + return self.server.remote_cmd(args, env, cwd, wait) def startup(self): """ @@ -220,8 +219,8 @@ class Veth(CoreInterface): :param str name: interface name :param str localname: interface local name :param mtu: interface mtu - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param bool start: start flag :raises CoreCommandError: when there is a command exception """ @@ -280,8 +279,8 @@ class TunTap(CoreInterface): :param str name: interface name :param str localname: local interface name :param mtu: interface mtu - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param bool start: start flag """ CoreInterface.__init__(self, node, name, mtu, server) @@ -463,8 +462,8 @@ class GreTap(CoreInterface): :param int ttl: ttl value :param int key: gre tap key :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ CoreInterface.__init__(self, node, name, mtu, server) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index afd36db2..eef3dc8f 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -5,7 +5,6 @@ import time from tempfile import NamedTemporaryFile from core import utils -from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode @@ -182,7 +181,7 @@ class LxcNode(CoreNode): if directory: self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) if self.server is not None: - distributed.remote_put(self.server, temp.name, temp.name) + self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) if self.server is not None: @@ -211,7 +210,7 @@ class LxcNode(CoreNode): else: temp = NamedTemporaryFile(delete=False) source = temp.name - distributed.remote_put(self.server, source, temp.name) + self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 4df7b28a..4d68ccaf 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -10,7 +10,6 @@ from socket import AF_INET, AF_INET6 from core import utils from core.constants import EBTABLES_BIN, TC_BIN -from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs from core.errors import CoreCommandError, CoreError @@ -257,8 +256,8 @@ class CoreNetwork(CoreNetworkBase): :param int _id: object id :param str name: object name :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param policy: network policy """ CoreNetworkBase.__init__(self, session, _id, name, start, server) @@ -289,9 +288,9 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - for server in self.session.servers: - conn = self.session.servers[server] - distributed.remote_cmd(conn, args, env, cwd, wait) + for host in self.session.servers: + server = self.session.servers[host] + server.remote_cmd(args, env, cwd, wait) return output def startup(self): @@ -632,8 +631,8 @@ class GreTapBridge(CoreNetwork): :param ttl: ttl value :param key: gre tap key :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ CoreNetwork.__init__(self, session, _id, name, False, server, policy) self.grekey = key @@ -753,8 +752,8 @@ class CtrlNet(CoreNetwork): :param prefix: control network ipv4 prefix :param hostid: host id :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param str assign_address: assigned address :param str updown_script: updown script :param serverintf: server interface @@ -1008,8 +1007,8 @@ class HubNode(CoreNetwork): :param int _id: node id :param str name: node namee :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ CoreNetwork.__init__(self, session, _id, name, start, server) @@ -1039,8 +1038,8 @@ class WlanNode(CoreNetwork): :param int _id: node id :param str name: node name :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param policy: wlan policy """ CoreNetwork.__init__(self, session, _id, name, start, server, policy) diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 9bd04c53..9daf4f35 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -242,8 +242,8 @@ class Rj45Node(CoreNodeBase, CoreInterface): :param str name: node name :param mtu: rj45 mtu :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ CoreNodeBase.__init__(self, session, _id, name, start, server) CoreInterface.__init__(self, node=self, name=name, mtu=mtu) diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 3b4fafef..0005c378 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -5,7 +5,6 @@ from tempfile import NamedTemporaryFile from lxml import etree from core import utils -from core.emulator import distributed from core.nodes.ipaddress import MacAddress from core.xml import corexml @@ -53,8 +52,8 @@ def create_file(xml_element, doc_name, file_path, server=None): :param lxml.etree.Element xml_element: root element to write to file :param str doc_name: name to use in the emane doctype :param str file_path: file path to write xml file to - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ doctype = ( @@ -65,7 +64,7 @@ def create_file(xml_element, doc_name, file_path, server=None): temp = NamedTemporaryFile(delete=False) create_file(xml_element, doc_name, temp.name) temp.close() - distributed.remote_put(server, temp.name, file_path) + server.remote_put(temp.name, file_path) os.unlink(temp.name) else: corexml.write_xml_file(xml_element, file_path, doctype=doctype) @@ -327,8 +326,8 @@ def create_phy_xml(emane_model, config, file_path, server): :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ phy_element = etree.Element("phy", name="%s PHY" % emane_model.name) @@ -355,8 +354,8 @@ def create_mac_xml(emane_model, config, file_path, server): :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ if not emane_model.mac_library: @@ -396,8 +395,8 @@ def create_nem_xml( :param str transport_definition: transport file definition path :param str mac_definition: mac file definition path :param str phy_definition: phy file definition path - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) @@ -424,8 +423,8 @@ def create_event_service_xml(group, port, device, file_directory, server=None): :param str port: event port :param str device: event device :param str file_directory: directory to create file in - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ event_element = etree.Element("emaneeventmsgsvc") From b2d2705849cb2f547586214790495ccb4bc19996 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 15 Oct 2019 14:13:42 -0700 Subject: [PATCH 28/38] removed broker from session, updated most places using broker to use alternative logic to compensate where needed --- daemon/core/__init__.py | 3 + daemon/core/api/tlv/coreapi.py | 18 -- daemon/core/api/tlv/corehandlers.py | 72 +++---- daemon/core/emane/emanemanager.py | 182 ++---------------- daemon/core/emulator/distributed.py | 8 +- daemon/core/emulator/session.py | 111 ++++------- daemon/core/location/mobility.py | 93 --------- daemon/core/nodes/base.py | 12 +- daemon/core/nodes/network.py | 4 +- daemon/core/plugins/sdt.py | 2 +- daemon/core/xml/corexmldeployment.py | 4 - daemon/core/xml/emanexml.py | 24 +-- daemon/examples/python/distributed.py | 16 +- daemon/examples/python/distributed_emane.py | 26 +-- daemon/examples/python/distributed_ptp.py | 13 +- .../examples/python/distributed_switches.py | 11 +- daemon/examples/python/distributed_wlan.py | 16 +- daemon/tests/conftest.py | 1 - daemon/tests/test_gui.py | 6 +- 19 files changed, 151 insertions(+), 471 deletions(-) diff --git a/daemon/core/__init__.py b/daemon/core/__init__.py index c847c8dc..40ca3604 100644 --- a/daemon/core/__init__.py +++ b/daemon/core/__init__.py @@ -2,3 +2,6 @@ import logging.config # setup default null handler logging.getLogger(__name__).addHandler(logging.NullHandler()) + +# disable paramiko logging +logging.getLogger("paramiko").setLevel(logging.WARNING) diff --git a/daemon/core/api/tlv/coreapi.py b/daemon/core/api/tlv/coreapi.py index 63747642..1e1de8be 100644 --- a/daemon/core/api/tlv/coreapi.py +++ b/daemon/core/api/tlv/coreapi.py @@ -15,7 +15,6 @@ from core.api.tlv import structutils from core.emulator.enumerations import ( ConfigTlvs, EventTlvs, - EventTypes, ExceptionTlvs, ExecuteTlvs, FileTlvs, @@ -1017,20 +1016,3 @@ def str_to_list(value): return None return value.split("|") - - -def state_name(value): - """ - Helper to convert state number into state name using event types. - - :param int value: state value to derive name from - :return: state name - :rtype: str - """ - - try: - value = EventTypes(value).name - except ValueError: - value = "unknown" - - return value diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 6ff7f55b..4b4e7c1e 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -86,6 +86,7 @@ class CoreHandler(socketserver.BaseRequestHandler): self.master = False self.session = None + self.session_clients = {} # core emulator self.coreemu = server.coreemu @@ -138,8 +139,9 @@ class CoreHandler(socketserver.BaseRequestHandler): if self.session: # remove client from session broker and shutdown if there are no clients self.remove_session_handlers() - self.session.broker.session_clients.remove(self) - if not self.session.broker.session_clients and not self.session.is_active(): + clients = self.session_clients[self.session.id] + clients.remove(self) + if not clients and not self.session.is_active(): logging.info( "no session clients left and not active, initiating shutdown" ) @@ -407,9 +409,7 @@ class CoreHandler(socketserver.BaseRequestHandler): tlv_data += coreapi.CoreRegisterTlv.pack( RegisterTlvs.EMULATION_SERVER.value, "core-daemon" ) - tlv_data += coreapi.CoreRegisterTlv.pack( - self.session.broker.config_type, self.session.broker.name - ) + tlv_data += coreapi.CoreRegisterTlv.pack(RegisterTlvs.UTILITY.value, "broker") tlv_data += coreapi.CoreRegisterTlv.pack( self.session.location.config_type, self.session.location.name ) @@ -533,10 +533,6 @@ class CoreHandler(socketserver.BaseRequestHandler): :param message: message to handle :return: nothing """ - if self.session and self.session.broker.handle_message(message): - logging.debug("message not being handled locally") - return - logging.debug( "%s handling message:\n%s", threading.currentThread().getName(), message ) @@ -606,12 +602,11 @@ class CoreHandler(socketserver.BaseRequestHandler): self.session = self.coreemu.create_session(port, master=False) logging.debug("created new session for client: %s", self.session.id) - # TODO: hack to associate this handler with this sessions broker for broadcasting - # TODO: broker needs to be pulled out of session to the server/handler level if self.master: logging.debug("session set to master") self.session.master = True - self.session.broker.session_clients.append(self) + clients = self.session_clients.setdefault(self.session.id, []) + clients.append(self) # add handlers for various data self.add_session_handlers() @@ -643,7 +638,8 @@ class CoreHandler(socketserver.BaseRequestHandler): ]: continue - for client in self.session.broker.session_clients: + clients = self.session_clients[self.session.id] + for client in clients: if client == self: continue @@ -734,6 +730,7 @@ class CoreHandler(socketserver.BaseRequestHandler): node_options.icon = message.get_tlv(NodeTlvs.ICON.value) node_options.canvas = message.get_tlv(NodeTlvs.CANVAS.value) node_options.opaque = message.get_tlv(NodeTlvs.OPAQUE.value) + node_options.emulation_server = message.get_tlv(NodeTlvs.EMULATION_SERVER.value) services = message.get_tlv(NodeTlvs.SERVICES.value) if services: @@ -1027,8 +1024,9 @@ class CoreHandler(socketserver.BaseRequestHandler): # find the session containing this client and set the session to master for _id in self.coreemu.sessions: - session = self.coreemu.sessions[_id] - if self in session.broker.session_clients: + clients = self.session_clients[_id] + if self in clients: + session = self.coreemu.sessions[_id] logging.debug("setting session to master: %s", session.id) session.master = True break @@ -1077,7 +1075,7 @@ class CoreHandler(socketserver.BaseRequestHandler): self.handle_config_location(message_type, config_data) elif config_data.object == self.session.metadata.name: replies = self.handle_config_metadata(message_type, config_data) - elif config_data.object == self.session.broker.name: + elif config_data.object == "broker": self.handle_config_broker(message_type, config_data) elif config_data.object == self.session.services.name: replies = self.handle_config_services(message_type, config_data) @@ -1182,7 +1180,6 @@ class CoreHandler(socketserver.BaseRequestHandler): def handle_config_broker(self, message_type, config_data): if message_type not in [ConfigFlags.REQUEST, ConfigFlags.RESET]: - session_id = config_data.session if not config_data.data_values: logging.info("emulation server data missing") else: @@ -1194,29 +1191,10 @@ class CoreHandler(socketserver.BaseRequestHandler): for server in server_list: server_items = server.split(":") - name, host, port = server_items[:3] - - if host == "": - host = None - - if port == "": - port = None - else: - port = int(port) - - if session_id is not None: - # receive session ID and my IP from master - self.session.broker.session_id_master = int( - session_id.split("|")[0] - ) - self.session.broker.myip = host - host = None - port = None - - # this connects to the server immediately; maybe we should wait - # or spin off a new "client" thread here - self.session.broker.addserver(name, host, port) - self.session.broker.setupserver(name) + name, host, _ = server_items[:3] + self.session.add_distributed(name, host) + elif message_type == ConfigFlags.RESET: + self.session.shutdown_distributed() def handle_config_services(self, message_type, config_data): replies = [] @@ -1842,11 +1820,9 @@ class CoreHandler(socketserver.BaseRequestHandler): # remove client from session broker and shutdown if needed self.remove_session_handlers() - self.session.broker.session_clients.remove(self) - if ( - not self.session.broker.session_clients - and not self.session.is_active() - ): + clients = self.session_clients[self.session.id] + clients.remove(self) + if not clients and not self.session.is_active(): self.coreemu.delete_session(self.session.id) # set session to join @@ -1855,7 +1831,8 @@ class CoreHandler(socketserver.BaseRequestHandler): # add client to session broker and set master if needed if self.master: self.session.master = True - self.session.broker.session_clients.append(self) + clients = self.session_clients.setdefault(self.session.id, []) + clients.append(self) # add broadcast handlers logging.info("adding session broadcast handlers") @@ -2139,7 +2116,8 @@ class CoreUdpHandler(CoreHandler): if not isinstance(message, (coreapi.CoreNodeMessage, coreapi.CoreLinkMessage)): return - for client in self.session.broker.session_clients: + clients = self.session_clients[self.session.id] + for client in clients: try: client.sendall(message.raw_message) except IOError: diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index f48d2e2e..e4208189 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -2,14 +2,12 @@ emane.py: definition of an Emane class for implementing configuration control of an EMANE emulation. """ -import copy import logging import os import threading from core import utils -from core.api.tlv import coreapi, dataconversion -from core.config import ConfigGroup, ConfigShim, Configuration, ModelManager +from core.config import ConfigGroup, Configuration, ModelManager from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel @@ -18,14 +16,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel -from core.emulator.enumerations import ( - ConfigDataTypes, - ConfigFlags, - ConfigTlvs, - MessageFlags, - MessageTypes, - RegisterTlvs, -) +from core.emulator.enumerations import ConfigDataTypes, RegisterTlvs from core.errors import CoreCommandError, CoreError from core.xml import emanexml @@ -75,8 +66,6 @@ class EmaneManager(ModelManager): self.session = session self._emane_nets = {} self._emane_node_lock = threading.Lock() - self._ifccounts = {} - self._ifccountslock = threading.Lock() # port numbers are allocated from these counters self.platformport = self.session.options.get_config_int( "emane_platform_port", 8100 @@ -91,7 +80,6 @@ class EmaneManager(ModelManager): self.emane_config = EmaneGlobalModel(session) self.set_configs(self.emane_config.default_values()) - session.broker.handlers.add(self.handledistributed) self.service = None self.event_device = None self.emane_check() @@ -154,8 +142,8 @@ class EmaneManager(ModelManager): args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - for host in self.session.servers: - server = self.session.servers[host] + for name in self.session.servers: + server = self.session.servers[name] server.remote_cmd(args) # load default emane models @@ -282,7 +270,6 @@ class EmaneManager(ModelManager): return EmaneManager.NOT_NEEDED # control network bridge required for EMANE 0.9.2 - # - needs to be configured before checkdistributed() for distributed # - needs to exist when eventservice binds to it (initeventservice) if self.session.master: otadev = self.get_config("otamanagerdevice") @@ -297,10 +284,9 @@ class EmaneManager(ModelManager): ) return EmaneManager.NOT_READY - ctrlnet = self.session.add_remove_control_net( + self.session.add_remove_control_net( net_index=netidx, remove=False, conf_required=False ) - self.distributedctrlnet(ctrlnet) eventdev = self.get_config("eventservicedevice") logging.debug("emane event service device: eventdev(%s)", eventdev) if eventdev != otadev: @@ -313,18 +299,9 @@ class EmaneManager(ModelManager): ) return EmaneManager.NOT_READY - ctrlnet = self.session.add_remove_control_net( + self.session.add_remove_control_net( net_index=netidx, remove=False, conf_required=False ) - self.distributedctrlnet(ctrlnet) - - if self.checkdistributed(): - # we are slave, but haven't received a platformid yet - platform_id_start = "platform_id_start" - default_values = self.emane_config.default_values() - value = self.get_config(platform_id_start) - if value == default_values[platform_id_start]: - return EmaneManager.NOT_READY self.check_node_models() return EmaneManager.SUCCESS @@ -413,9 +390,6 @@ class EmaneManager(ModelManager): """ stop all EMANE daemons """ - with self._ifccountslock: - self._ifccounts.clear() - with self._emane_node_lock: if not self._emane_nets: return @@ -424,92 +398,6 @@ class EmaneManager(ModelManager): self.stopdaemons() self.stopeventmonitor() - def handledistributed(self, message): - """ - Broker handler for processing CORE API messages as they are - received. This is used to snoop the Link add messages to get NEM - counts of NEMs that exist on other servers. - """ - if ( - message.message_type == MessageTypes.LINK.value - and message.flags & MessageFlags.ADD.value - ): - nn = message.node_numbers() - # first node is always link layer node in Link add message - if nn[0] in self.session.broker.network_nodes: - serverlist = self.session.broker.getserversbynode(nn[1]) - for server in serverlist: - with self._ifccountslock: - if server not in self._ifccounts: - self._ifccounts[server] = 1 - else: - self._ifccounts[server] += 1 - - def checkdistributed(self): - """ - Check for EMANE nodes that exist on multiple emulation servers and - coordinate the NEM id and port number space. - If we are the master EMANE node, return False so initialization will - proceed as normal; otherwise slaves return True here and - initialization is deferred. - """ - # check with the session if we are the "master" Emane object? - master = False - - with self._emane_node_lock: - if self._emane_nets: - master = self.session.master - logging.info("emane check distributed as master: %s.", master) - - # we are not the master Emane object, wait for nem id and ports - if not master: - return True - - nemcount = 0 - with self._emane_node_lock: - for key in self._emane_nets: - emane_node = self._emane_nets[key] - nemcount += emane_node.numnetif() - - nemid = int(self.get_config("nem_id_start")) - nemid += nemcount - - platformid = int(self.get_config("platform_id_start")) - - # build an ordered list of servers so platform ID is deterministic - servers = [] - for key in sorted(self._emane_nets): - for server in self.session.broker.getserversbynode(key): - if server not in servers: - servers.append(server) - - servers.sort(key=lambda x: x.name) - for server in servers: - if server.name == "localhost": - continue - - if server.sock is None: - continue - - platformid += 1 - - # create temporary config for updating distributed nodes - typeflags = ConfigFlags.UPDATE.value - config = copy.deepcopy(self.get_configs()) - config["platform_id_start"] = str(platformid) - config["nem_id_start"] = str(nemid) - config_data = ConfigShim.config_data( - 0, None, typeflags, self.emane_config, config - ) - message = dataconversion.convert_config(config_data) - server.sock.send(message) - # increment nemid for next server by number of interfaces - with self._ifccountslock: - if server in self._ifccounts: - nemid += self._ifccounts[server] - - return False - def buildxml(self): """ Build XML files required to run EMANE on each node. @@ -526,52 +414,6 @@ class EmaneManager(ModelManager): self.buildnemxml() self.buildeventservicexml() - # TODO: remove need for tlv messaging - def distributedctrlnet(self, ctrlnet): - """ - Distributed EMANE requires multiple control network prefixes to - be configured. This generates configuration for slave control nets - using the default list of prefixes. - """ - # slave server - session = self.session - if not session.master: - return - - # not distributed - servers = session.broker.getservernames() - if len(servers) < 2: - return - - # normal Config messaging will distribute controlnets - prefix = session.options.get_config("controlnet", default="") - prefixes = prefix.split() - if len(prefixes) < len(servers): - logging.info( - "setting up default controlnet prefixes for distributed (%d configured)", - len(prefixes), - ) - prefix = ctrlnet.DEFAULT_PREFIX_LIST[0] - prefixes = prefix.split() - servers.remove("localhost") - servers.insert(0, "localhost") - prefix = " ".join("%s:%s" % (s, prefixes[i]) for i, s in enumerate(servers)) - - # this generates a config message having controlnet prefix assignments - logging.info("setting up controlnet prefixes for distributed: %s", prefix) - vals = "controlnet=%s" % prefix - tlvdata = b"" - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.OBJECT.value, "session") - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.TYPE.value, 0) - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.VALUES.value, vals) - rawmsg = coreapi.CoreConfMessage.pack(0, tlvdata) - msghdr = rawmsg[: coreapi.CoreMessage.header_len] - msg = coreapi.CoreConfMessage( - flags=0, hdr=msghdr, data=rawmsg[coreapi.CoreMessage.header_len :] - ) - logging.debug("sending controlnet message:\n%s", msg) - self.session.broker.handle_message(msg) - def check_node_models(self): """ Associate EMANE model classes with EMANE network nodes. @@ -676,8 +518,8 @@ class EmaneManager(ModelManager): dev = self.get_config("eventservicedevice") emanexml.create_event_service_xml(group, port, dev, self.session.session_dir) - for server in self.session.servers: - conn = self.session.servers[server] + for name in self.session.servers: + conn = self.session.servers[name] emanexml.create_event_service_xml( group, port, dev, self.session.session_dir, conn ) @@ -756,8 +598,8 @@ class EmaneManager(ModelManager): emanecmd += " -f %s" % os.path.join(path, "emane.log") emanecmd += " %s" % os.path.join(path, "platform.xml") utils.check_cmd(emanecmd, cwd=path) - for host in self.session.servers: - server = self.session.servers[host] + for name in self.session.servers: + server = self.session.servers[name] server.remote_cmd(emanecmd, cwd=path) logging.info("host emane daemon running: %s", emanecmd) @@ -783,8 +625,8 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - for host in self.session.servers: - server = self.session[host] + for name in self.session.servers: + server = self.session[name] server.remote_cmd(kill_emaned) server.remote_cmd(kill_transortd) except CoreCommandError: diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 19594ae1..4e258937 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -20,12 +20,14 @@ class DistributedServer(object): Provides distributed server interactions. """ - def __init__(self, host): + def __init__(self, name, host): """ Create a DistributedServer instance. + :param str name: convenience name to associate with host :param str host: host to connect to """ + self.name = name self.host = host self.conn = Connection(host, user="root") self.lock = threading.Lock() @@ -36,8 +38,8 @@ class DistributedServer(object): :param str cmd: command to run :param dict env: environment for remote command, default is None - :param str cwd: directory to run command in, defaults to None, which is the user's - home directory + :param str cwd: directory to run command in, defaults to None, which is the + user's home directory :param bool wait: True to wait for status, False to background process :return: stdout when success :rtype: str diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index a0b4ec38..1445cb8f 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -15,8 +15,6 @@ import time from multiprocessing.pool import ThreadPool from core import constants, utils -from core.api.tlv import coreapi -from core.api.tlv.broker import CoreBroker from core.emane.emanemanager import EmaneManager from core.emane.nodes import EmaneNet from core.emulator.data import EventData, ExceptionData, NodeData @@ -142,10 +140,9 @@ class Session(object): # distributed servers self.servers = {} self.tunnels = {} - self.address = None + self.address = self.options.get_config("distributed_address", default=None) # initialize session feature helpers - self.broker = CoreBroker(session=self) self.location = CoreLocation() self.mobility = MobilityManager(session=self) self.services = CoreServices(session=self) @@ -161,9 +158,9 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } - def add_distributed(self, host): - server = DistributedServer(host) - self.servers[host] = server + def add_distributed(self, name, host): + server = DistributedServer(name, host) + self.servers[name] = server cmd = "mkdir -p %s" % self.session_dir server.remote_cmd(cmd) @@ -175,8 +172,8 @@ class Session(object): tunnel.shutdown() # remove all remote session directories - for host in self.servers: - server = self.servers[host] + for name in self.servers: + server = self.servers[name] cmd = "rm -rf %s" % self.session_dir server.remote_cmd(cmd) @@ -193,8 +190,9 @@ class Session(object): if isinstance(node, CtrlNet) and node.serverintf is not None: continue - for host in self.servers: - server = self.servers[host] + for name in self.servers: + server = self.servers[name] + host = server.host key = self.tunnelkey(node_id, IpAddress.to_int(host)) # local to server @@ -219,23 +217,35 @@ class Session(object): ) # save tunnels for shutdown - self.tunnels[key] = [local_tap, remote_tap] + self.tunnels[key] = (local_tap, remote_tap) - def tunnelkey(self, n1num, n2num): + def tunnelkey(self, n1_id, n2_id): """ Compute a 32-bit key used to uniquely identify a GRE tunnel. The hash(n1num), hash(n2num) values are used, so node numbers may be None or string values (used for e.g. "ctrlnet"). - :param int n1num: node one id - :param int n2num: node two id + :param int n1_id: node one id + :param int n2_id: node two id :return: tunnel key for the node pair :rtype: int """ - logging.debug("creating tunnel key for: %s, %s", n1num, n2num) - key = (self.id << 16) ^ utils.hashkey(n1num) ^ (utils.hashkey(n2num) << 8) + logging.debug("creating tunnel key for: %s, %s", n1_id, n2_id) + key = (self.id << 16) ^ utils.hashkey(n1_id) ^ (utils.hashkey(n2_id) << 8) return key & 0xFFFFFFFF + def gettunnel(self, n1_id, n2_id): + """ + Return the GreTap between two nodes if it exists. + + :param int n1_id: node one id + :param int n2_id: node two id + :return: gre tap between nodes or None + """ + key = self.tunnelkey(n1_id, n2_id) + logging.debug("checking for tunnel key(%s) in: %s", key, self.tunnels) + return self.tunnels.get(key) + @classmethod def get_node_class(cls, _type): """ @@ -285,7 +295,7 @@ class Session(object): node_two = self.get_node(node_two_id) # both node ids are provided - tunnel = self.broker.gettunnel(node_one_id, node_two_id) + tunnel = self.gettunnel(node_one_id, node_two_id) logging.debug("tunnel between nodes: %s", tunnel) if isinstance(tunnel, GreTapBridge): net_one = tunnel @@ -958,13 +968,13 @@ class Session(object): def clear(self): """ - Clear all CORE session data. (objects, hooks, broker) + Clear all CORE session data. (nodes, hooks, etc) :return: nothing """ self.delete_nodes() + self.shutdown_distributed() self.del_hooks() - self.broker.reset() self.emane.reset() def start_events(self): @@ -1038,17 +1048,16 @@ class Session(object): # shutdown/cleanup feature helpers self.emane.shutdown() - self.broker.shutdown() self.sdt.shutdown() - # delete all current nodes + # remove and shutdown all nodes and tunnels self.delete_nodes() + self.shutdown_distributed() # remove this sessions working directory preserve = self.options.get_config("preservedir") == "1" if not preserve: shutil.rmtree(self.session_dir, ignore_errors=True) - self.shutdown_distributed() # call session shutdown handlers for handler in self.shutdown_handlers: @@ -1160,7 +1169,7 @@ class Session(object): """ try: state_file = open(self._state_file, "w") - state_file.write("%d %s\n" % (state, coreapi.state_name(state))) + state_file.write("%d %s\n" % (state, EventTypes(self.state).name)) state_file.close() except IOError: logging.exception("error writing state file: %s", state) @@ -1278,7 +1287,7 @@ class Session(object): hook(state) except Exception: message = "exception occured when running %s state hook: %s" % ( - coreapi.state_name(state), + EventTypes(self.state).name, hook, ) logging.exception(message) @@ -1549,11 +1558,10 @@ class Session(object): # write current nodes out to session directory file self.write_nodes() - # create control net interfaces and broker network tunnels + # create control net interfaces and network tunnels # which need to exist for emane to sync on location events # in distributed scenarios self.add_remove_control_interface(node=None, remove=False) - self.broker.startup() # initialize distributed tunnels self.initialize_distributed() @@ -1566,9 +1574,6 @@ class Session(object): self.boot_nodes() self.mobility.startup() - # set broker local instantiation to complete - self.broker.local_instantiation_complete() - # notify listeners that instantiation is complete event = EventData(event_type=EventTypes.INSTANTIATION_COMPLETE.value) self.broadcast_event(event) @@ -1606,21 +1611,16 @@ class Session(object): have entered runtime (time=0). """ # this is called from instantiate() after receiving an event message - # for the instantiation state, and from the broker when distributed - # nodes have been started + # for the instantiation state logging.debug( "session(%s) checking if not in runtime state, current state: %s", self.id, - coreapi.state_name(self.state), + EventTypes(self.state).name, ) if self.state == EventTypes.RUNTIME_STATE.value: logging.info("valid runtime state found, returning") return - # check to verify that all nodes and networks are running - if not self.broker.instantiation_complete(): - return - # start event loop and set to runtime self.event_loop.run() self.set_state(EventTypes.RUNTIME_STATE, send_event=True) @@ -1830,37 +1830,11 @@ class Session(object): except IndexError: # no server name. possibly only one server prefix = prefixes[0] - else: - # slave servers have their name and localhost in the serverlist - servers = self.broker.getservernames() - servers.remove("localhost") - prefix = None - for server_prefix in prefixes: - try: - # split each entry into server and prefix - server, p = server_prefix.split(":") - except ValueError: - server = "" - p = None - - if server == servers[0]: - # the server name in the list matches this server - prefix = p - break - - if not prefix: - logging.error( - "control network prefix not found for server: %s", servers[0] - ) - assign_address = False - try: - prefix = prefixes[0].split(":", 1)[1] - except IndexError: - prefix = prefixes[0] # len(prefixes) == 1 else: - # TODO: can we get the server name from the servers.conf or from the node assignments? + # TODO: can we get the server name from the servers.conf or from the node + # assignments?o # with one prefix, only master gets a ctrlnet address assign_address = self.master prefix = prefixes[0] @@ -1882,13 +1856,6 @@ class Session(object): serverintf=server_interface, ) - # tunnels between controlnets will be built with Broker.addnettunnels() - # TODO: potentially remove documentation saying node ids are ints - # TODO: need to move broker code out of the session object - self.broker.addnet(_id) - for server in self.broker.getservers(): - self.broker.addnodemap(server, _id) - return control_net def add_remove_control_interface( diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index 2f323783..eae46ce4 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -19,13 +19,9 @@ from core.emulator.enumerations import ( EventTypes, LinkTypes, MessageFlags, - MessageTypes, - NodeTlvs, RegisterTlvs, ) from core.errors import CoreError -from core.nodes.base import CoreNodeBase -from core.nodes.ipaddress import IpAddress class MobilityManager(ModelManager): @@ -48,11 +44,6 @@ class MobilityManager(ModelManager): self.models[BasicRangeModel.name] = BasicRangeModel self.models[Ns2ScriptedMobility.name] = Ns2ScriptedMobility - # dummy node objects for tracking position of nodes on other servers - self.phys = {} - self.physnets = {} - self.session.broker.handlers.add(self.physnodehandlelink) - def reset(self): """ Clear out all current configurations. @@ -93,9 +84,6 @@ class MobilityManager(ModelManager): model_class = self.models[model_name] self.set_model(node, model_class, config) - if self.session.master: - self.installphysnodes(node) - if node.mobility: self.session.event_loop.add_event(0.0, node.mobility.startup) @@ -209,87 +197,6 @@ class MobilityManager(ModelManager): if node.model: node.model.update(moved, moved_netifs) - def addphys(self, netnum, node): - """ - Keep track of PhysicalNodes and which network they belong to. - - :param int netnum: network number - :param core.coreobj.PyCoreNode node: node to add physical network to - :return: nothing - """ - node_id = node.id - self.phys[node_id] = node - if netnum not in self.physnets: - self.physnets[netnum] = [node_id] - else: - self.physnets[netnum].append(node_id) - - # TODO: remove need for handling old style message - - def physnodehandlelink(self, message): - """ - Broker handler. Snoop Link add messages to get - node numbers of PhyiscalNodes and their nets. - Physical nodes exist only on other servers, but a shadow object is - created here for tracking node position. - - :param message: link message to handle - :return: nothing - """ - if ( - message.message_type == MessageTypes.LINK.value - and message.flags & MessageFlags.ADD.value - ): - nn = message.node_numbers() - # first node is always link layer node in Link add message - if nn[0] not in self.session.broker.network_nodes: - return - if nn[1] in self.session.broker.physical_nodes: - # record the fact that this PhysicalNode is linked to a net - dummy = CoreNodeBase( - session=self.session, _id=nn[1], name="n%d" % nn[1], start=False - ) - self.addphys(nn[0], dummy) - - # TODO: remove need to handling old style messages - def physnodeupdateposition(self, message): - """ - Snoop node messages belonging to physical nodes. The dummy object - in self.phys[] records the node position. - - :param message: message to handle - :return: nothing - """ - nodenum = message.node_numbers()[0] - try: - dummy = self.phys[nodenum] - nodexpos = message.get_tlv(NodeTlvs.X_POSITION.value) - nodeypos = message.get_tlv(NodeTlvs.Y_POSITION.value) - dummy.setposition(nodexpos, nodeypos, None) - except KeyError: - logging.exception("error retrieving physical node: %s", nodenum) - - def installphysnodes(self, net): - """ - After installing a mobility model on a net, include any physical - nodes that we have recorded. Use the GreTap tunnel to the physical node - as the node's interface. - - :param net: network to install - :return: nothing - """ - node_ids = self.physnets.get(net.id, []) - for node_id in node_ids: - node = self.phys[node_id] - # TODO: fix this bad logic, relating to depending on a break to get a valid server - for server in self.session.broker.getserversbynode(node_id): - break - netif = self.session.broker.gettunnel(net.id, IpAddress.to_int(server.host)) - node.addnetif(netif, 0) - netif.node = node - x, y, z = netif.node.position.get() - netif.poshook(netif, x, y, z) - class WirelessModel(ConfigurableOptions): """ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 7758e4af..a9e8dc43 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -199,7 +199,9 @@ class NodeBase(object): x, y, _ = self.getposition() model = self.type - emulation_server = self.server.host + emulation_server = None + if self.server is not None: + emulation_server = self.server.host services = self.services if services is not None: @@ -593,7 +595,13 @@ class CoreNode(CoreNodeBase): :param str sh: shell to execute command in :return: str """ - return self.client.create_cmd(sh) + terminal = self.client.create_cmd(sh) + if self.server is None: + return terminal + else: + return "ssh -X -f {host} xterm -e {terminal}".format( + host=self.server.host, terminal=terminal + ) def privatedir(self, path): """ diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 4d68ccaf..f25c800c 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -288,8 +288,8 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - for host in self.session.servers: - server = self.session.servers[host] + for name in self.session.servers: + server = self.session.servers[name] server.remote_cmd(args, env, cwd, wait) return output diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index 32800eea..fd674b45 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -76,7 +76,7 @@ class Sdt(object): # node information for remote nodes not in session._objs # local nodes also appear here since their obj may not exist yet self.remotes = {} - session.broker.handlers.add(self.handle_distributed) + # session.broker.handlers.add(self.handle_distributed) # add handler for node updates self.session.node_handlers.append(self.handle_node_update) diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index ee316ffc..0a81b75e 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -107,10 +107,6 @@ class CoreXmlDeployment(object): def add_deployment(self): physical_host = self.add_physical_host(socket.gethostname()) - # TODO: handle other servers - # servers = self.session.broker.getservernames() - # servers.remove("localhost") - for node_id in self.session.nodes: node = self.session.nodes[node_id] if isinstance(node, CoreNodeBase): diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 0005c378..881ff373 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -314,9 +314,9 @@ def build_transport_xml(emane_manager, node, transport_type): file_name = transport_file_name(node.id, transport_type) file_path = os.path.join(emane_manager.session.session_dir, file_name) create_file(transport_element, doc_name, file_path) - for server in emane_manager.session.servers: - conn = emane_manager.session.servers[server] - create_file(transport_element, doc_name, file_path, conn) + for name in emane_manager.session.servers: + server = emane_manager.session.servers[name] + create_file(transport_element, doc_name, file_path, server) def create_phy_xml(emane_model, config, file_path, server): @@ -342,9 +342,9 @@ def create_phy_xml(emane_model, config, file_path, server): create_file(phy_element, "phy", file_path, server) else: create_file(phy_element, "phy", file_path) - for server in emane_model.session.servers: - conn = emane_model.session.servers[server] - create_file(phy_element, "phy", file_path, conn) + for name in emane_model.session.servers: + server = emane_model.session.servers[name] + create_file(phy_element, "phy", file_path, server) def create_mac_xml(emane_model, config, file_path, server): @@ -372,9 +372,9 @@ def create_mac_xml(emane_model, config, file_path, server): create_file(mac_element, "mac", file_path, server) else: create_file(mac_element, "mac", file_path) - for server in emane_model.session.servers: - conn = emane_model.session.servers[server] - create_file(mac_element, "mac", file_path, conn) + for name in emane_model.session.servers: + server = emane_model.session.servers[name] + create_file(mac_element, "mac", file_path, server) def create_nem_xml( @@ -410,9 +410,9 @@ def create_nem_xml( create_file(nem_element, "nem", nem_file, server) else: create_file(nem_element, "nem", nem_file) - for server in emane_model.session.servers: - conn = emane_model.session.servers[server] - create_file(nem_element, "nem", nem_file, conn) + for name in emane_model.session.servers: + server = emane_model.session.servers[name] + create_file(nem_element, "nem", nem_file, server) def create_event_service_xml(group, port, device, file_directory, server=None): diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index ca9ca928..8bcf2972 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -8,21 +8,19 @@ from core.emulator.enumerations import EventTypes, NodeTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"controlnet": "172.16.0.0/24", "distributed_address": address}) session = coreemu.create_session() - # set controlnet - session.options.set_config("controlnet", "172.16.0.0/24") - # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -31,7 +29,7 @@ def main(): node_one = session.add_node() switch = session.add_node(_type=NodeTypes.SWITCH) options = NodeOptions() - options.emulation_server = remote + options.emulation_server = server_name node_two = session.add_node(node_options=options) # create node interfaces and link diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py index 1ffe5795..c64d1f0c 100644 --- a/daemon/examples/python/distributed_emane.py +++ b/daemon/examples/python/distributed_emane.py @@ -9,25 +9,25 @@ from core.emulator.enumerations import EventTypes, NodeTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu( + { + "controlnet": "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 " + "core4:172.16.4.0/24 core5:172.16.5.0/24", + "distributed_address": address, + } + ) session = coreemu.create_session() - # set controlnet - session.options.set_config( - "controlnet", - "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 " - "core4:172.16.4.0/24 core5:172.16.5.0/24", - ) - # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -38,7 +38,7 @@ def main(): node_one = session.add_node(node_options=options) emane_net = session.add_node(_type=NodeTypes.EMANE) session.emane.set_model(emane_net, EmaneIeee80211abgModel) - options.emulation_server = remote + options.emulation_server = server_name node_two = session.add_node(node_options=options) # create node interfaces and link diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index 2b611816..b0f27c28 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -8,18 +8,19 @@ from core.emulator.enumerations import EventTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"distributed_address": address}) session = coreemu.create_session() # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -27,7 +28,7 @@ def main(): # create local node, switch, and remote nodes options = NodeOptions() node_one = session.add_node(node_options=options) - options.emulation_server = remote + options.emulation_server = server_name node_two = session.add_node(node_options=options) # create node interfaces and link diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py index b7ed166b..bc13bf2c 100644 --- a/daemon/examples/python/distributed_switches.py +++ b/daemon/examples/python/distributed_switches.py @@ -7,15 +7,16 @@ from core.emulator.enumerations import EventTypes, NodeTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"distributed_address": address}) session = coreemu.create_session() # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_wlan.py b/daemon/examples/python/distributed_wlan.py index ca64ee01..f8af1f5f 100644 --- a/daemon/examples/python/distributed_wlan.py +++ b/daemon/examples/python/distributed_wlan.py @@ -9,21 +9,19 @@ from core.location.mobility import BasicRangeModel def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"distributed_address": address}) session = coreemu.create_session() - # set controlnet - # session.options.set_config("controlnet", "172.16.0.0/24") - # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -31,7 +29,7 @@ def main(): # create local node, switch, and remote nodes options = NodeOptions() options.set_position(0, 0) - options.emulation_server = remote + options.emulation_server = server_name node_one = session.add_node(node_options=options) wlan = session.add_node(_type=NodeTypes.WIRELESS_LAN) session.mobility.set_model(wlan, BasicRangeModel) diff --git a/daemon/tests/conftest.py b/daemon/tests/conftest.py index 001233bb..ead3c2b4 100644 --- a/daemon/tests/conftest.py +++ b/daemon/tests/conftest.py @@ -58,7 +58,6 @@ class CoreServerTest(object): self.request_handler = CoreHandler(request_mock, "", self.server) self.request_handler.session = self.session self.request_handler.add_session_handlers() - self.session.broker.session_clients.append(self.request_handler) # have broker handle a configuration state change self.session.set_state(EventTypes.DEFINITION_STATE) diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index caff15fe..02e634be 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -763,13 +763,11 @@ class TestGui: (ConfigTlvs.VALUES, "%s:%s:%s" % (server, host, port)), ], ) - coreserver.session.broker.addserver = mock.MagicMock() - coreserver.session.broker.setupserver = mock.MagicMock() + coreserver.session.add_distributed = mock.MagicMock() coreserver.request_handler.handle_message(message) - coreserver.session.broker.addserver.assert_called_once_with(server, host, port) - coreserver.session.broker.setupserver.assert_called_once_with(server) + coreserver.session.add_distributed.assert_called_once_with(server, host) def test_config_services_request_all(self, coreserver): message = coreapi.CoreConfMessage.create( From 0b8bc7bd1362e96d3089eb28b239afb2d0198b7e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 15 Oct 2019 15:02:38 -0700 Subject: [PATCH 29/38] updated corehandlers to allow sdt snooping to help mimic previous behavior --- daemon/core/api/tlv/corehandlers.py | 2 ++ daemon/core/plugins/sdt.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 4b4e7c1e..60ddfcca 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -2082,6 +2082,7 @@ class CoreUdpHandler(CoreHandler): logging.debug("session handling message: %s", session.session_id) self.session = session self.handle_message(message) + self.session.sdt.handle_distributed(message) self.broadcast(message) else: logging.error( @@ -2106,6 +2107,7 @@ class CoreUdpHandler(CoreHandler): if session or message.message_type == MessageTypes.REGISTER.value: self.session = session self.handle_message(message) + self.session.sdt.handle_distributed(message) self.broadcast(message) else: logging.error( diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index fd674b45..52635da3 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -76,7 +76,6 @@ class Sdt(object): # node information for remote nodes not in session._objs # local nodes also appear here since their obj may not exist yet self.remotes = {} - # session.broker.handlers.add(self.handle_distributed) # add handler for node updates self.session.node_handlers.append(self.handle_node_update) From 61a4e228a158ec9ffa5b4ce1b5df111f985c93e9 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 10:14:36 -0700 Subject: [PATCH 30/38] updated ctrlnets to assign unique addresses per server, fixed ovs command issue for interface specific commands --- daemon/core/nodes/base.py | 62 ++++++---------------------------- daemon/core/nodes/docker.py | 14 +++++--- daemon/core/nodes/interface.py | 33 +++++++++++------- daemon/core/nodes/netclient.py | 14 ++++++++ daemon/core/nodes/network.py | 35 ++++++++++++++----- daemon/core/nodes/physical.py | 2 +- 6 files changed, 81 insertions(+), 79 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index a9e8dc43..4e9de039 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -4,12 +4,9 @@ Defines the base logic for nodes used within core. import logging import os -import random import shutil import socket -import string import threading -from builtins import range from socket import AF_INET, AF_INET6 from core import utils @@ -18,8 +15,8 @@ from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.errors import CoreCommandError from core.nodes import client, ipaddress -from core.nodes.interface import CoreInterface, TunTap, Veth -from core.nodes.netclient import LinuxNetClient, OvsNetClient +from core.nodes.interface import TunTap, Veth +from core.nodes.netclient import get_net_client _DEFAULT_MTU = 1500 @@ -63,10 +60,8 @@ class NodeBase(object): self.opaque = None self.position = Position() - if session.options.get_config("ovs") == "True": - self.net_client = OvsNetClient(self.net_cmd) - else: - self.net_client = LinuxNetClient(self.net_cmd) + use_ovs = session.options.get_config("ovs") == "True" + self.net_client = get_net_client(use_ovs, self.net_cmd) def startup(self): """ @@ -461,15 +456,13 @@ class CoreNode(CoreNodeBase): def create_node_net_client(self, use_ovs): """ - Create a client for running network orchestration commands. + Create node network client for running network commands within the nodes + container. - :param bool use_ovs: True to use OVS bridges, False for Linux bridge - :return: network client + :param bool use_ovs: True for OVS bridges, False for Linux bridges + :return:node network client """ - if use_ovs: - return OvsNetClient(self.node_net_cmd) - else: - return LinuxNetClient(self.node_net_cmd) + return get_net_client(use_ovs, self.node_net_cmd) def alive(self): """ @@ -675,11 +668,7 @@ class CoreNode(CoreNodeBase): raise ValueError("interface name (%s) too long" % name) veth = Veth( - node=self, - name=name, - localname=localname, - start=self.up, - server=self.server, + self.session, self, name, localname, start=self.up, server=self.server ) if self.up: @@ -732,7 +721,7 @@ class CoreNode(CoreNodeBase): sessionid = self.session.short_session_id() localname = "tap%s.%s.%s" % (self.id, ifindex, sessionid) name = ifname - tuntap = TunTap(node=self, name=name, localname=localname, start=self.up) + tuntap = TunTap(self.session, self, name, localname, start=self.up) try: self.addnetif(tuntap, ifindex) @@ -849,35 +838,6 @@ class CoreNode(CoreNodeBase): self.ifup(ifindex) return ifindex - def connectnode(self, ifname, othernode, otherifname): - """ - Connect a node. - - :param str ifname: name of interface to connect - :param core.nodes.base.CoreNode othernode: node to connect to - :param str otherifname: interface name to connect to - :return: nothing - """ - tmplen = 8 - tmp1 = "tmp." + "".join( - [random.choice(string.ascii_lowercase) for _ in range(tmplen)] - ) - tmp2 = "tmp." + "".join( - [random.choice(string.ascii_lowercase) for _ in range(tmplen)] - ) - self.net_client.create_veth(tmp1, tmp2) - self.net_client.device_ns(tmp1, str(self.pid)) - self.node_net_client.device_name(tmp1, ifname) - interface = CoreInterface(node=self, name=ifname, mtu=_DEFAULT_MTU) - self.addnetif(interface, self.newifindex()) - - self.net_client.device_ns(tmp2, str(othernode.pid)) - othernode.node_net_client.device_name(tmp2, otherifname) - other_interface = CoreInterface( - node=othernode, name=otherifname, mtu=_DEFAULT_MTU - ) - othernode.addnetif(other_interface, othernode.newifindex()) - def addfile(self, srcname, filename): """ Add a file. diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 2679704f..b91e987e 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -7,7 +7,7 @@ from core import utils from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode -from core.nodes.netclient import LinuxNetClient, OvsNetClient +from core.nodes.netclient import get_net_client class DockerClient(object): @@ -107,10 +107,14 @@ class DockerNode(CoreNode): super(DockerNode, self).__init__(session, _id, name, nodedir, bootsh, start) def create_node_net_client(self, use_ovs): - if use_ovs: - return OvsNetClient(self.nsenter_cmd) - else: - return LinuxNetClient(self.nsenter_cmd) + """ + Create node network client for running network commands within the nodes + container. + + :param bool use_ovs: True for OVS bridges, False for Linux bridges + :return:node network client + """ + return get_net_client(use_ovs, self.nsenter_cmd) def alive(self): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index bfcb8583..a6e04eb5 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -8,7 +8,7 @@ from builtins import int, range from core import utils from core.errors import CoreCommandError -from core.nodes.netclient import LinuxNetClient +from core.nodes.netclient import get_net_client class CoreInterface(object): @@ -16,17 +16,18 @@ class CoreInterface(object): Base class for network interfaces. """ - def __init__(self, node, name, mtu, server=None): + def __init__(self, session, node, name, mtu, server=None): """ Creates a PyCoreNetIf instance. + :param core.emulator.session.Session session: core session instance :param core.nodes.base.CoreNode node: node for interface :param str name: interface name - :param mtu: mtu value + :param int mtu: mtu value :param core.emulator.distributed.DistributedServer server: remote server node will run on, default is None for localhost """ - + self.session = session self.node = node self.name = name if not isinstance(mtu, int): @@ -45,7 +46,8 @@ class CoreInterface(object): # index used to find flow data self.flow_id = None self.server = server - self.net_client = LinuxNetClient(self.net_cmd) + use_ovs = session.options.get_config("ovs") == "True" + self.net_client = get_net_client(use_ovs, self.net_cmd) def net_cmd(self, args, env=None, cwd=None, wait=True): """ @@ -211,21 +213,24 @@ class Veth(CoreInterface): Provides virtual ethernet functionality for core nodes. """ - def __init__(self, node, name, localname, mtu=1500, server=None, start=True): + def __init__( + self, session, node, name, localname, mtu=1500, server=None, start=True + ): """ Creates a VEth instance. + :param core.emulator.session.Session session: core session instance :param core.nodes.base.CoreNode node: related core node :param str name: interface name :param str localname: interface local name - :param mtu: interface mtu + :param int mtu: interface mtu :param core.emulator.distributed.DistributedServer server: remote server node will run on, default is None for localhost :param bool start: start flag :raises CoreCommandError: when there is a command exception """ # note that net arg is ignored - CoreInterface.__init__(self, node, name, mtu, server) + CoreInterface.__init__(self, session, node, name, mtu, server) self.localname = localname self.up = False if start: @@ -271,19 +276,22 @@ class TunTap(CoreInterface): TUN/TAP virtual device in TAP mode """ - def __init__(self, node, name, localname, mtu=1500, server=None, start=True): + def __init__( + self, session, node, name, localname, mtu=1500, server=None, start=True + ): """ Create a TunTap instance. + :param core.emulator.session.Session session: core session instance :param core.nodes.base.CoreNode node: related core node :param str name: interface name :param str localname: local interface name - :param mtu: interface mtu + :param int mtu: interface mtu :param core.emulator.distributed.DistributedServer server: remote server node will run on, default is None for localhost :param bool start: start flag """ - CoreInterface.__init__(self, node, name, mtu, server) + CoreInterface.__init__(self, session, node, name, mtu, server) self.localname = localname self.up = False self.transport_type = "virtual" @@ -466,8 +474,7 @@ class GreTap(CoreInterface): will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ - CoreInterface.__init__(self, node, name, mtu, server) - self.session = session + CoreInterface.__init__(self, session, node, name, mtu, server) if _id is None: # from PyCoreObj _id = ((id(self) >> 16) ^ (id(self) & 0xFFFF)) & 0xFFFF diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 6de5d698..9234bef5 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -7,6 +7,20 @@ import os from core.constants import BRCTL_BIN, ETHTOOL_BIN, IP_BIN, OVS_BIN, TC_BIN +def get_net_client(use_ovs, run): + """ + Retrieve desired net client for running network commands. + + :param bool use_ovs: True for OVS bridges, False for Linux bridges + :param func run: function used to run net client commands + :return: net client class + """ + if use_ovs: + return OvsNetClient(run) + else: + return LinuxNetClient(run) + + class LinuxNetClient(object): """ Client for creating Linux bridges and ip interfaces for nodes. diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index f25c800c..931622bb 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -16,6 +16,7 @@ from core.errors import CoreCommandError, CoreError from core.nodes import ipaddress from core.nodes.base import CoreNetworkBase from core.nodes.interface import GreTap, Veth +from core.nodes.netclient import get_net_client ebtables_lock = threading.Lock() @@ -558,7 +559,7 @@ class CoreNetwork(CoreNetworkBase): if len(name) >= 16: raise ValueError("interface name %s too long" % name) - netif = Veth(node=None, name=name, localname=localname, mtu=1500, start=self.up) + netif = Veth(self.session, None, name, localname, start=self.up) self.attach(netif) if net.up: # this is similar to net.attach() but uses netif.name instead of localname @@ -766,6 +767,24 @@ class CtrlNet(CoreNetwork): self.serverintf = serverintf CoreNetwork.__init__(self, session, _id, name, start, server) + def add_addresses(self, address): + """ + Add addresses used for created control networks, + + :param core.nodes.interfaces.IpAddress address: starting address to use + :return: + """ + use_ovs = self.session.options.get_config("ovs") == "True" + current = "%s/%s" % (address, self.prefix.prefixlen) + net_client = get_net_client(use_ovs, utils.check_cmd) + net_client.create_address(self.brname, current) + for name in self.session.servers: + server = self.session.servers[name] + address -= 1 + current = "%s/%s" % (address, self.prefix.prefixlen) + net_client = get_net_client(use_ovs, server.remote_cmd) + net_client.create_address(self.brname, current) + def startup(self): """ Startup functionality for the control network. @@ -778,16 +797,14 @@ class CtrlNet(CoreNetwork): CoreNetwork.startup(self) - if self.hostid: - addr = self.prefix.addr(self.hostid) - else: - addr = self.prefix.max_addr() - logging.info("added control network bridge: %s %s", self.brname, self.prefix) - if self.assign_address: - addrlist = ["%s/%s" % (addr, self.prefix.prefixlen)] - self.addrconfig(addrlist=addrlist) + if self.hostid and self.assign_address: + address = self.prefix.addr(self.hostid) + self.add_addresses(address) + elif self.assign_address: + address = self.prefix.max_addr() + self.add_addresses(address) if self.updown_script: logging.info( diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 9daf4f35..93e04c5e 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -246,7 +246,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): will run on, default is None for localhost """ CoreNodeBase.__init__(self, session, _id, name, start, server) - CoreInterface.__init__(self, node=self, name=name, mtu=mtu) + CoreInterface.__init__(self, session, self, name, mtu, server) self.up = False self.lock = threading.RLock() self.ifindex = None From 8aef9f273feb57da549b2246b7cb880676079e63 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 17:11:21 -0700 Subject: [PATCH 31/38] updates to clear broker from physical node --- daemon/core/emulator/session.py | 49 +++++++++++++++++---------------- daemon/core/nodes/physical.py | 43 +++++++++++++++-------------- gui/nodes.tcl | 2 +- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 1445cb8f..e864ec8b 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -153,7 +153,7 @@ class Session(object): self.services.default_services = { "mdr": ("zebra", "OSPFv3MDR", "IPForward"), "PC": ("DefaultRoute",), - "prouter": ("zebra", "OSPFv2", "OSPFv3", "IPForward"), + "prouter": (), "router": ("zebra", "OSPFv2", "OSPFv3", "IPForward"), "host": ("DefaultRoute", "SSH"), } @@ -192,32 +192,33 @@ class Session(object): for name in self.servers: server = self.servers[name] - host = server.host - key = self.tunnelkey(node_id, IpAddress.to_int(host)) + self.create_gre_tunnel(node, server) - # local to server - logging.info( - "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key - ) - local_tap = GreTap(session=self, remoteip=host, key=key) - local_tap.net_client.create_interface(node.brname, local_tap.localname) + def create_gre_tunnel(self, node, server): + host = server.host + key = self.tunnelkey(node.id, IpAddress.to_int(host)) + tunnel = self.tunnels.get(key) + if tunnel is not None: + return tunnel - # server to local - logging.info( - "remote tunnel node(%s) to local(%s) key(%s)", - node.name, - self.address, - key, - ) - remote_tap = GreTap( - session=self, remoteip=self.address, key=key, server=server - ) - remote_tap.net_client.create_interface( - node.brname, remote_tap.localname - ) + # local to server + logging.info( + "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key + ) + local_tap = GreTap(session=self, remoteip=host, key=key) + local_tap.net_client.create_interface(node.brname, local_tap.localname) - # save tunnels for shutdown - self.tunnels[key] = (local_tap, remote_tap) + # server to local + logging.info( + "remote tunnel node(%s) to local(%s) key(%s)", node.name, self.address, key + ) + remote_tap = GreTap(session=self, remoteip=self.address, key=key, server=server) + remote_tap.net_client.create_interface(node.brname, remote_tap.localname) + + # save tunnels for shutdown + tunnel = (local_tap, remote_tap) + self.tunnels[key] = tunnel + return tunnel def tunnelkey(self, n1_id, n2_id): """ diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 93e04c5e..ecbcf368 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -9,15 +9,19 @@ import threading from core import utils from core.constants import MOUNT_BIN, UMOUNT_BIN from core.emulator.enumerations import NodeTypes -from core.errors import CoreCommandError +from core.errors import CoreCommandError, CoreError from core.nodes.base import CoreNodeBase from core.nodes.interface import CoreInterface from core.nodes.network import CoreNetwork, GreTap class PhysicalNode(CoreNodeBase): - def __init__(self, session, _id=None, name=None, nodedir=None, start=True): - CoreNodeBase.__init__(self, session, _id, name, start=start) + def __init__( + self, session, _id=None, name=None, nodedir=None, start=True, server=None + ): + CoreNodeBase.__init__(self, session, _id, name, start, server) + if not self.server: + raise CoreError("physical nodes must be assigned to a remote server") self.nodedir = nodedir self.up = start self.lock = threading.RLock() @@ -86,7 +90,6 @@ class PhysicalNode(CoreNodeBase): def adoptnetif(self, netif, ifindex, hwaddr, addrlist): """ - The broker builds a GreTap tunnel device to this physical node. When a link message is received linking this node to another part of the emulation, no new interface is created; instead, adopt the GreTap netif as the node interface. @@ -157,26 +160,21 @@ class PhysicalNode(CoreNodeBase): if ifindex is None: ifindex = self.newifindex() - if self.up: - # this is reached when this node is linked to a network node - # tunnel to net not built yet, so build it now and adopt it - gt = self.session.broker.addnettunnel(net.id) - if gt is None or len(gt) != 1: - raise ValueError( - "error building tunnel from adding a new network interface: %s" % gt - ) - gt = gt[0] - net.detach(gt) - self.adoptnetif(gt, ifindex, hwaddr, addrlist) - return ifindex - - # this is reached when configuring services (self.up=False) if ifname is None: ifname = "gt%d" % ifindex - netif = GreTap(node=self, name=ifname, session=self.session, start=False) - self.adoptnetif(netif, ifindex, hwaddr, addrlist) - return ifindex + if self.up: + # this is reached when this node is linked to a network node + # tunnel to net not built yet, so build it now and adopt it + _, remote_tap = self.session.create_gre_tunnel(net, self.server) + # net.detach(remote_tap) + self.adoptnetif(remote_tap, ifindex, hwaddr, addrlist) + return ifindex + else: + # this is reached when configuring services (self.up=False) + netif = GreTap(node=self, name=ifname, session=self.session, start=False) + self.adoptnetif(netif, ifindex, hwaddr, addrlist) + return ifindex def privatedir(self, path): if path[0] != "/": @@ -223,6 +221,9 @@ class PhysicalNode(CoreNodeBase): os.chmod(node_file.name, mode) logging.info("created nodefile: '%s'; mode: 0%o", node_file.name, mode) + def node_net_cmd(self, args, wait=True): + return self.net_cmd(args, wait=wait) + class Rj45Node(CoreNodeBase, CoreInterface): """ diff --git a/gui/nodes.tcl b/gui/nodes.tcl index 00e52c5d..c8645f03 100644 --- a/gui/nodes.tcl +++ b/gui/nodes.tcl @@ -19,7 +19,7 @@ array set g_node_types_default { 4 {mdr mdr.gif mdr.gif {zebra OSPFv3MDR IPForward} \ netns {built-in type for wireless routers}} 5 {prouter router_green.gif router_green.gif \ - {zebra OSPFv2 OSPFv3 IPForward} \ + {} \ physical {built-in type for physical nodes}} } From 009ce8143efec337cb6afd22b27afd5217d12a11 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 20:19:51 -0700 Subject: [PATCH 32/38] removed lock for distributed commands and limited usage to uploads --- daemon/core/emulator/distributed.py | 13 ++++++------- daemon/core/nodes/physical.py | 1 - 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 4e258937..2df33541 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -53,16 +53,15 @@ class DistributedServer(object): "remote cmd server(%s) cwd(%s) wait(%s): %s", self.host, cwd, wait, cmd ) try: - with self.lock: - if cwd is None: + if cwd is None: + result = self.conn.run( + cmd, hide=False, env=env, replace_env=replace_env + ) + else: + with self.conn.cd(cwd): result = self.conn.run( cmd, hide=False, env=env, replace_env=replace_env ) - else: - with self.conn.cd(cwd): - result = self.conn.run( - cmd, hide=False, env=env, replace_env=replace_env - ) return result.stdout.strip() except UnexpectedExit as e: stdout, stderr = e.streams_for_display() diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index ecbcf368..37a2eb54 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -167,7 +167,6 @@ class PhysicalNode(CoreNodeBase): # this is reached when this node is linked to a network node # tunnel to net not built yet, so build it now and adopt it _, remote_tap = self.session.create_gre_tunnel(net, self.server) - # net.detach(remote_tap) self.adoptnetif(remote_tap, ifindex, hwaddr, addrlist) return ifindex else: From 774dd8330cd81c0a72ecdc7cbafddfe70427b1a6 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 20:26:14 -0700 Subject: [PATCH 33/38] removed broker.py --- daemon/core/api/tlv/broker.py | 1147 --------------------------------- 1 file changed, 1147 deletions(-) delete mode 100644 daemon/core/api/tlv/broker.py diff --git a/daemon/core/api/tlv/broker.py b/daemon/core/api/tlv/broker.py deleted file mode 100644 index d71d050c..00000000 --- a/daemon/core/api/tlv/broker.py +++ /dev/null @@ -1,1147 +0,0 @@ -""" -Broker class that is part of the session object. Handles distributing parts of the emulation out to -other emulation servers. The broker is consulted when handling messages to determine if messages -should be handled locally or forwarded on to another emulation server. -""" - -import logging -import os -import select -import socket -import threading - -from core import utils -from core.api.tlv import coreapi -from core.emane.nodes import EmaneNet -from core.emulator.enumerations import ( - ConfigDataTypes, - ConfigFlags, - ConfigTlvs, - EventTlvs, - EventTypes, - ExecuteTlvs, - FileTlvs, - LinkTlvs, - MessageFlags, - MessageTypes, - NodeTlvs, - NodeTypes, - RegisterTlvs, -) -from core.nodes.base import CoreNetworkBase, CoreNodeBase -from core.nodes.interface import GreTap -from core.nodes.ipaddress import IpAddress -from core.nodes.network import CtrlNet, GreTapBridge -from core.nodes.physical import PhysicalNode - - -class CoreDistributedServer(object): - """ - Represents CORE daemon servers for communication. - """ - - def __init__(self, name, host, port): - """ - Creates a CoreServer instance. - - :param str name: name of the CORE server - :param str host: server address - :param int port: server port - """ - self.name = name - self.host = host - self.port = port - self.sock = None - self.instantiation_complete = False - - def connect(self): - """ - Connect to CORE server and save connection. - - :return: nothing - """ - if self.sock: - raise ValueError("socket already connected") - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - try: - sock.connect((self.host, self.port)) - except IOError as e: - sock.close() - raise e - - self.sock = sock - - def close(self): - """ - Close connection with CORE server. - - :return: nothing - """ - if self.sock is not None: - self.sock.close() - self.sock = None - - -class CoreBroker(object): - """ - Helps with brokering messages between CORE daemon servers. - """ - - # configurable manager name - name = "broker" - - # configurable manager type - config_type = RegisterTlvs.UTILITY.value - - def __init__(self, session): - """ - Creates a CoreBroker instance. - - :param core.emulator.session.Session session: session this manager is tied to - :return: nothing - """ - - # ConfigurableManager.__init__(self) - self.session = session - self.session_clients = [] - self.session_id_master = None - self.myip = None - # dict containing tuples of (host, port, sock) - self.servers = {} - self.servers_lock = threading.Lock() - self.addserver("localhost", None, None) - # dict containing node number to server name mapping - self.nodemap = {} - # this lock also protects self.nodecounts - self.nodemap_lock = threading.Lock() - # reference counts of nodes on servers - self.nodecounts = {} - # set of node numbers that are link-layer nodes (networks) - self.network_nodes = set() - # set of node numbers that are PhysicalNode nodes - self.physical_nodes = set() - # allows for other message handlers to process API messages (e.g. EMANE) - self.handlers = set() - # dict with tunnel key to tunnel device mapping - self.tunnels = {} - self.dorecvloop = False - self.recvthread = None - self.bootcount = 0 - - def startup(self): - """ - Build tunnels between network-layer nodes now that all node - and link information has been received; called when session - enters the instantation state. - """ - self.addnettunnels() - self.writeservers() - - def shutdown(self): - """ - Close all active sockets; called when the session enters the - data collect state - """ - self.reset() - with self.servers_lock: - while len(self.servers) > 0: - name, server = self.servers.popitem() - if server.sock is not None: - logging.info( - "closing connection with %s: %s:%s", - name, - server.host, - server.port, - ) - server.close() - self.dorecvloop = False - if self.recvthread is not None: - self.recvthread.join() - - def reset(self): - """ - Reset to initial state. - """ - logging.debug("broker reset") - self.nodemap_lock.acquire() - self.nodemap.clear() - for server in self.nodecounts: - count = self.nodecounts[server] - if count < 1: - self.delserver(server) - self.nodecounts.clear() - self.bootcount = 0 - self.nodemap_lock.release() - self.network_nodes.clear() - self.physical_nodes.clear() - while len(self.tunnels) > 0: - _key, gt = self.tunnels.popitem() - gt.shutdown() - - def startrecvloop(self): - """ - Spawn the receive loop for receiving messages. - """ - if self.recvthread is not None: - logging.info("server receive loop already started") - if self.recvthread.isAlive(): - return - else: - self.recvthread.join() - # start reading data from connected sockets - logging.info("starting server receive loop") - self.dorecvloop = True - self.recvthread = threading.Thread(target=self.recvloop) - self.recvthread.daemon = True - self.recvthread.start() - - def recvloop(self): - """ - Receive loop for receiving messages from server sockets. - """ - self.dorecvloop = True - # note: this loop continues after emulation is stopped, - # even with 0 servers - while self.dorecvloop: - rlist = [] - with self.servers_lock: - # build a socket list for select call - for name in self.servers: - server = self.servers[name] - if server.sock is not None: - rlist.append(server.sock) - r, _w, _x = select.select(rlist, [], [], 1.0) - for sock in r: - server = self.getserverbysock(sock) - logging.info( - "attempting to receive from server: peer:%s remote:%s", - server.sock.getpeername(), - server.sock.getsockname(), - ) - if server is None: - # servers may have changed; loop again - continue - rcvlen = self.recv(server) - if rcvlen == 0: - logging.info( - "connection with server(%s) closed: %s:%s", - server.name, - server.host, - server.port, - ) - - def recv(self, server): - """ - Receive data on an emulation server socket and broadcast it to - all connected session handlers. Returns the length of data recevied - and forwarded. Return value of zero indicates the socket has closed - and should be removed from the self.servers dict. - - :param CoreDistributedServer server: server to receive from - :return: message length - :rtype: int - """ - msghdr = server.sock.recv(coreapi.CoreMessage.header_len) - if len(msghdr) == 0: - # server disconnected - logging.info("server disconnected, closing server") - server.close() - return 0 - - if len(msghdr) != coreapi.CoreMessage.header_len: - logging.warning( - "warning: broker received not enough data len=%s", len(msghdr) - ) - return len(msghdr) - - msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(msghdr) - msgdata = server.sock.recv(msglen) - data = msghdr + msgdata - count = None - logging.debug("received message type: %s", MessageTypes(msgtype)) - # snoop exec response for remote interactive TTYs - if msgtype == MessageTypes.EXECUTE.value and msgflags & MessageFlags.TTY.value: - data = self.fixupremotetty(msghdr, msgdata, server.host) - logging.debug("created remote tty message: %s", data) - elif msgtype == MessageTypes.NODE.value: - # snoop node delete response to decrement node counts - if msgflags & MessageFlags.DELETE.value: - msg = coreapi.CoreNodeMessage(msgflags, msghdr, msgdata) - nodenum = msg.get_tlv(NodeTlvs.NUMBER.value) - if nodenum is not None: - count = self.delnodemap(server, nodenum) - elif msgtype == MessageTypes.LINK.value: - # this allows green link lines for remote WLANs - msg = coreapi.CoreLinkMessage(msgflags, msghdr, msgdata) - self.session.sdt.handle_distributed(msg) - elif msgtype == MessageTypes.EVENT.value: - msg = coreapi.CoreEventMessage(msgflags, msghdr, msgdata) - eventtype = msg.get_tlv(EventTlvs.TYPE.value) - if eventtype == EventTypes.INSTANTIATION_COMPLETE.value: - server.instantiation_complete = True - if self.instantiation_complete(): - self.session.check_runtime() - else: - logging.error("unknown message type received: %s", msgtype) - - try: - for session_client in self.session_clients: - session_client.sendall(data) - except IOError: - logging.exception("error sending message") - - if count is not None and count < 1: - return 0 - else: - return len(data) - - def addserver(self, name, host, port): - """ - Add a new server, and try to connect to it. If we"re already connected to this - (host, port), then leave it alone. When host,port is None, do not try to connect. - - :param str name: name of server - :param str host: server address - :param int port: server port - :return: nothing - """ - with self.servers_lock: - server = self.servers.get(name) - if server is not None: - if ( - host == server.host - and port == server.port - and server.sock is not None - ): - # leave this socket connected - return - - logging.debug( - "closing connection with %s @ %s:%s", name, server.host, server.port - ) - server.close() - del self.servers[name] - - logging.debug("adding broker server(%s): %s:%s", name, host, port) - server = CoreDistributedServer(name, host, port) - if host is not None and port is not None: - try: - server.connect() - except IOError: - logging.exception( - "error connecting to server(%s): %s:%s", name, host, port - ) - if server.sock is not None: - self.startrecvloop() - self.servers[name] = server - - def delserver(self, server): - """ - Remove a server and hang up any connection. - - :param CoreDistributedServer server: server to delete - :return: nothing - """ - with self.servers_lock: - try: - s = self.servers.pop(server.name) - if s != server: - raise ValueError("server removed was not the server provided") - except KeyError: - logging.exception("error deleting server") - - if server.sock is not None: - logging.info( - "closing connection with %s @ %s:%s", - server.name, - server.host, - server.port, - ) - server.close() - - def getserverbyname(self, name): - """ - Return the server object having the given name, or None. - - :param str name: name of server to retrieve - :return: server for given name - :rtype: CoreDistributedServer - """ - with self.servers_lock: - return self.servers.get(name) - - def getserverbysock(self, sock): - """ - Return the server object corresponding to the given socket, or None. - - :param sock: socket associated with a server - :return: core server associated wit the socket - :rtype: CoreDistributedServer - """ - with self.servers_lock: - for name in self.servers: - server = self.servers[name] - if server.sock == sock: - return server - return None - - def getservers(self): - """ - Return a list of servers sorted by name. - - :return: sorted server list - :rtype: list - """ - with self.servers_lock: - return sorted(self.servers.values(), key=lambda x: x.name) - - def getservernames(self): - """ - Return a sorted list of server names (keys from self.servers). - - :return: sorted server names - :rtype: list - """ - with self.servers_lock: - return sorted(self.servers.keys()) - - def tunnelkey(self, n1num, n2num): - """ - Compute a 32-bit key used to uniquely identify a GRE tunnel. - The hash(n1num), hash(n2num) values are used, so node numbers may be - None or string values (used for e.g. "ctrlnet"). - - :param int n1num: node one id - :param int n2num: node two id - :return: tunnel key for the node pair - :rtype: int - """ - logging.debug("creating tunnel key for: %s, %s", n1num, n2num) - sid = self.session_id_master - if sid is None: - # this is the master session - sid = self.session.id - - key = (sid << 16) ^ utils.hashkey(n1num) ^ (utils.hashkey(n2num) << 8) - return key & 0xFFFFFFFF - - def addtunnel(self, remoteip, n1num, n2num, localnum): - """ - Adds a new GreTapBridge between nodes on two different machines. - - :param str remoteip: remote address for tunnel - :param int n1num: node one id - :param int n2num: node two id - :param int localnum: local id - :return: nothing - """ - key = self.tunnelkey(n1num, n2num) - if localnum == n2num: - remotenum = n1num - else: - remotenum = n2num - - if key in self.tunnels.keys(): - logging.warning( - "tunnel with key %s (%s-%s) already exists!", key, n1num, n2num - ) - else: - _id = key & ((1 << 16) - 1) - logging.info( - "adding tunnel for %s-%s to %s with key %s", n1num, n2num, remoteip, key - ) - if localnum in self.physical_nodes: - # no bridge is needed on physical nodes; use the GreTap directly - gt = GreTap( - node=None, - name=None, - session=self.session, - remoteip=remoteip, - key=key, - ) - else: - gt = self.session.create_node( - cls=GreTapBridge, - _id=_id, - policy="ACCEPT", - remoteip=remoteip, - key=key, - ) - gt.localnum = localnum - gt.remotenum = remotenum - self.tunnels[key] = gt - - def addnettunnels(self): - """ - Add GreTaps between network devices on different machines. - The GreTapBridge is not used since that would add an extra bridge. - """ - logging.debug("adding network tunnels for nodes: %s", self.network_nodes) - for n in self.network_nodes: - self.addnettunnel(n) - - def addnettunnel(self, node_id): - """ - Add network tunnel between node and broker. - - :param int node_id: node id of network to add tunnel to - :return: list of gre taps - :rtype: list - :raises core.CoreError: when node to add net tunnel to does not exist - """ - net = self.session.get_node(node_id) - logging.debug("adding net tunnel for: id(%s) %s", node_id, net.name) - - # add other nets here that do not require tunnels - if isinstance(net, EmaneNet): - logging.debug("emane network does not require a tunnel") - return None - - server_interface = getattr(net, "serverintf", None) - if isinstance(net, CtrlNet) and server_interface is not None: - logging.debug( - "control networks with server interfaces do not need a tunnel" - ) - return None - - servers = self.getserversbynode(node_id) - if len(servers) < 2: - logging.debug("not enough servers to create a tunnel for node: %s", node_id) - return None - - hosts = [] - for server in servers: - if server.host is None: - continue - logging.debug("adding server host for net tunnel: %s", server.host) - hosts.append(server.host) - - if len(hosts) == 0: - for session_client in self.session_clients: - # get IP address from API message sender (master) - if session_client.client_address != "": - address = session_client.client_address[0] - logging.debug("adding session_client host: %s", address) - hosts.append(address) - - r = [] - for host in hosts: - if self.myip: - # we are the remote emulation server - myip = self.myip - else: - # we are the session master - myip = host - key = self.tunnelkey(node_id, IpAddress.to_int(myip)) - if key in self.tunnels.keys(): - logging.debug( - "tunnel already exists, returning existing tunnel: %s", key - ) - gt = self.tunnels[key] - r.append(gt) - continue - logging.info( - "adding tunnel for net %s to %s with key %s", node_id, host, key - ) - gt = GreTap( - node=None, name=None, session=self.session, remoteip=host, key=key - ) - self.tunnels[key] = gt - r.append(gt) - # attaching to net will later allow gt to be destroyed - # during net.shutdown() - net.attach(gt) - - return r - - def deltunnel(self, n1num, n2num): - """ - Delete tunnel between nodes. - - :param int n1num: node one id - :param int n2num: node two id - :return: nothing - """ - key = self.tunnelkey(n1num, n2num) - try: - logging.info( - "deleting tunnel between %s - %s with key: %s", n1num, n2num, key - ) - gt = self.tunnels.pop(key) - except KeyError: - gt = None - if gt: - self.session.delete_node(gt.id) - del gt - - def gettunnel(self, n1num, n2num): - """ - Return the GreTap between two nodes if it exists. - - :param int n1num: node one id - :param int n2num: node two id - :return: gre tap between nodes or none - """ - key = self.tunnelkey(n1num, n2num) - logging.debug("checking for tunnel(%s) in: %s", key, self.tunnels.keys()) - if key in self.tunnels.keys(): - return self.tunnels[key] - else: - return None - - def addnodemap(self, server, nodenum): - """ - Record a node number to emulation server mapping. - - :param CoreDistributedServer server: core server to associate node with - :param int nodenum: node id - :return: nothing - """ - with self.nodemap_lock: - if nodenum in self.nodemap: - if server in self.nodemap[nodenum]: - return - self.nodemap[nodenum].add(server) - else: - self.nodemap[nodenum] = {server} - - if server in self.nodecounts: - self.nodecounts[server] += 1 - else: - self.nodecounts[server] = 1 - - def delnodemap(self, server, nodenum): - """ - Remove a node number to emulation server mapping. - Return the number of nodes left on this server. - - :param CoreDistributedServer server: server to remove from node map - :param int nodenum: node id - :return: number of nodes left on server - :rtype: int - """ - count = None - with self.nodemap_lock: - if nodenum not in self.nodemap: - return count - - self.nodemap[nodenum].remove(server) - if server in self.nodecounts: - count = self.nodecounts[server] - count -= 1 - self.nodecounts[server] = count - - return count - - def getserversbynode(self, nodenum): - """ - Retrieve a set of emulation servers given a node number. - - :param int nodenum: node id - :return: core server associated with node - :rtype: set - """ - with self.nodemap_lock: - if nodenum not in self.nodemap: - return set() - return self.nodemap[nodenum] - - def addnet(self, nodenum): - """ - Add a node number to the list of link-layer nodes. - - :param int nodenum: node id to add - :return: nothing - """ - logging.debug("adding net to broker: %s", nodenum) - self.network_nodes.add(nodenum) - logging.debug("broker network nodes: %s", self.network_nodes) - - def addphys(self, nodenum): - """ - Add a node number to the list of physical nodes. - - :param int nodenum: node id to add - :return: nothing - """ - self.physical_nodes.add(nodenum) - - def handle_message(self, message): - """ - Handle an API message. Determine whether this needs to be handled - by the local server or forwarded on to another one. - Returns True when message does not need to be handled locally, - and performs forwarding if required. - Returning False indicates this message should be handled locally. - - :param core.api.coreapi.CoreMessage message: message to handle - :return: true or false for handling locally - :rtype: bool - """ - servers = set() - handle_locally = False - # Do not forward messages when in definition state - # (for e.g. configuring services) - if self.session.state == EventTypes.DEFINITION_STATE.value: - return False - - # Decide whether message should be handled locally or forwarded, or both - if message.message_type == MessageTypes.NODE.value: - handle_locally, servers = self.handlenodemsg(message) - elif message.message_type == MessageTypes.EVENT.value: - # broadcast events everywhere - servers = self.getservers() - elif message.message_type == MessageTypes.CONFIG.value: - # broadcast location and services configuration everywhere - confobj = message.get_tlv(ConfigTlvs.OBJECT.value) - if ( - confobj == "location" - or confobj == "services" - or confobj == "session" - or confobj == "all" - ): - servers = self.getservers() - elif message.message_type == MessageTypes.FILE.value: - # broadcast hook scripts and custom service files everywhere - filetype = message.get_tlv(FileTlvs.TYPE.value) - if filetype is not None and ( - filetype[:5] == "hook:" or filetype[:8] == "service:" - ): - servers = self.getservers() - if message.message_type == MessageTypes.LINK.value: - # prepare a server list from two node numbers in link message - handle_locally, servers, message = self.handlelinkmsg(message) - elif len(servers) == 0: - # check for servers based on node numbers in all messages but link - nn = message.node_numbers() - if len(nn) == 0: - return False - servers = self.getserversbynode(nn[0]) - - # allow other handlers to process this message (this is used - # by e.g. EMANE to use the link add message to keep counts of - # interfaces on other servers) - for handler in self.handlers: - handler(message) - - # perform any message forwarding - handle_locally |= self.forwardmsg(message, servers) - return not handle_locally - - def setupserver(self, servername): - """ - Send the appropriate API messages for configuring the specified emulation server. - - :param str servername: name of server to configure - :return: nothing - """ - server = self.getserverbyname(servername) - if server is None: - logging.warning("ignoring unknown server: %s", servername) - return - - if server.sock is None or server.host is None or server.port is None: - logging.info("ignoring disconnected server: %s", servername) - return - - # communicate this session"s current state to the server - tlvdata = coreapi.CoreEventTlv.pack(EventTlvs.TYPE.value, self.session.state) - msg = coreapi.CoreEventMessage.pack(0, tlvdata) - server.sock.send(msg) - - # send a Configuration message for the broker object and inform the - # server of its local name - tlvdata = b"" - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.OBJECT.value, "broker") - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.TYPE.value, ConfigFlags.UPDATE.value - ) - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.DATA_TYPES.value, (ConfigDataTypes.STRING.value,) - ) - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.VALUES.value, - "%s:%s:%s" % (server.name, server.host, server.port), - ) - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.SESSION.value, "%s" % self.session.id - ) - msg = coreapi.CoreConfMessage.pack(0, tlvdata) - server.sock.send(msg) - - @staticmethod - def fixupremotetty(msghdr, msgdata, host): - """ - When an interactive TTY request comes from the GUI, snoop the reply - and add an SSH command to the appropriate remote server. - - :param msghdr: message header - :param msgdata: message data - :param str host: host address - :return: packed core execute tlv data - """ - msgtype, msgflags, _msglen = coreapi.CoreMessage.unpack_header(msghdr) - msgcls = coreapi.CLASS_MAP[msgtype] - msg = msgcls(msgflags, msghdr, msgdata) - - nodenum = msg.get_tlv(ExecuteTlvs.NODE.value) - execnum = msg.get_tlv(ExecuteTlvs.NUMBER.value) - cmd = msg.get_tlv(ExecuteTlvs.COMMAND.value) - res = msg.get_tlv(ExecuteTlvs.RESULT.value) - - tlvdata = b"" - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.NODE.value, nodenum) - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.NUMBER.value, execnum) - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.COMMAND.value, cmd) - res = "ssh -X -f " + host + " xterm -e " + res - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.RESULT.value, res) - - return coreapi.CoreExecMessage.pack(msgflags, tlvdata) - - def handlenodemsg(self, message): - """ - Determine and return the servers to which this node message should - be forwarded. Also keep track of link-layer nodes and the mapping of - nodes to servers. - - :param core.api.coreapi.CoreMessage message: message to handle - :return: boolean for handling locally and set of servers - :rtype: tuple - """ - servers = set() - handle_locally = False - serverfiletxt = None - - # snoop Node Message for emulation server TLV and record mapping - n = message.tlv_data[NodeTlvs.NUMBER.value] - - # replicate link-layer nodes on all servers - nodetype = message.get_tlv(NodeTlvs.TYPE.value) - if nodetype is not None: - try: - nodetype = NodeTypes(nodetype) - nodecls = self.session.get_node_class(nodetype) - except KeyError: - logging.warning("broker invalid node type %s", nodetype) - return handle_locally, servers - if nodecls is None: - logging.warning("broker unimplemented node type %s", nodetype) - return handle_locally, servers - if ( - issubclass(nodecls, CoreNetworkBase) - and nodetype != NodeTypes.WIRELESS_LAN.value - ): - # network node replicated on all servers; could be optimized - # don"t replicate WLANs, because ebtables rules won"t work - servers = self.getservers() - handle_locally = True - self.addnet(n) - for server in servers: - self.addnodemap(server, n) - # do not record server name for networks since network - # nodes are replicated across all server - return handle_locally, servers - elif issubclass(nodecls, CoreNodeBase): - name = message.get_tlv(NodeTlvs.NAME.value) - if name: - serverfiletxt = "%s %s %s" % (n, name, nodecls) - if issubclass(nodecls, PhysicalNode): - # remember physical nodes - self.addphys(n) - - # emulation server TLV specifies server - servername = message.get_tlv(NodeTlvs.EMULATION_SERVER.value) - server = self.getserverbyname(servername) - if server is not None: - self.addnodemap(server, n) - if server not in servers: - servers.add(server) - if serverfiletxt and self.session.master: - self.writenodeserver(serverfiletxt, server) - - # hook to update coordinates of physical nodes - if n in self.physical_nodes: - self.session.mobility.physnodeupdateposition(message) - - return handle_locally, servers - - def handlelinkmsg(self, message): - """ - Determine and return the servers to which this link message should - be forwarded. Also build tunnels between different servers or add - opaque data to the link message before forwarding. - - :param core.api.coreapi.CoreMessage message: message to handle - :return: boolean to handle locally, a set of server, and message - :rtype: tuple - """ - servers = set() - handle_locally = False - - # determine link message destination using non-network nodes - nn = message.node_numbers() - logging.debug( - "checking link nodes (%s) with network nodes (%s)", nn, self.network_nodes - ) - if nn[0] in self.network_nodes: - if nn[1] in self.network_nodes: - # two network nodes linked together - prevent loops caused by - # the automatic tunnelling - handle_locally = True - else: - servers = self.getserversbynode(nn[1]) - elif nn[1] in self.network_nodes: - servers = self.getserversbynode(nn[0]) - else: - logging.debug("link nodes are not network nodes") - servers1 = self.getserversbynode(nn[0]) - logging.debug("servers for node(%s): %s", nn[0], servers1) - servers2 = self.getserversbynode(nn[1]) - logging.debug("servers for node(%s): %s", nn[1], servers2) - # nodes are on two different servers, build tunnels as needed - if servers1 != servers2: - localn = None - if len(servers1) == 0 or len(servers2) == 0: - handle_locally = True - servers = servers1.union(servers2) - host = None - # get the IP of remote server and decide which node number - # is for a local node - for server in servers: - host = server.host - if host is None: - # server is local - handle_locally = True - if server in servers1: - localn = nn[0] - else: - localn = nn[1] - if handle_locally and localn is None: - # having no local node at this point indicates local node is - # the one with the empty server set - if len(servers1) == 0: - localn = nn[0] - elif len(servers2) == 0: - localn = nn[1] - if host is None: - host = self.getlinkendpoint(message, localn == nn[0]) - - logging.debug( - "handle locally(%s) and local node(%s)", handle_locally, localn - ) - if localn is None: - message = self.addlinkendpoints(message, servers1, servers2) - elif message.flags & MessageFlags.ADD.value: - self.addtunnel(host, nn[0], nn[1], localn) - elif message.flags & MessageFlags.DELETE.value: - self.deltunnel(nn[0], nn[1]) - handle_locally = False - else: - servers = servers1.union(servers2) - - return handle_locally, servers, message - - def addlinkendpoints(self, message, servers1, servers2): - """ - For a link message that is not handled locally, inform the remote - servers of the IP addresses used as tunnel endpoints by adding - opaque data to the link message. - - :param core.api.coreapi.CoreMessage message: message to link end points - :param servers1: - :param servers2: - :return: core link message - :rtype: coreapi.CoreLinkMessage - """ - ip1 = "" - for server in servers1: - if server.host is not None: - ip1 = server.host - break - ip2 = "" - for server in servers2: - if server.host is not None: - ip2 = server.host - break - tlvdata = message.raw_message[coreapi.CoreMessage.header_len :] - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.OPAQUE.value, "%s:%s" % (ip1, ip2)) - newraw = coreapi.CoreLinkMessage.pack(message.flags, tlvdata) - msghdr = newraw[: coreapi.CoreMessage.header_len] - return coreapi.CoreLinkMessage(message.flags, msghdr, tlvdata) - - def getlinkendpoint(self, msg, first_is_local): - """ - A link message between two different servers has been received, - and we need to determine the tunnel endpoint. First look for - opaque data in the link message, otherwise use the IP of the message - sender (the master server). - - :param core.api.tlv.coreapi.CoreLinkMessage msg: link message - :param bool first_is_local: is first local - :return: host address - :rtype: str - """ - host = None - opaque = msg.get_tlv(LinkTlvs.OPAQUE.value) - if opaque is not None: - if first_is_local: - host = opaque.split(":")[1] - else: - host = opaque.split(":")[0] - if host == "": - host = None - - if host is None: - for session_client in self.session_clients: - # get IP address from API message sender (master) - if session_client.client_address != "": - host = session_client.client_address[0] - break - - return host - - def handlerawmsg(self, msg): - """ - Helper to invoke message handler, using raw (packed) message bytes. - - :param msg: raw message butes - :return: should handle locally or not - :rtype: bool - """ - hdr = msg[: coreapi.CoreMessage.header_len] - msgtype, flags, _msglen = coreapi.CoreMessage.unpack_header(hdr) - msgcls = coreapi.CLASS_MAP[msgtype] - return self.handle_message( - msgcls(flags, hdr, msg[coreapi.CoreMessage.header_len :]) - ) - - def forwardmsg(self, message, servers): - """ - Forward API message to all given servers. - - Return True if an empty host/port is encountered, indicating - the message should be handled locally. - - :param core.api.coreapi.CoreMessage message: message to forward - :param list servers: server to forward message to - :return: handle locally value - :rtype: bool - """ - handle_locally = len(servers) == 0 - for server in servers: - if server.host is None and server.port is None: - # local emulation server, handle this locally - handle_locally = True - elif server.sock is None: - logging.info( - "server %s @ %s:%s is disconnected", - server.name, - server.host, - server.port, - ) - else: - logging.info( - "forwarding message to server(%s): %s:%s", - server.name, - server.host, - server.port, - ) - logging.debug("message being forwarded:\n%s", message) - server.sock.send(message.raw_message) - return handle_locally - - def writeservers(self): - """ - Write the server list to a text file in the session directory upon - startup: /tmp/pycore.nnnnn/servers - - :return: nothing - """ - servers = self.getservers() - filename = os.path.join(self.session.session_dir, "servers") - master = self.session_id_master - if master is None: - master = self.session.id - try: - with open(filename, "w") as f: - f.write("master=%s\n" % master) - for server in servers: - if server.name == "localhost": - continue - - lhost, lport = None, None - if server.sock: - lhost, lport = server.sock.getsockname() - f.write( - "%s %s %s %s %s\n" - % (server.name, server.host, server.port, lhost, lport) - ) - except IOError: - logging.exception("error writing server list to the file: %s", filename) - - def writenodeserver(self, nodestr, server): - """ - Creates a /tmp/pycore.nnnnn/nX.conf/server file having the node - and server info. This may be used by scripts for accessing nodes on - other machines, much like local nodes may be accessed via the - VnodeClient class. - - :param str nodestr: node string - :param CoreDistributedServer server: core server - :return: nothing - """ - serverstr = "%s %s %s" % (server.name, server.host, server.port) - name = nodestr.split()[1] - dirname = os.path.join(self.session.session_dir, name + ".conf") - filename = os.path.join(dirname, "server") - try: - os.makedirs(dirname) - except OSError: - # directory may already exist from previous distributed run - logging.exception("error creating directory: %s", dirname) - - try: - with open(filename, "w") as f: - f.write("%s\n%s\n" % (serverstr, nodestr)) - except IOError: - logging.exception( - "error writing server file %s for node %s", filename, name - ) - - def local_instantiation_complete(self): - """ - Set the local server"s instantiation-complete status to True. - - :return: nothing - """ - # TODO: do we really want to allow a localhost to not exist? - with self.servers_lock: - server = self.servers.get("localhost") - if server is not None: - server.instantiation_complete = True - - # broadcast out instantiate complete - tlvdata = b"" - tlvdata += coreapi.CoreEventTlv.pack( - EventTlvs.TYPE.value, EventTypes.INSTANTIATION_COMPLETE.value - ) - message = coreapi.CoreEventMessage.pack(0, tlvdata) - for session_client in self.session_clients: - session_client.sendall(message) - - def instantiation_complete(self): - """ - Return True if all servers have completed instantiation, False - otherwise. - - :return: have all server completed instantiation - :rtype: bool - """ - with self.servers_lock: - for name in self.servers: - server = self.servers[name] - if not server.instantiation_complete: - return False - return True From 7afaff8cbb28c9d7eb7b59bee780ee4af26b3d41 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 08:41:48 -0700 Subject: [PATCH 34/38] updated requirements and setup.py to include fabric/invoke --- daemon/requirements.txt | 2 ++ daemon/setup.py.in | 2 ++ 2 files changed, 4 insertions(+) diff --git a/daemon/requirements.txt b/daemon/requirements.txt index a32f12de..d9029923 100644 --- a/daemon/requirements.txt +++ b/daemon/requirements.txt @@ -1,7 +1,9 @@ configparser==4.0.2 +fabric==2.5.0 future==0.17.1 grpcio==1.23.0 grpcio-tools==1.21.1 +invoke==1.3.0 lxml==4.4.1 protobuf==3.9.1 six==1.12.0 diff --git a/daemon/setup.py.in b/daemon/setup.py.in index 49af9cfe..3a451fb4 100644 --- a/daemon/setup.py.in +++ b/daemon/setup.py.in @@ -35,8 +35,10 @@ setup( packages=find_packages(), install_requires=[ "configparser", + "fabric", "future", "grpcio", + "invoke", "lxml", "protobuf", ], From b7dd8ddb6670933bd9f3374ffc2a3353ca8602aa Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 09:09:03 -0700 Subject: [PATCH 35/38] fix for docker/lxd based nodes to use remote servers and example for lxd --- daemon/core/emulator/session.py | 1 + daemon/core/nodes/docker.py | 18 +++++++- daemon/core/nodes/lxd.py | 7 +++- daemon/examples/python/distributed_lxd.py | 51 +++++++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 daemon/examples/python/distributed_lxd.py diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index e864ec8b..5e3997eb 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -782,6 +782,7 @@ class Session(object): name=name, start=start, image=node_options.image, + server=server, ) else: node = self.create_node( diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index b91e987e..17d7578a 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -89,7 +89,17 @@ class DockerClient(object): class DockerNode(CoreNode): apitype = NodeTypes.DOCKER.value - def __init__(self, session, _id=None, name=None, nodedir=None, bootsh="boot.sh", start=True, image=None): + def __init__( + self, + session, + _id=None, + name=None, + nodedir=None, + bootsh="boot.sh", + start=True, + server=None, + image=None + ): """ Create a DockerNode instance. @@ -99,12 +109,16 @@ class DockerNode(CoreNode): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param str image: image to start container with """ if image is None: image = "ubuntu" self.image = image - super(DockerNode, self).__init__(session, _id, name, nodedir, bootsh, start) + super(DockerNode, self).__init__( + session, _id, name, nodedir, bootsh, start, server + ) def create_node_net_client(self, use_ovs): """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index eef3dc8f..b11086e7 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -74,6 +74,7 @@ class LxcNode(CoreNode): nodedir=None, bootsh="boot.sh", start=True, + server=None, image=None, ): """ @@ -85,12 +86,16 @@ class LxcNode(CoreNode): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param str image: image to start container with """ if image is None: image = "ubuntu" self.image = image - super(LxcNode, self).__init__(session, _id, name, nodedir, bootsh, start) + super(LxcNode, self).__init__( + session, _id, name, nodedir, bootsh, start, server + ) def alive(self): """ diff --git a/daemon/examples/python/distributed_lxd.py b/daemon/examples/python/distributed_lxd.py new file mode 100644 index 00000000..8bafeb7a --- /dev/null +++ b/daemon/examples/python/distributed_lxd.py @@ -0,0 +1,51 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes + + +def main(): + address = sys.argv[1] + remote = sys.argv[2] + + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu({"distributed_address": address}) + session = coreemu.create_session() + + # initialize distributed + server_name = "core2" + session.add_distributed(server_name, remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + options = NodeOptions(image="ubuntu:18.04") + node_one = session.add_node(_type=NodeTypes.LXC, node_options=options) + options.emulation_server = server_name + node_two = session.add_node(_type=NodeTypes.LXC, node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, node_two.id, interface_one, interface_two) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From 0ef06a0167a43092ae25090b871eb1ad9f162a31 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 09:32:32 -0700 Subject: [PATCH 36/38] added docs for session distributed commands --- daemon/core/emulator/session.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 5e3997eb..fe371c44 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -159,12 +159,25 @@ class Session(object): } def add_distributed(self, name, host): + """ + Add distributed server configuration. + + :param str name: distributed server name + :param str host: distributed server host address + :return: nothing + """ server = DistributedServer(name, host) self.servers[name] = server cmd = "mkdir -p %s" % self.session_dir server.remote_cmd(cmd) def shutdown_distributed(self): + """ + Shutdown logic for dealing with distributed tunnels and server session + directories. + + :return: nothing + """ # shutdown all tunnels for key in self.tunnels: tunnels = self.tunnels[key] @@ -180,7 +193,12 @@ class Session(object): # clear tunnels self.tunnels.clear() - def initialize_distributed(self): + def start_distributed(self): + """ + Start distributed network tunnels. + + :return: nothing + """ for node_id in self.nodes: node = self.nodes[node_id] @@ -195,6 +213,16 @@ class Session(object): self.create_gre_tunnel(node, server) def create_gre_tunnel(self, node, server): + """ + Create gre tunnel using a pair of gre taps between the local and remote server. + + + :param core.nodes.network.CoreNetwork node: node to create gre tunnel for + :param core.emulator.distributed.DistributedServer server: server to create + tunnel for + :return: local and remote gre taps created for tunnel + :rtype: tuple + """ host = server.host key = self.tunnelkey(node.id, IpAddress.to_int(host)) tunnel = self.tunnels.get(key) @@ -1566,7 +1594,7 @@ class Session(object): self.add_remove_control_interface(node=None, remove=False) # initialize distributed tunnels - self.initialize_distributed() + self.start_distributed() # instantiate will be invoked again upon Emane configure if self.emane.startup() == self.emane.NOT_READY: From e94a6d1afa7ec4014faa2287f79f0f2437374795 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 11:10:59 -0700 Subject: [PATCH 37/38] separated distributed session logic into its own class to help reduce session.py size as it is already too big --- daemon/core/api/tlv/corehandlers.py | 4 +- daemon/core/emane/emanemanager.py | 22 +-- daemon/core/emulator/distributed.py | 152 ++++++++++++++++++ daemon/core/emulator/session.py | 139 ++-------------- daemon/core/nodes/network.py | 9 +- daemon/core/nodes/physical.py | 2 +- daemon/core/xml/emanexml.py | 24 +-- daemon/examples/python/distributed.py | 2 +- daemon/examples/python/distributed_emane.py | 2 +- daemon/examples/python/distributed_lxd.py | 2 +- daemon/examples/python/distributed_ptp.py | 2 +- .../examples/python/distributed_switches.py | 2 +- daemon/examples/python/distributed_wlan.py | 2 +- daemon/tests/test_gui.py | 4 +- 14 files changed, 196 insertions(+), 172 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 60ddfcca..8f995920 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1192,9 +1192,9 @@ class CoreHandler(socketserver.BaseRequestHandler): for server in server_list: server_items = server.split(":") name, host, _ = server_items[:3] - self.session.add_distributed(name, host) + self.session.distributed.add_server(name, host) elif message_type == ConfigFlags.RESET: - self.session.shutdown_distributed() + self.session.distributed.shutdown() def handle_config_services(self, message_type, config_data): replies = [] diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index e4208189..91553b5a 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -142,9 +142,7 @@ class EmaneManager(ModelManager): args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - for name in self.session.servers: - server = self.session.servers[name] - server.remote_cmd(args) + self.session.distributed.execute(lambda x: x.remote_cmd(args)) # load default emane models self.load_models(EMANE_MODELS) @@ -518,11 +516,11 @@ class EmaneManager(ModelManager): dev = self.get_config("eventservicedevice") emanexml.create_event_service_xml(group, port, dev, self.session.session_dir) - for name in self.session.servers: - conn = self.session.servers[name] - emanexml.create_event_service_xml( - group, port, dev, self.session.session_dir, conn + self.session.distributed.execute( + lambda x: emanexml.create_event_service_xml( + group, port, dev, self.session.session_dir, x ) + ) def startdaemons(self): """ @@ -598,9 +596,7 @@ class EmaneManager(ModelManager): emanecmd += " -f %s" % os.path.join(path, "emane.log") emanecmd += " %s" % os.path.join(path, "platform.xml") utils.check_cmd(emanecmd, cwd=path) - for name in self.session.servers: - server = self.session.servers[name] - server.remote_cmd(emanecmd, cwd=path) + self.session.distributed.execute(lambda x: x.remote_cmd(emanecmd, cwd=path)) logging.info("host emane daemon running: %s", emanecmd) def stopdaemons(self): @@ -625,10 +621,8 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - for name in self.session.servers: - server = self.session[name] - server.remote_cmd(kill_emaned) - server.remote_cmd(kill_transortd) + self.session.distributed.execute(lambda x: x.remote_cmd(kill_emaned)) + self.session.distributed.execute(lambda x: x.remote_cmd(kill_transortd)) except CoreCommandError: logging.exception("error shutting down emane daemons") diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 2df33541..c6218441 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -5,12 +5,17 @@ Defines distributed server functionality. import logging import os import threading +from collections import OrderedDict from tempfile import NamedTemporaryFile from fabric import Connection from invoke import UnexpectedExit +from core import utils from core.errors import CoreCommandError +from core.nodes.interface import GreTap +from core.nodes.ipaddress import IpAddress +from core.nodes.network import CoreNetwork, CtrlNet LOCK = threading.Lock() @@ -93,3 +98,150 @@ class DistributedServer(object): temp.close() self.conn.put(temp.name, destination) os.unlink(temp.name) + + +class DistributedController(object): + def __init__(self, session): + """ + Create + + :param session: + """ + self.session = session + self.servers = OrderedDict() + self.tunnels = {} + self.address = self.session.options.get_config( + "distributed_address", default=None + ) + + def add_server(self, name, host): + """ + Add distributed server configuration. + + :param str name: distributed server name + :param str host: distributed server host address + :return: nothing + """ + server = DistributedServer(name, host) + self.servers[name] = server + cmd = "mkdir -p %s" % self.session.session_dir + server.remote_cmd(cmd) + + def execute(self, func): + """ + Convenience for executing logic against all distributed servers. + + :param func: function to run, that takes a DistributedServer as a parameter + :return: nothing + """ + for name in self.servers: + server = self.servers[name] + func(server) + + def shutdown(self): + """ + Shutdown logic for dealing with distributed tunnels and server session + directories. + + :return: nothing + """ + # shutdown all tunnels + for key in self.tunnels: + tunnels = self.tunnels[key] + for tunnel in tunnels: + tunnel.shutdown() + + # remove all remote session directories + for name in self.servers: + server = self.servers[name] + cmd = "rm -rf %s" % self.session.session_dir + server.remote_cmd(cmd) + + # clear tunnels + self.tunnels.clear() + + def start(self): + """ + Start distributed network tunnels. + + :return: nothing + """ + for node_id in self.session.nodes: + node = self.session.nodes[node_id] + + if not isinstance(node, CoreNetwork): + continue + + if isinstance(node, CtrlNet) and node.serverintf is not None: + continue + + for name in self.servers: + server = self.servers[name] + self.create_gre_tunnel(node, server) + + def create_gre_tunnel(self, node, server): + """ + Create gre tunnel using a pair of gre taps between the local and remote server. + + + :param core.nodes.network.CoreNetwork node: node to create gre tunnel for + :param core.emulator.distributed.DistributedServer server: server to create + tunnel for + :return: local and remote gre taps created for tunnel + :rtype: tuple + """ + host = server.host + key = self.tunnel_key(node.id, IpAddress.to_int(host)) + tunnel = self.tunnels.get(key) + if tunnel is not None: + return tunnel + + # local to server + logging.info( + "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key + ) + local_tap = GreTap(session=self.session, remoteip=host, key=key) + local_tap.net_client.create_interface(node.brname, local_tap.localname) + + # server to local + logging.info( + "remote tunnel node(%s) to local(%s) key(%s)", node.name, self.address, key + ) + remote_tap = GreTap( + session=self.session, remoteip=self.address, key=key, server=server + ) + remote_tap.net_client.create_interface(node.brname, remote_tap.localname) + + # save tunnels for shutdown + tunnel = (local_tap, remote_tap) + self.tunnels[key] = tunnel + return tunnel + + def tunnel_key(self, n1_id, n2_id): + """ + Compute a 32-bit key used to uniquely identify a GRE tunnel. + The hash(n1num), hash(n2num) values are used, so node numbers may be + None or string values (used for e.g. "ctrlnet"). + + :param int n1_id: node one id + :param int n2_id: node two id + :return: tunnel key for the node pair + :rtype: int + """ + logging.debug("creating tunnel key for: %s, %s", n1_id, n2_id) + key = ( + (self.session.id << 16) ^ utils.hashkey(n1_id) ^ (utils.hashkey(n2_id) << 8) + ) + return key & 0xFFFFFFFF + + def get_tunnel(self, n1_id, n2_id): + """ + Return the GreTap between two nodes if it exists. + + :param int n1_id: node one id + :param int n2_id: node two id + :return: gre tap between nodes or None + """ + key = self.tunnel_key(n1_id, n2_id) + logging.debug("checking for tunnel key(%s) in: %s", key, self.tunnels) + return self.tunnels.get(key) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index fe371c44..d962da28 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -18,7 +18,7 @@ from core import constants, utils from core.emane.emanemanager import EmaneManager from core.emane.nodes import EmaneNet from core.emulator.data import EventData, ExceptionData, NodeData -from core.emulator.distributed import DistributedServer +from core.emulator.distributed import DistributedController from core.emulator.emudata import ( IdGen, LinkOptions, @@ -34,11 +34,9 @@ from core.location.event import EventLoop from core.location.mobility import MobilityManager from core.nodes.base import CoreNetworkBase, CoreNode, CoreNodeBase from core.nodes.docker import DockerNode -from core.nodes.interface import GreTap -from core.nodes.ipaddress import IpAddress, MacAddress +from core.nodes.ipaddress import MacAddress from core.nodes.lxd import LxcNode from core.nodes.network import ( - CoreNetwork, CtrlNet, GreTapBridge, HubNode, @@ -137,10 +135,8 @@ class Session(object): self.options.set_config(key, value) self.metadata = SessionMetaData() - # distributed servers - self.servers = {} - self.tunnels = {} - self.address = self.options.get_config("distributed_address", default=None) + # distributed support and logic + self.distributed = DistributedController(self) # initialize session feature helpers self.location = CoreLocation() @@ -158,123 +154,6 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } - def add_distributed(self, name, host): - """ - Add distributed server configuration. - - :param str name: distributed server name - :param str host: distributed server host address - :return: nothing - """ - server = DistributedServer(name, host) - self.servers[name] = server - cmd = "mkdir -p %s" % self.session_dir - server.remote_cmd(cmd) - - def shutdown_distributed(self): - """ - Shutdown logic for dealing with distributed tunnels and server session - directories. - - :return: nothing - """ - # shutdown all tunnels - for key in self.tunnels: - tunnels = self.tunnels[key] - for tunnel in tunnels: - tunnel.shutdown() - - # remove all remote session directories - for name in self.servers: - server = self.servers[name] - cmd = "rm -rf %s" % self.session_dir - server.remote_cmd(cmd) - - # clear tunnels - self.tunnels.clear() - - def start_distributed(self): - """ - Start distributed network tunnels. - - :return: nothing - """ - for node_id in self.nodes: - node = self.nodes[node_id] - - if not isinstance(node, CoreNetwork): - continue - - if isinstance(node, CtrlNet) and node.serverintf is not None: - continue - - for name in self.servers: - server = self.servers[name] - self.create_gre_tunnel(node, server) - - def create_gre_tunnel(self, node, server): - """ - Create gre tunnel using a pair of gre taps between the local and remote server. - - - :param core.nodes.network.CoreNetwork node: node to create gre tunnel for - :param core.emulator.distributed.DistributedServer server: server to create - tunnel for - :return: local and remote gre taps created for tunnel - :rtype: tuple - """ - host = server.host - key = self.tunnelkey(node.id, IpAddress.to_int(host)) - tunnel = self.tunnels.get(key) - if tunnel is not None: - return tunnel - - # local to server - logging.info( - "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key - ) - local_tap = GreTap(session=self, remoteip=host, key=key) - local_tap.net_client.create_interface(node.brname, local_tap.localname) - - # server to local - logging.info( - "remote tunnel node(%s) to local(%s) key(%s)", node.name, self.address, key - ) - remote_tap = GreTap(session=self, remoteip=self.address, key=key, server=server) - remote_tap.net_client.create_interface(node.brname, remote_tap.localname) - - # save tunnels for shutdown - tunnel = (local_tap, remote_tap) - self.tunnels[key] = tunnel - return tunnel - - def tunnelkey(self, n1_id, n2_id): - """ - Compute a 32-bit key used to uniquely identify a GRE tunnel. - The hash(n1num), hash(n2num) values are used, so node numbers may be - None or string values (used for e.g. "ctrlnet"). - - :param int n1_id: node one id - :param int n2_id: node two id - :return: tunnel key for the node pair - :rtype: int - """ - logging.debug("creating tunnel key for: %s, %s", n1_id, n2_id) - key = (self.id << 16) ^ utils.hashkey(n1_id) ^ (utils.hashkey(n2_id) << 8) - return key & 0xFFFFFFFF - - def gettunnel(self, n1_id, n2_id): - """ - Return the GreTap between two nodes if it exists. - - :param int n1_id: node one id - :param int n2_id: node two id - :return: gre tap between nodes or None - """ - key = self.tunnelkey(n1_id, n2_id) - logging.debug("checking for tunnel key(%s) in: %s", key, self.tunnels) - return self.tunnels.get(key) - @classmethod def get_node_class(cls, _type): """ @@ -324,7 +203,7 @@ class Session(object): node_two = self.get_node(node_two_id) # both node ids are provided - tunnel = self.gettunnel(node_one_id, node_two_id) + tunnel = self.distributed.get_tunnel(node_one_id, node_two_id) logging.debug("tunnel between nodes: %s", tunnel) if isinstance(tunnel, GreTapBridge): net_one = tunnel @@ -789,7 +668,7 @@ class Session(object): name = "%s%s" % (node_class.__name__, _id) # verify distributed server - server = self.servers.get(node_options.emulation_server) + server = self.distributed.servers.get(node_options.emulation_server) if node_options.emulation_server is not None and server is None: raise CoreError( "invalid distributed server: %s" % node_options.emulation_server @@ -1003,7 +882,7 @@ class Session(object): :return: nothing """ self.delete_nodes() - self.shutdown_distributed() + self.distributed.shutdown() self.del_hooks() self.emane.reset() @@ -1082,7 +961,7 @@ class Session(object): # remove and shutdown all nodes and tunnels self.delete_nodes() - self.shutdown_distributed() + self.distributed.shutdown() # remove this sessions working directory preserve = self.options.get_config("preservedir") == "1" @@ -1594,7 +1473,7 @@ class Session(object): self.add_remove_control_interface(node=None, remove=False) # initialize distributed tunnels - self.start_distributed() + self.distributed.start() # instantiate will be invoked again upon Emane configure if self.emane.startup() == self.emane.NOT_READY: diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 931622bb..98bec198 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -289,9 +289,7 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - for name in self.session.servers: - server = self.session.servers[name] - server.remote_cmd(args, env, cwd, wait) + self.session.distributed.execute(lambda x: x.remote_cmd(args, env, cwd, wait)) return output def startup(self): @@ -778,8 +776,9 @@ class CtrlNet(CoreNetwork): current = "%s/%s" % (address, self.prefix.prefixlen) net_client = get_net_client(use_ovs, utils.check_cmd) net_client.create_address(self.brname, current) - for name in self.session.servers: - server = self.session.servers[name] + servers = self.session.distributed.servers + for name in servers: + server = servers[name] address -= 1 current = "%s/%s" % (address, self.prefix.prefixlen) net_client = get_net_client(use_ovs, server.remote_cmd) diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 37a2eb54..0f9e0217 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -166,7 +166,7 @@ class PhysicalNode(CoreNodeBase): if self.up: # this is reached when this node is linked to a network node # tunnel to net not built yet, so build it now and adopt it - _, remote_tap = self.session.create_gre_tunnel(net, self.server) + _, remote_tap = self.session.distributed.create_gre_tunnel(net, self.server) self.adoptnetif(remote_tap, ifindex, hwaddr, addrlist) return ifindex else: diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 881ff373..41319ea4 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -314,9 +314,9 @@ def build_transport_xml(emane_manager, node, transport_type): file_name = transport_file_name(node.id, transport_type) file_path = os.path.join(emane_manager.session.session_dir, file_name) create_file(transport_element, doc_name, file_path) - for name in emane_manager.session.servers: - server = emane_manager.session.servers[name] - create_file(transport_element, doc_name, file_path, server) + emane_manager.session.distributed.execute( + lambda x: create_file(transport_element, doc_name, file_path, x) + ) def create_phy_xml(emane_model, config, file_path, server): @@ -342,9 +342,9 @@ def create_phy_xml(emane_model, config, file_path, server): create_file(phy_element, "phy", file_path, server) else: create_file(phy_element, "phy", file_path) - for name in emane_model.session.servers: - server = emane_model.session.servers[name] - create_file(phy_element, "phy", file_path, server) + emane_model.session.distributed.execute( + lambda x: create_file(phy_element, "phy", file_path, x) + ) def create_mac_xml(emane_model, config, file_path, server): @@ -372,9 +372,9 @@ def create_mac_xml(emane_model, config, file_path, server): create_file(mac_element, "mac", file_path, server) else: create_file(mac_element, "mac", file_path) - for name in emane_model.session.servers: - server = emane_model.session.servers[name] - create_file(mac_element, "mac", file_path, server) + emane_model.session.distributed.execute( + lambda x: create_file(mac_element, "mac", file_path, x) + ) def create_nem_xml( @@ -410,9 +410,9 @@ def create_nem_xml( create_file(nem_element, "nem", nem_file, server) else: create_file(nem_element, "nem", nem_file) - for name in emane_model.session.servers: - server = emane_model.session.servers[name] - create_file(nem_element, "nem", nem_file, server) + emane_model.session.distributed.execute( + lambda x: create_file(nem_element, "nem", nem_file, x) + ) def create_event_service_xml(group, port, device, file_directory, server=None): diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index 8bcf2972..8eb23b2c 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -20,7 +20,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py index c64d1f0c..4ef50ccb 100644 --- a/daemon/examples/python/distributed_emane.py +++ b/daemon/examples/python/distributed_emane.py @@ -27,7 +27,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_lxd.py b/daemon/examples/python/distributed_lxd.py index 8bafeb7a..130942ea 100644 --- a/daemon/examples/python/distributed_lxd.py +++ b/daemon/examples/python/distributed_lxd.py @@ -20,7 +20,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index b0f27c28..62e7df64 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -20,7 +20,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py index bc13bf2c..f9b69757 100644 --- a/daemon/examples/python/distributed_switches.py +++ b/daemon/examples/python/distributed_switches.py @@ -16,7 +16,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_wlan.py b/daemon/examples/python/distributed_wlan.py index f8af1f5f..10f25aa8 100644 --- a/daemon/examples/python/distributed_wlan.py +++ b/daemon/examples/python/distributed_wlan.py @@ -21,7 +21,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index 02e634be..c07e2bd3 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -763,11 +763,11 @@ class TestGui: (ConfigTlvs.VALUES, "%s:%s:%s" % (server, host, port)), ], ) - coreserver.session.add_distributed = mock.MagicMock() + coreserver.session.distributed.add_server = mock.MagicMock() coreserver.request_handler.handle_message(message) - coreserver.session.add_distributed.assert_called_once_with(server, host) + coreserver.session.distributed.add_server.assert_called_once_with(server, host) def test_config_services_request_all(self, coreserver): message = coreapi.CoreConfMessage.create( From 4746fe67ef78adde410ff00e0c07f7af63e783eb Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 11:35:48 -0700 Subject: [PATCH 38/38] added docs for distributed.py --- daemon/core/emulator/distributed.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index c6218441..83576434 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -101,6 +101,10 @@ class DistributedServer(object): class DistributedController(object): + """ + Provides logic for dealing with remote tunnels and distributed servers. + """ + def __init__(self, session): """ Create