From d9bd0fa434351e1192804c855c856dcacb0ef783 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2019 23:59:50 +0000 Subject: [PATCH 001/113] Bump jackson.version from 2.9.9 to 2.10.0.pr2 in /corefx Bumps `jackson.version` from 2.9.9 to 2.10.0.pr2. Updates `jackson-core` from 2.9.9 to 2.10.0.pr2 - [Release notes](https://github.com/FasterXML/jackson-core/releases) - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.9.9...jackson-core-2.10.0.pr2) Updates `jackson-databind` from 2.9.9 to 2.10.0.pr2 - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) Updates `jackson-annotations` from 2.9.9 to 2.10.0.pr2 - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) Signed-off-by: dependabot[bot] --- corefx/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corefx/pom.xml b/corefx/pom.xml index af957aa2..ed5454e0 100644 --- a/corefx/pom.xml +++ b/corefx/pom.xml @@ -13,7 +13,7 @@ 1.8 1.8 2.1.1 - 2.9.9 + 2.10.0.pr2 1.20.0 2.9.0 From d5c257fc9bdbdbab67b6ed0e9ae50df0a40704fc Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 23 Sep 2019 22:02:09 -0700 Subject: [PATCH 002/113] fixed flake8 config to match other tooling --- daemon/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/setup.cfg b/daemon/setup.cfg index 576aa9f0..a3084b8b 100644 --- a/daemon/setup.cfg +++ b/daemon/setup.cfg @@ -11,7 +11,7 @@ line_length=88 [flake8] ignore=E501,W503,E203 -max-line-length=100 +max-line-length=88 max-complexity=26 select=B,C,E,F,W,T4 exclude=*_pb2*.py,utm.py,doc,build From 212b8ccb0233862e3d345179b62f1dc1183e4b95 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 23 Sep 2019 22:15:20 -0700 Subject: [PATCH 003/113] bumping version for release --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 64f291b1..4f774f6d 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. # this defines the CORE version number, must be static for AC_INIT -AC_INIT(core, 5.3.1) +AC_INIT(core, 5.4.0) # autoconf and automake initialization AC_CONFIG_SRCDIR([netns/version.h.in]) From 9c4459e7e79de4e9ccd0dd264e2503d744abf19f Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 23 Sep 2019 22:45:27 -0700 Subject: [PATCH 004/113] update to devguide --- docs/devguide.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/devguide.md b/docs/devguide.md index 5bd02095..ca207a82 100644 --- a/docs/devguide.md +++ b/docs/devguide.md @@ -32,6 +32,9 @@ To leverage the dev environment you need python 3.6+. # change to daemon directory cd $REPO/daemon +# copy setup.py for installation +cp setup.py.in setup.py + # install pipenv pip3 install pipenv From 1c11c6c573eccc666492f25091043ec923a79f09 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 23 Sep 2019 23:40:59 -0700 Subject: [PATCH 005/113] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0af9390..099467f0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# CORE [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d94eb0244ade4510a106b4af76077a92)](https://www.codacy.com/app/blakeharnden/core?utm_source=github.com&utm_medium=referral&utm_content=coreemu/core&utm_campaign=Badge_Grade) +# CORE CORE: Common Open Research Emulator From b0c30056f9d3a9e3cc0b2b072d664b57ebc946d3 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 25 Sep 2019 10:46:41 -0700 Subject: [PATCH 006/113] removed ovs node from gui --- gui/api.tcl | 3 +-- gui/editor.tcl | 13 +++++-------- gui/ipv4.tcl | 2 +- gui/linkcfg.tcl | 33 +++++++++++---------------------- gui/nodes.tcl | 9 +++------ 5 files changed, 21 insertions(+), 39 deletions(-) diff --git a/gui/api.tcl b/gui/api.tcl index 310e5ddc..1801f82c 100644 --- a/gui/api.tcl +++ b/gui/api.tcl @@ -2706,7 +2706,7 @@ proc sendNodeTypeInfo { sock reset } { set typesinuse "" foreach node $node_list { set type [nodeType $node] - if { $type != "router" && $type != "OVS" } { continue } + if { $type != "router" } { continue } set model [getNodeModel $node] if { [lsearch $typesinuse $model] < 0 } { lappend typesinuse $model } } @@ -2910,7 +2910,6 @@ proc getNodeTypeAPI { node } { router { return 0x0 } netns { return 0x0 } jail { return 0x0 } - OVS { return 0x0 } physical { return 0x1 } tbd { return 0x3 } lanswitch { return 0x4 } diff --git a/gui/editor.tcl b/gui/editor.tcl index fec8a499..f93d85c1 100644 --- a/gui/editor.tcl +++ b/gui/editor.tcl @@ -333,7 +333,7 @@ proc redrawAll {} { proc drawNode { c node } { global showNodeLabels - global router pc host lanswitch rj45 hub pseudo OVS + global router pc host lanswitch rj45 hub pseudo global curcanvas zoom global wlan if { $c == "" } { set c .c } ;# default canvas @@ -348,7 +348,7 @@ proc drawNode { c node } { set cimg "" set imgzoom $zoom if { $zoom == 0.75 || $zoom == 1.5 } { set imgzoom 1.0 } - if { $type == "router" || $type == "OVS" } { + if { $type == "router" } { set model [getNodeModel $node] set cimg [getNodeTypeImage $model normal] } @@ -1535,7 +1535,7 @@ proc raiseAll {c} { proc button1 { c x y button } { global node_list plot_list curcanvas zoom global activetool activetoolp newlink curobj changed def_router_model - global router pc host lanswitch rj45 hub OVS + global router pc host lanswitch rj45 hub global oval rectangle text global lastX lastY global background selectbox @@ -1607,10 +1607,7 @@ proc button1 { c x y button } { rectangle text} $activetool] < 0 } { if { $g_view_locked == 1 } { return } if { $activetoolp == "routers" } { - if {$activetool != "OVS"} { - set node [newNode router] - } else { - set node [newNode OVS]} + set node [newNode router] setNodeModel $node $activetool } else { set node [newNode $activetool] @@ -2550,7 +2547,7 @@ proc popupConfigDialog { c } { -side right -padx 4 -pady 4 # end Boeing pack $wi.ftop -side top - if { $type == "router" || $type == "OVS"} { + if { $type == "router" } { ttk::frame $wi.model -borderwidth 4 ttk::label $wi.model.label -text "Type:" diff --git a/gui/ipv4.tcl b/gui/ipv4.tcl index 76901060..145b0988 100644 --- a/gui/ipv4.tcl +++ b/gui/ipv4.tcl @@ -108,7 +108,7 @@ proc autoIPv4addr { node iface } { set peer_node [logicalPeerByIfc $node $iface] # find addresses of NETWORK layer peer nodes - if { [[typemodel $peer_node].layer] == "LINK" || [nodeType $peer_node] == "OVS" } { + if { [[typemodel $peer_node].layer] == "LINK" } { foreach l2node [listLANnodes $peer_node {}] { foreach ifc [ifcList $l2node] { set peer [logicalPeerByIfc $l2node $ifc] diff --git a/gui/linkcfg.tcl b/gui/linkcfg.tcl index 5c773097..9c7172a1 100644 --- a/gui/linkcfg.tcl +++ b/gui/linkcfg.tcl @@ -747,15 +747,12 @@ proc newLink { lnode1 lnode2 } { global defLinkColor defLinkWidth global curcanvas global systype - if { ([nodeType $lnode1] == "lanswitch" ||[nodeType $lnode1] == "OVS") && \ + if { [nodeType $lnode1] == "lanswitch" && \ [nodeType $lnode2] != "router" && \ - ([nodeType $lnode2] != "lanswitch" || [nodeType $lnode2] != "OVS") } { - set regular no } - if { ([nodeType $lnode2] == "lanswitch" || [nodeType $lnode2] == "OVS") && \ + [nodeType $lnode2] != "lanswitch" } { set regular no } + if { [nodeType $lnode2] == "lanswitch" && \ [nodeType $lnode1] != "router" && \ - ([nodeType $lnode1] != "lanswitch" || [nodeType $lnode1] != "OVS" )} { - #Khaled: puts "connecting '$lnode1' (type: '[nodeType $lnode1]') to '$lnode2' (type: '[nodeType $lnode2]') " - set regular no } + [nodeType $lnode1] != "lanswitch" } { set regular no } if { [nodeType $lnode1] == "hub" && \ [nodeType $lnode2] == "hub" } { set regular no } # Boeing: added tunnel, ktunnel types to behave as rj45 @@ -771,7 +768,7 @@ proc newLink { lnode1 lnode2 } { set othernode $lnode1 } # only allowed to link with certain types - if { [lsearch {router lanswitch hub pc host wlan OVS} \ + if { [lsearch {router lanswitch hub pc host wlan} \ [nodeType $othernode]] < 0} { return } @@ -836,18 +833,13 @@ proc newLink { lnode1 lnode2 } { } elseif {$delay != ""} { lappend $link "delay $delay" } - # Exclude OVS from network layer nodes IP address asignments - if { ([[typemodel $lnode2].layer] == "NETWORK") && ([nodeType $lnode2] != "OVS") } { - - #Khaled: puts "Assigning '$lnode2' (type: '[nodeType $lnode2]') an automatic IP address" - + if { [[typemodel $lnode2].layer] == "NETWORK" } { if { $ipv4_addr2 == "" } { autoIPv4addr $lnode2 $ifname2 } if { $ipv6_addr2 == "" } { autoIPv6addr $lnode2 $ifname2 } } # tunnels also excluded from link settings - # OVS and Lanswitch should go side by side - } elseif { (([nodeType $lnode1] == "lanswitch" || [nodeType $lnode1] == "OVS" )|| \ - ([nodeType $lnode2] == "lanswitch"|| [nodeType $lnode2] == "OVS") || \ + } elseif { ([nodeType $lnode1] == "lanswitch" || \ + [nodeType $lnode2] == "lanswitch" || \ [string first eth "$ifname1 $ifname2"] != -1) && \ [nodeType $lnode1] != "rj45" && [nodeType $lnode2] != "rj45" && \ [nodeType $lnode1] != "tunnel" && [nodeType $lnode2] != "tunnel" && \ @@ -859,13 +851,11 @@ proc newLink { lnode1 lnode2 } { } lappend link_list $link - # Exclude OVS from Network layer node configs if { [nodeType $lnode2] != "pseudo" && [nodeType $lnode1] != "wlan" && - ([[typemodel $lnode1].layer] == "NETWORK" && [nodeType $lnode1] != "OVS") } { - + [[typemodel $lnode1].layer] == "NETWORK" } { if { $ipv4_addr1 == "" && $do_auto_addressing } { - autoIPv4addr $lnode1 $ifname1 + autoIPv4addr $lnode1 $ifname1 } if { $ipv6_addr1 == "" && $do_auto_addressing } { autoIPv6addr $lnode1 $ifname1 @@ -874,8 +864,7 @@ proc newLink { lnode1 lnode2 } { # assume wlan is always lnode1 if { [nodeType $lnode1] != "pseudo" && [nodeType $lnode1] != "wlan" && - ([[typemodel $lnode2].layer] == "NETWORK" && [nodeType $lnode2] != "OVS") } { - + [[typemodel $lnode2].layer] == "NETWORK" } { if { $ipv4_addr2 == "" && $do_auto_addressing } { autoIPv4addr $lnode2 $ifname2 } diff --git a/gui/nodes.tcl b/gui/nodes.tcl index cabad158..00e52c5d 100644 --- a/gui/nodes.tcl +++ b/gui/nodes.tcl @@ -21,12 +21,10 @@ array set g_node_types_default { 5 {prouter router_green.gif router_green.gif \ {zebra OSPFv2 OSPFv3 IPForward} \ physical {built-in type for physical nodes}} - 6 {OVS lanswitch.gif lanswitch.gif {DefaultRoute SSH OvsService} OVS {} } - } # possible machine types for nodes -set MACHINE_TYPES "netns physical OVS" +set MACHINE_TYPES "netns physical" # array populated from nodes.conf file array set g_node_types { } @@ -184,7 +182,7 @@ proc getNodeTypeServices { type } { # node type from the toolbar proc getNodeTypeMachineType { type } { global MACHINE_TYPES g_node_types - set default_machine_type [lindex $MACHINE_TYPES 3] + set default_machine_type [lindex $MACHINE_TYPES 0] set i [getNodeTypeIndex $type] if { $i < 0 } { return $default_machine_type }; # failsafe return [lindex $g_node_types($i) 4] @@ -208,7 +206,7 @@ proc getNodeTypeProfile { type } { # node type from the toolbar proc getNodeTypeMachineType { type } { global MACHINE_TYPES g_node_types - set default_machine_type [lindex $MACHINE_TYPES 3] + set default_machine_type [lindex $MACHINE_TYPES 0] set i [getNodeTypeIndex $type] if { $i < 0 } { return $default_machine_type }; # failsafe return [lindex $g_node_types($i) 4] @@ -714,7 +712,6 @@ proc lanswitch.layer {} { return LINK } proc hub.layer {} { return LINK } proc tunnel.layer {} { return LINK } proc wlan.layer {} { return LINK } -proc OVS.layer {} { return NETWORK } proc router.layer {} { return NETWORK } proc router.shellcmd { n } { return "vtysh" } From b14012e5e96206f14b7079668ebac1ed36bbab06 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 26 Sep 2019 13:00:12 -0700 Subject: [PATCH 007/113] removed node map file and utilities, added formal class methods to session objects and added checks for specific classes where needed --- daemon/Pipfile | 5 +- daemon/core/api/grpc/server.py | 14 +-- daemon/core/api/tlv/broker.py | 14 ++- daemon/core/api/tlv/corehandlers.py | 6 +- daemon/core/emane/emanemanager.py | 5 +- daemon/core/emane/nodes.py | 3 +- daemon/core/emulator/coreemu.py | 14 --- daemon/core/emulator/emudata.py | 7 +- daemon/core/emulator/session.py | 120 ++++++++++++++++++-------- daemon/core/nodes/base.py | 7 +- daemon/core/nodes/interface.py | 7 +- daemon/core/nodes/nodemaps.py | 32 ------- daemon/core/nodes/nodeutils.py | 97 --------------------- daemon/core/plugins/sdt.py | 21 +++-- daemon/core/services/emaneservices.py | 5 +- daemon/core/services/frr.py | 12 +-- daemon/core/services/quagga.py | 17 ++-- daemon/core/xml/corexml.py | 7 +- daemon/core/xml/corexmldeployment.py | 6 +- daemon/scripts/core-daemon | 3 - ns3/examples/ns3lte.py | 5 +- ns3/examples/ns3wifi.py | 5 +- ns3/examples/ns3wifirandomwalk.py | 6 +- ns3/examples/ns3wimax.py | 7 +- 24 files changed, 161 insertions(+), 264 deletions(-) delete mode 100644 daemon/core/nodes/nodemaps.py delete mode 100644 daemon/core/nodes/nodeutils.py diff --git a/daemon/Pipfile b/daemon/Pipfile index e8fd5954..a689f4dc 100644 --- a/daemon/Pipfile +++ b/daemon/Pipfile @@ -4,8 +4,9 @@ url = "https://pypi.org/simple" verify_ssl = true [scripts] -coredev = "python scripts/core-daemon -f data/core.conf -l data/logging.conf" -coretest = "python -m pytest -v tests" +core = "python scripts/core-daemon -f data/core.conf -l data/logging.conf" +test = "pytest -v tests" +test_emane = "pytest -v tests/emane" [dev-packages] grpcio-tools = "*" diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 0a15f8f0..ecb391ce 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -12,6 +12,7 @@ import grpc from core import CoreError from core.api.grpc import core_pb2, core_pb2_grpc +from core.emane.nodes import EmaneNode from core.emulator.data import ( ConfigData, EventData, @@ -23,7 +24,6 @@ from core.emulator.data import ( from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions from core.emulator.enumerations import EventTypes, LinkTypes, NodeTypes from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility -from core.nodes import nodeutils from core.nodes.base import CoreNetworkBase from core.nodes.docker import DockerNode from core.nodes.ipaddress import MacAddress @@ -444,7 +444,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): if not isinstance(node.id, int): continue - node_type = nodeutils.get_node_type(node.__class__).value + node_type = session.get_node_type(node.__class__) model = getattr(node, "type", None) position = core_pb2.Position( x=node.position.x, y=node.position.y, z=node.position.z @@ -456,7 +456,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): services = [x.name for x in services] emane_model = None - if nodeutils.is_node(node, NodeTypes.EMANE): + if isinstance(node, EmaneNode): emane_model = node.model.name node_proto = core_pb2.Node( @@ -464,7 +464,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): name=node.name, emane=emane_model, model=model, - type=node_type, + type=node_type.value, position=position, services=services, ) @@ -809,18 +809,18 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): interfaces.append(interface_proto) emane_model = None - if nodeutils.is_node(node, NodeTypes.EMANE): + if isinstance(node, EmaneNode): emane_model = node.model.name services = [x.name for x in getattr(node, "services", [])] position = core_pb2.Position( x=node.position.x, y=node.position.y, z=node.position.z ) - node_type = nodeutils.get_node_type(node.__class__).value + node_type = session.get_node_type(node.__class__) node_proto = core_pb2.Node( id=node.id, name=node.name, - type=node_type, + type=node_type.value, emane=emane_model, model=node.type, position=position, diff --git a/daemon/core/api/tlv/broker.py b/daemon/core/api/tlv/broker.py index b5fd5d0b..7c0b3992 100644 --- a/daemon/core/api/tlv/broker.py +++ b/daemon/core/api/tlv/broker.py @@ -12,6 +12,7 @@ import threading from core import utils from core.api.tlv import coreapi +from core.emane.nodes import EmaneNet from core.emulator.enumerations import ( ConfigDataTypes, ConfigFlags, @@ -27,11 +28,10 @@ from core.emulator.enumerations import ( NodeTypes, RegisterTlvs, ) -from core.nodes import nodeutils from core.nodes.base import CoreNetworkBase, CoreNodeBase from core.nodes.interface import GreTap from core.nodes.ipaddress import IpAddress -from core.nodes.network import GreTapBridge +from core.nodes.network import CtrlNet, GreTapBridge from core.nodes.physical import PhysicalNode @@ -495,15 +495,12 @@ class CoreBroker(object): logging.info("adding net tunnel for: id(%s) %s", node_id, net) # add other nets here that do not require tunnels - if nodeutils.is_node(net, NodeTypes.EMANE_NET): + if isinstance(net, EmaneNet): logging.warning("emane network does not require a tunnel") return None server_interface = getattr(net, "serverintf", None) - if ( - nodeutils.is_node(net, NodeTypes.CONTROL_NET) - and server_interface is not None - ): + if isinstance(net, CtrlNet) and server_interface is not None: logging.warning( "control networks with server interfaces do not need a tunnel" ) @@ -824,7 +821,8 @@ class CoreBroker(object): nodetype = message.get_tlv(NodeTlvs.TYPE.value) if nodetype is not None: try: - nodecls = nodeutils.get_node_class(NodeTypes(nodetype)) + nodetype = NodeTypes(nodetype) + nodecls = self.session.get_node_class(nodetype) except KeyError: logging.warning("broker invalid node type %s", nodetype) return handle_locally, servers diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index bcbb0f57..24e29169 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -38,7 +38,7 @@ from core.emulator.enumerations import ( SessionTlvs, ) from core.location.mobility import BasicRangeModel -from core.nodes import nodeutils +from core.nodes.network import WlanNode from core.services.coreservices import ServiceManager, ServiceShim @@ -1603,8 +1603,8 @@ class CoreHandler(socketserver.BaseRequestHandler): node = self.session.get_node(node_id) # configure mobility models for WLAN added during runtime - if event_type == EventTypes.INSTANTIATION_STATE and nodeutils.is_node( - node, NodeTypes.WIRELESS_LAN + if event_type == EventTypes.INSTANTIATION_STATE and isinstance( + node, WlanNode ): self.session.start_mobility(node_ids=(node.id,)) return () diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index be8ca368..c93d28dd 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -15,6 +15,7 @@ from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel from core.emane.emanemodel import EmaneModel from core.emane.ieee80211abg import EmaneIeee80211abgModel +from core.emane.nodes import EmaneNode from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel from core.emulator.enumerations import ( @@ -23,10 +24,8 @@ from core.emulator.enumerations import ( ConfigTlvs, MessageFlags, MessageTypes, - NodeTypes, RegisterTlvs, ) -from core.nodes import nodeutils from core.xml import emanexml try: @@ -266,7 +265,7 @@ class EmaneManager(ModelManager): with self.session._nodes_lock: for node_id in self.session.nodes: node = self.session.nodes[node_id] - if nodeutils.is_node(node, NodeTypes.EMANE): + if isinstance(node, EmaneNode): logging.debug( "adding emane node: id(%s) name(%s)", node.id, node.name ) diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 7b987098..79b767d1 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -25,7 +25,6 @@ class EmaneNet(CoreNetworkBase): apitype = NodeTypes.EMANE.value linktype = LinkTypes.WIRELESS.value - # icon used type = "wlan" @@ -36,6 +35,8 @@ class EmaneNode(EmaneNet): Emane controller object that exists in a session. """ + is_emane = True + def __init__(self, session, _id=None, name=None, start=True): super(EmaneNode, self).__init__(session, _id, name, start) self.conf = "" diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 9a039627..c0eae5cd 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -7,7 +7,6 @@ import sys import core.services from core.emulator.emudata import IdGen from core.emulator.session import Session -from core.nodes import nodemaps, nodeutils from core.services.coreservices import ServiceManager @@ -53,10 +52,6 @@ class CoreEmu(object): self.session_id_gen = IdGen(_id=0) self.sessions = {} - # set default nodes - node_map = nodemaps.NODES - nodeutils.set_node_map(node_map) - # load services self.service_errors = [] self.load_services() @@ -77,15 +72,6 @@ class CoreEmu(object): custom_service_errors = ServiceManager.add_services(service_path) self.service_errors.extend(custom_service_errors) - def update_nodes(self, node_map): - """ - Updates node map used by core. - - :param dict node_map: node map to update existing node map with - :return: nothing - """ - nodeutils.update_node_map(node_map) - def shutdown(self): """ Shutdown all CORE session. diff --git a/daemon/core/emulator/emudata.py b/daemon/core/emulator/emudata.py index 2cbd3007..0f7a6af6 100644 --- a/daemon/core/emulator/emudata.py +++ b/daemon/core/emulator/emudata.py @@ -1,7 +1,8 @@ -from core.emulator.enumerations import LinkTypes, NodeTypes -from core.nodes import nodeutils +from core.emane.nodes import EmaneNode +from core.emulator.enumerations import LinkTypes from core.nodes.base import CoreNetworkBase from core.nodes.ipaddress import Ipv4Prefix, Ipv6Prefix, MacAddress +from core.nodes.physical import PhysicalNode class IdGen(object): @@ -65,7 +66,7 @@ def link_config(network, interface, link_options, devname=None, interface_two=No } # hacky check here, because physical and emane nodes do not conform to the same linkconfig interface - if not nodeutils.is_node(network, [NodeTypes.EMANE, NodeTypes.PHYSICAL]): + if not isinstance(network, (EmaneNode, PhysicalNode)): config["devname"] = devname network.linkconfig(**config) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index e7a8e3f5..375e6f24 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -14,11 +14,11 @@ import threading import time from multiprocessing.pool import ThreadPool -import core.nodes.base from core import CoreError, constants, utils from core.api.tlv import coreapi from core.api.tlv.broker import CoreBroker from core.emane.emanemanager import EmaneManager +from core.emane.nodes import EmaneNet, EmaneNode from core.emulator.data import EventData, ExceptionData, NodeData from core.emulator.emudata import ( IdGen, @@ -33,14 +33,46 @@ from core.emulator.sessionconfig import SessionConfig, SessionMetaData from core.location.corelocation import CoreLocation from core.location.event import EventLoop from core.location.mobility import MobilityManager -from core.nodes import nodeutils -from core.nodes.base import CoreNodeBase +from core.nodes.base import CoreNode, CoreNodeBase +from core.nodes.docker import DockerNode from core.nodes.ipaddress import MacAddress +from core.nodes.lxd import LxcNode +from core.nodes.network import ( + CtrlNet, + GreTapBridge, + HubNode, + PtpNet, + SwitchNode, + TunnelNode, + WlanNode, +) +from core.nodes.physical import PhysicalNode, Rj45Node from core.plugins.sdt import Sdt from core.services.coreservices import CoreServices from core.xml import corexml, corexmldeployment from core.xml.corexml import CoreXmlReader, CoreXmlWriter +# maps for converting from API call node type values to classes and vice versa +NODES = { + NodeTypes.DEFAULT: CoreNode, + NodeTypes.PHYSICAL: PhysicalNode, + NodeTypes.TBD: None, + NodeTypes.SWITCH: SwitchNode, + NodeTypes.HUB: HubNode, + NodeTypes.WIRELESS_LAN: WlanNode, + NodeTypes.RJ45: Rj45Node, + NodeTypes.TUNNEL: TunnelNode, + NodeTypes.KTUNNEL: None, + NodeTypes.EMANE: EmaneNode, + NodeTypes.EMANE_NET: EmaneNet, + NodeTypes.TAP_BRIDGE: GreTapBridge, + NodeTypes.PEER_TO_PEER: PtpNet, + NodeTypes.CONTROL_NET: CtrlNet, + NodeTypes.DOCKER: DockerNode, + NodeTypes.LXC: LxcNode, +} +NODES_TYPE = {NODES[x]: x for x in NODES} + class Session(object): """ @@ -121,6 +153,33 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } + @classmethod + def get_node_class(cls, _type): + """ + Retrieve the class for a given node type. + + :param core.emulator.enumerations.NodeTypes _type: node type to get class for + :return: node class + """ + node_class = NODES.get(_type) + if node_class is None: + raise CoreError("invalid node type: %s" % _type) + return node_class + + @classmethod + def get_node_type(cls, _class): + """ + Retrieve node type for a given node class. + + :param _class: node class to get a node type for + :return: node type + :rtype: core.emulator.enumerations.NodeTypes + """ + node_type = NODES_TYPE.get(_class) + if node_type is None: + raise CoreError("invalid node class: %s" % _class) + return node_type + def _link_nodes(self, node_one_id, node_two_id): """ Convenience method for retrieving nodes within link data. @@ -145,7 +204,7 @@ class Session(object): # both node ids are provided tunnel = self.broker.gettunnel(node_one_id, node_two_id) logging.debug("tunnel between nodes: %s", tunnel) - if nodeutils.is_node(tunnel, NodeTypes.TAP_BRIDGE): + if isinstance(tunnel, GreTapBridge): net_one = tunnel if tunnel.remotenum == node_one_id: node_one = None @@ -203,9 +262,7 @@ class Session(object): raise CoreError("no common network found for wireless link/unlink") for common_network, interface_one, interface_two in common_networks: - if not nodeutils.is_node( - common_network, [NodeTypes.WIRELESS_LAN, NodeTypes.EMANE] - ): + if not isinstance(common_network, (WlanNode, EmaneNode)): logging.info( "skipping common network that is not wireless/emane: %s", common_network, @@ -268,9 +325,8 @@ class Session(object): node_one.name, node_two.name, ) - ptp_class = nodeutils.get_node_class(NodeTypes.PEER_TO_PEER) start = self.state > EventTypes.DEFINITION_STATE.value - net_one = self.create_node(cls=ptp_class, start=start) + net_one = self.create_node(cls=PtpNet, start=start) # node to network if node_one and net_one: @@ -300,7 +356,7 @@ class Session(object): net_one.name, net_two.name, ) - if nodeutils.is_node(net_two, NodeTypes.RJ45): + if isinstance(net_two, Rj45Node): interface = net_two.linknet(net_one) else: interface = net_one.linknet(net_two) @@ -324,12 +380,12 @@ class Session(object): # tunnel node logic key = link_options.key - if key and nodeutils.is_node(net_one, NodeTypes.TUNNEL): + if key and isinstance(net_one, TunnelNode): logging.info("setting tunnel key for: %s", net_one.name) net_one.setkey(key) if addresses: net_one.addrconfig(addresses) - if key and nodeutils.is_node(net_two, NodeTypes.TUNNEL): + if key and isinstance(net_two, TunnelNode): logging.info("setting tunnel key for: %s", net_two.name) net_two.setkey(key) if addresses: @@ -337,14 +393,14 @@ class Session(object): # physical node connected with tunnel if not net_one and not net_two and (node_one or node_two): - if node_one and nodeutils.is_node(node_one, NodeTypes.PHYSICAL): + if node_one and isinstance(node_one, PhysicalNode): logging.info("adding link for physical node: %s", node_one.name) addresses = interface_one.get_addresses() node_one.adoptnetif( tunnel, interface_one.id, interface_one.mac, addresses ) link_config(node_one, tunnel, link_options) - elif node_two and nodeutils.is_node(node_two, NodeTypes.PHYSICAL): + elif node_two and isinstance(node_two, PhysicalNode): logging.info("adding link for physical node: %s", node_two.name) addresses = interface_two.get_addresses() node_two.adoptnetif( @@ -584,14 +640,11 @@ class Session(object): :param int _id: id for node, defaults to None for generated id :param core.emulator.emudata.NodeOptions node_options: data to create node with :return: created node + :raises core.CoreError: when an invalid node type is given """ - # retrieve node class for given node type - try: - node_class = nodeutils.get_node_class(_type) - except KeyError: - logging.error("invalid node type to create: %s", _type) - return None + # validate node type, get class, or throw error + node_class = self.get_node_class(_type) # set node start based on current session state, override and check when rj45 start = self.state > EventTypes.DEFINITION_STATE.value @@ -651,10 +704,8 @@ class Session(object): logging.debug("set node type: %s", node.type) self.services.add_services(node, node.type, node_options.services) - # boot nodes if created after runtime, LcxNodes, Physical, and RJ45 are all PyCoreNodes - is_boot_node = isinstance(node, CoreNodeBase) and not nodeutils.is_node( - node, NodeTypes.RJ45 - ) + # boot nodes if created after runtime, CoreNodes, Physical, and RJ45 are all nodes + is_boot_node = isinstance(node, CoreNodeBase) and not isinstance(node, Rj45Node) if self.state == EventTypes.RUNTIME_STATE.value and is_boot_node: self.write_nodes() self.add_remove_control_interface(node=node, remove=False) @@ -1441,12 +1492,10 @@ class Session(object): count = 0 for node_id in self.nodes: node = self.nodes[node_id] - is_p2p_ctrlnet = nodeutils.is_node( - node, (NodeTypes.PEER_TO_PEER, NodeTypes.CONTROL_NET) + is_p2p_ctrlnet = isinstance(node, (PtpNet, CtrlNet)) + is_tap = isinstance(node, GreTapBridge) and not isinstance( + node, TunnelNode ) - is_tap = nodeutils.is_node( - node, NodeTypes.TAP_BRIDGE - ) and not nodeutils.is_node(node, NodeTypes.TUNNEL) if is_p2p_ctrlnet or is_tap: continue @@ -1493,7 +1542,7 @@ class Session(object): for node_id in self.nodes: node = self.nodes[node_id] # TODO: determine if checking for CoreNode alone is ok - if isinstance(node, core.nodes.base.CoreNodeBase): + if isinstance(node, CoreNodeBase): self.services.stop_services(node) # shutdown emane @@ -1546,10 +1595,7 @@ class Session(object): start = time.time() for _id in self.nodes: node = self.nodes[_id] - # TODO: PyCoreNode is not the type to check - if isinstance(node, CoreNodeBase) and not nodeutils.is_node( - node, NodeTypes.RJ45 - ): + if isinstance(node, CoreNodeBase) and not isinstance(node, Rj45Node): # add a control interface if configured logging.info( "booting node(%s): %s", @@ -1648,8 +1694,7 @@ class Session(object): # no controlnet needed return None else: - control_net_class = nodeutils.get_node_class(NodeTypes.CONTROL_NET) - prefix_spec = control_net_class.DEFAULT_PREFIX_LIST[net_index] + prefix_spec = CtrlNet.DEFAULT_PREFIX_LIST[net_index] logging.debug("prefix spec: %s", prefix_spec) server_interface = self.get_control_net_server_interfaces()[net_index] @@ -1725,9 +1770,8 @@ class Session(object): prefix = prefixes[0] logging.info("controlnet prefix: %s - %s", type(prefix), prefix) - control_net_class = nodeutils.get_node_class(NodeTypes.CONTROL_NET) control_net = self.create_node( - cls=control_net_class, + cls=CtrlNet, _id=_id, prefix=prefix, assign_address=assign_address, diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 2f3d7b64..6d06dd79 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -17,7 +17,7 @@ from socket import AF_INET, AF_INET6 from core import CoreCommandError, constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes -from core.nodes import client, ipaddress, nodeutils +from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth _DEFAULT_MTU = 1500 @@ -886,8 +886,8 @@ class CoreNode(CoreNodeBase): addrlist = [] with self.lock: - # TODO: see if you can move this to emane specific code - if nodeutils.is_node(net, NodeTypes.EMANE): + # TODO: emane specific code + if net.is_emane is True: ifindex = self.newtuntap(ifindex=ifindex, ifname=ifname, net=net) # TUN/TAP is not ready for addressing yet; the device may # take some time to appear, and installing it into a @@ -1050,6 +1050,7 @@ class CoreNetworkBase(NodeBase): """ linktype = LinkTypes.WIRED.value + is_emane = False def __init__(self, session, _id, name, start=True): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 0db3c6c4..8720d69a 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -7,8 +7,6 @@ import time from builtins import int, range from core import CoreCommandError, constants, utils -from core.emulator.enumerations import NodeTypes -from core.nodes import nodeutils utils.check_executables([constants.IP_BIN]) @@ -387,13 +385,12 @@ class TunTap(CoreInterface): if result: break + # TODO: emane specific code # check if this is an EMANE interface; if so, continue # waiting if EMANE is still running - # TODO: remove emane code should_retry = count < 5 - is_emane_node = nodeutils.is_node(self.net, NodeTypes.EMANE) is_emane_running = self.node.session.emane.emanerunning(self.node) - if all([should_retry, is_emane_node, is_emane_running]): + if all([should_retry, self.net.is_emane, is_emane_running]): count += 1 else: raise RuntimeError("node device failed to exist") diff --git a/daemon/core/nodes/nodemaps.py b/daemon/core/nodes/nodemaps.py deleted file mode 100644 index d9d0c9ad..00000000 --- a/daemon/core/nodes/nodemaps.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Provides default node maps that can be used to run core with. -""" -import core.nodes.base -import core.nodes.docker -import core.nodes.lxd -import core.nodes.network -import core.nodes.physical -from core.emane.nodes import EmaneNet, EmaneNode -from core.emulator.enumerations import NodeTypes -from core.nodes import physical -from core.nodes.network import GreTapBridge - -# legacy core nodes, that leverage linux bridges -NODES = { - NodeTypes.DEFAULT: core.nodes.base.CoreNode, - NodeTypes.PHYSICAL: physical.PhysicalNode, - NodeTypes.TBD: None, - NodeTypes.SWITCH: core.nodes.network.SwitchNode, - NodeTypes.HUB: core.nodes.network.HubNode, - NodeTypes.WIRELESS_LAN: core.nodes.network.WlanNode, - NodeTypes.RJ45: core.nodes.physical.Rj45Node, - NodeTypes.TUNNEL: core.nodes.network.TunnelNode, - NodeTypes.KTUNNEL: None, - NodeTypes.EMANE: EmaneNode, - NodeTypes.EMANE_NET: EmaneNet, - NodeTypes.TAP_BRIDGE: GreTapBridge, - NodeTypes.PEER_TO_PEER: core.nodes.network.PtpNet, - NodeTypes.CONTROL_NET: core.nodes.network.CtrlNet, - NodeTypes.DOCKER: core.nodes.docker.DockerNode, - NodeTypes.LXC: core.nodes.lxd.LxcNode, -} diff --git a/daemon/core/nodes/nodeutils.py b/daemon/core/nodes/nodeutils.py deleted file mode 100644 index 95c53165..00000000 --- a/daemon/core/nodes/nodeutils.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Serves as a global point for storing and retrieving node types needed during simulation. -""" - -import logging - -_NODE_MAP = None - - -def _log_map(): - global _NODE_MAP - for key in _NODE_MAP: - value = _NODE_MAP[key] - name = None - if value: - name = value.__name__ - logging.debug("node type (%s) - class (%s)", key.name, name) - - -def _convert_map(x, y): - """ - Convenience method to create a human readable version of the node map to log. - - :param dict x: dictionary to reduce node items into - :param tuple y: current node item - :return: human readable name mapping of the node map - """ - x[y[0].name] = y[1] - return x - - -def update_node_map(node_map): - """ - Update the current node map with the provided node map values. - - :param dict node_map: node map to update with - """ - global _NODE_MAP - _NODE_MAP.update(node_map) - _log_map() - - -def set_node_map(node_map): - """ - Set the global node map that proides a consistent way to retrieve differently configured nodes. - - :param dict node_map: node map to set to - :return: nothing - """ - global _NODE_MAP - _NODE_MAP = node_map - _log_map() - - -def get_node_class(node_type): - """ - Retrieve the node class for a given node type. - - :param int node_type: node type to retrieve class for - :return: node class - """ - global _NODE_MAP - return _NODE_MAP[node_type] - - -def get_node_type(node_class): - """ - Retrieve the node type given a node class. - - :param class node_class: node class to get type for - :return: node type - :rtype: core.emulator.enumerations.NodeTypes - """ - global _NODE_MAP - node_type_map = {_NODE_MAP[x]: x for x in _NODE_MAP} - return node_type_map.get(node_class) - - -def is_node(obj, node_types): - """ - Validates if an object is one of the provided node types. - - :param obj: object to check type for - :param int|tuple|list node_types: node type(s) to check against - :return: True if the object is one of the node types, False otherwise - :rtype: bool - """ - type_classes = [] - if isinstance(node_types, (tuple, list)): - for node_type in node_types: - type_class = get_node_class(node_type) - type_classes.append(type_class) - else: - type_class = get_node_class(node_types) - type_classes.append(type_class) - - return isinstance(obj, tuple(type_classes)) diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index 3eaba44a..ce60ccac 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -8,6 +8,7 @@ import socket from future.moves.urllib.parse import urlparse from core import CoreError, constants +from core.emane.nodes import EmaneNode from core.emulator.enumerations import ( EventTypes, LinkTlvs, @@ -17,8 +18,8 @@ from core.emulator.enumerations import ( NodeTlvs, NodeTypes, ) -from core.nodes import nodeutils from core.nodes.base import CoreNetworkBase, NodeBase +from core.nodes.network import WlanNode # TODO: A named tuple may be more appropriate, than abusing a class dict like this @@ -27,14 +28,13 @@ class Bunch(object): Helper class for recording a collection of attributes. """ - def __init__(self, **kwds): + def __init__(self, **kwargs): """ Create a Bunch instance. - :param dict kwds: keyword arguments - :return: + :param dict kwargs: keyword arguments """ - self.__dict__.update(kwds) + self.__dict__.update(kwargs) class Sdt(object): @@ -365,9 +365,7 @@ class Sdt(object): for net in nets: all_links = net.all_link_data(flags=MessageFlags.ADD.value) for link_data in all_links: - is_wireless = nodeutils.is_node( - net, (NodeTypes.WIRELESS_LAN, NodeTypes.EMANE) - ) + is_wireless = isinstance(net, (WlanNode, EmaneNode)) wireless_link = link_data.message_type == LinkTypes.WIRELESS.value if is_wireless and link_data.node1_id == net.id: continue @@ -401,7 +399,7 @@ class Sdt(object): def handlenodemsg(self, msg): """ Process a Node Message to add/delete or move a node on - the SDT display. Node properties are found in session._objs or + the SDT display. Node properties are found in a session or self.remotes for remote nodes (or those not yet instantiated). :param msg: node message to handle @@ -430,7 +428,8 @@ class Sdt(object): model = "router" nodetype = model elif nodetype is not None: - nodetype = nodeutils.get_node_class(NodeTypes(nodetype)).type + nodetype = NodeTypes(nodetype) + nodetype = self.session.get_node_class(nodetype).type net = True else: nodetype = None @@ -509,6 +508,6 @@ class Sdt(object): n = self.session.get_node(nodenum) except CoreError: return False - if nodeutils.is_node(n, (NodeTypes.WIRELESS_LAN, NodeTypes.EMANE)): + if isinstance(n, (WlanNode, EmaneNode)): return True return False diff --git a/daemon/core/services/emaneservices.py b/daemon/core/services/emaneservices.py index 19560579..5cb7ec5c 100644 --- a/daemon/core/services/emaneservices.py +++ b/daemon/core/services/emaneservices.py @@ -1,5 +1,4 @@ -from core.emulator.enumerations import NodeTypes -from core.nodes import nodeutils +from core.emane.nodes import EmaneNode from core.services.coreservices import CoreService from core.xml import emanexml @@ -22,7 +21,7 @@ class EmaneTransportService(CoreService): transport_commands = [] for interface in node.netifs(sort=True): network_node = node.session.get_node(interface.net.id) - if nodeutils.is_node(network_node, NodeTypes.EMANE): + if isinstance(network_node, EmaneNode): config = node.session.emane.get_configs( network_node.id, network_node.model.name ) diff --git a/daemon/core/services/frr.py b/daemon/core/services/frr.py index 45516a55..b4332009 100644 --- a/daemon/core/services/frr.py +++ b/daemon/core/services/frr.py @@ -4,8 +4,10 @@ Assumes installation of FRR via https://deb.frrouting.org/ """ from core import constants -from core.emulator.enumerations import LinkTypes, NodeTypes -from core.nodes import ipaddress, nodeutils +from core.emulator.enumerations import LinkTypes +from core.nodes import ipaddress +from core.nodes.network import PtpNet +from core.nodes.physical import Rj45Node from core.services.coreservices import CoreService @@ -341,7 +343,7 @@ class FrrService(CoreService): for peerifc in ifc.net.netifs(): if peerifc == ifc: continue - if nodeutils.is_node(peerifc, NodeTypes.RJ45): + if isinstance(peerifc, Rj45Node): return True return False @@ -395,7 +397,7 @@ class FRROspfv2(FrrService): Helper to detect whether interface is connected to a notional point-to-point link. """ - if nodeutils.is_node(ifc.net, NodeTypes.PEER_TO_PEER): + if isinstance(ifc.net, PtpNet): return " ip ospf network point-to-point\n" return "" @@ -481,7 +483,7 @@ class FRROspfv3(FrrService): Helper to detect whether interface is connected to a notional point-to-point link. """ - if nodeutils.is_node(ifc.net, NodeTypes.PEER_TO_PEER): + if isinstance(ifc.net, PtpNet): return " ipv6 ospf6 network point-to-point\n" return "" diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index 7d8e8d16..0f213b7b 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -3,8 +3,11 @@ quagga.py: defines routing services provided by Quagga. """ from core import constants -from core.emulator.enumerations import LinkTypes, NodeTypes -from core.nodes import ipaddress, nodeutils +from core.emane.nodes import EmaneNode +from core.emulator.enumerations import LinkTypes +from core.nodes import ipaddress +from core.nodes.network import PtpNet, WlanNode +from core.nodes.physical import Rj45Node from core.services.coreservices import CoreService @@ -267,7 +270,7 @@ class QuaggaService(CoreService): for peerifc in ifc.net.netifs(): if peerifc == ifc: continue - if nodeutils.is_node(peerifc, NodeTypes.RJ45): + if isinstance(peerifc, Rj45Node): return True return False @@ -321,7 +324,7 @@ class Ospfv2(QuaggaService): Helper to detect whether interface is connected to a notional point-to-point link. """ - if nodeutils.is_node(ifc.net, NodeTypes.PEER_TO_PEER): + if isinstance(ifc.net, PtpNet): return " ip ospf network point-to-point\n" return "" @@ -407,7 +410,7 @@ class Ospfv3(QuaggaService): Helper to detect whether interface is connected to a notional point-to-point link. """ - if nodeutils.is_node(ifc.net, NodeTypes.PEER_TO_PEER): + if isinstance(ifc.net, PtpNet): return " ipv6 ospf6 network point-to-point\n" return "" @@ -457,9 +460,7 @@ class Ospfv3mdr(Ospfv3): cfg = cls.mtucheck(ifc) # Uncomment the following line to use Address Family Translation for IPv4 cfg += " ipv6 ospf6 instance-id 65\n" - if ifc.net is not None and nodeutils.is_node( - ifc.net, (NodeTypes.WIRELESS_LAN, NodeTypes.EMANE) - ): + if ifc.net is not None and isinstance(ifc.net, (WlanNode, EmaneNode)): return ( cfg + """\ diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 31618277..ad4de155 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -4,11 +4,12 @@ from lxml import etree import core.nodes.base import core.nodes.physical +from core.emane.nodes import EmaneNode from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions from core.emulator.enumerations import NodeTypes -from core.nodes import nodeutils from core.nodes.base import CoreNetworkBase from core.nodes.ipaddress import MacAddress +from core.nodes.network import CtrlNet def write_xml_file(xml_element, file_path, doctype=None): @@ -408,7 +409,7 @@ class CoreXmlWriter(object): is_network_or_rj45 = isinstance( node, (core.nodes.base.CoreNetworkBase, core.nodes.physical.Rj45Node) ) - is_controlnet = nodeutils.is_node(node, NodeTypes.CONTROL_NET) + is_controlnet = isinstance(node, CtrlNet) if is_network_or_rj45 and not is_controlnet: self.write_network(node) # device node @@ -457,7 +458,7 @@ class CoreXmlWriter(object): interface_name = node_interface.name # check if emane interface - if nodeutils.is_node(node_interface.net, NodeTypes.EMANE): + if isinstance(node_interface.net, EmaneNode): nem = node_interface.net.getnemid(node_interface) add_attribute(interface, "nem", nem) diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index 8d926a8c..a5c966d5 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -4,8 +4,8 @@ import socket from lxml import etree from core import constants, utils -from core.emulator.enumerations import NodeTypes -from core.nodes import ipaddress, nodeutils +from core.emane.nodes import EmaneNode +from core.nodes import ipaddress from core.nodes.base import CoreNodeBase @@ -144,7 +144,7 @@ class CoreXmlDeployment(object): for netif in node.netifs(): emane_element = None - if nodeutils.is_node(netif.net, NodeTypes.EMANE): + if isinstance(netif.net, EmaneNode): emane_element = add_emane_interface(host_element, netif) parent_element = host_element diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index ea49bb7f..58540ef9 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -60,9 +60,6 @@ def cored(cfg, use_ovs): try: address = (host, port) server = CoreServer(address, CoreHandler, cfg) - if use_ovs: - from core.nodes.openvswitch import OVS_NODES - server.coreemu.update_nodes(OVS_NODES) except: logging.exception("error starting main server on: %s:%s", host, port) sys.exit(1) diff --git a/ns3/examples/ns3lte.py b/ns3/examples/ns3lte.py index 2cb17407..52355552 100644 --- a/ns3/examples/ns3lte.py +++ b/ns3/examples/ns3lte.py @@ -10,17 +10,16 @@ import sys import ns.core import ns.mobility - -from core.nodes import nodeutils, nodemaps, ipaddress from corens3.obj import Ns3LteNet from corens3.obj import Ns3Session +from core.nodes import ipaddress + def ltesession(opt): """ Run a test LTE session. """ - nodeutils.set_node_map(nodemaps.NODES) session = Ns3Session(1, persistent=True, duration=opt.duration) lte = session.create_node(cls=Ns3LteNet, name="wlan1") lte.setsubchannels(range(25), range(50, 100)) diff --git a/ns3/examples/ns3wifi.py b/ns3/examples/ns3wifi.py index 747b184b..e06358a4 100644 --- a/ns3/examples/ns3wifi.py +++ b/ns3/examples/ns3wifi.py @@ -27,11 +27,11 @@ import optparse import sys import ns.core - -from core.nodes import nodeutils, nodemaps, ipaddress from corens3.obj import Ns3Session from corens3.obj import Ns3WifiNet +from core.nodes import ipaddress + def add_to_server(session): """ @@ -50,7 +50,6 @@ def wifisession(opt): """ Run a test wifi session. """ - nodeutils.set_node_map(nodemaps.NODES) session = Ns3Session(1, persistent=True, duration=opt.duration) session.name = "ns3wifi" session.filename = session.name + ".py" diff --git a/ns3/examples/ns3wifirandomwalk.py b/ns3/examples/ns3wifirandomwalk.py index a96f37f6..6371a3f1 100644 --- a/ns3/examples/ns3wifirandomwalk.py +++ b/ns3/examples/ns3wifirandomwalk.py @@ -14,13 +14,14 @@ How to run this: import logging import optparse import sys +from builtins import range import ns.core import ns.network from corens3.obj import Ns3Session from corens3.obj import Ns3WifiNet -from core.nodes import nodeutils, nodemaps, ipaddress +from core.nodes import ipaddress def add_to_server(session): @@ -40,7 +41,6 @@ def wifisession(opt): """ Run a random walk wifi session. """ - nodeutils.set_node_map(nodemaps.NODES) session = Ns3Session(1, persistent=True, duration=opt.duration) session.name = "ns3wifirandomwalk" session.filename = session.name + ".py" @@ -54,7 +54,7 @@ def wifisession(opt): prefix = ipaddress.Ipv4Prefix("10.0.0.0/16") services_str = "zebra|OSPFv3MDR|IPForward" nodes = [] - for i in xrange(1, opt.numnodes + 1): + for i in range(1, opt.numnodes + 1): node = session.addnode(name="n%d" % i) node.newnetif(wifi, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) nodes.append(node) diff --git a/ns3/examples/ns3wimax.py b/ns3/examples/ns3wimax.py index b1d7212c..e38034e5 100644 --- a/ns3/examples/ns3wimax.py +++ b/ns3/examples/ns3wimax.py @@ -12,17 +12,18 @@ Current issues: import logging import optparse import sys +from builtins import range -from core.nodes import nodeutils, nodemaps, ipaddress from corens3.obj import Ns3Session from corens3.obj import Ns3WimaxNet +from core.nodes import ipaddress + def wimaxsession(opt): """ Run a test wimax session. """ - nodeutils.set_node_map(nodemaps.NODES) session = Ns3Session(1, persistent=True, duration=opt.duration) wimax = session.create_node(cls=Ns3WimaxNet, name="wlan1") # wimax.wimax.EnableLogComponents() @@ -33,7 +34,7 @@ def wimaxsession(opt): # classifier = (0, 65000, 0, 65000, 1, 1) classifier = (0, 65000, 0, 65000, 17, 1) nodes = [] - for i in xrange(1, opt.numnodes + 1): + for i in range(1, opt.numnodes + 1): node = session.addnode(name="n%d" % i) if i == 1: wimax.setbasestation(node) From bdf288ff1d347f3f32dfcd2f3489ce594f5978a8 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 26 Sep 2019 13:15:46 -0700 Subject: [PATCH 008/113] removed is_net_node utility to leverage isinstance as other places are consistently --- daemon/core/emulator/emudata.py | 12 ------------ daemon/core/emulator/session.py | 7 +++---- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/daemon/core/emulator/emudata.py b/daemon/core/emulator/emudata.py index 0f7a6af6..467fe2e4 100644 --- a/daemon/core/emulator/emudata.py +++ b/daemon/core/emulator/emudata.py @@ -1,6 +1,5 @@ from core.emane.nodes import EmaneNode from core.emulator.enumerations import LinkTypes -from core.nodes.base import CoreNetworkBase from core.nodes.ipaddress import Ipv4Prefix, Ipv6Prefix, MacAddress from core.nodes.physical import PhysicalNode @@ -14,17 +13,6 @@ class IdGen(object): return self.id -def is_net_node(node): - """ - Convenience method for testing if a legacy core node is considered a network node. - - :param object node: object to test against - :return: True if object is an instance of a network node, False otherwise - :rtype: bool - """ - return isinstance(node, CoreNetworkBase) - - def create_interface(node, network, interface_data): """ Create an interface for a node on a network using provided interface data. diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 375e6f24..4ffb2391 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -25,7 +25,6 @@ from core.emulator.emudata import ( LinkOptions, NodeOptions, create_interface, - is_net_node, link_config, ) from core.emulator.enumerations import EventTypes, ExceptionLevels, LinkTypes, NodeTypes @@ -33,7 +32,7 @@ from core.emulator.sessionconfig import SessionConfig, SessionMetaData from core.location.corelocation import CoreLocation from core.location.event import EventLoop from core.location.mobility import MobilityManager -from core.nodes.base import CoreNode, CoreNodeBase +from core.nodes.base import CoreNetworkBase, CoreNode, CoreNodeBase from core.nodes.docker import DockerNode from core.nodes.ipaddress import MacAddress from core.nodes.lxd import LxcNode @@ -217,14 +216,14 @@ class Session(object): else: node_two = None - if is_net_node(node_one): + if isinstance(node_one, CoreNetworkBase): if not net_one: net_one = node_one else: net_two = node_one node_one = None - if is_net_node(node_two): + if isinstance(node_two, CoreNetworkBase): if not net_one: net_one = node_two else: From b449729a319ad7000735819fa6da3af6d8804e1a Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 26 Sep 2019 15:20:32 -0700 Subject: [PATCH 009/113] added net client to consolidate bridge and interface creation --- .editorconfig | 1 + daemon/core/emulator/coreemu.py | 2 +- daemon/core/nodes/base.py | 5 ++ daemon/core/nodes/netclient.py | 74 ++++++++++++++++++++++++++++ daemon/core/nodes/network.py | 87 ++++++++++++--------------------- daemon/scripts/core-daemon | 8 +-- 6 files changed, 113 insertions(+), 64 deletions(-) create mode 100644 daemon/core/nodes/netclient.py diff --git a/.editorconfig b/.editorconfig index d0466b2f..f1b09263 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,7 @@ insert_final_newline = true [*.py] indent_style = space indent_size = 4 +max_line_length = 88 [*.am] indent_style = tab diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index c0eae5cd..1c0923de 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -44,7 +44,7 @@ class CoreEmu(object): os.umask(0) # configuration - if not config: + if config is None: config = {} self.config = config diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 6d06dd79..556f5232 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -19,6 +19,7 @@ from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth +from core.nodes.netclient import BrctlClient, OvsClient _DEFAULT_MTU = 1500 @@ -1064,6 +1065,10 @@ class CoreNetworkBase(NodeBase): super(CoreNetworkBase, self).__init__(session, _id, name, start=start) self._linked = {} self._linked_lock = threading.Lock() + if session.options.get_config("ovs"): + self.net_client = OvsClient() + else: + self.net_client = BrctlClient() def startup(self): """ diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py new file mode 100644 index 00000000..7b298863 --- /dev/null +++ b/daemon/core/nodes/netclient.py @@ -0,0 +1,74 @@ +""" +Clients for dealing with bridge and interface commands. +""" + +import abc +import os + +from core import constants, utils + + +class NetworkClient(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def create_bridge(self, name): + return NotImplemented + + +class BrctlClient(object): + def create_bridge(self, name): + utils.check_cmd([constants.BRCTL_BIN, "addbr", name]) + # disable spanning tree protocol and set forward delay to 0 + utils.check_cmd([constants.BRCTL_BIN, "stp", name, "off"]) + utils.check_cmd([constants.BRCTL_BIN, "setfd", name, "0"]) + utils.check_cmd([constants.IP_BIN, "link", "set", name, "up"]) + + # turn off multicast snooping so forwarding occurs w/o IGMP joins + snoop = "/sys/devices/virtual/net/%s/bridge/multicast_snooping" % name + if os.path.exists(snoop): + with open(snoop, "w") as f: + f.write("0") + + def delete_bridge(self, name): + utils.check_cmd([constants.IP_BIN, "link", "set", name, "down"]) + utils.check_cmd([constants.BRCTL_BIN, "delbr", name]) + + def create_interface(self, bridge_name, interface_name): + utils.check_cmd([constants.BRCTL_BIN, "addif", bridge_name, interface_name]) + utils.check_cmd([constants.IP_BIN, "link", "set", interface_name, "up"]) + + def delete_interface(self, bridge_name, interface_name): + utils.check_cmd([constants.BRCTL_BIN, "delif", bridge_name, interface_name]) + + def get_bridges(self): + return utils.check_cmd([constants.OVS_BIN, "list-br"]) + + def disable_mac_learning(self, name): + utils.check_cmd([constants.BRCTL_BIN, "setageing", name, "0"]) + + +class OvsClient(object): + def create_bridge(self, name): + # turn off spanning tree protocol and forwarding delay + # TODO: verify stp and rstp are always off by default + # TODO: ovs only supports rstp forward delay and again it's off by default + utils.check_cmd([constants.OVS_BIN, "add-br", name]) + utils.check_cmd([constants.IP_BIN, "link", "set", name, "up"]) + + def delete_bridge(self, name): + utils.check_cmd([constants.IP_BIN, "link", "set", name, "down"]) + utils.check_cmd([constants.OVS_BIN, "del-br", name]) + + def create_interface(self, bridge_name, interface_name): + utils.check_cmd([constants.OVS_BIN, "add-port", bridge_name, interface_name]) + utils.check_cmd([constants.IP_BIN, "link", "set", interface_name, "up"]) + + def delete_interface(self, bridge_name, interface_name): + utils.check_cmd([constants.OVS_BIN, "del-port", bridge_name, interface_name]) + + def get_bridges(self): + utils.check_cmd([constants.BRCTL_BIN, "show"]) + + def disable_mac_learning(self, name): + pass diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 6ba4ebd4..852f8110 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -314,12 +314,8 @@ class CoreNetwork(CoreNetworkBase): :return: nothing :raises CoreCommandError: when there is a command exception """ - utils.check_cmd([constants.BRCTL_BIN, "addbr", self.brname]) + self.net_client.create_bridge(self.brname) - # turn off spanning tree protocol and forwarding delay - utils.check_cmd([constants.BRCTL_BIN, "stp", self.brname, "off"]) - utils.check_cmd([constants.BRCTL_BIN, "setfd", self.brname, "0"]) - utils.check_cmd([constants.IP_BIN, "link", "set", self.brname, "up"]) # create a new ebtables chain for this bridge ebtablescmds( utils.check_cmd, @@ -336,11 +332,6 @@ class CoreNetwork(CoreNetworkBase): ], ], ) - # turn off multicast snooping so mcast forwarding occurs w/o IGMP joins - snoop = "/sys/devices/virtual/net/%s/bridge/multicast_snooping" % self.brname - if os.path.exists(snoop): - with open(snoop, "w") as snoop_file: - snoop_file.write("0") self.up = True @@ -356,8 +347,7 @@ class CoreNetwork(CoreNetworkBase): ebq.stopupdateloop(self) try: - utils.check_cmd([constants.IP_BIN, "link", "set", self.brname, "down"]) - utils.check_cmd([constants.BRCTL_BIN, "delbr", self.brname]) + self.net_client.delete_bridge(self.brname) ebtablescmds( utils.check_cmd, [ @@ -385,7 +375,8 @@ class CoreNetwork(CoreNetworkBase): del self.session self.up = False - # TODO: this depends on a subtype with localname defined, seems like the wrong place for this to live + # TODO: this depends on a subtype with localname defined, seems like the + # wrong place for this to live def attach(self, netif): """ Attach a network interface. @@ -394,10 +385,7 @@ class CoreNetwork(CoreNetworkBase): :return: nothing """ if self.up: - utils.check_cmd( - [constants.BRCTL_BIN, "addif", self.brname, netif.localname] - ) - utils.check_cmd([constants.IP_BIN, "link", "set", netif.localname, "up"]) + self.net_client.create_interface(self.brname, netif.localname) CoreNetworkBase.attach(self, netif) @@ -409,9 +397,7 @@ class CoreNetwork(CoreNetworkBase): :return: nothing """ if self.up: - utils.check_cmd( - [constants.BRCTL_BIN, "delif", self.brname, netif.localname] - ) + self.net_client.delete_interface(self.brname, netif.localname) CoreNetworkBase.detach(self, netif) @@ -610,10 +596,8 @@ class CoreNetwork(CoreNetworkBase): ) self.attach(netif) if net.up: - # this is similar to net.attach() but uses netif.name instead - # of localname - utils.check_cmd([constants.BRCTL_BIN, "addif", net.brname, netif.name]) - utils.check_cmd([constants.IP_BIN, "link", "set", netif.name, "up"]) + # this is similar to net.attach() but uses netif.name instead of localname + self.net_client.create_interface(net.brname, netif.name) i = net.newifindex() net._netif[i] = netif with net._linked_lock: @@ -848,41 +832,32 @@ class CtrlNet(CoreNetwork): utils.check_cmd([self.updown_script, self.brname, "startup"]) if self.serverintf: - # sets the interface as a port of the bridge - utils.check_cmd( - [constants.BRCTL_BIN, "addif", self.brname, self.serverintf] - ) - - # bring interface up - utils.check_cmd([constants.IP_BIN, "link", "set", self.serverintf, "up"]) + self.net_client.create_interface(self.brname, self.serverintf) def detectoldbridge(self): """ - Occasionally, control net bridges from previously closed sessions are not cleaned up. - Check if there are old control net bridges and delete them + Occasionally, control net bridges from previously closed sessions are not + cleaned up. Check if there are old control net bridges and delete them :return: True if an old bridge was detected, False otherwise :rtype: bool """ - status, output = utils.cmd_output([constants.BRCTL_BIN, "show"]) - if status != 0: - logging.error("Unable to retrieve list of installed bridges") - else: - lines = output.split("\n") - for line in lines[1:]: - cols = line.split("\t") - oldbr = cols[0] - flds = cols[0].split(".") - if len(flds) == 3: - if flds[0] == "b" and flds[1] == self.id: - logging.error( - "error: An active control net bridge (%s) found. " - "An older session might still be running. " - "Stop all sessions and, if needed, delete %s to continue.", - oldbr, - oldbr, - ) - return True + output = self.net_client.get_bridges() + lines = output.split("\n") + for line in lines[1:]: + cols = line.split("\t") + oldbr = cols[0] + flds = cols[0].split(".") + if len(flds) == 3: + if flds[0] == "b" and flds[1] == self.id: + logging.error( + "error: An active control net bridge (%s) found. " + "An older session might still be running. " + "Stop all sessions and, if needed, delete %s to continue.", + oldbr, + oldbr, + ) + return True return False def shutdown(self): @@ -893,9 +868,7 @@ class CtrlNet(CoreNetwork): """ if self.serverintf is not None: try: - utils.check_cmd( - [constants.BRCTL_BIN, "delif", self.brname, self.serverintf] - ) + self.net_client.delete_interface(self.brname, self.serverintf) except CoreCommandError: logging.exception( "error deleting server interface %s from bridge %s", @@ -1100,7 +1073,7 @@ class HubNode(CoreNetwork): # TODO: move to startup method if start: - utils.check_cmd([constants.BRCTL_BIN, "setageing", self.brname, "0"]) + self.net_client.disable_mac_learning(self.brname) class WlanNode(CoreNetwork): @@ -1131,7 +1104,7 @@ class WlanNode(CoreNetwork): # TODO: move to startup method if start: - utils.check_cmd([constants.BRCTL_BIN, "setageing", self.brname, "0"]) + self.net_client.disable_mac_learning(self.brname) def attach(self, netif): """ diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 58540ef9..13e6e334 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -44,12 +44,11 @@ def start_udp(mainserver, server_address): mainserver.udpthread.start() -def cored(cfg, use_ovs): +def cored(cfg): """ Start the CoreServer object and enter the server loop. :param dict cfg: core configuration - :param bool use_ovs: flag to determine if ovs nodes should be used :return: nothing """ host = cfg["listenaddr"] @@ -153,11 +152,8 @@ def main(): cfg = get_merged_config("%s/core.conf" % constants.CORE_CONF_DIR) banner() - # check if ovs flag was provided - use_ovs = len(sys.argv) == 2 and sys.argv[1] == "ovs" - try: - cored(cfg, use_ovs) + cored(cfg) except KeyboardInterrupt: logging.info("keyboard interrupt, stopping core daemon") From 0c002bb4914b21db905d9de717f78fae00e190e0 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 26 Sep 2019 22:04:29 -0700 Subject: [PATCH 010/113] implemented net client for linux and ovs --- daemon/core/nodes/base.py | 8 +- daemon/core/nodes/netclient.py | 136 ++++++++++++++++++++++++++------- daemon/core/nodes/network.py | 32 +------- 3 files changed, 114 insertions(+), 62 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 556f5232..4861f93c 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -19,7 +19,7 @@ from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth -from core.nodes.netclient import BrctlClient, OvsClient +from core.nodes.netclient import LinuxNetClient, OvsNetClient _DEFAULT_MTU = 1500 @@ -1065,10 +1065,10 @@ class CoreNetworkBase(NodeBase): super(CoreNetworkBase, self).__init__(session, _id, name, start=start) self._linked = {} self._linked_lock = threading.Lock() - if session.options.get_config("ovs"): - self.net_client = OvsClient() + if session.options.get_config("ovs") == "True": + self.net_client = OvsNetClient() else: - self.net_client = BrctlClient() + self.net_client = LinuxNetClient() def startup(self): """ diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 7b298863..0ec2c389 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -1,28 +1,90 @@ """ -Clients for dealing with bridge and interface commands. +Clients for dealing with bridge/interface commands. """ import abc import os -from core import constants, utils +from future.utils import with_metaclass + +from core.constants import BRCTL_BIN, IP_BIN, OVS_BIN +from core.utils import check_cmd -class NetworkClient(object): - __metaclass__ = abc.ABCMeta +class NetClientBase(with_metaclass(abc.ABCMeta)): + """ + Base client for running command line bridge/interface commands. + """ @abc.abstractmethod def create_bridge(self, name): - return NotImplemented + """ + Create a network bridge to connect interfaces to. + + :param str name: bridge name + :return: nothing + """ + pass + + @abc.abstractmethod + def delete_bridge(self, name): + """ + Delete a network bridge. + + :param str name: bridge name + :return: nothing + """ + pass + + @abc.abstractmethod + def create_interface(self, bridge_name, interface_name): + """ + Create an interface associated with a network bridge. + + :param str bridge_name: bridge name + :param str interface_name: interface name + :return: nothing + """ + pass + + @abc.abstractmethod + def delete_interface(self, bridge_name, interface_name): + """ + Delete an interface associated with a network bridge. + + :param str bridge_name: bridge name + :param str interface_name: interface name + :return: nothing + """ + pass + + @abc.abstractmethod + def existing_bridges(self, _id): + """ + Checks if there are any existing bridges for a node. + + :param _id: node id to check bridges for + """ + pass + + @abc.abstractmethod + def disable_mac_learning(self, name): + """ + Disable mac learning for a branch. + + :param str name: branch name + :return: nothing + """ + pass -class BrctlClient(object): +class LinuxNetClient(NetClientBase): def create_bridge(self, name): - utils.check_cmd([constants.BRCTL_BIN, "addbr", name]) + check_cmd([BRCTL_BIN, "addbr", name]) # disable spanning tree protocol and set forward delay to 0 - utils.check_cmd([constants.BRCTL_BIN, "stp", name, "off"]) - utils.check_cmd([constants.BRCTL_BIN, "setfd", name, "0"]) - utils.check_cmd([constants.IP_BIN, "link", "set", name, "up"]) + check_cmd([BRCTL_BIN, "stp", name, "off"]) + check_cmd([BRCTL_BIN, "setfd", name, "0"]) + check_cmd([IP_BIN, "link", "set", name, "up"]) # turn off multicast snooping so forwarding occurs w/o IGMP joins snoop = "/sys/devices/virtual/net/%s/bridge/multicast_snooping" % name @@ -31,44 +93,60 @@ class BrctlClient(object): f.write("0") def delete_bridge(self, name): - utils.check_cmd([constants.IP_BIN, "link", "set", name, "down"]) - utils.check_cmd([constants.BRCTL_BIN, "delbr", name]) + check_cmd([IP_BIN, "link", "set", name, "down"]) + check_cmd([BRCTL_BIN, "delbr", name]) def create_interface(self, bridge_name, interface_name): - utils.check_cmd([constants.BRCTL_BIN, "addif", bridge_name, interface_name]) - utils.check_cmd([constants.IP_BIN, "link", "set", interface_name, "up"]) + check_cmd([BRCTL_BIN, "addif", bridge_name, interface_name]) + check_cmd([IP_BIN, "link", "set", interface_name, "up"]) def delete_interface(self, bridge_name, interface_name): - utils.check_cmd([constants.BRCTL_BIN, "delif", bridge_name, interface_name]) + check_cmd([BRCTL_BIN, "delif", bridge_name, interface_name]) - def get_bridges(self): - return utils.check_cmd([constants.OVS_BIN, "list-br"]) + def existing_bridges(self, _id): + output = check_cmd([BRCTL_BIN, "show"]) + lines = output.split("\n") + for line in lines[1:]: + columns = line.split() + name = columns[0] + fields = name.split(".") + if len(fields) != 3: + continue + if fields[0] == "b" and fields[1] == _id: + return True + return False def disable_mac_learning(self, name): - utils.check_cmd([constants.BRCTL_BIN, "setageing", name, "0"]) + check_cmd([BRCTL_BIN, "setageing", name, "0"]) -class OvsClient(object): +class OvsNetClient(NetClientBase): def create_bridge(self, name): # turn off spanning tree protocol and forwarding delay # TODO: verify stp and rstp are always off by default # TODO: ovs only supports rstp forward delay and again it's off by default - utils.check_cmd([constants.OVS_BIN, "add-br", name]) - utils.check_cmd([constants.IP_BIN, "link", "set", name, "up"]) + check_cmd([OVS_BIN, "add-br", name]) + check_cmd([IP_BIN, "link", "set", name, "up"]) def delete_bridge(self, name): - utils.check_cmd([constants.IP_BIN, "link", "set", name, "down"]) - utils.check_cmd([constants.OVS_BIN, "del-br", name]) + check_cmd([IP_BIN, "link", "set", name, "down"]) + check_cmd([OVS_BIN, "del-br", name]) def create_interface(self, bridge_name, interface_name): - utils.check_cmd([constants.OVS_BIN, "add-port", bridge_name, interface_name]) - utils.check_cmd([constants.IP_BIN, "link", "set", interface_name, "up"]) + check_cmd([OVS_BIN, "add-port", bridge_name, interface_name]) + check_cmd([IP_BIN, "link", "set", interface_name, "up"]) def delete_interface(self, bridge_name, interface_name): - utils.check_cmd([constants.OVS_BIN, "del-port", bridge_name, interface_name]) + check_cmd([OVS_BIN, "del-port", bridge_name, interface_name]) - def get_bridges(self): - utils.check_cmd([constants.BRCTL_BIN, "show"]) + def existing_bridges(self, _id): + output = check_cmd([OVS_BIN, "list-br"]) + if output: + for line in output.split("\n"): + fields = line.split(".") + if fields[0] == "b" and fields[1] == _id: + return True + return False def disable_mac_learning(self, name): - pass + check_cmd([OVS_BIN, "set", "bridge", name, "other_config:mac-aging-time=0"]) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 852f8110..9a81b8e1 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -9,7 +9,7 @@ import threading import time from socket import AF_INET, AF_INET6 -from core import CoreCommandError, constants, utils +from core import CoreCommandError, CoreError, constants, utils from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs from core.nodes import ipaddress @@ -806,8 +806,8 @@ class CtrlNet(CoreNetwork): :return: nothing :raises CoreCommandError: when there is a command exception """ - if self.detectoldbridge(): - return + if self.net_client.existing_bridges(self.id): + raise CoreError("old bridges exist for node: %s" % self.id) CoreNetwork.startup(self) @@ -834,32 +834,6 @@ class CtrlNet(CoreNetwork): if self.serverintf: self.net_client.create_interface(self.brname, self.serverintf) - def detectoldbridge(self): - """ - Occasionally, control net bridges from previously closed sessions are not - cleaned up. Check if there are old control net bridges and delete them - - :return: True if an old bridge was detected, False otherwise - :rtype: bool - """ - output = self.net_client.get_bridges() - lines = output.split("\n") - for line in lines[1:]: - cols = line.split("\t") - oldbr = cols[0] - flds = cols[0].split(".") - if len(flds) == 3: - if flds[0] == "b" and flds[1] == self.id: - logging.error( - "error: An active control net bridge (%s) found. " - "An older session might still be running. " - "Stop all sessions and, if needed, delete %s to continue.", - oldbr, - oldbr, - ) - return True - return False - def shutdown(self): """ Control network shutdown. From 1277ae46864cac274fb9b9bc0fed1dd5d1723e61 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 10:48:24 -0700 Subject: [PATCH 011/113] fleshed out documentation for netclient.py and removed openvswitch.py --- daemon/core/nodes/netclient.py | 93 +++- daemon/core/nodes/openvswitch.py | 825 ------------------------------- 2 files changed, 87 insertions(+), 831 deletions(-) delete mode 100644 daemon/core/nodes/openvswitch.py diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 0ec2c389..292b6c95 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -70,18 +70,27 @@ class NetClientBase(with_metaclass(abc.ABCMeta)): @abc.abstractmethod def disable_mac_learning(self, name): """ - Disable mac learning for a branch. + Disable mac learning for a bridge. - :param str name: branch name + :param str name: bridge name :return: nothing """ pass class LinuxNetClient(NetClientBase): + """ + Client for creating Linux bridges and ip interfaces for nodes. + """ + def create_bridge(self, name): + """ + Create a Linux bridge and bring it up. + + :param str name: bridge name + :return: nothing + """ check_cmd([BRCTL_BIN, "addbr", name]) - # disable spanning tree protocol and set forward delay to 0 check_cmd([BRCTL_BIN, "stp", name, "off"]) check_cmd([BRCTL_BIN, "setfd", name, "0"]) check_cmd([IP_BIN, "link", "set", name, "up"]) @@ -93,17 +102,42 @@ class LinuxNetClient(NetClientBase): f.write("0") def delete_bridge(self, name): + """ + Bring down and delete a Linux bridge. + + :param str name: bridge name + :return: nothing + """ check_cmd([IP_BIN, "link", "set", name, "down"]) check_cmd([BRCTL_BIN, "delbr", name]) def create_interface(self, bridge_name, interface_name): + """ + Create an interface associated with a Linux bridge. + + :param str bridge_name: bridge name + :param str interface_name: interface name + :return: nothing + """ check_cmd([BRCTL_BIN, "addif", bridge_name, interface_name]) check_cmd([IP_BIN, "link", "set", interface_name, "up"]) def delete_interface(self, bridge_name, interface_name): + """ + Delete an interface associated with a Linux bridge. + + :param str bridge_name: bridge name + :param str interface_name: interface name + :return: nothing + """ check_cmd([BRCTL_BIN, "delif", bridge_name, interface_name]) def existing_bridges(self, _id): + """ + Checks if there are any existing Linux bridges for a node. + + :param _id: node id to check bridges for + """ output = check_cmd([BRCTL_BIN, "show"]) lines = output.split("\n") for line in lines[1:]: @@ -117,29 +151,70 @@ class LinuxNetClient(NetClientBase): return False def disable_mac_learning(self, name): + """ + Disable mac learning for a Linux bridge. + + :param str name: bridge name + :return: nothing + """ check_cmd([BRCTL_BIN, "setageing", name, "0"]) class OvsNetClient(NetClientBase): + """ + Client for creating OVS bridges and ip interfaces for nodes. + """ + def create_bridge(self, name): - # turn off spanning tree protocol and forwarding delay - # TODO: verify stp and rstp are always off by default - # TODO: ovs only supports rstp forward delay and again it's off by default + """ + Create a OVS bridge and bring it up. + + :param str name: bridge name + :return: nothing + """ check_cmd([OVS_BIN, "add-br", name]) + check_cmd([OVS_BIN, "set", "bridge", name, "stp_enable=false"]) + check_cmd([OVS_BIN, "set", "bridge", name, "other_config:stp-max-age=6"]) + check_cmd([OVS_BIN, "set", "bridge", name, "other_config:stp-forward-delay=4"]) check_cmd([IP_BIN, "link", "set", name, "up"]) def delete_bridge(self, name): + """ + Bring down and delete a OVS bridge. + + :param str name: bridge name + :return: nothing + """ check_cmd([IP_BIN, "link", "set", name, "down"]) check_cmd([OVS_BIN, "del-br", name]) def create_interface(self, bridge_name, interface_name): + """ + Create an interface associated with a network bridge. + + :param str bridge_name: bridge name + :param str interface_name: interface name + :return: nothing + """ check_cmd([OVS_BIN, "add-port", bridge_name, interface_name]) check_cmd([IP_BIN, "link", "set", interface_name, "up"]) def delete_interface(self, bridge_name, interface_name): + """ + Delete an interface associated with a OVS bridge. + + :param str bridge_name: bridge name + :param str interface_name: interface name + :return: nothing + """ check_cmd([OVS_BIN, "del-port", bridge_name, interface_name]) def existing_bridges(self, _id): + """ + Checks if there are any existing OVS bridges for a node. + + :param _id: node id to check bridges for + """ output = check_cmd([OVS_BIN, "list-br"]) if output: for line in output.split("\n"): @@ -149,4 +224,10 @@ class OvsNetClient(NetClientBase): return False def disable_mac_learning(self, name): + """ + Disable mac learning for a OVS bridge. + + :param str name: bridge name + :return: nothing + """ check_cmd([OVS_BIN, "set", "bridge", name, "other_config:mac-aging-time=0"]) diff --git a/daemon/core/nodes/openvswitch.py b/daemon/core/nodes/openvswitch.py deleted file mode 100644 index 6a95687d..00000000 --- a/daemon/core/nodes/openvswitch.py +++ /dev/null @@ -1,825 +0,0 @@ -""" -TODO: probably goes away, or implement the usage of "unshare", or docker formal. -""" - -import logging -import socket -import threading -from socket import AF_INET, AF_INET6 - -from core import CoreCommandError, constants, utils -from core.emulator.data import LinkData -from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs -from core.nodes import ipaddress -from core.nodes.base import CoreNetworkBase -from core.nodes.interface import GreTap, Veth -from core.nodes.network import EbtablesQueue, GreTapBridge - -# a global object because all WLANs share the same queue -# cannot have multiple threads invoking the ebtables commnd -ebtables_queue = EbtablesQueue() - -ebtables_lock = threading.Lock() - -utils.check_executables([constants.IP_BIN, constants.EBTABLES_BIN, constants.TC_BIN]) - - -def ebtables_commands(call, commands): - with ebtables_lock: - for command in commands: - call(command) - - -class OvsNet(CoreNetworkBase): - """ - Used to be LxBrNet. - - Base class for providing Openvswitch functionality to objects that create bridges. - """ - - policy = "DROP" - - def __init__(self, session, _id=None, name=None, start=True, policy=None): - """ - Creates an OvsNet instance. - - :param core.emulator.session.Session session: session this object is a part of - :param int _id: object id - :param str name: object name - :param bool start: start flag - :param policy: network policy - """ - - CoreNetworkBase.__init__(self, session, _id, name, start) - - if policy: - self.policy = policy - else: - self.policy = self.__class__.policy - - session_id = self.session.short_session_id() - self.bridge_name = "b.%s.%s" % (str(self.id), session_id) - self.up = False - - if start: - self.startup() - ebtables_queue.startupdateloop(self) - - def startup(self): - """ - - :return: - :raises CoreCommandError: when there is a command exception - """ - utils.check_cmd([constants.OVS_BIN, "add-br", self.bridge_name]) - - # turn off spanning tree protocol and forwarding delay - # TODO: appears stp and rstp are off by default, make sure this always holds true - # TODO: apears ovs only supports rstp forward delay and again it's off by default - utils.check_cmd([constants.IP_BIN, "link", "set", self.bridge_name, "up"]) - - # create a new ebtables chain for this bridge - ebtables_commands( - utils.check_cmd, - [ - [constants.EBTABLES_BIN, "-N", self.bridge_name, "-P", self.policy], - [ - constants.EBTABLES_BIN, - "-A", - "FORWARD", - "--logical-in", - self.bridge_name, - "-j", - self.bridge_name, - ], - ], - ) - - self.up = True - - def shutdown(self): - if not self.up: - logging.info("exiting shutdown, object is not up") - return - - ebtables_queue.stopupdateloop(self) - - try: - utils.check_cmd([constants.IP_BIN, "link", "set", self.bridge_name, "down"]) - utils.check_cmd([constants.OVS_BIN, "del-br", self.bridge_name]) - ebtables_commands( - utils.check_cmd, - [ - [ - constants.EBTABLES_BIN, - "-D", - "FORWARD", - "--logical-in", - self.bridge_name, - "-j", - self.bridge_name, - ], - [constants.EBTABLES_BIN, "-X", self.bridge_name], - ], - ) - except CoreCommandError: - logging.exception("error bringing bridge down and removing it") - - # removes veth pairs used for bridge-to-bridge connections - for interface in self.netifs(): - interface.shutdown() - - self._netif.clear() - self._linked.clear() - del self.session - self.up = False - - def attach(self, interface): - if self.up: - utils.check_cmd( - [constants.OVS_BIN, "add-port", self.bridge_name, interface.localname] - ) - utils.check_cmd( - [constants.IP_BIN, "link", "set", interface.localname, "up"] - ) - - CoreNetworkBase.attach(self, interface) - - def detach(self, interface): - if self.up: - utils.check_cmd( - [constants.OVS_BIN, "del-port", self.bridge_name, interface.localname] - ) - - CoreNetworkBase.detach(self, interface) - - def linked(self, interface_one, interface_two): - # check if the network interfaces are attached to this network - if self._netif[interface_one.netifi] != interface_one: - raise ValueError("inconsistency for interface %s" % interface_one.name) - - if self._netif[interface_two.netifi] != interface_two: - raise ValueError("inconsistency for interface %s" % interface_two.name) - - try: - linked = self._linked[interface_one][interface_two] - except KeyError: - if self.policy == "ACCEPT": - linked = True - elif self.policy == "DROP": - linked = False - else: - raise ValueError("unknown policy: %s" % self.policy) - - self._linked[interface_one][interface_two] = linked - - return linked - - def unlink(self, interface_one, interface_two): - """ - Unlink two PyCoreNetIfs, resulting in adding or removing ebtables - filtering rules. - """ - with self._linked_lock: - if not self.linked(interface_one, interface_two): - return - - self._linked[interface_one][interface_two] = False - - ebtables_queue.ebchange(self) - - def link(self, interface_one, interface_two): - """ - Link two interfaces together, resulting in adding or removing - ebtables filtering rules. - """ - with self._linked_lock: - if self.linked(interface_one, interface_two): - return - - self._linked[interface_one][interface_two] = True - - ebtables_queue.ebchange(self) - - def linkconfig( - self, - netif, - bw=None, - delay=None, - loss=None, - duplicate=None, - jitter=None, - netif2=None, - devname=None, - ): - """ - Configure link parameters by applying tc queuing disciplines on the - interface. - """ - if not devname: - devname = netif.localname - - tc = [constants.TC_BIN, "qdisc", "replace", "dev", devname] - parent = ["root"] - - # attempt to set bandwidth and update as needed if value changed - bandwidth_changed = netif.setparam("bw", bw) - if bandwidth_changed: - # from tc-tbf(8): minimum value for burst is rate / kernel_hz - if bw > 0: - if self.up: - burst = max(2 * netif.mtu, bw / 1000) - limit = 0xFFFF # max IP payload - tbf = [ - "tbf", - "rate", - str(bw), - "burst", - str(burst), - "limit", - str(limit), - ] - logging.info( - "linkconfig: %s" % [tc + parent + ["handle", "1:"] + tbf] - ) - utils.check_cmd(tc + parent + ["handle", "1:"] + tbf) - netif.setparam("has_tbf", True) - elif netif.getparam("has_tbf") and bw <= 0: - tcd = [] + tc - tcd[2] = "delete" - - if self.up: - utils.check_cmd(tcd + parent) - - netif.setparam("has_tbf", False) - # removing the parent removes the child - netif.setparam("has_netem", False) - - if netif.getparam("has_tbf"): - parent = ["parent", "1:1"] - - netem = ["netem"] - delay_changed = netif.setparam("delay", delay) - - if loss is not None: - loss = float(loss) - loss_changed = netif.setparam("loss", loss) - - if duplicate is not None: - duplicate = int(duplicate) - duplicate_changed = netif.setparam("duplicate", duplicate) - jitter_changed = netif.setparam("jitter", jitter) - - # if nothing changed return - if not any( - [ - bandwidth_changed, - delay_changed, - loss_changed, - duplicate_changed, - jitter_changed, - ] - ): - return - - # jitter and delay use the same delay statement - if delay is not None: - netem += ["delay", "%sus" % delay] - else: - netem += ["delay", "0us"] - - if jitter is not None: - netem += ["%sus" % jitter, "25%"] - - if loss is not None and loss > 0: - netem += ["loss", "%s%%" % min(loss, 100)] - - if duplicate is not None and duplicate > 0: - netem += ["duplicate", "%s%%" % min(duplicate, 100)] - - if delay <= 0 and jitter <= 0 and loss <= 0 and duplicate <= 0: - # possibly remove netem if it exists and parent queue wasn"t removed - if not netif.getparam("has_netem"): - return - - tc[2] = "delete" - - if self.up: - logging.info("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) - utils.check_cmd(tc + parent + ["handle", "10:"]) - netif.setparam("has_netem", False) - elif len(netem) > 1: - if self.up: - logging.info( - "linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],) - ) - utils.check_cmd(tc + parent + ["handle", "10:"] + netem) - netif.setparam("has_netem", True) - - def linknet(self, network): - """ - Link this bridge with another by creating a veth pair and installing - each device into each bridge. - """ - session_id = self.session.short_session_id() - - try: - _id = "%x" % self.id - except TypeError: - _id = "%s" % self.id - - try: - network_id = "%x" % network.id - except TypeError: - network_id = "%s" % network.id - - localname = "veth%s.%s.%s" % (_id, network_id, session_id) - - if len(localname) >= 16: - raise ValueError("interface local name %s too long" % localname) - - name = "veth%s.%s.%s" % (network_id, _id, session_id) - if len(name) >= 16: - raise ValueError("interface name %s too long" % name) - - interface = Veth( - node=None, name=name, localname=localname, mtu=1500, net=self, start=self.up - ) - self.attach(interface) - if network.up: - # this is similar to net.attach() but uses netif.name instead of localname - utils.check_cmd( - [constants.OVS_BIN, "add-port", network.bridge_name, interface.name] - ) - utils.check_cmd([constants.IP_BIN, "link", "set", interface.name, "up"]) - - network.attach(interface) - interface.net = self - interface.othernet = network - return interface - - def getlinknetif(self, network): - """ - Return the interface of that links this net with another net - (that were linked using linknet()). - """ - for interface in self.netifs(): - if hasattr(interface, "othernet") and interface.othernet == network: - return interface - - return None - - def addrconfig(self, addresses): - """ - Set addresses on the bridge. - """ - if not self.up: - return - - for address in addresses: - utils.check_cmd( - [constants.IP_BIN, "addr", "add", str(address), "dev", self.bridge_name] - ) - - -class OvsCtrlNet(OvsNet): - policy = "ACCEPT" - CTRLIF_IDX_BASE = 99 # base control interface index - DEFAULT_PREFIX_LIST = [ - "172.16.0.0/24 172.16.1.0/24 172.16.2.0/24 172.16.3.0/24 172.16.4.0/24", - "172.17.0.0/24 172.17.1.0/24 172.17.2.0/24 172.17.3.0/24 172.17.4.0/24", - "172.18.0.0/24 172.18.1.0/24 172.18.2.0/24 172.18.3.0/24 172.18.4.0/24", - "172.19.0.0/24 172.19.1.0/24 172.19.2.0/24 172.19.3.0/24 172.19.4.0/24", - ] - - def __init__( - self, - session, - _id="ctrlnet", - name=None, - prefix=None, - hostid=None, - start=True, - assign_address=True, - updown_script=None, - serverintf=None, - ): - self.prefix = ipaddress.Ipv4Prefix(prefix) - self.hostid = hostid - self.assign_address = assign_address - self.updown_script = updown_script - self.serverintf = serverintf - OvsNet.__init__(self, session, _id=_id, name=name, start=start) - - def startup(self): - if self.detectoldbridge(): - return - - OvsNet.startup(self) - if self.hostid: - addr = self.prefix.addr(self.hostid) - else: - addr = self.prefix.max_addr() - - message = "Added control network bridge: %s %s" % ( - self.bridge_name, - self.prefix, - ) - addresses = ["%s/%s" % (addr, self.prefix.prefixlen)] - if self.assign_address: - self.addrconfig(addresses=addresses) - message += " address %s" % addr - logging.info(message) - - if self.updown_script: - logging.info( - "interface %s updown script %s startup called" - % (self.bridge_name, self.updown_script) - ) - utils.check_cmd([self.updown_script, self.bridge_name, "startup"]) - - if self.serverintf: - utils.check_cmd( - [constants.OVS_BIN, "add-port", self.bridge_name, self.serverintf] - ) - utils.check_cmd([constants.IP_BIN, "link", "set", self.serverintf, "up"]) - - def detectoldbridge(self): - """ - Occasionally, control net bridges from previously closed sessions are not cleaned up. - Check if there are old control net bridges and delete them - """ - - output = utils.check_cmd([constants.OVS_BIN, "list-br"]) - output = output.strip() - if output: - for line in output.split("\n"): - bride_name = line.split(".") - if bride_name[0] == "b" and bride_name[1] == self.id: - logging.error( - "older session may still be running with conflicting id for bridge: %s", - line, - ) - return True - - return False - - def shutdown(self): - if self.serverintf: - try: - utils.check_cmd( - [constants.OVS_BIN, "del-port", self.bridge_name, self.serverintf] - ) - except CoreCommandError: - logging.exception( - "error deleting server interface %s to controlnet bridge %s", - self.serverintf, - self.bridge_name, - ) - - if self.updown_script: - try: - logging.info( - "interface %s updown script (%s shutdown) called", - self.bridge_name, - self.updown_script, - ) - utils.check_cmd([self.updown_script, self.bridge_name, "shutdown"]) - except CoreCommandError: - logging.exception("error during updown script shutdown") - - OvsNet.shutdown(self) - - def all_link_data(self, flags): - """ - Do not include CtrlNet in link messages describing this session. - """ - return [] - - -class OvsPtpNet(OvsNet): - policy = "ACCEPT" - - def attach(self, interface): - if len(self._netif) >= 2: - raise ValueError( - "point-to-point links support at most 2 network interfaces" - ) - OvsNet.attach(self, interface) - - def data(self, message_type, lat=None, lon=None, alt=None): - """ - Do not generate a Node Message for point-to-point links. They are - built using a link message instead. - """ - return None - - def all_link_data(self, flags): - """ - Build CORE API TLVs for a point-to-point link. One Link message describes this network. - """ - - all_links = [] - - if len(self._netif) != 2: - return all_links - - if1, if2 = self._netif.values() - - unidirectional = 0 - if if1.getparams() != if2.getparams(): - unidirectional = 1 - - interface1_ip4 = None - interface1_ip4_mask = None - interface1_ip6 = None - interface1_ip6_mask = None - for address in if1.addrlist: - ip, _sep, mask = address.partition("/") - mask = int(mask) - if ipaddress.is_ipv4_address(ip): - family = AF_INET - ipl = socket.inet_pton(family, ip) - interface1_ip4 = ipaddress.IpAddress(af=family, address=ipl) - interface1_ip4_mask = mask - else: - family = AF_INET6 - ipl = socket.inet_pton(family, ip) - interface1_ip6 = ipaddress.IpAddress(af=family, address=ipl) - interface1_ip6_mask = mask - - interface2_ip4 = None - interface2_ip4_mask = None - interface2_ip6 = None - interface2_ip6_mask = None - for address in if2.addrlist: - ip, _sep, mask = address.partition("/") - mask = int(mask) - if ipaddress.is_ipv4_address(ip): - family = AF_INET - ipl = socket.inet_pton(family, ip) - interface2_ip4 = ipaddress.IpAddress(af=family, address=ipl) - interface2_ip4_mask = mask - else: - family = AF_INET6 - ipl = socket.inet_pton(family, ip) - interface2_ip6 = ipaddress.IpAddress(af=family, address=ipl) - interface2_ip6_mask = mask - - # TODO: not currently used - # loss=netif.getparam("loss") - link_data = LinkData( - message_type=flags, - node1_id=if1.node.id, - node2_id=if2.node.id, - link_type=self.linktype, - unidirectional=unidirectional, - delay=if1.getparam("delay"), - bandwidth=if1.getparam("bw"), - dup=if1.getparam("duplicate"), - jitter=if1.getparam("jitter"), - interface1_id=if1.node.getifindex(if1), - interface1_mac=if1.hwaddr, - interface1_ip4=interface1_ip4, - interface1_ip4_mask=interface1_ip4_mask, - interface1_ip6=interface1_ip6, - interface1_ip6_mask=interface1_ip6_mask, - interface2_id=if2.node.getifindex(if2), - interface2_mac=if2.hwaddr, - interface2_ip4=interface2_ip4, - interface2_ip4_mask=interface2_ip4_mask, - interface2_ip6=interface2_ip6, - interface2_ip6_mask=interface2_ip6_mask, - ) - - all_links.append(link_data) - - # build a 2nd link message for the upstream link parameters - # (swap if1 and if2) - if unidirectional: - link_data = LinkData( - message_type=0, - node1_id=if2.node.id, - node2_id=if1.node.id, - delay=if1.getparam("delay"), - bandwidth=if1.getparam("bw"), - dup=if1.getparam("duplicate"), - jitter=if1.getparam("jitter"), - unidirectional=1, - interface1_id=if2.node.getifindex(if2), - interface2_id=if1.node.getifindex(if1), - ) - all_links.append(link_data) - - return all_links - - -class OvsSwitchNode(OvsNet): - apitype = NodeTypes.SWITCH.value - policy = "ACCEPT" - type = "lanswitch" - - -class OvsHubNode(OvsNet): - apitype = NodeTypes.HUB.value - policy = "ACCEPT" - type = "hub" - - def __init__(self, session, _id=None, name=None, start=True): - """ - the Hub node forwards packets to all bridge ports by turning off - the MAC address learning - """ - OvsNet.__init__(self, session, _id, name, start) - - if start: - # TODO: verify that the below flow accomplishes what is desired for a "HUB" - # TODO: replace "brctl setageing 0" - utils.check_cmd( - [constants.OVS_FLOW_BIN, "add-flow", self.bridge_name, "action=flood"] - ) - - -class OvsWlanNode(OvsNet): - apitype = NodeTypes.WIRELESS_LAN.value - linktype = LinkTypes.WIRELESS.value - policy = "DROP" - type = "wlan" - - def __init__(self, session, _id=None, name=None, start=True, policy=None): - OvsNet.__init__(self, session, _id, name, start, policy) - - # wireless model such as basic range - self.model = None - # mobility model such as scripted - self.mobility = None - - def attach(self, interface): - OvsNet.attach(self, interface) - - if self.model: - interface.poshook = self.model.position_callback - - if interface.node is None: - return - - x, y, z = interface.node.position.get() - # invokes any netif.poshook - interface.setposition(x, y, z) - # self.model.setlinkparams() - - def setmodel(self, model, config=None): - """ - Mobility and wireless model. - """ - logging.info("adding model %s", model.name) - - if model.type == RegisterTlvs.WIRELESS.value: - self.model = model(session=self.session, _id=self.id, config=config) - if self.model.position_callback: - for interface in self.netifs(): - interface.poshook = self.model.position_callback - if interface.node is not None: - x, y, z = interface.node.position.get() - interface.poshook(interface, x, y, z) - self.model.setlinkparams() - elif model.type == RegisterTlvs.MOBILITY.value: - self.mobility = model(session=self.session, _id=self.id, config=config) - - def updatemodel(self, config): - if not self.model: - raise ValueError("no model set to update for node(%s)", self.id) - logging.info( - "node(%s) updating model(%s): %s", self.id, self.model.name, config - ) - self.model.set_configs(config, node_id=self.id) - if self.model.position_callback: - for netif in self.netifs(): - netif.poshook = self.model.position_callback - if netif.node is not None: - x, y, z = netif.node.position.get() - netif.poshook(netif, x, y, z) - self.model.updateconfig() - - def all_link_data(self, flags): - all_links = OvsNet.all_link_data(self, flags) - - if self.model: - all_links.extend(self.model.all_link_data(flags)) - - return all_links - - -class OvsTunnelNode(GreTapBridge): - apitype = NodeTypes.TUNNEL.value - policy = "ACCEPT" - type = "tunnel" - - -class OvsGreTapBridge(OvsNet): - """ - A network consisting of a bridge with a gretap device for tunneling to - another system. - """ - - def __init__( - self, - session, - remoteip=None, - _id=None, - name=None, - policy="ACCEPT", - localip=None, - ttl=255, - key=None, - start=True, - ): - OvsNet.__init__( - self, session=session, _id=_id, name=name, policy=policy, start=False - ) - self.grekey = key - if self.grekey is None: - self.grekey = self.session.id ^ self.id - - self.localnum = None - self.remotenum = None - self.remoteip = remoteip - self.localip = localip - self.ttl = ttl - - if remoteip is None: - self.gretap = None - else: - self.gretap = GreTap( - node=self, - session=session, - remoteip=remoteip, - localip=localip, - ttl=ttl, - key=self.grekey, - ) - if start: - self.startup() - - def startup(self): - """ - Creates a bridge and adds the gretap device to it. - """ - OvsNet.startup(self) - - if self.gretap: - self.attach(self.gretap) - - def shutdown(self): - """ - Detach the gretap device and remove the bridge. - """ - if self.gretap: - self.detach(self.gretap) - self.gretap.shutdown() - self.gretap = None - - OvsNet.shutdown(self) - - def addrconfig(self, addresses): - """ - Set the remote tunnel endpoint. This is a one-time method for - creating the GreTap device, which requires the remoteip at startup. - The 1st address in the provided list is remoteip, 2nd optionally - specifies localip. - """ - if self.gretap: - raise ValueError("gretap already exists for %s" % self.name) - - remoteip = addresses[0].split("/")[0] - localip = None - - if len(addresses) > 1: - localip = addresses[1].split("/")[0] - - self.gretap = GreTap( - session=self.session, - remoteip=remoteip, - localip=localip, - ttl=self.ttl, - key=self.grekey, - ) - self.attach(self.gretap) - - def setkey(self, key): - """ - Set the GRE key used for the GreTap device. This needs to be set - prior to instantiating the GreTap device (before addrconfig). - """ - self.grekey = key - - -OVS_NODES = { - NodeTypes.SWITCH: OvsSwitchNode, - NodeTypes.HUB: OvsHubNode, - NodeTypes.WIRELESS_LAN: OvsWlanNode, - NodeTypes.TUNNEL: OvsTunnelNode, - NodeTypes.TAP_BRIDGE: OvsGreTapBridge, - NodeTypes.PEER_TO_PEER: OvsPtpNet, - NodeTypes.CONTROL_NET: OvsCtrlNet, -} From a01ea35f7c4f71c295558cf58eef3eb21cfe1adc Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 11:01:19 -0700 Subject: [PATCH 012/113] bumping configparser version in requirements.txt due to previous version removal --- daemon/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/requirements.txt b/daemon/requirements.txt index ec738953..a32f12de 100644 --- a/daemon/requirements.txt +++ b/daemon/requirements.txt @@ -1,4 +1,4 @@ -configparser==4.0.1 +configparser==4.0.2 future==0.17.1 grpcio==1.23.0 grpcio-tools==1.21.1 From 4b028661aba80c1e5147498ea71d53ce8a3f8a87 Mon Sep 17 00:00:00 2001 From: Jeff Ahrenholz Date: Fri, 27 Sep 2019 11:02:01 -0700 Subject: [PATCH 013/113] fix #290 use pythondir var as base for setting PYTHONLIBDIR --- daemon/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/Makefile.am b/daemon/Makefile.am index 2f112b85..0182b291 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -15,7 +15,7 @@ DOCS = doc endif if PYTHON3 -PYTHONLIBDIR=$(libdir)/python3/dist-packages +PYTHONLIBDIR=$(shell echo $(pythondir) | sed -e 's/site-packages/dist-packages/') else PYTHONLIBDIR=$(pythondir) endif From cd747515ea7e8e85034ce201ea94823af3f17123 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 12:11:14 -0700 Subject: [PATCH 014/113] added python path change to ns3 and simplified substitution, which is not version dependent --- daemon/Makefile.am | 6 +----- ns3/Makefile.am | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/daemon/Makefile.am b/daemon/Makefile.am index 0182b291..a6503cc0 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -14,11 +14,7 @@ if WANT_DOCS DOCS = doc endif -if PYTHON3 -PYTHONLIBDIR=$(shell echo $(pythondir) | sed -e 's/site-packages/dist-packages/') -else -PYTHONLIBDIR=$(pythondir) -endif +PYTHONLIBDIR=$(subst site-packages,dist-packages,$(pythondir)) SUBDIRS = proto $(DOCS) diff --git a/ns3/Makefile.am b/ns3/Makefile.am index c1858665..115a7008 100644 --- a/ns3/Makefile.am +++ b/ns3/Makefile.am @@ -9,11 +9,7 @@ if WANT_PYTHON -if PYTHON3 -PYTHONLIBDIR=$(libdir)/python3/dist-packages -else -PYTHONLIBDIR=$(pythondir) -endif +PYTHONLIBDIR=$(subst site-packages,dist-packages,$(pythondir)) SETUPPY = setup.py SETUPPYFLAGS = -v From 03cf401639a96fd27b9fc39583bf6f47bc3a41c2 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 12:41:34 -0700 Subject: [PATCH 015/113] moved AM_CONDITIONAL out of if statement --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 4f774f6d..80b76cec 100644 --- a/configure.ac +++ b/configure.ac @@ -110,7 +110,6 @@ if test "x$enable_daemon" = "xyes"; then AC_CHECK_FUNCS([atexit dup2 gettimeofday memset socket strerror uname]) AM_PATH_PYTHON(2.7) - AM_CONDITIONAL([PYTHON3], [test "x$PYTHON" == "xpython3"]) AS_IF([$PYTHON -m grpc_tools.protoc -h &> /dev/null], [], [AC_MSG_ERROR([please install python grpcio-tools])]) AC_CHECK_PROG(brctl_path, brctl, $as_dir, no, $SEARCHPATH) @@ -157,6 +156,8 @@ if test "x$enable_daemon" = "xyes"; then CFLAGS=$CFLAGS_save CPPFLAGS=$CPPFLAGS_save fi +AM_CONDITIONAL([PYTHON3], [test "x$PYTHON" == "xpython3"]) + if [ test "x$enable_daemon" = "xyes" || test "x$enable_vnodedonly" = "xyes" ] ; then want_linux_netns=yes PKG_CHECK_MODULES(libev, libev, From 3cf557024c21e9aa8b9b703c928843ffb3154d8f Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 13:08:59 -0700 Subject: [PATCH 016/113] added more formal configure option for enabling python3, avoids issues with not having a very specific name for PYTHON being set --- configure.ac | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 80b76cec..fdd04f16 100644 --- a/configure.ac +++ b/configure.ac @@ -54,6 +54,13 @@ if test "x$enable_python" = "xyes" ; then else want_python=no fi + +AC_ARG_ENABLE([python3], + [AS_HELP_STRING([--enable-python3], + [enable python3 default is python2])], + [enable_python3=yes], [enable_python3=no]) +AM_CONDITIONAL([PYTHON3], [test "x$enable_python3" == "xyes"]) + AC_ARG_ENABLE([daemon], [AS_HELP_STRING([--enable-daemon[=ARG]], [build and install the daemon with Python modules @@ -156,7 +163,6 @@ if test "x$enable_daemon" = "xyes"; then CFLAGS=$CFLAGS_save CPPFLAGS=$CPPFLAGS_save fi -AM_CONDITIONAL([PYTHON3], [test "x$PYTHON" == "xpython3"]) if [ test "x$enable_daemon" = "xyes" || test "x$enable_vnodedonly" = "xyes" ] ; then want_linux_netns=yes From f5aa74ed06bc0cb9be761e7a09dd1382c5a8d2f6 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 13:15:20 -0700 Subject: [PATCH 017/113] updated install doc to refer to new configure option for building packages --- docs/install.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/install.md b/docs/install.md index 761c589c..291c62ab 100644 --- a/docs/install.md +++ b/docs/install.md @@ -249,7 +249,7 @@ sudo yum -y install automake gcc python-devel libev-devel tk ```shell ./bootstrap.sh -# use python2 or python3 depending on desired version +# $VERSION should be path to python2/3 PYTHON=$VERSION ./configure make sudo make install @@ -268,7 +268,7 @@ sudo apt install python3-sphinx sudo yum install python3-sphinx ./bootstrap.sh -# use python2 or python3 depending on desired version +# $VERSION should be path to python2/3 PYTHON=$VERSION ./configure make doc ``` @@ -282,8 +282,10 @@ Build package commands, DESTDIR is used to make install into and then for packag ```shell ./bootstrap.sh -# use python2 or python3 depending on desired version -PYTHON=$VERSION ./configure +# for python2 +PYTHON=python2 ./configure +# for python3 +PYTHON=python3 ./configure --enable-python3 make mkdir /tmp/core-build make fpm DESTDIR=/tmp/core-build From 07c28e8a93dcc308bd8ec4db627006bfce385fe0 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 14:22:50 -0700 Subject: [PATCH 018/113] updated configure.ac python3 argument help line to be more specific --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index fdd04f16..07c8150a 100644 --- a/configure.ac +++ b/configure.ac @@ -57,7 +57,7 @@ fi AC_ARG_ENABLE([python3], [AS_HELP_STRING([--enable-python3], - [enable python3 default is python2])], + [sets python3 flag for building packages])], [enable_python3=yes], [enable_python3=no]) AM_CONDITIONAL([PYTHON3], [test "x$enable_python3" == "xyes"]) From ce27a43486289ef4e15b0a7c7fe6dd50f575194f Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 15:07:41 -0700 Subject: [PATCH 019/113] updated install instruction to denote installing ethtool --- docs/install.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/install.md b/docs/install.md index 291c62ab..dbe6ab35 100644 --- a/docs/install.md +++ b/docs/install.md @@ -230,19 +230,19 @@ pip3 install grpcio-tools ### Ubuntu 18.04 Requirements ```shell -sudo apt install automake pkg-config gcc libev-dev bridge-utils ebtables python-dev python-setuptools tk libtk-img +sudo apt install automake pkg-config gcc libev-dev bridge-utils ebtables python-dev python-setuptools tk libtk-img ethtool ``` ### Ubuntu 16.04 Requirements ```shell -sudo apt-get install automake bridge-utils ebtables python-dev libev-dev python-setuptools libtk-img +sudo apt-get install automake bridge-utils ebtables python-dev libev-dev python-setuptools libtk-img ethtool ``` ### CentOS 7 with Gnome Desktop Requirements ```shell -sudo yum -y install automake gcc python-devel libev-devel tk +sudo yum -y install automake gcc python-devel libev-devel tk ethtool ``` ## Build and Install From b961a8a959b42cbe3a907da9e5579bd7532bd0e2 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 15:31:19 -0700 Subject: [PATCH 020/113] updated broker.py logging to not be as noisy by default --- daemon/core/api/tlv/broker.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/daemon/core/api/tlv/broker.py b/daemon/core/api/tlv/broker.py index 7c0b3992..7b3087b4 100644 --- a/daemon/core/api/tlv/broker.py +++ b/daemon/core/api/tlv/broker.py @@ -318,13 +318,13 @@ class CoreBroker(object): # leave this socket connected return - logging.info( + logging.debug( "closing connection with %s @ %s:%s", name, server.host, server.port ) server.close() del self.servers[name] - logging.info("adding broker server(%s): %s:%s", name, host, port) + logging.debug("adding broker server(%s): %s:%s", name, host, port) server = CoreDistributedServer(name, host, port) if host is not None and port is not None: try: @@ -492,7 +492,7 @@ class CoreBroker(object): :raises core.CoreError: when node to add net tunnel to does not exist """ net = self.session.get_node(node_id) - logging.info("adding net tunnel for: id(%s) %s", node_id, net) + logging.debug("adding net tunnel for: id(%s) %s", node_id, net.name) # add other nets here that do not require tunnels if isinstance(net, EmaneNet): @@ -501,21 +501,21 @@ class CoreBroker(object): server_interface = getattr(net, "serverintf", None) if isinstance(net, CtrlNet) and server_interface is not None: - logging.warning( + logging.debug( "control networks with server interfaces do not need a tunnel" ) return None servers = self.getserversbynode(node_id) if len(servers) < 2: - logging.warning("not enough servers to create a tunnel: %s", servers) + logging.debug("not enough servers to create a tunnel for node: %s", node_id) return None hosts = [] for server in servers: if server.host is None: continue - logging.info("adding server host for net tunnel: %s", server.host) + logging.debug("adding server host for net tunnel: %s", server.host) hosts.append(server.host) if len(hosts) == 0: @@ -523,7 +523,7 @@ class CoreBroker(object): # get IP address from API message sender (master) if session_client.client_address != "": address = session_client.client_address[0] - logging.info("adding session_client host: %s", address) + logging.debug("adding session_client host: %s", address) hosts.append(address) r = [] @@ -536,7 +536,7 @@ class CoreBroker(object): myip = host key = self.tunnelkey(node_id, IpAddress.to_int(myip)) if key in self.tunnels.keys(): - logging.info( + logging.debug( "tunnel already exists, returning existing tunnel: %s", key ) gt = self.tunnels[key] @@ -655,9 +655,9 @@ class CoreBroker(object): :param int nodenum: node id to add :return: nothing """ - logging.info("adding net to broker: %s", nodenum) + logging.debug("adding net to broker: %s", nodenum) self.network_nodes.add(nodenum) - logging.info("broker network nodes: %s", self.network_nodes) + logging.debug("broker network nodes: %s", self.network_nodes) def addphys(self, nodenum): """ From 050297587d8fc4ae384115e1a0a6a058f40e3fea Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 22:31:56 -0700 Subject: [PATCH 021/113] consolidated EmaneNode and EmaneNet, renaming EmaneNode to EmaneNet --- daemon/core/api/grpc/server.py | 6 +- daemon/core/emane/emanemanager.py | 110 +++++++++++++------------- daemon/core/emane/nodes.py | 18 ++--- daemon/core/emulator/emudata.py | 20 +++-- daemon/core/emulator/session.py | 8 +- daemon/core/plugins/sdt.py | 8 +- daemon/core/services/emaneservices.py | 4 +- daemon/core/services/quagga.py | 4 +- daemon/core/xml/corexml.py | 4 +- daemon/core/xml/corexmldeployment.py | 4 +- daemon/core/xml/emanexml.py | 41 +++++----- 11 files changed, 115 insertions(+), 112 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index ecb391ce..93ef1935 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -12,7 +12,7 @@ import grpc from core import CoreError from core.api.grpc import core_pb2, core_pb2_grpc -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.emulator.data import ( ConfigData, EventData, @@ -456,7 +456,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): services = [x.name for x in services] emane_model = None - if isinstance(node, EmaneNode): + if isinstance(node, EmaneNet): emane_model = node.model.name node_proto = core_pb2.Node( @@ -809,7 +809,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): interfaces.append(interface_proto) emane_model = None - if isinstance(node, EmaneNode): + if isinstance(node, EmaneNet): emane_model = node.model.name services = [x.name for x in getattr(node, "services", [])] diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index c93d28dd..934fa3a1 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -15,7 +15,7 @@ from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel from core.emane.emanemodel import EmaneModel from core.emane.ieee80211abg import EmaneIeee80211abgModel -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel from core.emulator.enumerations import ( @@ -53,8 +53,8 @@ DEFAULT_EMANE_PREFIX = "/usr" class EmaneManager(ModelManager): """ EMANE controller object. Lives in a Session instance and is used for - building EMANE config files from all of the EmaneNode objects in this - emulation, and for controlling the EMANE daemons. + building EMANE config files for all EMANE networks in this emulation, and for + controlling the EMANE daemons. """ name = "emane" @@ -72,7 +72,7 @@ class EmaneManager(ModelManager): """ super(EmaneManager, self).__init__() self.session = session - self._emane_nodes = {} + self._emane_nets = {} self._emane_node_lock = threading.Lock() self._ifccounts = {} self._ifccountslock = threading.Lock() @@ -226,38 +226,39 @@ class EmaneManager(ModelManager): emane_model.load(emane_prefix) self.models[emane_model.name] = emane_model - def add_node(self, emane_node): + def add_node(self, emane_net): """ - Add a new EmaneNode object to this Emane controller object + Add EMANE network object to this manager. - :param core.emane.nodes.EmaneNode emane_node: emane node to add + :param core.emane.nodes.EmaneNet emane_net: emane node to add :return: nothing """ with self._emane_node_lock: - if emane_node.id in self._emane_nodes: + if emane_net.id in self._emane_nets: raise KeyError( - "non-unique EMANE object id %s for %s" % (emane_node.id, emane_node) + "non-unique EMANE object id %s for %s" % (emane_net.id, emane_net) ) - self._emane_nodes[emane_node.id] = emane_node + self._emane_nets[emane_net.id] = emane_net def getnodes(self): """ - Return a set of CoreNodes that are linked to an EmaneNode, + Return a set of CoreNodes that are linked to an EMANE network, e.g. containers having one or more radio interfaces. """ # assumes self._objslock already held nodes = set() - for emane_node in self._emane_nodes.values(): - for netif in emane_node.netifs(): + for emane_net in self._emane_nets.values(): + for netif in emane_net.netifs(): nodes.add(netif.node) return nodes def setup(self): """ - Populate self._objs with EmaneNodes; perform distributed setup; - associate models with EmaneNodes from self.config. Returns - Emane.(SUCCESS, NOT_NEEDED, NOT_READY) in order to delay session - instantiation. + Setup duties for EMANE manager. + + :return: SUCCESS, NOT_NEEDED, NOT_READY in order to delay session + instantiation + :rtype: int """ logging.debug("emane setup") @@ -265,13 +266,13 @@ class EmaneManager(ModelManager): with self.session._nodes_lock: for node_id in self.session.nodes: node = self.session.nodes[node_id] - if isinstance(node, EmaneNode): + if isinstance(node, EmaneNet): logging.debug( "adding emane node: id(%s) name(%s)", node.id, node.name ) self.add_node(node) - if not self._emane_nodes: + if not self._emane_nets: logging.debug("no emane nodes in session") return EmaneManager.NOT_NEEDED @@ -325,9 +326,12 @@ class EmaneManager(ModelManager): def startup(self): """ - After all the EmaneNode objects have been added, build XML files - and start the daemons. Returns Emane.(SUCCESS, NOT_NEEDED, or - NOT_READY) which is used to delay session instantiation. + After all the EMANE networks have been added, build XML files + and start the daemons. + + :return: SUCCESS, NOT_NEEDED, NOT_READY in order to delay session + instantiation + :rtype: int """ self.reset() r = self.setup() @@ -346,8 +350,8 @@ class EmaneManager(ModelManager): self.startdaemons() self.installnetifs() - for node_id in self._emane_nodes: - emane_node = self._emane_nodes[node_id] + for node_id in self._emane_nets: + emane_node = self._emane_nets[node_id] for netif in emane_node.netifs(): nems.append( (netif.node.name, netif.name, emane_node.getnemid(netif)) @@ -372,8 +376,8 @@ class EmaneManager(ModelManager): return with self._emane_node_lock: - for key in sorted(self._emane_nodes.keys()): - emane_node = self._emane_nodes[key] + for key in sorted(self._emane_nets.keys()): + emane_node = self._emane_nets[key] logging.debug( "post startup for emane node: %s - %s", emane_node.id, @@ -386,11 +390,11 @@ class EmaneManager(ModelManager): def reset(self): """ - remove all EmaneNode objects from the dictionary, - reset port numbers and nem id counters + Remove all EMANE networks from the dictionary, reset port numbers and + nem id counters """ with self._emane_node_lock: - self._emane_nodes.clear() + self._emane_nets.clear() # don't clear self._ifccounts here; NEM counts are needed for buildxml self.platformport = self.session.options.get_config_int( @@ -408,7 +412,7 @@ class EmaneManager(ModelManager): self._ifccounts.clear() with self._emane_node_lock: - if not self._emane_nodes: + if not self._emane_nets: return logging.info("stopping EMANE daemons.") self.deinstallnetifs() @@ -448,7 +452,7 @@ class EmaneManager(ModelManager): master = False with self._emane_node_lock: - if self._emane_nodes: + if self._emane_nets: master = self.session.master logging.info("emane check distributed as master: %s.", master) @@ -458,8 +462,8 @@ class EmaneManager(ModelManager): nemcount = 0 with self._emane_node_lock: - for key in self._emane_nodes: - emane_node = self._emane_nodes[key] + for key in self._emane_nets: + emane_node = self._emane_nets[key] nemcount += emane_node.numnetif() nemid = int(self.get_config("nem_id_start")) @@ -469,7 +473,7 @@ class EmaneManager(ModelManager): # build an ordered list of servers so platform ID is deterministic servers = [] - for key in sorted(self._emane_nodes): + for key in sorted(self._emane_nets): for server in self.session.broker.getserversbynode(key): if server not in servers: servers.append(server) @@ -565,11 +569,10 @@ class EmaneManager(ModelManager): def check_node_models(self): """ - Associate EmaneModel classes with EmaneNode nodes. The model - configurations are stored in self.configs. + Associate EMANE model classes with EMANE network nodes. """ - for node_id in self._emane_nodes: - emane_node = self._emane_nodes[node_id] + for node_id in self._emane_nets: + emane_node = self._emane_nets[node_id] logging.debug("checking emane model for node: %s", node_id) # skip nodes that already have a model set @@ -595,13 +598,13 @@ class EmaneManager(ModelManager): def nemlookup(self, nemid): """ Look for the given numerical NEM ID and return the first matching - EmaneNode and NEM interface. + EMANE network and NEM interface. """ emane_node = None netif = None - for node_id in self._emane_nodes: - emane_node = self._emane_nodes[node_id] + for node_id in self._emane_nets: + emane_node = self._emane_nets[node_id] netif = emane_node.getnemnetif(nemid) if netif is not None: break @@ -615,8 +618,8 @@ class EmaneManager(ModelManager): Return the number of NEMs emulated locally. """ count = 0 - for node_id in self._emane_nodes: - emane_node = self._emane_nodes[node_id] + for node_id in self._emane_nets: + emane_node = self._emane_nets[node_id] count += len(emane_node.netifs()) return count @@ -628,20 +631,19 @@ class EmaneManager(ModelManager): platform_xmls = {} # assume self._objslock is already held here - for key in sorted(self._emane_nodes.keys()): - emane_node = self._emane_nodes[key] + for key in sorted(self._emane_nets.keys()): + emane_node = self._emane_nets[key] nemid = emanexml.build_node_platform_xml( self, ctrlnet, emane_node, nemid, platform_xmls ) def buildnemxml(self): """ - Builds the xxxnem.xml, xxxmac.xml, and xxxphy.xml files which - are defined on a per-EmaneNode basis. + Builds the nem, mac, and phy xml files for each EMANE network. """ - for key in sorted(self._emane_nodes.keys()): - emane_node = self._emane_nodes[key] - emanexml.build_xml_files(self, emane_node) + for key in sorted(self._emane_nets): + emane_net = self._emane_nets[key] + emanexml.build_xml_files(self, emane_net) def buildtransportxml(self): """ @@ -785,8 +787,8 @@ class EmaneManager(ModelManager): Install TUN/TAP virtual interfaces into their proper namespaces now that the EMANE daemons are running. """ - for key in sorted(self._emane_nodes.keys()): - emane_node = self._emane_nodes[key] + for key in sorted(self._emane_nets.keys()): + emane_node = self._emane_nets[key] logging.info("emane install netifs for node: %d", key) emane_node.installnetifs() @@ -794,8 +796,8 @@ class EmaneManager(ModelManager): """ Uninstall TUN/TAP virtual interfaces. """ - for key in sorted(self._emane_nodes.keys()): - emane_node = self._emane_nodes[key] + for key in sorted(self._emane_nets.keys()): + emane_node = self._emane_nets[key] emane_node.deinstallnetifs() def doeventmonitor(self): diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 79b767d1..89c97b6b 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -1,6 +1,5 @@ """ -nodes.py: definition of an EmaneNode class for implementing configuration -control of an EMANE emulation. An EmaneNode has several attached NEMs that +Provides an EMANE network node class, which has several attached NEMs that share the same MAC+PHY model. """ @@ -19,26 +18,19 @@ except ImportError: class EmaneNet(CoreNetworkBase): - """ - EMANE network base class. - """ - - apitype = NodeTypes.EMANE.value - linktype = LinkTypes.WIRELESS.value - type = "wlan" - - -class EmaneNode(EmaneNet): """ EMANE node contains NEM configuration and causes connected nodes to have TAP interfaces (instead of VEth). These are managed by the Emane controller object that exists in a session. """ + apitype = NodeTypes.EMANE.value + linktype = LinkTypes.WIRELESS.value + type = "wlan" is_emane = True def __init__(self, session, _id=None, name=None, start=True): - super(EmaneNode, self).__init__(session, _id, name, start) + super(EmaneNet, self).__init__(session, _id, name, start) self.conf = "" self.up = False self.nemidmap = {} diff --git a/daemon/core/emulator/emudata.py b/daemon/core/emulator/emudata.py index 467fe2e4..d11d1e0e 100644 --- a/daemon/core/emulator/emudata.py +++ b/daemon/core/emulator/emudata.py @@ -1,4 +1,4 @@ -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.emulator.enumerations import LinkTypes from core.nodes.ipaddress import Ipv4Prefix, Ipv6Prefix, MacAddress from core.nodes.physical import PhysicalNode @@ -53,8 +53,9 @@ def link_config(network, interface, link_options, devname=None, interface_two=No "netif2": interface_two, } - # hacky check here, because physical and emane nodes do not conform to the same linkconfig interface - if not isinstance(network, (EmaneNode, PhysicalNode)): + # hacky check here, because physical and emane nodes do not conform to the same + # linkconfig interface + if not isinstance(network, (EmaneNet, PhysicalNode)): config["devname"] = devname network.linkconfig(**config) @@ -70,7 +71,8 @@ class NodeOptions(object): Create a NodeOptions object. :param str name: name of node, defaults to node class name postfix with its id - :param str model: defines services for default and physical nodes, defaults to "router" + :param str model: defines services for default and physical nodes, defaults to + "router" :param str image: image to use for docker nodes """ self.name = name @@ -122,7 +124,8 @@ class LinkOptions(object): """ Create a LinkOptions object. - :param core.emulator.enumerations.LinkTypes _type: type of link, defaults to wired + :param core.emulator.enumerations.LinkTypes _type: type of link, defaults to + wired """ self.type = _type self.session = None @@ -191,12 +194,13 @@ class IpPrefixes(object): def create_interface(self, node, name=None, mac=None): """ - Creates interface data for linking nodes, using the nodes unique id for generation, along with a random - mac address, unless provided. + Creates interface data for linking nodes, using the nodes unique id for + generation, along with a random mac address, unless provided. :param core.nodes.base.CoreNode node: node to create interface for :param str name: name to set for interface, default is eth{id} - :param str mac: mac address to use for this interface, default is random generation + :param str mac: mac address to use for this interface, default is random + generation :return: new interface data for the provided node :rtype: InterfaceData """ diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 4ffb2391..519e5204 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -18,7 +18,7 @@ from core import CoreError, constants, utils from core.api.tlv import coreapi from core.api.tlv.broker import CoreBroker from core.emane.emanemanager import EmaneManager -from core.emane.nodes import EmaneNet, EmaneNode +from core.emane.nodes import EmaneNet from core.emulator.data import EventData, ExceptionData, NodeData from core.emulator.emudata import ( IdGen, @@ -62,8 +62,8 @@ NODES = { NodeTypes.RJ45: Rj45Node, NodeTypes.TUNNEL: TunnelNode, NodeTypes.KTUNNEL: None, - NodeTypes.EMANE: EmaneNode, - NodeTypes.EMANE_NET: EmaneNet, + NodeTypes.EMANE: EmaneNet, + NodeTypes.EMANE_NET: None, NodeTypes.TAP_BRIDGE: GreTapBridge, NodeTypes.PEER_TO_PEER: PtpNet, NodeTypes.CONTROL_NET: CtrlNet, @@ -261,7 +261,7 @@ class Session(object): raise CoreError("no common network found for wireless link/unlink") for common_network, interface_one, interface_two in common_networks: - if not isinstance(common_network, (WlanNode, EmaneNode)): + if not isinstance(common_network, (WlanNode, EmaneNet)): logging.info( "skipping common network that is not wireless/emane: %s", common_network, diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index ce60ccac..79ffdc6c 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -8,7 +8,7 @@ import socket from future.moves.urllib.parse import urlparse from core import CoreError, constants -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.emulator.enumerations import ( EventTypes, LinkTlvs, @@ -365,7 +365,7 @@ class Sdt(object): for net in nets: all_links = net.all_link_data(flags=MessageFlags.ADD.value) for link_data in all_links: - is_wireless = isinstance(net, (WlanNode, EmaneNode)) + is_wireless = isinstance(net, (WlanNode, EmaneNet)) wireless_link = link_data.message_type == LinkTypes.WIRELESS.value if is_wireless and link_data.node1_id == net.id: continue @@ -493,7 +493,7 @@ class Sdt(object): def wlancheck(self, nodenum): """ - Helper returns True if a node number corresponds to a WlanNode or EmaneNode. + Helper returns True if a node number corresponds to a WLAN or EMANE node. :param int nodenum: node id to check :return: True if node is wlan or emane, False otherwise @@ -508,6 +508,6 @@ class Sdt(object): n = self.session.get_node(nodenum) except CoreError: return False - if isinstance(n, (WlanNode, EmaneNode)): + if isinstance(n, (WlanNode, EmaneNet)): return True return False diff --git a/daemon/core/services/emaneservices.py b/daemon/core/services/emaneservices.py index 5cb7ec5c..e145e842 100644 --- a/daemon/core/services/emaneservices.py +++ b/daemon/core/services/emaneservices.py @@ -1,4 +1,4 @@ -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.services.coreservices import CoreService from core.xml import emanexml @@ -21,7 +21,7 @@ class EmaneTransportService(CoreService): transport_commands = [] for interface in node.netifs(sort=True): network_node = node.session.get_node(interface.net.id) - if isinstance(network_node, EmaneNode): + if isinstance(network_node, EmaneNet): config = node.session.emane.get_configs( network_node.id, network_node.model.name ) diff --git a/daemon/core/services/quagga.py b/daemon/core/services/quagga.py index 0f213b7b..5b5cf0ba 100644 --- a/daemon/core/services/quagga.py +++ b/daemon/core/services/quagga.py @@ -3,7 +3,7 @@ quagga.py: defines routing services provided by Quagga. """ from core import constants -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.emulator.enumerations import LinkTypes from core.nodes import ipaddress from core.nodes.network import PtpNet, WlanNode @@ -460,7 +460,7 @@ class Ospfv3mdr(Ospfv3): cfg = cls.mtucheck(ifc) # Uncomment the following line to use Address Family Translation for IPv4 cfg += " ipv6 ospf6 instance-id 65\n" - if ifc.net is not None and isinstance(ifc.net, (WlanNode, EmaneNode)): + if ifc.net is not None and isinstance(ifc.net, (WlanNode, EmaneNet)): return ( cfg + """\ diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index ad4de155..78d1c488 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -4,7 +4,7 @@ from lxml import etree import core.nodes.base import core.nodes.physical -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions from core.emulator.enumerations import NodeTypes from core.nodes.base import CoreNetworkBase @@ -458,7 +458,7 @@ class CoreXmlWriter(object): interface_name = node_interface.name # check if emane interface - if isinstance(node_interface.net, EmaneNode): + if isinstance(node_interface.net, EmaneNet): nem = node_interface.net.getnemid(node_interface) add_attribute(interface, "nem", nem) diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index a5c966d5..c410ef5f 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -4,7 +4,7 @@ import socket from lxml import etree from core import constants, utils -from core.emane.nodes import EmaneNode +from core.emane.nodes import EmaneNet from core.nodes import ipaddress from core.nodes.base import CoreNodeBase @@ -144,7 +144,7 @@ class CoreXmlDeployment(object): for netif in node.netifs(): emane_element = None - if isinstance(netif.net, EmaneNode): + if isinstance(netif.net, EmaneNet): emane_element = add_emane_interface(host_element, netif) parent_element = host_element diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 952ed41b..d73f3d5b 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -103,9 +103,11 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x """ Create platform xml for a specific node. - :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane configurations - :param core.nodes.network.CtrlNet control_net: control net node for this emane network - :param core.emane.nodes.EmaneNode node: node to write platform xml for + :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane + configurations + :param core.nodes.network.CtrlNet control_net: control net node for this emane + network + :param core.emane.nodes.EmaneNet node: node to write platform xml for :param int nem_id: nem id to use for interfaces for this node :param dict platform_xmls: stores platform xml elements to append nem entries to :return: the next nem id that can be used for creating platform xml files @@ -120,7 +122,7 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x nem_entries = {} if node.model is None: - logging.warning("warning: EmaneNode %s has no associated model", node.name) + logging.warning("warning: EMANE network %s has no associated model", node.name) return nem_entries for netif in node.netifs(): @@ -133,7 +135,8 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x "nem", id=str(nem_id), name=netif.localname, definition=nem_definition ) - # check if this is an external transport, get default config if an interface specific one does not exist + # check if this is an external transport, get default config if an interface + # specific one does not exist config = emane_manager.getifcconfig(node.model.id, netif, node.model.name) if is_external(config): @@ -220,8 +223,9 @@ def build_xml_files(emane_manager, node): """ Generate emane xml files required for node. - :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane configurations - :param core.emane.nodes.EmaneNode node: node to write platform xml for + :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane + configurations + :param core.emane.nodes.EmaneNet node: node to write platform xml for :return: nothing """ logging.debug("building all emane xml for node(%s): %s", node, node.name) @@ -233,7 +237,7 @@ def build_xml_files(emane_manager, node): if not config: return - # build XML for overall network (EmaneNode) configs + # build XML for overall network EMANE configs node.model.build_xml_files(config) # build XML for specific interface (NEM) configs @@ -243,7 +247,7 @@ def build_xml_files(emane_manager, node): rtype = "raw" for netif in node.netifs(): - # check for interface specific emane configuration and write xml files, if needed + # check for interface specific emane configuration and write xml files config = emane_manager.getifcconfig(node.model.id, netif, node.model.name) if config: node.model.build_xml_files(config, netif) @@ -267,8 +271,9 @@ def build_transport_xml(emane_manager, node, transport_type): """ Build transport xml file for node and transport type. - :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane configurations - :param core.emane.nodes.EmaneNode node: node to write platform xml for + :param core.emane.emanemanager.EmaneManager emane_manager: emane manager with emane + configurations + :param core.emane.nodes.EmaneNet node: node to write platform xml for :param str transport_type: transport type to build xml for :return: nothing """ @@ -304,7 +309,7 @@ def create_phy_xml(emane_model, config, file_path): """ Create the phy xml document. - :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to :return: nothing @@ -323,7 +328,7 @@ def create_mac_xml(emane_model, config, file_path): """ Create the mac xml document. - :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to :return: nothing @@ -346,7 +351,7 @@ def create_nem_xml( """ Create the nem xml document. - :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str nem_file: nem file path to write :param str transport_definition: transport file definition path @@ -422,7 +427,7 @@ def nem_file_name(emane_model, interface=None): """ Return the string name for the NEM XML file, e.g. "n3rfpipenem.xml" - :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create file :param interface: interface for this model :return: nem xml filename :rtype: str @@ -438,7 +443,7 @@ def shim_file_name(emane_model, interface=None): """ Return the string name for the SHIM XML file, e.g. "commeffectshim.xml" - :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create file :param interface: interface for this model :return: shim xml filename :rtype: str @@ -450,7 +455,7 @@ def mac_file_name(emane_model, interface=None): """ Return the string name for the MAC XML file, e.g. "n3rfpipemac.xml" - :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create file :param interface: interface for this model :return: mac xml filename :rtype: str @@ -462,7 +467,7 @@ def phy_file_name(emane_model, interface=None): """ Return the string name for the PHY XML file, e.g. "n3rfpipephy.xml" - :param core.emane.emanemodel.EmaneModel emane_model: emane model to create phy xml for + :param core.emane.emanemodel.EmaneModel emane_model: emane model to create file :param interface: interface for this model :return: phy xml filename :rtype: str From 503fa754a91014d6dabf75d772142dd64c49783d Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 Sep 2019 23:29:15 -0700 Subject: [PATCH 022/113] updates to fail early for required executables that are not in PATH --- daemon/core/__init__.py | 37 ------------- daemon/core/api/grpc/server.py | 2 +- daemon/core/api/tlv/corehandlers.py | 3 +- daemon/core/constants.py.in | 37 +++++-------- daemon/core/emane/emanemanager.py | 3 +- daemon/core/emane/emanemodel.py | 2 +- daemon/core/emulator/session.py | 3 +- daemon/core/errors.py | 21 ++++++++ daemon/core/location/mobility.py | 3 +- daemon/core/nodes/base.py | 5 +- daemon/core/nodes/client.py | 3 +- daemon/core/nodes/docker.py | 3 +- daemon/core/nodes/interface.py | 5 +- daemon/core/nodes/lxd.py | 3 +- daemon/core/nodes/network.py | 7 +-- daemon/core/nodes/physical.py | 3 +- daemon/core/plugins/sdt.py | 3 +- daemon/core/services/coreservices.py | 13 ++--- daemon/core/services/utility.py | 3 +- daemon/core/utils.py | 78 +++++++++++++++++----------- daemon/examples/api/emane80211.py | 2 +- daemon/examples/api/switch.py | 2 +- daemon/examples/api/switch_inject.py | 2 +- daemon/examples/api/wlan.py | 2 +- daemon/scripts/core-daemon | 4 +- daemon/tests/emane/test_emane.py | 2 +- daemon/tests/test_grpc.py | 2 +- daemon/tests/test_gui.py | 2 +- daemon/tests/test_nodes.py | 3 +- daemon/tests/test_xml.py | 2 +- 30 files changed, 128 insertions(+), 132 deletions(-) create mode 100644 daemon/core/errors.py diff --git a/daemon/core/__init__.py b/daemon/core/__init__.py index 6696aec7..c847c8dc 100644 --- a/daemon/core/__init__.py +++ b/daemon/core/__init__.py @@ -1,41 +1,4 @@ -import json import logging.config -import os -import subprocess - -from core import constants # setup default null handler logging.getLogger(__name__).addHandler(logging.NullHandler()) - - -def load_logging_config(config_path=None): - """ - Load CORE logging configuration file. - - :param str config_path: path to logging config file, - when None defaults to /etc/core/logging.conf - :return: nothing - """ - if not config_path: - config_path = os.path.join(constants.CORE_CONF_DIR, "logging.conf") - with open(config_path, "r") as log_config_file: - log_config = json.load(log_config_file) - logging.config.dictConfig(log_config) - - -class CoreCommandError(subprocess.CalledProcessError): - """ - Used when encountering internal CORE command errors. - """ - - def __str__(self): - return "Command(%s), Status(%s):\n%s" % (self.cmd, self.returncode, self.output) - - -class CoreError(Exception): - """ - Used for errors when dealing with CoreEmu and Sessions. - """ - - pass diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 93ef1935..7fb6ed31 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,7 +10,6 @@ from queue import Empty, Queue import grpc -from core import CoreError from core.api.grpc import core_pb2, core_pb2_grpc from core.emane.nodes import EmaneNet from core.emulator.data import ( @@ -23,6 +22,7 @@ from core.emulator.data import ( ) from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions from core.emulator.enumerations import EventTypes, LinkTypes, NodeTypes +from core.errors import CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility from core.nodes.base import CoreNetworkBase from core.nodes.docker import DockerNode diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 24e29169..bd84049c 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -14,7 +14,7 @@ from builtins import range from itertools import repeat from queue import Empty, Queue -from core import CoreError, utils +from core import utils from core.api.tlv import coreapi, dataconversion, structutils from core.config import ConfigShim from core.emulator.data import ConfigData, EventData, ExceptionData, FileData @@ -37,6 +37,7 @@ from core.emulator.enumerations import ( RegisterTlvs, SessionTlvs, ) +from core.errors import CoreError from core.location.mobility import BasicRangeModel from core.nodes.network import WlanNode from core.services.coreservices import ServiceManager, ServiceShim diff --git a/daemon/core/constants.py.in b/daemon/core/constants.py.in index a84a375a..38e2b67f 100644 --- a/daemon/core/constants.py.in +++ b/daemon/core/constants.py.in @@ -1,29 +1,20 @@ -import os +from core.utils import which -COREDPY_VERSION = "@PACKAGE_VERSION@" -CORE_STATE_DIR = "@CORE_STATE_DIR@" +COREDPY_VERSION = "@PACKAGE_VERSION@" CORE_CONF_DIR = "@CORE_CONF_DIR@" CORE_DATA_DIR = "@CORE_DATA_DIR@" QUAGGA_STATE_DIR = "@CORE_STATE_DIR@/run/quagga" FRR_STATE_DIR = "@CORE_STATE_DIR@/run/frr" - -def which(command): - for path in os.environ["PATH"].split(os.pathsep): - command_path = os.path.join(path, command) - if os.path.isfile(command_path) and os.access(command_path, os.X_OK): - return command_path - - -VNODED_BIN = which("vnoded") -VCMD_BIN = which("vcmd") -BRCTL_BIN = which("brctl") -SYSCTL_BIN = which("sysctl") -IP_BIN = which("ip") -ETHTOOL_BIN = which("ethtool") -TC_BIN = which("tc") -EBTABLES_BIN = which("ebtables") -MOUNT_BIN = which("mount") -UMOUNT_BIN = which("umount") -OVS_BIN = which("ovs-vsctl") -OVS_FLOW_BIN = which("ovs-ofctl") +VNODED_BIN = which("vnoded", required=True) +VCMD_BIN = which("vcmd", required=True) +BRCTL_BIN = which("brctl", required=True) +SYSCTL_BIN = which("sysctl", required=True) +IP_BIN = which("ip", required=True) +ETHTOOL_BIN = which("ethtool", required=True) +TC_BIN = which("tc", required=True) +EBTABLES_BIN = which("ebtables", required=True) +MOUNT_BIN = which("mount", required=True) +UMOUNT_BIN = which("umount", required=True) +OVS_BIN = which("ovs-vsctl", required=False) +OVS_FLOW_BIN = which("ovs-ofctl", required=False) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 934fa3a1..cc734e21 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -7,7 +7,7 @@ import logging import os import threading -from core import CoreCommandError, CoreError, constants, utils +from core import constants, utils from core.api.tlv import coreapi, dataconversion from core.config import ConfigGroup, ConfigShim, Configuration, ModelManager from core.emane import emanemanifest @@ -26,6 +26,7 @@ from core.emulator.enumerations import ( MessageTypes, RegisterTlvs, ) +from core.errors import CoreCommandError, CoreError from core.xml import emanexml try: diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 974329f2..56eee289 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -4,10 +4,10 @@ Defines Emane Models used within CORE. import logging import os -from core import CoreError from core.config import ConfigGroup, Configuration from core.emane import emanemanifest from core.emulator.enumerations import ConfigDataTypes +from core.errors import CoreError from core.location.mobility import WirelessModel from core.xml import emanexml diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 519e5204..3e5877a5 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -14,7 +14,7 @@ import threading import time from multiprocessing.pool import ThreadPool -from core import CoreError, constants, utils +from core import constants, utils from core.api.tlv import coreapi from core.api.tlv.broker import CoreBroker from core.emane.emanemanager import EmaneManager @@ -29,6 +29,7 @@ from core.emulator.emudata import ( ) from core.emulator.enumerations import EventTypes, ExceptionLevels, LinkTypes, NodeTypes from core.emulator.sessionconfig import SessionConfig, SessionMetaData +from core.errors import CoreError from core.location.corelocation import CoreLocation from core.location.event import EventLoop from core.location.mobility import MobilityManager diff --git a/daemon/core/errors.py b/daemon/core/errors.py new file mode 100644 index 00000000..bb124434 --- /dev/null +++ b/daemon/core/errors.py @@ -0,0 +1,21 @@ +""" +Provides CORE specific errors. +""" +import subprocess + + +class CoreCommandError(subprocess.CalledProcessError): + """ + Used when encountering internal CORE command errors. + """ + + def __str__(self): + return "Command(%s), Status(%s):\n%s" % (self.cmd, self.returncode, self.output) + + +class CoreError(Exception): + """ + Used for errors when dealing with CoreEmu and Sessions. + """ + + pass diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index ef719e86..f6ce60ca 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -11,7 +11,7 @@ import time from builtins import int from functools import total_ordering -from core import CoreError, utils +from core import utils from core.config import ConfigGroup, ConfigurableOptions, Configuration, ModelManager from core.emulator.data import EventData, LinkData from core.emulator.enumerations import ( @@ -23,6 +23,7 @@ from core.emulator.enumerations import ( NodeTlvs, RegisterTlvs, ) +from core.errors import CoreError from core.nodes.base import CoreNodeBase from core.nodes.ipaddress import IpAddress diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 4861f93c..2773745c 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,17 +14,16 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 -from core import CoreCommandError, constants, utils +from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes +from core.errors import CoreCommandError from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, OvsNetClient _DEFAULT_MTU = 1500 -utils.check_executables([constants.IP_BIN]) - class NodeBase(object): """ diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 499a805e..4bfef967 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -8,7 +8,8 @@ import logging import os from subprocess import PIPE, Popen -from core import CoreCommandError, constants, utils +from core import constants, utils +from core.errors import CoreCommandError class VnodeClient(object): diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 36ab64d2..eecc1175 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -2,8 +2,9 @@ import json import logging import os -from core import CoreCommandError, utils +from core import utils from core.emulator.enumerations import NodeTypes +from core.errors import CoreCommandError from core.nodes.base import CoreNode diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 8720d69a..210b92b9 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -6,9 +6,8 @@ import logging import time from builtins import int, range -from core import CoreCommandError, constants, utils - -utils.check_executables([constants.IP_BIN]) +from core import constants, utils +from core.errors import CoreCommandError class CoreInterface(object): diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index a8d0d5a5..ff1e60b8 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -3,8 +3,9 @@ import logging import os import time -from core import CoreCommandError, utils +from core import utils from core.emulator.enumerations import NodeTypes +from core.errors import CoreCommandError from core.nodes.base import CoreNode diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 9a81b8e1..1d20bc35 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -9,17 +9,14 @@ import threading import time from socket import AF_INET, AF_INET6 -from core import CoreCommandError, CoreError, constants, utils +from core import constants, utils from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs +from core.errors import CoreCommandError, CoreError from core.nodes import ipaddress from core.nodes.base import CoreNetworkBase from core.nodes.interface import GreTap, Veth -utils.check_executables( - [constants.BRCTL_BIN, constants.IP_BIN, constants.EBTABLES_BIN, constants.TC_BIN] -) - ebtables_lock = threading.Lock() diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 41ae444e..92a9616c 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -7,8 +7,9 @@ import os import subprocess import threading -from core import CoreCommandError, constants, utils +from core import constants, utils from core.emulator.enumerations import NodeTypes +from core.errors import CoreCommandError from core.nodes.base import CoreNodeBase from core.nodes.interface import CoreInterface from core.nodes.network import CoreNetwork, GreTap diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index 79ffdc6c..32800eea 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -7,7 +7,7 @@ import socket from future.moves.urllib.parse import urlparse -from core import CoreError, constants +from core import constants from core.emane.nodes import EmaneNet from core.emulator.enumerations import ( EventTypes, @@ -18,6 +18,7 @@ from core.emulator.enumerations import ( NodeTlvs, NodeTypes, ) +from core.errors import CoreError from core.nodes.base import CoreNetworkBase, NodeBase from core.nodes.network import WlanNode diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index f3ad9e80..42718f5e 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -12,10 +12,11 @@ import logging import time from multiprocessing.pool import ThreadPool -from core import CoreCommandError, utils +from core import utils from core.constants import which from core.emulator.data import FileData from core.emulator.enumerations import MessageFlags, RegisterTlvs +from core.errors import CoreCommandError class ServiceBootError(Exception): @@ -258,13 +259,7 @@ class ServiceManager(object): # validate dependent executables are present for executable in service.executables: - if not which(executable): - logging.debug( - "service(%s) missing executable: %s", service.name, executable - ) - raise ValueError( - "service(%s) missing executable: %s" % (service.name, executable) - ) + which(executable, required=True) # make service available cls.services[name] = service @@ -300,7 +295,7 @@ class ServiceManager(object): cls.add(service) except ValueError as e: service_errors.append(service.name) - logging.debug("not loading service: %s", e) + logging.debug("not loading service(%s): %s", service.name, e) return service_errors diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 283e3eaa..8088b149 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -4,7 +4,8 @@ utility.py: defines miscellaneous utility services. import os -from core import CoreCommandError, constants, utils +from core import constants, utils +from core.errors import CoreCommandError from core.nodes.ipaddress import Ipv4Prefix, Ipv6Prefix from core.services.coreservices import CoreService diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 76d9abd2..425fc0ac 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -6,7 +6,9 @@ import fcntl import hashlib import importlib import inspect +import json import logging +import logging.config import os import shlex import subprocess @@ -14,7 +16,8 @@ import sys from past.builtins import basestring -from core import CoreCommandError +from core import constants +from core.errors import CoreCommandError DEVNULL = open(os.devnull, "wb") @@ -109,17 +112,6 @@ def _is_class(module, member, clazz): return True -def _is_exe(file_path): - """ - Check if a given file path exists and is an executable file. - - :param str file_path: file path to check - :return: True if the file is considered and executable file, False otherwise - :rtype: bool - """ - return os.path.isfile(file_path) and os.access(file_path, os.X_OK) - - def close_onexec(fd): """ Close on execution of a shell process. @@ -131,17 +123,26 @@ def close_onexec(fd): fcntl.fcntl(fd, fcntl.F_SETFD, fdflags | fcntl.FD_CLOEXEC) -def check_executables(executables): +def which(command, required): """ - Check executables, verify they exist and are executable. + Find location of desired executable within current PATH. - :param list[str] executables: executable to check - :return: nothing - :raises EnvironmentError: when an executable doesn't exist or is not executable + :param str command: command to find location for + :param bool required: command is required to be found, false otherwise + :return: command location or None + :raises ValueError: when not found and required """ - for executable in executables: - if not _is_exe(executable): - raise EnvironmentError("executable not found: %s" % executable) + found_path = None + for path in os.environ["PATH"].split(os.pathsep): + command_path = os.path.join(path, command) + if os.path.isfile(command_path) and os.access(command_path, os.X_OK): + found_path = command_path + break + + if found_path is None and required: + raise ValueError("failed to find required executable(%s) in path" % command) + + return found_path def make_tuple(obj): @@ -167,7 +168,8 @@ def make_tuple_fromstr(s, value_type): :return: tuple from string :rtype: tuple """ - # remove tuple braces and strip commands and space from all values in the tuple string + # remove tuple braces and strip commands and space from all values in the tuple + # string values = [] for x in s.strip("(), ").split(","): x = x.strip("' ") @@ -178,7 +180,8 @@ def make_tuple_fromstr(s, value_type): def split_args(args): """ - Convenience method for splitting potential string commands into a shell-like syntax list. + Convenience method for splitting potential string commands into a shell-like + syntax list. :param list/str args: command list or string :return: shell-like syntax list @@ -227,8 +230,8 @@ def cmd(args, wait=True): def cmd_output(args): """ - Execute a command on the host and return a tuple containing the exit status and result string. stderr output - is folded into the stdout result string. + Execute a command on the host and return a tuple containing the exit status and + result string. stderr output is folded into the stdout result string. :param list[str]|str args: command arguments :return: command status and stdout @@ -248,14 +251,15 @@ def cmd_output(args): def check_cmd(args, **kwargs): """ - Execute a command on the host and return a tuple containing the exit status and result string. stderr output - is folded into the stdout result string. + Execute a command on the host and return a tuple containing the exit status and + result string. stderr output is folded into the stdout result string. :param list[str]|str args: command arguments :param dict kwargs: keyword arguments to pass to subprocess.Popen :return: combined stdout and stderr :rtype: str - :raises CoreCommandError: when there is a non-zero exit status or the file to execute is not found + :raises CoreCommandError: when there is a non-zero exit status or the file to + execute is not found """ kwargs["stdout"] = subprocess.PIPE kwargs["stderr"] = subprocess.STDOUT @@ -350,7 +354,7 @@ def expand_corepath(pathname, session=None, node=None): Expand a file path given session information. :param str pathname: file path to expand - :param core.emulator.session.Session session: core session object to expand path with + :param core.emulator.session.Session session: core session object to expand path :param core.nodes.base.CoreNode node: node to expand path with :return: expanded path :rtype: str @@ -383,7 +387,8 @@ def sysctl_devname(devname): def load_config(filename, d): """ - Read key=value pairs from a file, into a dict. Skip comments; strip newline characters and spacing. + Read key=value pairs from a file, into a dict. Skip comments; strip newline + characters and spacing. :param str filename: file to read into a dictionary :param dict d: dictionary to read file into @@ -444,3 +449,18 @@ def load_classes(path, clazz): ) return classes + + +def load_logging_config(config_path=None): + """ + Load CORE logging configuration file. + + :param str config_path: path to logging config file, + when None defaults to /etc/core/logging.conf + :return: nothing + """ + if not config_path: + config_path = os.path.join(constants.CORE_CONF_DIR, "logging.conf") + with open(config_path, "r") as log_config_file: + log_config = json.load(log_config_file) + logging.config.dictConfig(log_config) diff --git a/daemon/examples/api/emane80211.py b/daemon/examples/api/emane80211.py index 50c2f1ff..7cbfd586 100644 --- a/daemon/examples/api/emane80211.py +++ b/daemon/examples/api/emane80211.py @@ -6,11 +6,11 @@ import datetime import parser from builtins import range -from core import load_logging_config from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes from core.emulator.enumerations import EventTypes +from core.utils import load_logging_config load_logging_config() diff --git a/daemon/examples/api/switch.py b/daemon/examples/api/switch.py index 1e3b19e9..5b086d6a 100644 --- a/daemon/examples/api/switch.py +++ b/daemon/examples/api/switch.py @@ -9,10 +9,10 @@ import datetime import parser from builtins import range -from core import load_logging_config from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes from core.emulator.enumerations import EventTypes, NodeTypes +from core.utils import load_logging_config load_logging_config() diff --git a/daemon/examples/api/switch_inject.py b/daemon/examples/api/switch_inject.py index 105c888e..b8587760 100644 --- a/daemon/examples/api/switch_inject.py +++ b/daemon/examples/api/switch_inject.py @@ -6,9 +6,9 @@ # nodestep from builtins import range -from core import load_logging_config from core.emulator.emudata import IpPrefixes from core.emulator.enumerations import EventTypes, NodeTypes +from core.utils import load_logging_config load_logging_config() diff --git a/daemon/examples/api/wlan.py b/daemon/examples/api/wlan.py index 5db306c5..f94fd528 100644 --- a/daemon/examples/api/wlan.py +++ b/daemon/examples/api/wlan.py @@ -9,11 +9,11 @@ import datetime import parser from builtins import range -from core import load_logging_config from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes from core.location.mobility import BasicRangeModel +from core.utils import load_logging_config load_logging_config() diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 13e6e334..6614d784 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -12,12 +12,12 @@ import threading import time from configparser import ConfigParser -from core import constants, load_logging_config +from core import constants from core.api.grpc.server import CoreGrpcServer from core.api.tlv.corehandlers import CoreHandler, CoreUdpHandler from core.api.tlv.coreserver import CoreServer, CoreUdpServer from core.emulator import enumerations -from core.utils import close_onexec +from core.utils import close_onexec, load_logging_config def banner(): diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 4a327475..85836605 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -6,13 +6,13 @@ from xml.etree import ElementTree import pytest -from core import CoreError from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel from core.emulator.emudata import NodeOptions +from core.errors import CoreError _EMANE_MODELS = [ EmaneIeee80211abgModel, diff --git a/daemon/tests/test_grpc.py b/daemon/tests/test_grpc.py index ce9f5da8..f9604229 100644 --- a/daemon/tests/test_grpc.py +++ b/daemon/tests/test_grpc.py @@ -5,7 +5,6 @@ from queue import Queue import grpc import pytest -from core import CoreError from core.api.grpc import core_pb2 from core.api.grpc.client import CoreGrpcClient from core.config import ConfigShim @@ -18,6 +17,7 @@ from core.emulator.enumerations import ( ExceptionLevels, NodeTypes, ) +from core.errors import CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index 9de2914b..caff15fe 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -7,7 +7,6 @@ import time import mock import pytest -from core import CoreError from core.api.tlv import coreapi from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emulator.enumerations import ( @@ -24,6 +23,7 @@ from core.emulator.enumerations import ( RegisterTlvs, SessionTlvs, ) +from core.errors import CoreError from core.location.mobility import BasicRangeModel from core.nodes.ipaddress import Ipv4Prefix diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index 5bd7bdc7..baf0c20c 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -3,9 +3,10 @@ import time import pytest -from core import CoreError, utils +from core import utils from core.emulator.emudata import NodeOptions from core.emulator.enumerations import NodeTypes +from core.errors import CoreError MODELS = ["router", "host", "PC", "mdr"] diff --git a/daemon/tests/test_xml.py b/daemon/tests/test_xml.py index 20e84d3e..dc01c09d 100644 --- a/daemon/tests/test_xml.py +++ b/daemon/tests/test_xml.py @@ -2,9 +2,9 @@ from xml.etree import ElementTree import pytest -from core import CoreError from core.emulator.emudata import LinkOptions, NodeOptions from core.emulator.enumerations import NodeTypes +from core.errors import CoreError from core.location.mobility import BasicRangeModel from core.services.utility import SshService From d6cf09e4b8e79bcb5adf330b5c500504e60488de Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 30 Sep 2019 09:33:17 -0700 Subject: [PATCH 023/113] updated service loading to catch on_load exceptions and ignore service --- daemon/core/services/coreservices.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index 42718f5e..4563d4b7 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -249,6 +249,7 @@ class ServiceManager(object): :param CoreService service: service to add :return: nothing + :raises ValueError: when service cannot be loaded """ name = service.name logging.debug("loading service: class(%s) name(%s)", service.__name__, name) @@ -261,6 +262,13 @@ class ServiceManager(object): for executable in service.executables: which(executable, required=True) + # validate service on load succeeds + try: + service.on_load() + except Exception as e: + logging.exception("error during service(%s) on load", service.name) + raise ValueError(e) + # make service available cls.services[name] = service @@ -289,7 +297,6 @@ class ServiceManager(object): for service in services: if not service.name: continue - service.on_load() try: cls.add(service) From 1fb99e86904d06eefe7afd69d042ab0641033f71 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 30 Sep 2019 10:06:10 -0700 Subject: [PATCH 024/113] converted info to debug for some corehandlers logging --- daemon/core/api/tlv/corehandlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index bd84049c..e520ce95 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1248,10 +1248,10 @@ class CoreHandler(socketserver.BaseRequestHandler): values = [] group_strings = [] start_index = 1 - logging.info("sorted groups: %s", groups) + logging.debug("sorted groups: %s", groups) for group in groups: services = sorted(group_map[group], key=lambda x: x.name.lower()) - logging.info("sorted services for group(%s): %s", group, services) + logging.debug("sorted services for group(%s): %s", group, services) end_index = start_index + len(services) - 1 group_strings.append("%s:%s-%s" % (group, start_index, end_index)) start_index += len(services) From 6b43c00031bd6d1b435a502bdbf59bdfc6ce13b4 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 30 Sep 2019 10:36:27 -0700 Subject: [PATCH 025/113] removed xmlfilever from config, since not used --- daemon/core/emulator/session.py | 12 ++++++------ daemon/data/core.conf | 1 - daemon/scripts/core-daemon | 1 - 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 3e5877a5..1b67935c 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -1229,12 +1229,12 @@ class Session(object): """ if state == EventTypes.RUNTIME_STATE.value: self.emane.poststartup() - xml_file_version = self.options.get_config("xmlfilever") - if xml_file_version in ("1.0",): - xml_file_name = os.path.join(self.session_dir, "session-deployed.xml") - xml_writer = corexml.CoreXmlWriter(self) - corexmldeployment.CoreXmlDeployment(self, xml_writer.scenario) - xml_writer.write(xml_file_name) + + # create session deployed xml + xml_file_name = os.path.join(self.session_dir, "session-deployed.xml") + xml_writer = corexml.CoreXmlWriter(self) + corexmldeployment.CoreXmlDeployment(self, xml_writer.scenario) + xml_writer.write(xml_file_name) def get_environment(self, state=True): """ diff --git a/daemon/data/core.conf b/daemon/data/core.conf index 27fa698e..056372f1 100644 --- a/daemon/data/core.conf +++ b/daemon/data/core.conf @@ -6,7 +6,6 @@ ### core-daemon configuration options ### [core-daemon] -xmlfilever = 1.0 listenaddr = localhost port = 4038 numthreads = 1 diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 6614d784..0fa2ce69 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -93,7 +93,6 @@ def get_merged_config(filename): defaults = { "port": "%d" % enumerations.CORE_API_PORT, "listenaddr": "localhost", - "xmlfilever": "1.0", "numthreads": "1", "grpcport": "50051", "grpcaddress": "localhost", From 3f1ff113caecc14402b06f3939a5fd5cdfc82bc1 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 30 Sep 2019 11:51:40 -0700 Subject: [PATCH 026/113] small core configuration cleanup --- daemon/data/core.conf | 16 ++++------------ daemon/scripts/core-daemon | 6 +++--- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/daemon/data/core.conf b/daemon/data/core.conf index 056372f1..3a111ba0 100644 --- a/daemon/data/core.conf +++ b/daemon/data/core.conf @@ -1,10 +1,3 @@ -# Configuration file for CORE (core-gui, core-daemon) - -### GUI configuration options ### -[core-gui] -# no options are presently defined; see the ~/.core preferences file - -### core-daemon configuration options ### [core-daemon] listenaddr = localhost port = 4038 @@ -15,15 +8,14 @@ frr_bin_search = "/usr/local/bin /usr/bin /usr/lib/frr" frr_sbin_search = "/usr/local/sbin /usr/sbin /usr/lib/frr" # uncomment the following line to load custom services from the specified dir -# this may be a comma-separated list, and directory names should be unique -# and not named 'services' +# this may be a comma-separated list, and directory names should be unique +# and not named 'services' #custom_services_dir = /home/username/.core/myservices -# + # uncomment to establish a standalone control backchannel for accessing nodes # (overriden by the session option of the same name) #controlnet = 172.16.0.0/24 -# -# + # uncomment and edit to establish a distributed control backchannel #controlnet = core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 core4:172.16.4.0/24 core5:172.16.5.0/24 diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 0fa2ce69..4becdaa1 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -132,9 +132,9 @@ def get_merged_config(filename): if not cfg.has_section(section): cfg.add_section(section) - # merge command line with config file - for opt in args.__dict__: - val = args.__dict__[opt] + # merge argparse with configparser + for opt in vars(args): + val = getattr(args, opt) if val is not None: cfg.set(section, opt, str(val)) From a798774f182b10862db14b3a4c781c53227d6ba9 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 30 Sep 2019 12:58:13 -0700 Subject: [PATCH 027/113] updates to tone down warning logs to debug in broker.py and session.py --- daemon/core/api/tlv/broker.py | 2 +- daemon/core/emulator/session.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/core/api/tlv/broker.py b/daemon/core/api/tlv/broker.py index 7b3087b4..d71d050c 100644 --- a/daemon/core/api/tlv/broker.py +++ b/daemon/core/api/tlv/broker.py @@ -496,7 +496,7 @@ class CoreBroker(object): # add other nets here that do not require tunnels if isinstance(net, EmaneNet): - logging.warning("emane network does not require a tunnel") + logging.debug("emane network does not require a tunnel") return None server_interface = getattr(net, "serverintf", None) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 1b67935c..7b2a03b9 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -1721,7 +1721,7 @@ class Session(object): if net_index == 0: updown_script = self.options.get_config("controlnet_updown_script") if not updown_script: - logging.warning("controlnet updown script not configured") + logging.debug("controlnet updown script not configured") prefixes = prefix_spec.split() if len(prefixes) > 1: From e9364676a975ce62ff01e5dad374c953a3b85538 Mon Sep 17 00:00:00 2001 From: Kevin Larson Date: Tue, 1 Oct 2019 11:31:15 -0700 Subject: [PATCH 028/113] Update install.md --- docs/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index 761c589c..382f02b6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -91,7 +91,7 @@ Requires building from source, from the latest nightly snapshot. ```shell # packages needed beyond what's normally required to build core on ubuntu -sudo apt install libtool libreadline-dev +sudo apt install libtool libreadline-dev autoconf wget https://downloads.pf.itd.nrl.navy.mil/ospf-manet/nightly_snapshots/quagga-svnsnap.tgz tar xzf quagga-svnsnap.tgz From e4bb315c14dbeb26262d500d908192d9adcbcf9d Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Oct 2019 12:14:37 -0700 Subject: [PATCH 029/113] updates to netclient, providing commonly used network commands in more convenient functions --- daemon/core/emane/emanemanager.py | 8 +- daemon/core/nodes/base.py | 137 +++++++-------------- daemon/core/nodes/interface.py | 85 ++++--------- daemon/core/nodes/netclient.py | 197 ++++++++++++++++-------------- daemon/core/nodes/network.py | 4 +- daemon/core/nodes/physical.py | 96 +++++---------- 6 files changed, 203 insertions(+), 324 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index cc734e21..8d5cb54a 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -7,7 +7,7 @@ import logging import os import threading -from core import constants, utils +from core import utils from core.api.tlv import coreapi, dataconversion from core.config import ConfigGroup, ConfigShim, Configuration, ModelManager from core.emane import emanemanifest @@ -733,13 +733,11 @@ class EmaneManager(ModelManager): ) # multicast route is needed for OTA data - args = [constants.IP_BIN, "route", "add", otagroup, "dev", otadev] - node.network_cmd(args) + node.node_net_client.add_route(otagroup, otadev) # multicast route is also needed for event data if on control network if eventservicenetidx >= 0 and eventgroup != otagroup: - args = [constants.IP_BIN, "route", "add", eventgroup, "dev", eventdev] - node.network_cmd(args) + node.node_net_client.add_route(eventgroup, eventdev) # start emane args = emanecmd + [ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 2773745c..8ab5adea 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -62,6 +62,11 @@ class NodeBase(object): self.opaque = None self.position = Position() + if session.options.get_config("ovs") == "True": + self.net_client = OvsNetClient(utils.check_cmd) + else: + self.net_client = LinuxNetClient(utils.check_cmd) + def startup(self): """ Each object implements its own startup method. @@ -434,6 +439,12 @@ class CoreNode(CoreNodeBase): self.lock = threading.RLock() self._mounts = [] self.bootsh = bootsh + + if session.options.get_config("ovs") == "True": + self.node_net_client = OvsNetClient(self.network_cmd) + else: + self.node_net_client = LinuxNetClient(self.network_cmd) + if start: self.startup() @@ -489,11 +500,11 @@ class CoreNode(CoreNodeBase): # bring up the loopback interface logging.debug("bringing up loopback interface") - self.network_cmd([constants.IP_BIN, "link", "set", "lo", "up"]) + self.node_net_client.device_up("lo") # set hostname for node logging.debug("setting hostname: %s", self.name) - self.network_cmd(["hostname", self.name]) + self.node_net_client.set_hostname(self.name) # mark node as up self.up = True @@ -682,22 +693,16 @@ class CoreNode(CoreNodeBase): ) if self.up: - utils.check_cmd( - [constants.IP_BIN, "link", "set", veth.name, "netns", str(self.pid)] - ) - self.network_cmd( - [constants.IP_BIN, "link", "set", veth.name, "name", ifname] - ) - self.network_cmd( - [constants.ETHTOOL_BIN, "-K", ifname, "rx", "off", "tx", "off"] - ) + self.net_client.device_ns(veth.name, str(self.pid)) + self.node_net_client.device_name(veth.name, ifname) + self.node_net_client.checksums_off(ifname) veth.name = ifname if self.up: # TODO: potentially find better way to query interface ID # retrieve interface information - output = self.network_cmd([constants.IP_BIN, "link", "show", veth.name]) + output = self.node_net_client.device_show(veth.name) logging.debug("interface command output: %s", output) output = output.split("\n") veth.flow_id = int(output[0].strip().split(":")[0]) + 1 @@ -707,7 +712,8 @@ class CoreNode(CoreNodeBase): logging.debug("interface mac: %s - %s", veth.name, veth.hwaddr) try: - # add network interface to the node. If unsuccessful, destroy the network interface and raise exception. + # add network interface to the node. If unsuccessful, destroy the + # network interface and raise exception. self.addnetif(veth, ifindex) except ValueError as e: veth.shutdown() @@ -758,79 +764,47 @@ class CoreNode(CoreNodeBase): :return: nothing :raises CoreCommandError: when a non-zero exit status occurs """ - self._netif[ifindex].sethwaddr(addr) + interface = self._netif[ifindex] + interface.sethwaddr(addr) if self.up: - args = [ - constants.IP_BIN, - "link", - "set", - "dev", - self.ifname(ifindex), - "address", - str(addr), - ] - self.network_cmd(args) + self.node_net_client.device_mac(interface.name, str(addr)) def addaddr(self, ifindex, addr): """ Add interface address. :param int ifindex: index of interface to add address to - :param str addr: address to add to interface + :param core.nodes.ipaddress.IpAddress addr: address to add to interface :return: nothing """ + interface = self._netif[ifindex] + interface.addaddr(addr) if self.up: - # check if addr is ipv6 - if ":" in str(addr): - args = [ - constants.IP_BIN, - "addr", - "add", - str(addr), - "dev", - self.ifname(ifindex), - ] - self.network_cmd(args) - else: - args = [ - constants.IP_BIN, - "addr", - "add", - str(addr), - "broadcast", - "+", - "dev", - self.ifname(ifindex), - ] - self.network_cmd(args) - - self._netif[ifindex].addaddr(addr) + address = str(addr) + # ipv6 check + broadcast = None + if ":" not in address: + broadcast = "+" + self.node_net_client.create_address(interface.name, address, broadcast) def deladdr(self, ifindex, addr): """ Delete address from an interface. :param int ifindex: index of interface to delete address from - :param str addr: address to delete from interface + :param core.nodes.ipaddress.IpAddress addr: address to delete from interface :return: nothing :raises CoreCommandError: when a non-zero exit status occurs """ + interface = self._netif[ifindex] + try: - self._netif[ifindex].deladdr(addr) + interface.deladdr(addr) except ValueError: logging.exception("trying to delete unknown address: %s" % addr) if self.up: - self.network_cmd( - [ - constants.IP_BIN, - "addr", - "del", - str(addr), - "dev", - self.ifname(ifindex), - ] - ) + self.node_net_client.delete_address(interface.name, str(addr)) def delalladdr(self, ifindex, address_types=None): """ @@ -866,9 +840,8 @@ class CoreNode(CoreNodeBase): :return: nothing """ if self.up: - self.network_cmd( - [constants.IP_BIN, "link", "set", self.ifname(ifindex), "up"] - ) + interface_name = self.ifname(ifindex) + self.node_net_client.device_up(interface_name) def newnetif(self, net=None, addrlist=None, hwaddr=None, ifindex=None, ifname=None): """ @@ -919,7 +892,7 @@ class CoreNode(CoreNodeBase): Connect a node. :param str ifname: name of interface to connect - :param core.nodes.CoreNodeBase othernode: node to connect to + :param core.nodes.base.CoreNode othernode: node to connect to :param str otherifname: interface name to connect to :return: nothing """ @@ -930,32 +903,14 @@ class CoreNode(CoreNodeBase): tmp2 = "tmp." + "".join( [random.choice(string.ascii_lowercase) for _ in range(tmplen)] ) - utils.check_cmd( - [ - constants.IP_BIN, - "link", - "add", - "name", - tmp1, - "type", - "veth", - "peer", - "name", - tmp2, - ] - ) - - utils.check_cmd([constants.IP_BIN, "link", "set", tmp1, "netns", str(self.pid)]) - self.network_cmd([constants.IP_BIN, "link", "set", tmp1, "name", ifname]) + self.net_client.create_veth(tmp1, tmp2) + self.net_client.device_ns(tmp1, str(self.pid)) + self.node_net_client.device_name(tmp1, ifname) interface = CoreInterface(node=self, name=ifname, mtu=_DEFAULT_MTU) self.addnetif(interface, self.newifindex()) - utils.check_cmd( - [constants.IP_BIN, "link", "set", tmp2, "netns", str(othernode.pid)] - ) - othernode.network_cmd( - [constants.IP_BIN, "link", "set", tmp2, "name", otherifname] - ) + self.net_client.device_ns(tmp2, str(othernode.pid)) + othernode.node_net_client.device_name(tmp2, otherifname) other_interface = CoreInterface( node=othernode, name=otherifname, mtu=_DEFAULT_MTU ) @@ -1064,10 +1019,6 @@ class CoreNetworkBase(NodeBase): super(CoreNetworkBase, self).__init__(session, _id, name, start=start) self._linked = {} self._linked_lock = threading.Lock() - if session.options.get_config("ovs") == "True": - self.net_client = OvsNetClient() - else: - self.net_client = LinuxNetClient() def startup(self): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 210b92b9..51859e3a 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -6,8 +6,9 @@ import logging import time from builtins import int, range -from core import constants, utils +from core import utils from core.errors import CoreCommandError +from core.nodes.netclient import LinuxNetClient class CoreInterface(object): @@ -41,6 +42,7 @@ class CoreInterface(object): self.netindex = None # index used to find flow data self.flow_id = None + self.net_client = LinuxNetClient(utils.check_cmd) def startup(self): """ @@ -216,21 +218,8 @@ class Veth(CoreInterface): :return: nothing :raises CoreCommandError: when there is a command exception """ - utils.check_cmd( - [ - constants.IP_BIN, - "link", - "add", - "name", - self.localname, - "type", - "veth", - "peer", - "name", - self.name, - ] - ) - utils.check_cmd([constants.IP_BIN, "link", "set", self.localname, "up"]) + self.net_client.create_veth(self.localname, self.name) + self.net_client.device_up(self.localname) self.up = True def shutdown(self): @@ -244,15 +233,13 @@ class Veth(CoreInterface): if self.node: try: - self.node.network_cmd( - [constants.IP_BIN, "-6", "addr", "flush", "dev", self.name] - ) + self.node.node_net_client.device_flush(self.name) except CoreCommandError: logging.exception("error shutting down interface") if self.localname: try: - utils.check_cmd([constants.IP_BIN, "link", "delete", self.localname]) + self.net_client.delete_device(self.localname) except CoreCommandError: logging.info("link already removed: %s", self.localname) @@ -307,9 +294,7 @@ class TunTap(CoreInterface): return try: - self.node.network_cmd( - [constants.IP_BIN, "-6", "addr", "flush", "dev", self.name] - ) + self.node.node_net_client.device_flush(self.name) except CoreCommandError: logging.exception("error shutting down tunnel tap") @@ -357,8 +342,11 @@ class TunTap(CoreInterface): logging.debug("waiting for device local: %s", self.localname) def localdevexists(): - args = [constants.IP_BIN, "link", "show", self.localname] - return utils.cmd(args) + try: + self.net_client.device_show(self.localname) + return 0 + except CoreCommandError: + return 1 self.waitfor(localdevexists) @@ -371,9 +359,8 @@ class TunTap(CoreInterface): logging.debug("waiting for device node: %s", self.name) def nodedevexists(): - args = [constants.IP_BIN, "link", "show", self.name] try: - self.node.network_cmd(args) + self.node.node_net_client.device_show(self.name) return 0 except CoreCommandError: return 1 @@ -406,13 +393,9 @@ class TunTap(CoreInterface): """ self.waitfordevicelocal() netns = str(self.node.pid) - utils.check_cmd( - [constants.IP_BIN, "link", "set", self.localname, "netns", netns] - ) - self.node.network_cmd( - [constants.IP_BIN, "link", "set", self.localname, "name", self.name] - ) - self.node.network_cmd([constants.IP_BIN, "link", "set", self.name, "up"]) + self.net_client.device_ns(self.localname, netns) + self.node.node_net_client.device_name(self.localname, self.name) + self.node.node_net_client.device_up(self.name) def setaddrs(self): """ @@ -422,9 +405,7 @@ class TunTap(CoreInterface): """ self.waitfordevicenode() for addr in self.addrlist: - self.node.network_cmd( - [constants.IP_BIN, "addr", "add", str(addr), "dev", self.name] - ) + self.node.node_net_client.create_address(self.name, str(addr)) class GreTap(CoreInterface): @@ -478,25 +459,11 @@ class GreTap(CoreInterface): if remoteip is None: raise ValueError("missing remote IP required for GRE TAP device") - args = [ - constants.IP_BIN, - "link", - "add", - self.localname, - "type", - "gretap", - "remote", - str(remoteip), - ] - if localip: - args += ["local", str(localip)] - if ttl: - args += ["ttl", str(ttl)] - if key: - args += ["key", str(key)] - utils.check_cmd(args) - args = [constants.IP_BIN, "link", "set", self.localname, "up"] - utils.check_cmd(args) + + self.net_client.create_gretap( + self.localname, str(remoteip), str(localip), str(ttl), str(key) + ) + self.net_client.device_up(self.localname) self.up = True def shutdown(self): @@ -507,10 +474,8 @@ class GreTap(CoreInterface): """ if self.localname: try: - args = [constants.IP_BIN, "link", "set", self.localname, "down"] - utils.check_cmd(args) - args = [constants.IP_BIN, "link", "del", self.localname] - utils.check_cmd(args) + self.net_client.device_down(self.localname) + self.net_client.delete_device(self.localname) except CoreCommandError: logging.exception("error during shutdown") diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 292b6c95..adcf008d 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -2,87 +2,94 @@ Clients for dealing with bridge/interface commands. """ -import abc import os -from future.utils import with_metaclass - -from core.constants import BRCTL_BIN, IP_BIN, OVS_BIN +from core.constants import BRCTL_BIN, ETHTOOL_BIN, IP_BIN, OVS_BIN, TC_BIN from core.utils import check_cmd -class NetClientBase(with_metaclass(abc.ABCMeta)): - """ - Base client for running command line bridge/interface commands. - """ - - @abc.abstractmethod - def create_bridge(self, name): - """ - Create a network bridge to connect interfaces to. - - :param str name: bridge name - :return: nothing - """ - pass - - @abc.abstractmethod - def delete_bridge(self, name): - """ - Delete a network bridge. - - :param str name: bridge name - :return: nothing - """ - pass - - @abc.abstractmethod - def create_interface(self, bridge_name, interface_name): - """ - Create an interface associated with a network bridge. - - :param str bridge_name: bridge name - :param str interface_name: interface name - :return: nothing - """ - pass - - @abc.abstractmethod - def delete_interface(self, bridge_name, interface_name): - """ - Delete an interface associated with a network bridge. - - :param str bridge_name: bridge name - :param str interface_name: interface name - :return: nothing - """ - pass - - @abc.abstractmethod - def existing_bridges(self, _id): - """ - Checks if there are any existing bridges for a node. - - :param _id: node id to check bridges for - """ - pass - - @abc.abstractmethod - def disable_mac_learning(self, name): - """ - Disable mac learning for a bridge. - - :param str name: bridge name - :return: nothing - """ - pass - - -class LinuxNetClient(NetClientBase): +class LinuxNetClient(object): """ Client for creating Linux bridges and ip interfaces for nodes. """ + def __init__(self, run_func): + self.run_func = run_func + + def run(self, cmd): + return self.run_func(cmd) + + def set_hostname(self, name): + self.run(["hostname", name]) + + def add_route(self, route, device): + self.run([IP_BIN, "route", "add", route, "dev", device]) + + def device_up(self, device): + self.run([IP_BIN, "link", "set", device, "up"]) + + def device_down(self, device): + self.run([IP_BIN, "link", "set", device, "down"]) + + def device_name(self, device, name): + self.run([IP_BIN, "link", "set", device, "name", name]) + + def device_show(self, device): + return self.run([IP_BIN, "link", "show", device]) + + def device_ns(self, device, namespace): + self.run([IP_BIN, "link", "set", device, "netns", namespace]) + + def device_flush(self, device): + self.run([IP_BIN, "-6", "address", "flush", "dev", device]) + + def device_mac(self, device, mac): + self.run([IP_BIN, "link", "set", "dev", device, "address", mac]) + + def delete_device(self, device): + self.run([IP_BIN, "link", "delete", device]) + + def delete_tc(self, device): + self.run([TC_BIN, "qdisc", "del", "dev", device, "root"]) + + def checksums_off(self, interface_name): + self.run([ETHTOOL_BIN, "-K", interface_name, "rx", "off", "tx", "off"]) + + def delete_address(self, device, address): + self.run([IP_BIN, "address", "delete", address, "dev", device]) + + def create_veth(self, name, peer): + self.run( + [IP_BIN, "link", "add", "name", name, "type", "veth", "peer", "name", peer] + ) + + def create_gretap(self, device, address, local, ttl, key): + cmd = [IP_BIN, "link", "add", device, "type", "gretap", "remote", address] + if local is not None: + cmd.extend(["local", local]) + if ttl is not None: + cmd.extend(["ttl", ttl]) + if key is not None: + cmd.extend(["key", key]) + self.run(cmd) + + def create_address(self, device, address, broadcast=None): + if broadcast is not None: + self.run( + [ + IP_BIN, + "address", + "add", + address, + "broadcast", + broadcast, + "dev", + device, + ] + ) + else: + self.run([IP_BIN, "address", "add", address, "dev", device]) + def create_bridge(self, name): """ Create a Linux bridge and bring it up. @@ -90,10 +97,10 @@ class LinuxNetClient(NetClientBase): :param str name: bridge name :return: nothing """ - check_cmd([BRCTL_BIN, "addbr", name]) - check_cmd([BRCTL_BIN, "stp", name, "off"]) - check_cmd([BRCTL_BIN, "setfd", name, "0"]) - check_cmd([IP_BIN, "link", "set", name, "up"]) + self.run([BRCTL_BIN, "addbr", name]) + self.run([BRCTL_BIN, "stp", name, "off"]) + self.run([BRCTL_BIN, "setfd", name, "0"]) + self.device_up(name) # turn off multicast snooping so forwarding occurs w/o IGMP joins snoop = "/sys/devices/virtual/net/%s/bridge/multicast_snooping" % name @@ -108,8 +115,8 @@ class LinuxNetClient(NetClientBase): :param str name: bridge name :return: nothing """ - check_cmd([IP_BIN, "link", "set", name, "down"]) - check_cmd([BRCTL_BIN, "delbr", name]) + self.device_down(name) + self.run([BRCTL_BIN, "delbr", name]) def create_interface(self, bridge_name, interface_name): """ @@ -119,8 +126,8 @@ class LinuxNetClient(NetClientBase): :param str interface_name: interface name :return: nothing """ - check_cmd([BRCTL_BIN, "addif", bridge_name, interface_name]) - check_cmd([IP_BIN, "link", "set", interface_name, "up"]) + self.run([BRCTL_BIN, "addif", bridge_name, interface_name]) + self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): """ @@ -130,7 +137,7 @@ class LinuxNetClient(NetClientBase): :param str interface_name: interface name :return: nothing """ - check_cmd([BRCTL_BIN, "delif", bridge_name, interface_name]) + self.run([BRCTL_BIN, "delif", bridge_name, interface_name]) def existing_bridges(self, _id): """ @@ -138,7 +145,7 @@ class LinuxNetClient(NetClientBase): :param _id: node id to check bridges for """ - output = check_cmd([BRCTL_BIN, "show"]) + output = self.run([BRCTL_BIN, "show"]) lines = output.split("\n") for line in lines[1:]: columns = line.split() @@ -160,7 +167,7 @@ class LinuxNetClient(NetClientBase): check_cmd([BRCTL_BIN, "setageing", name, "0"]) -class OvsNetClient(NetClientBase): +class OvsNetClient(LinuxNetClient): """ Client for creating OVS bridges and ip interfaces for nodes. """ @@ -172,11 +179,11 @@ class OvsNetClient(NetClientBase): :param str name: bridge name :return: nothing """ - check_cmd([OVS_BIN, "add-br", name]) - check_cmd([OVS_BIN, "set", "bridge", name, "stp_enable=false"]) - check_cmd([OVS_BIN, "set", "bridge", name, "other_config:stp-max-age=6"]) - check_cmd([OVS_BIN, "set", "bridge", name, "other_config:stp-forward-delay=4"]) - check_cmd([IP_BIN, "link", "set", name, "up"]) + self.run([OVS_BIN, "add-br", name]) + self.run([OVS_BIN, "set", "bridge", name, "stp_enable=false"]) + self.run([OVS_BIN, "set", "bridge", name, "other_config:stp-max-age=6"]) + self.run([OVS_BIN, "set", "bridge", name, "other_config:stp-forward-delay=4"]) + self.device_up(name) def delete_bridge(self, name): """ @@ -185,8 +192,8 @@ class OvsNetClient(NetClientBase): :param str name: bridge name :return: nothing """ - check_cmd([IP_BIN, "link", "set", name, "down"]) - check_cmd([OVS_BIN, "del-br", name]) + self.device_down(name) + self.run([OVS_BIN, "del-br", name]) def create_interface(self, bridge_name, interface_name): """ @@ -196,8 +203,8 @@ class OvsNetClient(NetClientBase): :param str interface_name: interface name :return: nothing """ - check_cmd([OVS_BIN, "add-port", bridge_name, interface_name]) - check_cmd([IP_BIN, "link", "set", interface_name, "up"]) + self.run([OVS_BIN, "add-port", bridge_name, interface_name]) + self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): """ @@ -207,7 +214,7 @@ class OvsNetClient(NetClientBase): :param str interface_name: interface name :return: nothing """ - check_cmd([OVS_BIN, "del-port", bridge_name, interface_name]) + self.run([OVS_BIN, "del-port", bridge_name, interface_name]) def existing_bridges(self, _id): """ @@ -215,7 +222,7 @@ class OvsNetClient(NetClientBase): :param _id: node id to check bridges for """ - output = check_cmd([OVS_BIN, "list-br"]) + output = self.run([OVS_BIN, "list-br"]) if output: for line in output.split("\n"): fields = line.split(".") @@ -230,4 +237,4 @@ class OvsNetClient(NetClientBase): :param str name: bridge name :return: nothing """ - check_cmd([OVS_BIN, "set", "bridge", name, "other_config:mac-aging-time=0"]) + self.run([OVS_BIN, "set", "bridge", name, "other_config:mac-aging-time=0"]) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 1d20bc35..6af1ed9e 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -629,9 +629,7 @@ class CoreNetwork(CoreNetworkBase): return for addr in addrlist: - utils.check_cmd( - [constants.IP_BIN, "addr", "add", str(addr), "dev", self.brname] - ) + self.net_client.create_address(self.brname, str(addr)) class GreTapBridge(CoreNetwork): diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 92a9616c..0035f97a 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -100,51 +100,33 @@ class PhysicalNode(CoreNodeBase): """ Set hardware address for an interface. """ - self._netif[ifindex].sethwaddr(addr) - ifname = self.ifname(ifindex) + interface = self._netif[ifindex] + interface.sethwaddr(addr) if self.up: - self.check_cmd( - [constants.IP_BIN, "link", "set", "dev", ifname, "address", str(addr)] - ) + self.net_client.device_mac(interface.name, str(addr)) def addaddr(self, ifindex, addr): """ Add an address to an interface. """ + interface = self._netif[ifindex] if self.up: - self.check_cmd( - [ - constants.IP_BIN, - "addr", - "add", - str(addr), - "dev", - self.ifname(ifindex), - ] - ) - - self._netif[ifindex].addaddr(addr) + self.net_client.create_address(interface.name, str(addr)) + interface.addaddr(addr) def deladdr(self, ifindex, addr): """ Delete an address from an interface. """ + interface = self._netif[ifindex] + try: - self._netif[ifindex].deladdr(addr) + interface.deladdr(addr) except ValueError: logging.exception("trying to delete unknown address: %s", addr) if self.up: - self.check_cmd( - [ - constants.IP_BIN, - "addr", - "del", - str(addr), - "dev", - self.ifname(ifindex), - ] - ) + self.net_client.delete_address(interface.name, str(addr)) def adoptnetif(self, netif, ifindex, hwaddr, addrlist): """ @@ -159,12 +141,8 @@ class PhysicalNode(CoreNodeBase): # use a more reasonable name, e.g. "gt0" instead of "gt.56286.150" if self.up: - self.check_cmd( - [constants.IP_BIN, "link", "set", "dev", netif.localname, "down"] - ) - self.check_cmd( - [constants.IP_BIN, "link", "set", netif.localname, "name", netif.name] - ) + self.net_client.device_down(netif.localname) + self.net_client.device_name(netif.localname, netif.name) netif.localname = netif.name @@ -175,9 +153,7 @@ class PhysicalNode(CoreNodeBase): self.addaddr(ifindex, addr) if self.up: - self.check_cmd( - [constants.IP_BIN, "link", "set", "dev", netif.localname, "up"] - ) + self.net_client.device_up(netif.localname) def linkconfig( self, @@ -335,7 +311,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): """ # interface will also be marked up during net.attach() self.savestate() - utils.check_cmd([constants.IP_BIN, "link", "set", self.localname, "up"]) + self.net_client.device_up(self.localname) self.up = True def shutdown(self): @@ -349,18 +325,17 @@ class Rj45Node(CoreNodeBase, CoreInterface): return try: - utils.check_cmd([constants.IP_BIN, "link", "set", self.localname, "down"]) - utils.check_cmd([constants.IP_BIN, "addr", "flush", "dev", self.localname]) - utils.check_cmd( - [constants.TC_BIN, "qdisc", "del", "dev", self.localname, "root"] - ) + self.net_client.device_down(self.localname) + self.net_client.device_flush(self.localname) + self.net_client.delete_tc(self.localname) except CoreCommandError: logging.exception("error shutting down") self.up = False self.restorestate() - # TODO: issue in that both classes inherited from provide the same method with different signatures + # TODO: issue in that both classes inherited from provide the same method with + # different signatures def attachnet(self, net): """ Attach a network. @@ -370,7 +345,8 @@ class Rj45Node(CoreNodeBase, CoreInterface): """ CoreInterface.attachnet(self, net) - # TODO: issue in that both classes inherited from provide the same method with different signatures + # TODO: issue in that both classes inherited from provide the same method with + # different signatures def detachnet(self): """ Detach a network. @@ -476,9 +452,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): :raises CoreCommandError: when there is a command exception """ if self.up: - utils.check_cmd( - [constants.IP_BIN, "addr", "add", str(addr), "dev", self.name] - ) + self.net_client.create_address(self.name, str(addr)) CoreInterface.addaddr(self, addr) @@ -491,9 +465,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): :raises CoreCommandError: when there is a command exception """ if self.up: - utils.check_cmd( - [constants.IP_BIN, "addr", "del", str(addr), "dev", self.name] - ) + self.net_client.delete_address(self.name, str(addr)) CoreInterface.deladdr(self, addr) @@ -507,8 +479,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): """ self.old_up = False self.old_addrs = [] - args = [constants.IP_BIN, "addr", "show", "dev", self.localname] - output = utils.check_cmd(args) + output = self.net_client.device_show(self.localname) for line in output.split("\n"): items = line.split() if len(items) < 2: @@ -534,25 +505,14 @@ class Rj45Node(CoreNodeBase, CoreInterface): """ for addr in self.old_addrs: if addr[1] is None: - utils.check_cmd( - [constants.IP_BIN, "addr", "add", addr[0], "dev", self.localname] - ) + self.net_client.create_address(self.localname, addr[0]) else: - utils.check_cmd( - [ - constants.IP_BIN, - "addr", - "add", - addr[0], - "brd", - addr[1], - "dev", - self.localname, - ] + self.net_client.create_address( + self.localname, addr[0], broadcast=addr[1] ) if self.old_up: - utils.check_cmd([constants.IP_BIN, "link", "set", self.localname, "up"]) + self.net_client.device_up(self.localname) def setposition(self, x=None, y=None, z=None): """ From d3d70ecc54b64e76b6e60316417c3ec5ff6d52d9 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Oct 2019 12:30:32 -0700 Subject: [PATCH 030/113] removed getaddr from node clients, since it was not being used --- daemon/core/nodes/base.py | 26 ---------------------- daemon/core/nodes/client.py | 43 ------------------------------------- daemon/core/nodes/docker.py | 34 ----------------------------- daemon/core/nodes/lxd.py | 34 ----------------------------- daemon/tests/test_core.py | 1 - 5 files changed, 138 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 8ab5adea..69aa575c 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -806,32 +806,6 @@ class CoreNode(CoreNodeBase): if self.up: self.node_net_client.delete_address(interface.name, str(addr)) - def delalladdr(self, ifindex, address_types=None): - """ - Delete all addresses from an interface. - - :param int ifindex: index of interface to delete address types from - :param tuple[str] address_types: address types to delete - :return: nothing - :raises CoreCommandError: when a non-zero exit status occurs - """ - if not address_types: - address_types = self.valid_address_types - - interface_name = self.ifname(ifindex) - addresses = self.client.getaddr(interface_name, rescan=True) - - for address_type in address_types: - if address_type not in self.valid_address_types: - raise ValueError( - "addr type must be in: %s" % " ".join(self.valid_address_types) - ) - for address in addresses[address_type]: - self.deladdr(ifindex, address) - - # update cached information - self.client.getaddr(interface_name, rescan=True) - def ifup(self, ifindex): """ Bring an interface up. diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 4bfef967..00833ef4 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -26,7 +26,6 @@ class VnodeClient(object): """ self.name = name self.ctrlchnlname = ctrlchnlname - self._addr = {} def _verify_connection(self): """ @@ -238,48 +237,6 @@ class VnodeClient(object): """ return self.cmd_output([sh, "-c", cmd]) - def getaddr(self, ifname, rescan=False): - """ - Get address for interface on node. - - :param str ifname: interface name to get address for - :param bool rescan: rescan flag - :return: interface information - :rtype: dict - """ - if ifname in self._addr and not rescan: - return self._addr[ifname] - - interface = {"ether": [], "inet": [], "inet6": [], "inet6link": []} - args = [constants.IP_BIN, "addr", "show", "dev", ifname] - p, stdin, stdout, stderr = self.popen(args) - stdin.close() - - for line in stdout: - line = line.strip().split() - if line[0] == "link/ether": - interface["ether"].append(line[1]) - elif line[0] == "inet": - interface["inet"].append(line[1]) - elif line[0] == "inet6": - if line[3] == "global": - interface["inet6"].append(line[1]) - elif line[3] == "link": - interface["inet6link"].append(line[1]) - else: - logging.warning("unknown scope: %s" % line[3]) - - err = stderr.read() - stdout.close() - stderr.close() - status = p.wait() - if status: - logging.warning("nonzero exist status (%s) for cmd: %s", status, args) - if err: - logging.warning("error output: %s", err) - self._addr[ifname] = interface - return interface - def netifstats(self, ifname=None): """ Retrieve network interface state. diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index eecc1175..7bf041e0 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -95,40 +95,6 @@ class DockerClient(object): if status: raise CoreCommandError(status, args, output) - def getaddr(self, ifname, rescan=False): - """ - Get address for interface on node. - - :param str ifname: interface name to get address for - :param bool rescan: rescan flag - :return: interface information - :rtype: dict - """ - if ifname in self._addr and not rescan: - return self._addr[ifname] - - interface = {"ether": [], "inet": [], "inet6": [], "inet6link": []} - args = ["ip", "addr", "show", "dev", ifname] - status, output = self.ns_cmd(args) - for line in output: - line = line.strip().split() - if line[0] == "link/ether": - interface["ether"].append(line[1]) - elif line[0] == "inet": - interface["inet"].append(line[1]) - elif line[0] == "inet6": - if line[3] == "global": - interface["inet6"].append(line[1]) - elif line[3] == "link": - interface["inet6link"].append(line[1]) - else: - logging.warning("unknown scope: %s" % line[3]) - - if status: - logging.warning("nonzero exist status (%s) for cmd: %s", status, args) - self._addr[ifname] = interface - return interface - class DockerNode(CoreNode): apitype = NodeTypes.DOCKER.value diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index ff1e60b8..30c021ce 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -91,40 +91,6 @@ class LxdClient(object): if status: raise CoreCommandError(status, args, output) - def getaddr(self, ifname, rescan=False): - """ - Get address for interface on node. - - :param str ifname: interface name to get address for - :param bool rescan: rescan flag - :return: interface information - :rtype: dict - """ - if ifname in self._addr and not rescan: - return self._addr[ifname] - - interface = {"ether": [], "inet": [], "inet6": [], "inet6link": []} - args = ["ip", "addr", "show", "dev", ifname] - status, output = self.ns_cmd_output(args) - for line in output: - line = line.strip().split() - if line[0] == "link/ether": - interface["ether"].append(line[1]) - elif line[0] == "inet": - interface["inet"].append(line[1]) - elif line[0] == "inet6": - if line[3] == "global": - interface["inet6"].append(line[1]) - elif line[3] == "link": - interface["inet6link"].append(line[1]) - else: - logging.warning("unknown scope: %s" % line[3]) - - if status: - logging.warning("nonzero exist status (%s) for cmd: %s", status, args) - self._addr[ifname] = interface - return interface - class LxcNode(CoreNode): apitype = NodeTypes.LXC.value diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index f5f2c3e3..47c2a716 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -127,7 +127,6 @@ class TestCore: assert createclients(session.session_dir) # check convenience methods for interface information - assert client.getaddr("eth0") assert client.netifstats() def test_netif(self, session, ip_prefixes): From 223590c8fbf93a0c4c50257264c721ac32344d34 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Oct 2019 12:52:25 -0700 Subject: [PATCH 031/113] removed vnode client netif stats, since was not being used --- daemon/core/nodes/client.py | 44 ------------------------------------- daemon/tests/test_core.py | 3 --- 2 files changed, 47 deletions(-) diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 00833ef4..65dd77e0 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -236,47 +236,3 @@ class VnodeClient(object): :rtype: tuple[int, str] """ return self.cmd_output([sh, "-c", cmd]) - - def netifstats(self, ifname=None): - """ - Retrieve network interface state. - - :param str ifname: name of interface to get state for - :return: interface state information - :rtype: dict - """ - stats = {} - args = ["cat", "/proc/net/dev"] - p, stdin, stdout, stderr = self.popen(args) - stdin.close() - # ignore first line - stdout.readline() - # second line has count names - tmp = stdout.readline().decode("utf-8").strip().split("|") - rxkeys = tmp[1].split() - txkeys = tmp[2].split() - for line in stdout: - line = line.decode("utf-8").strip().split() - devname, tmp = line[0].split(":") - if tmp: - line.insert(1, tmp) - stats[devname] = {"rx": {}, "tx": {}} - field = 1 - for count in rxkeys: - stats[devname]["rx"][count] = int(line[field]) - field += 1 - for count in txkeys: - stats[devname]["tx"][count] = int(line[field]) - field += 1 - err = stderr.read() - stdout.close() - stderr.close() - status = p.wait() - if status: - logging.warning("nonzero exist status (%s) for cmd: %s", status, args) - if err: - logging.warning("error output: %s", err) - if ifname is not None: - return stats[ifname] - else: - return stats diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 47c2a716..5c3a627a 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -126,9 +126,6 @@ class TestCore: # check module methods assert createclients(session.session_dir) - # check convenience methods for interface information - assert client.netifstats() - def test_netif(self, session, ip_prefixes): """ Test netif methods. From af7faa85df2590c3330180d2cab0e48c82b5bb1c Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Oct 2019 14:40:24 -0700 Subject: [PATCH 032/113] removed redircmd, shcmd, shcmd_result, getaddr, netifstats from VnodeClient --- daemon/core/nodes/base.py | 20 ++++---------- daemon/core/nodes/client.py | 52 ------------------------------------- daemon/core/nodes/docker.py | 1 - daemon/core/nodes/lxd.py | 1 - daemon/tests/test_core.py | 6 ----- 5 files changed, 5 insertions(+), 75 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 69aa575c..5e665dd8 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -17,7 +17,6 @@ from socket import AF_INET, AF_INET6 from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes -from core.errors import CoreCommandError from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, OvsNetClient @@ -636,15 +635,8 @@ class CoreNode(CoreNodeBase): """ source = os.path.abspath(source) logging.debug("node(%s) mounting: %s at %s", self.name, source, target) - cmd = 'mkdir -p "%s" && %s -n --bind "%s" "%s"' % ( - target, - constants.MOUNT_BIN, - source, - target, - ) - status, output = self.client.shcmd_result(cmd) - if status: - raise CoreCommandError(status, cmd, output) + self.client.check_cmd(["mkdir", "-p", target]) + self.client.check_cmd([constants.MOUNT_BIN, "-n", "--bind", source, target]) self._mounts.append((source, target)) def newifindex(self): @@ -901,11 +893,9 @@ class CoreNode(CoreNodeBase): """ logging.info("adding file from %s to %s", srcname, filename) directory = os.path.dirname(filename) - - cmd = 'mkdir -p "%s" && mv "%s" "%s" && sync' % (directory, srcname, filename) - status, output = self.client.shcmd_result(cmd) - if status: - raise CoreCommandError(status, cmd, output) + self.client.check_cmd(["mkdir", "-p", directory]) + self.client.check_cmd(["mv", srcname, filename]) + self.client.check_cmd(["sync"]) def hostfilename(self, filename): """ diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 65dd77e0..6b7fa44c 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -145,36 +145,6 @@ class VnodeClient(object): *args ) - def redircmd(self, infd, outfd, errfd, args, wait=True): - """ - Execute a command on a node with standard input, output, and - error redirected according to the given file descriptors. - - :param infd: stdin file descriptor - :param outfd: stdout file descriptor - :param errfd: stderr file descriptor - :param list[str]|str args: command arguments - :param bool wait: wait flag - :return: command status - :rtype: int - """ - self._verify_connection() - - # run command, return process when not waiting - args = utils.split_args(args) - cmd = self._cmd_args() + args - logging.debug("redircmd: %s", cmd) - p = Popen(cmd, stdin=infd, stdout=outfd, stderr=errfd) - - if not wait: - return p - - # wait for and return exit status - status = p.wait() - if status: - logging.warning("cmd exited with status %s: %s", status, args) - return status - def term(self, sh="/bin/sh"): """ Open a terminal on a node. @@ -214,25 +184,3 @@ class VnodeClient(object): :return: str """ return "%s -c %s -- %s" % (constants.VCMD_BIN, self.ctrlchnlname, sh) - - def shcmd(self, cmd, sh="/bin/sh"): - """ - Execute a shell command. - - :param str cmd: command string - :param str sh: shell to run command in - :return: command result - :rtype: int - """ - return self.cmd([sh, "-c", cmd]) - - def shcmd_result(self, cmd, sh="/bin/sh"): - """ - Execute a shell command and return the exist status and combined output. - - :param str cmd: shell command to run - :param str sh: shell to run command in - :return: exist status and combined output - :rtype: tuple[int, str] - """ - return self.cmd_output([sh, "-c", cmd]) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 7bf041e0..2587177b 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -13,7 +13,6 @@ class DockerClient(object): self.name = name self.image = image self.pid = None - self._addr = {} def create_container(self): utils.check_cmd( diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 30c021ce..2af071fa 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -14,7 +14,6 @@ class LxdClient(object): self.name = name self.image = image self.pid = None - self._addr = {} def create_container(self): utils.check_cmd( diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 5c3a627a..e120dcc8 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -4,7 +4,6 @@ Unit tests for testing basic CORE networks. import os import stat -import subprocess import threading import pytest @@ -109,10 +108,6 @@ class TestCore: p, stdin, stdout, stderr = client.popen(command) assert not p.wait() assert not client.icmd(command) - assert not client.redircmd( - subprocess.PIPE, subprocess.PIPE, subprocess.PIPE, command - ) - assert not client.shcmd(command[0]) # check various command using command line assert not client.cmd(command) @@ -121,7 +116,6 @@ class TestCore: p, stdin, stdout, stderr = client.popen(command) assert not p.wait() assert not client.icmd(command) - assert not client.shcmd(command[0]) # check module methods assert createclients(session.session_dir) From 4079df973905ae0f81b08eb5397b472f1ad06e45 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Oct 2019 15:38:23 -0700 Subject: [PATCH 033/113] added pydocs for netclient --- daemon/core/emane/emanemanager.py | 4 +- daemon/core/nodes/base.py | 13 ++- daemon/core/nodes/netclient.py | 157 +++++++++++++++++++++++++----- 3 files changed, 148 insertions(+), 26 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 8d5cb54a..746016f9 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -733,11 +733,11 @@ class EmaneManager(ModelManager): ) # multicast route is needed for OTA data - node.node_net_client.add_route(otagroup, otadev) + node.node_net_client.create_route(otagroup, otadev) # multicast route is also needed for event data if on control network if eventservicenetidx >= 0 and eventgroup != otagroup: - node.node_net_client.add_route(eventgroup, eventdev) + node.node_net_client.create_route(eventgroup, eventdev) # start emane args = emanecmd + [ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 5e665dd8..c261460f 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -53,7 +53,7 @@ class NodeBase(object): self.type = None self.server = None self.services = None - # ifindex is key, PyCoreNetIf instance is value + # ifindex is key, CoreInterface instance is value self._netif = {} self.ifindex = 0 self.canvas = None @@ -364,6 +364,17 @@ class CoreNodeBase(NodeBase): return common + def network_cmd(self, args): + """ + Runs a command for a node that is used to configure and setup network interfaces. + + :param list[str]|str args: command to run + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + raise NotImplementedError + def check_cmd(self, args): """ Runs shell command on node. diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index adcf008d..34fee343 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -13,67 +13,136 @@ class LinuxNetClient(object): Client for creating Linux bridges and ip interfaces for nodes. """ - def __init__(self, run_func): - self.run_func = run_func + def __init__(self, run): + """ + Create LinuxNetClient instance. - def run(self, cmd): - return self.run_func(cmd) + :param run: function to run commands with + """ + self.run = run def set_hostname(self, name): + """ + Set network hostname. + + :param str name: name for hostname + :return: nothing + """ self.run(["hostname", name]) - def add_route(self, route, device): + def create_route(self, route, device): + """ + Create a new route for a device. + + :param str route: route to create + :param str device: device to add route to + :return: nothing + """ self.run([IP_BIN, "route", "add", route, "dev", device]) def device_up(self, device): + """ + Bring a device up. + + :param str device: device to bring up + :return: nothing + """ self.run([IP_BIN, "link", "set", device, "up"]) def device_down(self, device): + """ + Bring a device down. + + :param str device: device to bring down + :return: nothing + """ self.run([IP_BIN, "link", "set", device, "down"]) def device_name(self, device, name): + """ + Set a device name. + + :param str device: device to set name for + :param str name: name to set + :return: nothing + """ self.run([IP_BIN, "link", "set", device, "name", name]) def device_show(self, device): + """ + Show information for a device. + + :param str device: device to get information for + :return: device information + :rtype: str + """ return self.run([IP_BIN, "link", "show", device]) def device_ns(self, device, namespace): + """ + Set netns for a device. + + :param str device: device to setns for + :param str namespace: namespace to set device to + :return: nothing + """ self.run([IP_BIN, "link", "set", device, "netns", namespace]) def device_flush(self, device): + """ + Flush device addresses. + + :param str device: device to flush + :return: nothing + """ self.run([IP_BIN, "-6", "address", "flush", "dev", device]) def device_mac(self, device, mac): + """ + Set MAC address for a device. + + :param str device: device to set mac for + :param str mac: mac to set + :return: nothing + """ self.run([IP_BIN, "link", "set", "dev", device, "address", mac]) def delete_device(self, device): + """ + Delete device. + + :param str device: device to delete + :return: nothing + """ self.run([IP_BIN, "link", "delete", device]) def delete_tc(self, device): + """ + Remove traffic control settings for a device. + + :param str device: device to remove tc + :return: nothing + """ self.run([TC_BIN, "qdisc", "del", "dev", device, "root"]) def checksums_off(self, interface_name): + """ + Turns interface checksums off. + + :param str interface_name: interface to update + :return: nothing + """ self.run([ETHTOOL_BIN, "-K", interface_name, "rx", "off", "tx", "off"]) - def delete_address(self, device, address): - self.run([IP_BIN, "address", "delete", address, "dev", device]) - - def create_veth(self, name, peer): - self.run( - [IP_BIN, "link", "add", "name", name, "type", "veth", "peer", "name", peer] - ) - - def create_gretap(self, device, address, local, ttl, key): - cmd = [IP_BIN, "link", "add", device, "type", "gretap", "remote", address] - if local is not None: - cmd.extend(["local", local]) - if ttl is not None: - cmd.extend(["ttl", ttl]) - if key is not None: - cmd.extend(["key", key]) - self.run(cmd) - def create_address(self, device, address, broadcast=None): + """ + Create address for a device. + + :param str device: device to add address to + :param str address: address to add + :param str broadcast: broadcast address to use, default is None + :return: nothing + """ if broadcast is not None: self.run( [ @@ -90,6 +159,48 @@ class LinuxNetClient(object): else: self.run([IP_BIN, "address", "add", address, "dev", device]) + def delete_address(self, device, address): + """ + Delete an address from a device. + + :param str device: targeted device + :param str address: address to remove + :return: nothing + """ + self.run([IP_BIN, "address", "delete", address, "dev", device]) + + def create_veth(self, name, peer): + """ + Create a veth pair. + + :param str name: veth name + :param str peer: peer name + :return: nothing + """ + self.run( + [IP_BIN, "link", "add", "name", name, "type", "veth", "peer", "name", peer] + ) + + def create_gretap(self, device, address, local, ttl, key): + """ + Create a GRE tap on a device. + + :param str device: device to add tap to + :param str address: address to add tap for + :param str local: local address to tie to + :param str ttl: time to live value + :param str key: key for tap + :return: nothing + """ + cmd = [IP_BIN, "link", "add", device, "type", "gretap", "remote", address] + if local is not None: + cmd.extend(["local", local]) + if ttl is not None: + cmd.extend(["ttl", ttl]) + if key is not None: + cmd.extend(["key", key]) + self.run(cmd) + def create_bridge(self, name): """ Create a Linux bridge and bring it up. From 0fe8bea25b92c2152f04145a50323f50b209fd20 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Oct 2019 21:06:11 -0700 Subject: [PATCH 034/113] added net_cmd to NodeBase for encapsulating network configuration commands --- daemon/core/nodes/base.py | 30 ++++++++++++++++++++++-------- daemon/core/nodes/docker.py | 2 +- daemon/core/nodes/lxd.py | 2 +- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index c261460f..cb1169f8 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -62,9 +62,9 @@ class NodeBase(object): self.position = Position() if session.options.get_config("ovs") == "True": - self.net_client = OvsNetClient(utils.check_cmd) + self.net_client = OvsNetClient(self.net_cmd) else: - self.net_client = LinuxNetClient(utils.check_cmd) + self.net_client = LinuxNetClient(self.net_cmd) def startup(self): """ @@ -82,6 +82,18 @@ class NodeBase(object): """ raise NotImplementedError + def net_cmd(self, args): + """ + Runs a command that is used to configure and setup the network on the host + system. + + :param list[str]|str args: command to run + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + utils.check_cmd(args) + def setposition(self, x=None, y=None, z=None): """ Set the (x,y,z) position of the object. @@ -364,9 +376,10 @@ class CoreNodeBase(NodeBase): return common - def network_cmd(self, args): + def node_net_cmd(self, args): """ - Runs a command for a node that is used to configure and setup network interfaces. + Runs a command that is used to configure and setup the network within a + node. :param list[str]|str args: command to run :return: combined stdout and stderr @@ -451,9 +464,9 @@ class CoreNode(CoreNodeBase): self.bootsh = bootsh if session.options.get_config("ovs") == "True": - self.node_net_client = OvsNetClient(self.network_cmd) + self.node_net_client = OvsNetClient(self.node_net_cmd) else: - self.node_net_client = LinuxNetClient(self.network_cmd) + self.node_net_client = LinuxNetClient(self.node_net_cmd) if start: self.startup() @@ -589,9 +602,10 @@ class CoreNode(CoreNodeBase): """ return self.client.cmd_output(args) - def network_cmd(self, args): + def node_net_cmd(self, args): """ - Runs a command for a node that is used to configure and setup network interfaces. + Runs a command that is used to configure and setup the network within a + node. :param list[str]|str args: command to run :return: combined stdout and stderr diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 2587177b..ad7deff2 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -190,7 +190,7 @@ class DockerNode(CoreNode): raise CoreCommandError(status, args, output) return output - def network_cmd(self, args): + def node_net_cmd(self, args): if not self.up: logging.debug("node down, not running network command: %s", args) return 0 diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 2af071fa..86ca192e 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -193,7 +193,7 @@ class LxcNode(CoreNode): raise CoreCommandError(status, args, output) return output - def network_cmd(self, args): + def node_net_cmd(self, args): if not self.up: logging.debug("node down, not running network command: %s", args) return 0 From 6c3066e7520e8125e5a136d26b3cd6e595d604b8 Mon Sep 17 00:00:00 2001 From: Jeff Ahrenholz Date: Wed, 2 Oct 2019 07:55:51 -0700 Subject: [PATCH 035/113] cherry-pick commit e56c001 for Session() instead of EmuSession() class --- daemon/core/emulator/coreemu.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 1c0923de..9c8b35ee 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -85,12 +85,13 @@ class CoreEmu(object): session = sessions[_id] session.shutdown() - def create_session(self, _id=None, master=True): + def create_session(self, _id=None, master=True, _cls=Session): """ Create a new CORE session, set to master if running standalone. :param int _id: session id for new session :param bool master: sets session to master + :param class _cls: Session class to use :return: created session :rtype: EmuSession """ @@ -100,7 +101,7 @@ class CoreEmu(object): if _id not in self.sessions: break - session = Session(_id, config=self.config) + session = _cls(_id, config=self.config) logging.info("created session: %s", _id) if master: session.master = True From 572ed48241f19a36e2c312531152c2a1ebdc7746 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 2 Oct 2019 08:58:25 -0700 Subject: [PATCH 036/113] added ethtool to configure.ac, updated other program checks to fail when not found and required --- configure.ac | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 07c8150a..db8b0294 100644 --- a/configure.ac +++ b/configure.ac @@ -123,32 +123,55 @@ if test "x$enable_daemon" = "xyes"; then if test "x$brctl_path" = "xno" ; then AC_MSG_ERROR([Could not locate brctl (from bridge-utils package).]) fi + AC_CHECK_PROG(sysctl_path, sysctl, $as_dir, no, $SEARCHPATH) + if test "x$sysctl_path" = "xno" ; then + AC_MSG_ERROR([Could not locate sysctl (from procps package).]) + fi + AC_CHECK_PROG(ebtables_path, ebtables, $as_dir, no, $SEARCHPATH) if test "x$ebtables_path" = "xno" ; then AC_MSG_ERROR([Could not locate ebtables (from ebtables package).]) fi + AC_CHECK_PROG(ip_path, ip, $as_dir, no, $SEARCHPATH) if test "x$ip_path" = "xno" ; then AC_MSG_ERROR([Could not locate ip (from iproute package).]) fi + AC_CHECK_PROG(tc_path, tc, $as_dir, no, $SEARCHPATH) if test "x$tc_path" = "xno" ; then AC_MSG_ERROR([Could not locate tc (from iproute package).]) fi + + AC_CHECK_PROG(ethtool_path, ethtool, $as_dir, no, $SEARCHPATH) + if test "x$ethtool_path" = "xno" ; then + AC_MSG_ERROR([Could not locate ethtool (from package ethtool)]) + fi + AC_CHECK_PROG(mount_path, mount, $as_dir, no, $SEARCHPATH) + if test "x$mount_path" = "xno" ; then + AC_MSG_ERROR([Could not locate mount (from package mount)]) + fi + AC_CHECK_PROG(umount_path, umount, $as_dir, no, $SEARCHPATH) + if test "x$umount_path" = "xno" ; then + AC_MSG_ERROR([Could not locate umount (from package mount)]) + fi + AC_CHECK_PROG(convert, convert, yes, no, $SEARCHPATH) if test "x$convert" = "xno" ; then AC_MSG_WARN([Could not locate ImageMagick convert.]) fi + AC_CHECK_PROG(ovs_vs_path, ovs-vsctl, $as_dir, no, $SEARCHPATH) if test "x$ovs_vs_path" = "xno" ; then - AC_MSG_WARN([Could not locate ovs-vsctl cannot use OVS nodes]) + AC_MSG_WARN([Could not locate ovs-vsctl cannot use OVS mode]) fi + AC_CHECK_PROG(ovs_of_path, ovs-ofctl, $as_dir, no, $SEARCHPATH) if test "x$ovs_of_path" = "xno" ; then - AC_MSG_WARN([Could not locate ovs-ofctl cannot use OVS nodes]) + AC_MSG_WARN([Could not locate ovs-ofctl cannot use OVS mode]) fi CFLAGS_save=$CFLAGS From 6d0d183788856724d6c79e471c84e4e768ca1e7a Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 11:59:45 -0700 Subject: [PATCH 037/113] fixed returning output from NodeBase.net_cmd --- daemon/core/nodes/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index cb1169f8..ff34984d 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -92,7 +92,7 @@ class NodeBase(object): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - utils.check_cmd(args) + return utils.check_cmd(args) def setposition(self, x=None, y=None, z=None): """ From 319c10aa341e71860e85b3e7a09078fd13d4b84c Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 12:11:12 -0700 Subject: [PATCH 038/113] added notes about installing python/pip to install doc --- docs/install.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/install.md b/docs/install.md index fa8018e9..822642fa 100644 --- a/docs/install.md +++ b/docs/install.md @@ -42,6 +42,21 @@ Install Path | Description /etc/init.d/core-daemon|SysV startup script for daemon /etc/systemd/system/core-daemon.service|Systemd startup script for daemon +# Pre-Req Installing Python + +You may already have these installed, and can ignore this step if so, but if + needed you can run the following to install python and pip. + +```shell +# python 2 +sudo apt install python +sudo apt install python-pip + +# python 3 +sudo apt install python3 +sudo apt install python3-pip +``` + # Pre-Req Python Requirements The newly added gRPC API which depends on python library grpcio is not commonly found within system repos. From 41f7f46988640ecba942a64b69d75c542446084a Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 12:41:56 -0700 Subject: [PATCH 039/113] removed old examples, updated examples api to denote they are python script examples --- README.md | 2 +- daemon/examples/eventloop.py | 44 -- daemon/examples/netns/daemonnodes.py | 212 ------ daemon/examples/netns/distributed.py | 151 ----- daemon/examples/netns/howmanynodes.py | 247 ------- daemon/examples/netns/ospfmanetmdrtest.py | 637 ------------------ daemon/examples/{api => python}/emane80211.py | 0 daemon/examples/{api => python}/parser.py | 0 daemon/examples/{api => python}/switch.py | 0 .../examples/{api => python}/switch_inject.py | 0 daemon/examples/{api => python}/wlan.py | 0 daemon/examples/stopsession.py | 50 -- 12 files changed, 1 insertion(+), 1342 deletions(-) delete mode 100644 daemon/examples/eventloop.py delete mode 100755 daemon/examples/netns/daemonnodes.py delete mode 100755 daemon/examples/netns/distributed.py delete mode 100755 daemon/examples/netns/howmanynodes.py delete mode 100755 daemon/examples/netns/ospfmanetmdrtest.py rename daemon/examples/{api => python}/emane80211.py (100%) rename daemon/examples/{api => python}/parser.py (100%) rename daemon/examples/{api => python}/switch.py (100%) rename daemon/examples/{api => python}/switch_inject.py (100%) rename daemon/examples/{api => python}/wlan.py (100%) delete mode 100755 daemon/examples/stopsession.py diff --git a/README.md b/README.md index 099467f0..cd77ef4d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ scripting network emulation. * Documentation hosted on GitHub * * Basic Script Examples - * [Examples](daemon/examples/api) + * [Examples](daemon/examples/python) * Custom Service Example * [sample.py](daemon/examples/myservices/sample.py) * Custom Emane Model Example diff --git a/daemon/examples/eventloop.py b/daemon/examples/eventloop.py deleted file mode 100644 index d602ee50..00000000 --- a/daemon/examples/eventloop.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging -import time - -from core.location.event import EventLoop - - -def main(): - loop = EventLoop() - - def msg(arg): - delta = time.time() - loop.start - logging.debug("%s arg: %s", delta, arg) - - def repeat(interval, count): - count -= 1 - msg("repeat: interval: %s; remaining: %s" % (interval, count)) - if count > 0: - loop.add_event(interval, repeat, interval, count) - - def sleep(delay): - msg("sleep %s" % delay) - time.sleep(delay) - msg("sleep done") - - def stop(arg): - msg(arg) - loop.stop() - - loop.add_event(0, msg, "start") - loop.add_event(0, msg, "time zero") - - for delay in 5, 4, 10, -1, 0, 9, 3, 7, 3.14: - loop.add_event(delay, msg, "time %s" % delay) - - loop.run() - - loop.add_event(0, repeat, 1, 5) - loop.add_event(12, sleep, 10) - - loop.add_event(15.75, stop, "stop time: 15.75") - - -if __name__ == "__main__": - main() diff --git a/daemon/examples/netns/daemonnodes.py b/daemon/examples/netns/daemonnodes.py deleted file mode 100755 index a17c34d5..00000000 --- a/daemon/examples/netns/daemonnodes.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/python -i - -# Copyright (c)2010-2013 the Boeing Company. -# See the LICENSE file included in this distribution. - -# A distributed example where CORE API messaging is used to create a session -# on a daemon server. The daemon server defaults to 127.0.0.1:4038 -# to target a remote machine specify "-d " parameter, it needs to be -# running the daemon with listenaddr=0.0.0.0 in the core.conf file. -# This script creates no nodes locally and therefore can be run as an -# unprivileged user. - -import datetime -import optparse -import sys -from builtins import range - -import core.nodes.base -import core.nodes.network -from core.api.tlv import coreapi, dataconversion -from core.api.tlv.coreapi import CoreExecuteTlv -from core.emulator.enumerations import ( - CORE_API_PORT, - EventTlvs, - EventTypes, - ExecuteTlvs, - LinkTlvs, - LinkTypes, - MessageFlags, - MessageTypes, -) -from core.emulator.session import Session -from core.nodes import ipaddress - -# declare classes for use with Broker - -# node list (count from 1) -n = [None] -exec_num = 1 - - -def cmd(node, exec_cmd): - """ - :param node: The node the command should be issued too - :param exec_cmd: A string with the command to be run - :return: Returns the result of the command - """ - global exec_num - - # Set up the command api message - tlvdata = CoreExecuteTlv.pack(ExecuteTlvs.NODE.value, node.id) - tlvdata += CoreExecuteTlv.pack(ExecuteTlvs.NUMBER.value, exec_num) - tlvdata += CoreExecuteTlv.pack(ExecuteTlvs.COMMAND.value, exec_cmd) - msg = coreapi.CoreExecMessage.pack( - MessageFlags.STRING.value | MessageFlags.TEXT.value, tlvdata - ) - node.session.broker.handlerawmsg(msg) - exec_num += 1 - - # Now wait for the response - server = node.session.broker.servers["localhost"] - server.sock.settimeout(50.0) - - # receive messages until we get our execute response - result = None - while True: - msghdr = server.sock.recv(coreapi.CoreMessage.header_len) - msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(msghdr) - msgdata = server.sock.recv(msglen) - - # If we get the right response return the results - print("received response message: %s" % MessageTypes(msgtype)) - if msgtype == MessageTypes.EXECUTE.value: - msg = coreapi.CoreExecMessage(msgflags, msghdr, msgdata) - result = msg.get_tlv(ExecuteTlvs.RESULT.value) - break - - return result - - -def main(): - usagestr = "usage: %prog [-n] number of nodes [-d] daemon address" - parser = optparse.OptionParser(usage=usagestr) - parser.set_defaults(numnodes=5, daemon="127.0.0.1:" + str(CORE_API_PORT)) - - parser.add_option( - "-n", "--numnodes", dest="numnodes", type=int, help="number of nodes" - ) - parser.add_option( - "-d", - "--daemon-server", - dest="daemon", - type=str, - help="daemon server IP address", - ) - - def usage(msg=None, err=0): - sys.stdout.write("\n") - if msg: - sys.stdout.write(msg + "\n\n") - parser.print_help() - sys.exit(err) - - # parse command line options - (options, args) = parser.parse_args() - - if options.numnodes < 1: - usage("invalid number of nodes: %s" % options.numnodes) - if not options.daemon: - usage("daemon server IP address (-d) is a required argument") - - for a in args: - sys.stderr.write("ignoring command line argument: %s\n" % a) - - start = datetime.datetime.now() - - prefix = ipaddress.Ipv4Prefix("10.83.0.0/16") - session = Session(1) - server = globals().get("server") - if server: - server.addsession(session) - - # distributed setup - connect to daemon server - daemonport = options.daemon.split(":") - daemonip = daemonport[0] - - # Localhost is already set in the session but we change it to be the remote daemon - # This stops the remote daemon trying to build a tunnel back which would fail - daemon = "localhost" - if len(daemonport) > 1: - port = int(daemonport[1]) - else: - port = CORE_API_PORT - print("connecting to daemon at %s:%d" % (daemon, port)) - session.broker.addserver(daemon, daemonip, port) - - # Set the local session id to match the port. - # Not necessary but seems neater. - session.broker.setupserver(daemon) - - # We do not want the recvloop running as we will deal ourselves - session.broker.dorecvloop = False - - # Change to configuration state on both machines - session.set_state(EventTypes.CONFIGURATION_STATE) - tlvdata = coreapi.CoreEventTlv.pack( - EventTlvs.TYPE.value, EventTypes.CONFIGURATION_STATE.value - ) - session.broker.handlerawmsg(coreapi.CoreEventMessage.pack(0, tlvdata)) - - flags = MessageFlags.ADD.value - switch = core.nodes.network.SwitchNode(session=session, name="switch", start=False) - switch.setposition(x=80, y=50) - switch.server = daemon - switch_data = switch.data(flags) - switch_message = dataconversion.convert_node(switch_data) - session.broker.handlerawmsg(switch_message) - - number_of_nodes = options.numnodes - - print( - "creating %d remote nodes with addresses from %s" % (options.numnodes, prefix) - ) - - # create remote nodes via API - for i in range(1, number_of_nodes + 1): - node = core.nodes.base.CoreNode( - session=session, _id=i, name="n%d" % i, start=False - ) - node.setposition(x=150 * i, y=150) - node.server = daemon - node_data = node.data(flags) - node_message = dataconversion.convert_node(node_data) - session.broker.handlerawmsg(node_message) - n.append(node) - - # create remote links via API - for i in range(1, number_of_nodes + 1): - tlvdata = coreapi.CoreLinkTlv.pack(LinkTlvs.N1_NUMBER.value, switch.id) - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.N2_NUMBER.value, i) - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.TYPE.value, LinkTypes.WIRED.value) - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.INTERFACE2_NUMBER.value, 0) - tlvdata += coreapi.CoreLinkTlv.pack( - LinkTlvs.INTERFACE2_IP4.value, prefix.addr(i) - ) - tlvdata += coreapi.CoreLinkTlv.pack( - LinkTlvs.INTERFACE2_IP4_MASK.value, prefix.prefixlen - ) - msg = coreapi.CoreLinkMessage.pack(flags, tlvdata) - session.broker.handlerawmsg(msg) - - # We change the daemon to Instantiation state - # We do not change the local session as it would try and build a tunnel and fail - tlvdata = coreapi.CoreEventTlv.pack( - EventTlvs.TYPE.value, EventTypes.INSTANTIATION_STATE.value - ) - msg = coreapi.CoreEventMessage.pack(0, tlvdata) - session.broker.handlerawmsg(msg) - - # Get the ip or last node and ping it from the first - print("Pinging from the first to the last node") - pingip = cmd(n[-1], "ip -4 -o addr show dev eth0").split()[3].split("/")[0] - print(cmd(n[1], "ping -c 5 " + pingip)) - print("elapsed time: %s" % (datetime.datetime.now() - start)) - print( - "To stop this session, use the core-cleanup script on the remote daemon server." - ) - input("press enter to exit") - - -if __name__ == "__main__" or __name__ == "__builtin__": - main() diff --git a/daemon/examples/netns/distributed.py b/daemon/examples/netns/distributed.py deleted file mode 100755 index 3d66e8c7..00000000 --- a/daemon/examples/netns/distributed.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/python -i - -# Copyright (c)2010-2013 the Boeing Company. -# See the LICENSE file included in this distribution. - -# A distributed example where CORE API messaging is used to create a session -# distributed across the local server and one slave server. The slave server -# must be specified using the '-s ' parameter, and needs to be -# running the daemon with listenaddr=0.0.0.0 in the core.conf file. -# - -import datetime -import optparse -import sys -from builtins import range - -import core.nodes.base -import core.nodes.network -from core import constants -from core.api.tlv import coreapi, dataconversion -from core.emulator.enumerations import ( - CORE_API_PORT, - EventTlvs, - EventTypes, - LinkTlvs, - LinkTypes, - MessageFlags, -) -from core.emulator.session import Session -from core.nodes import ipaddress - -# node list (count from 1) -n = [None] - - -def main(): - usagestr = "usage: %prog [-h] [options] [args]" - parser = optparse.OptionParser(usage=usagestr) - parser.set_defaults(numnodes=5, slave=None) - - parser.add_option( - "-n", "--numnodes", dest="numnodes", type=int, help="number of nodes" - ) - parser.add_option( - "-s", "--slave-server", dest="slave", type=str, help="slave server IP address" - ) - - def usage(msg=None, err=0): - sys.stdout.write("\n") - if msg: - sys.stdout.write(msg + "\n\n") - parser.print_help() - sys.exit(err) - - # parse command line options - (options, args) = parser.parse_args() - - if options.numnodes < 1: - usage("invalid number of nodes: %s" % options.numnodes) - if not options.slave: - usage("slave server IP address (-s) is a required argument") - - for a in args: - sys.stderr.write("ignoring command line argument: '%s'\n" % a) - - start = datetime.datetime.now() - - prefix = ipaddress.Ipv4Prefix("10.83.0.0/16") - session = Session(1) - server = globals().get("server") - if server is not None: - server.addsession(session) - - # distributed setup - connect to slave server - slaveport = options.slave.split(":") - slave = slaveport[0] - if len(slaveport) > 1: - port = int(slaveport[1]) - else: - port = CORE_API_PORT - print("connecting to slave at %s:%d" % (slave, port)) - session.broker.addserver(slave, slave, port) - session.broker.setupserver(slave) - session.set_state(EventTypes.CONFIGURATION_STATE) - tlvdata = coreapi.CoreEventTlv.pack( - EventTlvs.TYPE.value, EventTypes.CONFIGURATION_STATE.value - ) - session.broker.handlerawmsg(coreapi.CoreEventMessage.pack(0, tlvdata)) - - switch = session.create_node(cls=core.nodes.network.SwitchNode, name="switch") - switch.setposition(x=80, y=50) - num_local = options.numnodes / 2 - num_remote = options.numnodes / 2 + options.numnodes % 2 - print( - "creating %d (%d local / %d remote) nodes with addresses from %s" - % (options.numnodes, num_local, num_remote, prefix) - ) - for i in range(1, num_local + 1): - node = session.create_node(cls=core.nodes.base.CoreNode, name="n%d" % i, _id=i) - node.newnetif(switch, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) - node.cmd([constants.SYSCTL_BIN, "net.ipv4.icmp_echo_ignore_broadcasts=0"]) - node.setposition(x=150 * i, y=150) - n.append(node) - - flags = MessageFlags.ADD.value - session.broker.handlerawmsg(switch.tonodemsg(flags=flags)) - - # create remote nodes via API - for i in range(num_local + 1, options.numnodes + 1): - node = core.nodes.base.CoreNode( - session=session, _id=i, name="n%d" % i, start=False - ) - node.setposition(x=150 * i, y=150) - node.server = slave - n.append(node) - node_data = node.data(flags) - node_message = dataconversion.convert_node(node_data) - session.broker.handlerawmsg(node_message) - - # create remote links via API - for i in range(num_local + 1, options.numnodes + 1): - tlvdata = coreapi.CoreLinkTlv.pack(LinkTlvs.N1_NUMBER.value, switch.id) - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.N2_NUMBER.value, i) - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.TYPE.value, LinkTypes.WIRED.value) - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.INTERFACE2_NUMBER.value, 0) - tlvdata += coreapi.CoreLinkTlv.pack( - LinkTlvs.INTERFACE2_IP4.value, prefix.addr(i) - ) - tlvdata += coreapi.CoreLinkTlv.pack( - LinkTlvs.INTERFACE2_IP4_MASK.value, prefix.prefixlen - ) - msg = coreapi.CoreLinkMessage.pack(flags, tlvdata) - session.broker.handlerawmsg(msg) - - session.instantiate() - tlvdata = coreapi.CoreEventTlv.pack( - EventTlvs.TYPE.value, EventTypes.INSTANTIATION_STATE.value - ) - msg = coreapi.CoreEventMessage.pack(0, tlvdata) - session.broker.handlerawmsg(msg) - - # start a shell on node 1 - n[1].client.term("bash") - - print("elapsed time: %s" % (datetime.datetime.now() - start)) - print("To stop this session, use the 'core-cleanup' script on this server") - print("and on the remote slave server.") - - -if __name__ == "__main__" or __name__ == "__builtin__": - main() diff --git a/daemon/examples/netns/howmanynodes.py b/daemon/examples/netns/howmanynodes.py deleted file mode 100755 index a32e8f38..00000000 --- a/daemon/examples/netns/howmanynodes.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/python - -# Copyright (c)2010-2012 the Boeing Company. -# See the LICENSE file included in this distribution. -# -# author: Jeff Ahrenholz -# - -""" -howmanynodes.py - This is a CORE script that creates network namespace nodes -having one virtual Ethernet interface connected to a bridge. It continues to -add nodes until an exception occurs. The number of nodes per bridge can be -specified. -""" - -import datetime -import optparse -import shutil -import sys -import time - -import core.nodes.base -import core.nodes.network -from core import constants -from core.emulator.session import Session -from core.nodes import ipaddress - -GBD = 1024.0 * 1024.0 - - -def linuxversion(): - """ Return a string having the Linux kernel version. - """ - f = open("/proc/version", "r") - v = f.readline().split() - version_str = " ".join(v[:3]) - f.close() - return version_str - - -MEMKEYS = ("total", "free", "buff", "cached", "stotal", "sfree") - - -def memfree(): - """ Returns kilobytes memory [total, free, buff, cached, stotal, sfree]. - useful stats are: - free memory = free + buff + cached - swap used = stotal - sfree - """ - f = open("/proc/meminfo", "r") - lines = f.readlines() - f.close() - kbs = {} - for k in MEMKEYS: - kbs[k] = 0 - for l in lines: - if l[:9] == "MemTotal:": - kbs["total"] = int(l.split()[1]) - elif l[:8] == "MemFree:": - kbs["free"] = int(l.split()[1]) - elif l[:8] == "Buffers:": - kbs["buff"] = int(l.split()[1]) - elif l[:8] == "Cached:": - kbs["cache"] = int(l.split()[1]) - elif l[:10] == "SwapTotal:": - kbs["stotal"] = int(l.split()[1]) - elif l[:9] == "SwapFree:": - kbs["sfree"] = int(l.split()[1]) - break - return kbs - - -# node list (count from 1) -nodelist = [None] -switchlist = [] - - -def main(): - usagestr = "usage: %prog [-h] [options] [args]" - parser = optparse.OptionParser(usage=usagestr) - parser.set_defaults( - waittime=0.2, numnodes=0, bridges=0, retries=0, logfile=None, services=None - ) - - parser.add_option( - "-w", - "--waittime", - dest="waittime", - type=float, - help="number of seconds to wait between node creation" - " (default = %s)" % parser.defaults["waittime"], - ) - parser.add_option( - "-n", - "--numnodes", - dest="numnodes", - type=int, - help="number of nodes (default = unlimited)", - ) - parser.add_option( - "-b", - "--bridges", - dest="bridges", - type=int, - help="number of nodes per bridge; 0 = one bridge " - "(def. = %s)" % parser.defaults["bridges"], - ) - parser.add_option( - "-r", - "--retry", - dest="retries", - type=int, - help="number of retries on error (default = %s)" % parser.defaults["retries"], - ) - parser.add_option( - "-l", - "--log", - dest="logfile", - type=str, - help="log memory usage to this file (default = %s)" - % parser.defaults["logfile"], - ) - parser.add_option( - "-s", - "--services", - dest="services", - type=str, - help="pipe-delimited list of services added to each " - "node (default = %s)\n(Example: zebra|OSPFv2|OSPFv3|" - "IPForward)" % parser.defaults["services"], - ) - - def usage(msg=None, err=0): - sys.stdout.write("\n") - if msg: - sys.stdout.write(msg + "\n\n") - parser.print_help() - sys.exit(err) - - options, args = parser.parse_args() - - for a in args: - sys.stderr.write("ignoring command line argument: %s\n" % a) - - start = datetime.datetime.now() - prefix = ipaddress.Ipv4Prefix("10.83.0.0/16") - - print("Testing how many network namespace nodes this machine can create.") - print(" - %s" % linuxversion()) - mem = memfree() - print( - " - %.02f GB total memory (%.02f GB swap)" - % (mem["total"] / GBD, mem["stotal"] / GBD) - ) - print(" - using IPv4 network prefix %s" % prefix) - print(" - using wait time of %s" % options.waittime) - print(" - using %d nodes per bridge" % options.bridges) - print(" - will retry %d times on failure" % options.retries) - print(" - adding these services to each node: %s" % options.services) - print(" ") - - lfp = None - if options.logfile is not None: - # initialize a csv log file header - lfp = open(options.logfile, "a") - lfp.write("# log from howmanynodes.py %s\n" % time.ctime()) - lfp.write("# options = %s\n#\n" % options) - lfp.write("# numnodes,%s\n" % ",".join(MEMKEYS)) - lfp.flush() - - session = Session(1) - switch = session.create_node(cls=core.nodes.network.SwitchNode) - switchlist.append(switch) - print("Added bridge %s (%d)." % (switch.brname, len(switchlist))) - - i = 0 - retry_count = options.retries - while True: - i += 1 - # optionally add a bridge (options.bridges nodes per bridge) - try: - if 0 < options.bridges <= switch.numnetif(): - switch = session.create_node(cls=core.nodes.network.SwitchNode) - switchlist.append(switch) - print( - "\nAdded bridge %s (%d) for node %d." - % (switch.brname, len(switchlist), i) - ) - except Exception as e: - print( - "At %d bridges (%d nodes) caught exception:\n%s\n" - % (len(switchlist), i - 1, e) - ) - break - - # create a node - try: - n = session.create_node(cls=core.nodes.base.CoreNode, name="n%d" % i) - n.newnetif(switch, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)]) - n.cmd([constants.SYSCTL_BIN, "net.ipv4.icmp_echo_ignore_broadcasts=0"]) - if options.services is not None: - session.services.add_services(n, "", options.services.split("|")) - session.services.boot_services(n) - nodelist.append(n) - if i % 25 == 0: - print("\n%s nodes created " % i) - mem = memfree() - free = mem["free"] + mem["buff"] + mem["cached"] - swap = mem["stotal"] - mem["sfree"] - print("(%.02f/%.02f GB free/swap)" % (free / GBD, swap / GBD)) - if lfp: - lfp.write("%d," % i) - lfp.write("%s\n" % ",".join(str(mem[x]) for x in MEMKEYS)) - lfp.flush() - else: - sys.stdout.write(".") - sys.stdout.flush() - time.sleep(options.waittime) - except Exception as e: - print("At %d nodes caught exception:\n" % i, e) - if retry_count > 0: - print("\nWill retry creating node %d." % i) - shutil.rmtree(n.nodedir, ignore_errors=True) - retry_count -= 1 - i -= 1 - time.sleep(options.waittime) - continue - else: - print("Stopping at %d nodes!" % i) - break - - if i == options.numnodes: - print("Stopping at %d nodes due to numnodes option." % i) - break - # node creation was successful at this point - retry_count = options.retries - - if lfp: - lfp.flush() - lfp.close() - - print("elapsed time: %s" % (datetime.datetime.now() - start)) - print("Use the core-cleanup script to remove nodes and bridges.") - - -if __name__ == "__main__": - main() diff --git a/daemon/examples/netns/ospfmanetmdrtest.py b/daemon/examples/netns/ospfmanetmdrtest.py deleted file mode 100755 index 88a8472f..00000000 --- a/daemon/examples/netns/ospfmanetmdrtest.py +++ /dev/null @@ -1,637 +0,0 @@ -#!/usr/bin/python - -# Copyright (c)2011-2014 the Boeing Company. -# See the LICENSE file included in this distribution. - -# create a random topology running OSPFv3 MDR, wait and then check -# that all neighbor states are either full or two-way, and check the routes -# in zebra vs those installed in the kernel. - -import datetime -import optparse -import os -import random -import sys -import time -from builtins import range -from string import Template - -import core.nodes.base -import core.nodes.network -from core.constants import QUAGGA_STATE_DIR -from core.emulator.session import Session -from core.nodes import ipaddress -from core.utils import check_cmd - -quagga_sbin_search = ("/usr/local/sbin", "/usr/sbin", "/usr/lib/quagga") -quagga_path = "zebra" - -# sanity check that zebra is installed -try: - for p in quagga_sbin_search: - if os.path.exists(os.path.join(p, "zebra")): - quagga_path = p - break - check_cmd([os.path.join(quagga_path, "zebra"), "-u", "root", "-g", "root", "-v"]) -except OSError: - sys.stderr.write("ERROR: running zebra failed\n") - sys.exit(1) - - -class ManetNode(core.nodes.base.CoreNode): - """ An Lxc namespace node configured for Quagga OSPFv3 MANET MDR - """ - - conftemp = Template( - """\ -interface eth0 - ip address $ipaddr - ipv6 ospf6 instance-id 65 - ipv6 ospf6 hello-interval 2 - ipv6 ospf6 dead-interval 6 - ipv6 ospf6 retransmit-interval 5 - ipv6 ospf6 network manet-designated-router - ipv6 ospf6 diffhellos - ipv6 ospf6 adjacencyconnectivity biconnected - ipv6 ospf6 lsafullness mincostlsa -! -router ospf6 - router-id $routerid - interface eth0 area 0.0.0.0 -! -ip forwarding -""" - ) - - confdir = "/usr/local/etc/quagga" - - def __init__(self, core, ipaddr, routerid=None, _id=None, name=None, nodedir=None): - if routerid is None: - routerid = ipaddr.split("/")[0] - self.ipaddr = ipaddr - self.routerid = routerid - core.nodes.base.CoreBaseNode.__init__(self, core, _id, name, nodedir) - self.privatedir(self.confdir) - self.privatedir(QUAGGA_STATE_DIR) - - def qconf(self): - return self.conftemp.substitute(ipaddr=self.ipaddr, routerid=self.routerid) - - def config(self): - filename = os.path.join(self.confdir, "Quagga.conf") - f = self.opennodefile(filename, "w") - f.write(self.qconf()) - f.close() - tmp = self.bootscript() - if tmp: - self.nodefile(self.bootsh, tmp, mode=0o755) - - def boot(self): - self.config() - self.session.services.boot_services(self) - - def bootscript(self): - return """\ -#!/bin/sh -e - -STATEDIR=%s - -waitfile() -{ - fname=$1 - - i=0 - until [ -e $fname ]; do - i=$(($i + 1)) - if [ $i -eq 10 ]; then - echo "file not found: $fname" >&2 - exit 1 - fi - sleep 0.1 - done -} - -mkdir -p $STATEDIR - -%s/zebra -d -u root -g root -waitfile $STATEDIR/zebra.vty - -%s/ospf6d -d -u root -g root -waitfile $STATEDIR/ospf6d.vty - -vtysh -b -""" % ( - QUAGGA_STATE_DIR, - quagga_path, - quagga_path, - ) - - -class Route(object): - """ Helper class for organzing routing table entries. """ - - def __init__(self, prefix=None, gw=None, metric=None): - try: - self.prefix = ipaddress.Ipv4Prefix(prefix) - except Exception as e: - raise ValueError( - "Invalid prefix given to Route object: %s\n%s" % (prefix, e) - ) - self.gw = gw - self.metric = metric - - def __eq__(self, other): - try: - return ( - self.prefix == other.prefix - and self.gw == other.gw - and self.metric == other.metric - ) - except Exception: - return False - - def __str__(self): - return "(%s,%s,%s)" % (self.prefix, self.gw, self.metric) - - @staticmethod - def key(r): - if not r.prefix: - return 0 - return r.prefix.prefix - - -class ManetExperiment(object): - """ A class for building an MDR network and checking and logging its state. - """ - - def __init__(self, options, start): - """ Initialize with options and start time. """ - self.session = None - # node list - self.nodes = [] - # WLAN network - self.net = None - self.verbose = options.verbose - # dict from OptionParser - self.options = options - self.start = start - self.logbegin() - - def info(self, msg): - """ Utility method for writing output to stdout. """ - print(msg) - sys.stdout.flush() - self.log(msg) - - def warn(self, msg): - """ Utility method for writing output to stderr. """ - sys.stderr.write(msg) - sys.stderr.flush() - self.log(msg) - - def logbegin(self): - """ Start logging. """ - self.logfp = None - if not self.options.logfile: - return - self.logfp = open(self.options.logfile, "w") - self.log("ospfmanetmdrtest begin: %s\n" % self.start.ctime()) - - def logend(self): - """ End logging. """ - if not self.logfp: - return - end = datetime.datetime.now() - self.log("ospfmanetmdrtest end: %s (%s)\n" % (end.ctime(), end - self.start)) - self.logfp.flush() - self.logfp.close() - self.logfp = None - - def log(self, msg): - """ Write to the log file, if any. """ - if not self.logfp: - return - self.logfp.write(msg) - - def logdata(self, nbrs, mdrs, lsdbs, krs, zrs): - """ Dump experiment parameters and data to the log file. """ - self.log("ospfmantetmdrtest data:") - self.log("----- parameters -----") - self.log("%s" % self.options) - self.log("----- neighbors -----") - for rtrid in sorted(nbrs.keys()): - self.log("%s: %s" % (rtrid, nbrs[rtrid])) - self.log("----- mdr levels -----") - self.log(mdrs) - self.log("----- link state databases -----") - for rtrid in sorted(lsdbs.keys()): - self.log("%s lsdb:" % rtrid) - for line in lsdbs[rtrid].split("\n"): - self.log(line) - self.log("----- kernel routes -----") - for rtrid in sorted(krs.keys()): - msg = rtrid + ": " - for rt in krs[rtrid]: - msg += "%s" % rt - self.log(msg) - self.log("----- zebra routes -----") - for rtrid in sorted(zrs.keys()): - msg = rtrid + ": " - for rt in zrs[rtrid]: - msg += "%s" % rt - self.log(msg) - - def topology(self, numnodes, linkprob, verbose=False): - """ Build a topology consisting of the given number of ManetNodes - connected to a WLAN and probabilty of links and set - the session, WLAN, and node list objects. - """ - # IP subnet - prefix = ipaddress.Ipv4Prefix("10.14.0.0/16") - self.session = Session(1) - # emulated network - self.net = self.session.create_node(cls=core.nodes.network.WlanNode) - for i in range(1, numnodes + 1): - addr = "%s/%s" % (prefix.addr(i), 32) - tmp = self.session.create_node( - cls=ManetNode, ipaddr=addr, _id="%d" % i, name="n%d" % i - ) - tmp.newnetif(self.net, [addr]) - self.nodes.append(tmp) - # connect nodes with probability linkprob - for i in range(numnodes): - for j in range(i + 1, numnodes): - r = random.random() - if r < linkprob: - if self.verbose: - self.info("linking (%d,%d)" % (i, j)) - self.net.link(self.nodes[i].netif(0), self.nodes[j].netif(0)) - # force one link to avoid partitions (should check if this is needed) - j = i - while j == i: - j = random.randint(0, numnodes - 1) - if self.verbose: - self.info("linking (%d,%d)" % (i, j)) - self.net.link(self.nodes[i].netif(0), self.nodes[j].netif(0)) - self.nodes[i].boot() - # run the boot.sh script on all nodes to start Quagga - for i in range(numnodes): - self.nodes[i].cmd(["./%s" % self.nodes[i].bootsh]) - - def compareroutes(self, node, kr, zr): - """ Compare two lists of Route objects. - """ - kr.sort(key=Route.key) - zr.sort(key=Route.key) - if kr != zr: - self.warn("kernel and zebra routes differ") - if self.verbose: - msg = "kernel: " - for r in kr: - msg += "%s " % r - msg += "\nzebra: " - for r in zr: - msg += "%s " % r - self.warn(msg) - else: - self.info(" kernel and zebra routes match") - - def comparemdrlevels(self, nbrs, mdrs): - """ Check that all routers form a connected dominating set, i.e. all - routers are either MDR, BMDR, or adjacent to one. - """ - msg = "All routers form a CDS" - for n in self.nodes: - if mdrs[n.routerid] != "OTHER": - continue - connected = False - for nbr in nbrs[n.routerid]: - if mdrs[nbr] == "MDR" or mdrs[nbr] == "BMDR": - connected = True - break - if not connected: - msg = "All routers do not form a CDS" - self.warn( - "XXX %s: not in CDS; neighbors: %s" % (n.routerid, nbrs[n.routerid]) - ) - if self.verbose: - self.info(msg) - - def comparelsdbs(self, lsdbs): - """ Check LSDBs for consistency. - """ - msg = "LSDBs of all routers are consistent" - prev = self.nodes[0] - for n in self.nodes: - db = lsdbs[n.routerid] - if lsdbs[prev.routerid] != db: - msg = "LSDBs of all routers are not consistent" - self.warn( - "XXX LSDBs inconsistent for %s and %s" % (n.routerid, prev.routerid) - ) - i = 0 - for entry in lsdbs[n.routerid].split("\n"): - preventries = lsdbs[prev.routerid].split("\n") - try: - preventry = preventries[i] - except IndexError: - preventry = None - if entry != preventry: - self.warn("%s: %s" % (n.routerid, entry)) - self.warn("%s: %s" % (prev.routerid, preventry)) - i += 1 - prev = n - if self.verbose: - self.info(msg) - - def checknodes(self): - """ Check the neighbor state and routing tables of all nodes. """ - nbrs = {} - mdrs = {} - lsdbs = {} - krs = {} - zrs = {} - v = self.verbose - for n in self.nodes: - self.info("checking %s" % n.name) - nbrs[n.routerid] = Ospf6NeighState(n, verbose=v).run() - krs[n.routerid] = KernelRoutes(n, verbose=v).run() - zrs[n.routerid] = ZebraRoutes(n, verbose=v).run() - self.compareroutes(n, krs[n.routerid], zrs[n.routerid]) - mdrs[n.routerid] = Ospf6MdrLevel(n, verbose=v).run() - lsdbs[n.routerid] = Ospf6Database(n, verbose=v).run() - self.comparemdrlevels(nbrs, mdrs) - self.comparelsdbs(lsdbs) - self.logdata(nbrs, mdrs, lsdbs, krs, zrs) - - -class Cmd: - """ Helper class for running a command on a node and parsing the result. """ - - args = "" - - def __init__(self, node, verbose=False): - """ Initialize with a CoreNode (LxcNode) """ - self.id = None - self.stdin = None - self.out = None - self.node = node - self.verbose = verbose - - def info(self, msg): - """ Utility method for writing output to stdout.""" - print(msg) - sys.stdout.flush() - - def warn(self, msg): - """ Utility method for writing output to stderr. """ - sys.stderr.write("XXX %s:" % self.node.routerid, msg) - sys.stderr.flush() - - def run(self): - """ This is the primary method used for running this command. """ - self.open() - r = self.parse() - self.cleanup() - return r - - def open(self): - """ Exceute call to node.popen(). """ - self.id, self.stdin, self.out, self.err = self.node.client.popen(self.args) - - def parse(self): - """ This method is overloaded by child classes and should return some - result. - """ - return None - - def cleanup(self): - """ Close the Popen channels.""" - self.stdin.close() - self.out.close() - self.err.close() - tmp = self.id.wait() - if tmp: - self.warn("nonzero exit status:", tmp) - - -class VtyshCmd(Cmd): - """ Runs a vtysh command. """ - - def open(self): - args = ("vtysh", "-c", self.args) - self.id, self.stdin, self.out, self.err = self.node.client.popen(args) - - -class Ospf6NeighState(VtyshCmd): - """ Check a node for OSPFv3 neighbors in the full/two-way states. """ - - args = "show ipv6 ospf6 neighbor" - - def parse(self): - # skip first line - self.out.readline() - nbrlist = [] - for line in self.out: - field = line.split() - nbr = field[0] - state = field[3].split("/")[0] - if not state.lower() in ("full", "twoway"): - self.warn("neighbor %s state: %s" % (nbr, state)) - nbrlist.append(nbr) - - if len(nbrlist) == 0: - self.warn("no neighbors") - if self.verbose: - self.info(" %s has %d neighbors" % (self.node.routerid, len(nbrlist))) - return nbrlist - - -class Ospf6MdrLevel(VtyshCmd): - """ Retrieve the OSPFv3 MDR level for a node. """ - - args = "show ipv6 ospf6 mdrlevel" - - def parse(self): - line = self.out.readline() - # TODO: handle multiple interfaces - field = line.split() - mdrlevel = field[4] - if mdrlevel not in ("MDR", "BMDR", "OTHER"): - self.warn("mdrlevel: %s" % mdrlevel) - if self.verbose: - self.info(" %s is %s" % (self.node.routerid, mdrlevel)) - return mdrlevel - - -class Ospf6Database(VtyshCmd): - """ Retrieve the OSPFv3 LSDB summary for a node. """ - - args = "show ipv6 ospf6 database" - - def parse(self): - db = "" - for line in self.out: - field = line.split() - if len(field) < 8: - continue - # filter out Age and Duration columns - filtered = field[:3] + field[4:7] - db += " ".join(filtered) + "\n" - return db - - -class ZebraRoutes(VtyshCmd): - """ Return a list of Route objects for a node based on its zebra - routing table. - """ - - args = "show ip route" - - def parse(self): - for i in range(0, 3): - # skip first three lines - self.out.readline() - r = [] - prefix = None - for line in self.out: - field = line.split() - if len(field) < 1: - continue - # only use OSPFv3 selected FIB routes - elif field[0][:2] == "o>": - prefix = field[1] - metric = field[2].split("/")[1][:-1] - if field[0][2:] != "*": - continue - if field[3] == "via": - gw = field[4][:-1] - else: - gw = field[6][:-1] - r.append(Route(prefix, gw, metric)) - prefix = None - elif prefix and field[0] == "*": - # already have prefix and metric from previous line - gw = field[2][:-1] - r.append(Route(prefix, gw, metric)) - prefix = None - - if len(r) == 0: - self.warn("no zebra routes") - if self.verbose: - self.info(" %s has %d zebra routes" % (self.node.routerid, len(r))) - return r - - -class KernelRoutes(Cmd): - """ Return a list of Route objects for a node based on its kernel - routing table. - """ - - args = ("/sbin/ip", "route", "show") - - def parse(self): - r = [] - prefix = None - for line in self.out: - field = line.split() - if field[0] == "nexthop": - if not prefix: - # this saves only the first nexthop entry if multiple exist - continue - else: - prefix = field[0] - metric = field[-1] - tmp = prefix.split("/") - if len(tmp) < 2: - prefix += "/32" - if field[1] == "proto": - # nexthop entry is on the next line - continue - # nexthop IP or interface - gw = field[2] - r.append(Route(prefix, gw, metric)) - prefix = None - - if len(r) == 0: - self.warn("no kernel routes") - if self.verbose: - self.info(" %s has %d kernel routes" % (self.node.routerid, len(r))) - return r - - -def main(): - usagestr = "usage: %prog [-h] [options] [args]" - parser = optparse.OptionParser(usage=usagestr) - parser.set_defaults(numnodes=10, linkprob=0.35, delay=20, seed=None) - - parser.add_option( - "-n", "--numnodes", dest="numnodes", type=int, help="number of nodes" - ) - parser.add_option( - "-p", "--linkprob", dest="linkprob", type=float, help="link probabilty" - ) - parser.add_option( - "-d", "--delay", dest="delay", type=float, help="wait time before checking" - ) - parser.add_option( - "-s", - "--seed", - dest="seed", - type=int, - help="specify integer to use for random seed", - ) - parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="be more verbose" - ) - parser.add_option( - "-l", - "--logfile", - dest="logfile", - type=str, - help="log detailed output to the specified file", - ) - - def usage(msg=None, err=0): - sys.stdout.write("\n") - if msg: - sys.stdout.write(msg + "\n\n") - parser.print_help() - sys.exit(err) - - # parse command line options - (options, args) = parser.parse_args() - - if options.numnodes < 2: - usage("invalid numnodes: %s" % options.numnodes) - if options.linkprob <= 0.0 or options.linkprob > 1.0: - usage("invalid linkprob: %s" % options.linkprob) - if options.delay < 0.0: - usage("invalid delay: %s" % options.delay) - - for a in args: - sys.stderr.write("ignoring command line argument: '%s'\n" % a) - - if options.seed: - random.seed(options.seed) - - me = ManetExperiment(options=options, start=datetime.datetime.now()) - me.info( - "creating topology: numnodes = %s; linkprob = %s" - % (options.numnodes, options.linkprob) - ) - me.topology(options.numnodes, options.linkprob) - - me.info("waiting %s sec" % options.delay) - time.sleep(options.delay) - me.info("checking neighbor state and routes") - me.checknodes() - me.info("done") - me.info("elapsed time: %s" % (datetime.datetime.now() - me.start)) - me.logend() - - return me - - -if __name__ == "__main__": - me = main() diff --git a/daemon/examples/api/emane80211.py b/daemon/examples/python/emane80211.py similarity index 100% rename from daemon/examples/api/emane80211.py rename to daemon/examples/python/emane80211.py diff --git a/daemon/examples/api/parser.py b/daemon/examples/python/parser.py similarity index 100% rename from daemon/examples/api/parser.py rename to daemon/examples/python/parser.py diff --git a/daemon/examples/api/switch.py b/daemon/examples/python/switch.py similarity index 100% rename from daemon/examples/api/switch.py rename to daemon/examples/python/switch.py diff --git a/daemon/examples/api/switch_inject.py b/daemon/examples/python/switch_inject.py similarity index 100% rename from daemon/examples/api/switch_inject.py rename to daemon/examples/python/switch_inject.py diff --git a/daemon/examples/api/wlan.py b/daemon/examples/python/wlan.py similarity index 100% rename from daemon/examples/api/wlan.py rename to daemon/examples/python/wlan.py diff --git a/daemon/examples/stopsession.py b/daemon/examples/stopsession.py deleted file mode 100755 index f75ce7d2..00000000 --- a/daemon/examples/stopsession.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# (c)2010-2012 the Boeing Company -# author: Jeff Ahrenholz -# -# List and stop CORE sessions from the command line. -# - -import optparse -import socket - -from core.api.tlv import coreapi -from core.emulator.enumerations import CORE_API_PORT, MessageFlags, SessionTlvs - - -def main(): - parser = optparse.OptionParser(usage="usage: %prog [-l] ") - parser.add_option( - "-l", "--list", dest="list", action="store_true", help="list running sessions" - ) - (options, args) = parser.parse_args() - - if options.list is True: - num = "0" - flags = MessageFlags.STRING.value - else: - num = args[0] - flags = MessageFlags.DELETE.value - tlvdata = coreapi.CoreSessionTlv.pack(SessionTlvs.NUMBER.value, num) - message = coreapi.CoreSessionMessage.pack(flags, tlvdata) - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(("localhost", CORE_API_PORT)) - sock.send(message) - - # receive and print a session list - if options.list is True: - hdr = sock.recv(coreapi.CoreMessage.header_len) - msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(hdr) - data = "" - if msglen: - data = sock.recv(msglen) - message = coreapi.CoreMessage(msgflags, hdr, data) - sessions = message.get_tlv(coreapi.SessionTlvs.NUMBER.value) - print("sessions: {}".format(sessions)) - - sock.close() - - -if __name__ == "__main__": - main() From 0bba23da69aa199017a5045360895762266a9903 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 12:54:56 -0700 Subject: [PATCH 040/113] set version to 5.5.0 for release --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index db8b0294..a4734b9e 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. # this defines the CORE version number, must be static for AC_INIT -AC_INIT(core, 5.4.0) +AC_INIT(core, 5.5.0) # autoconf and automake initialization AC_CONFIG_SRCDIR([netns/version.h.in]) From 8ce6f650c493aca2e7f831481730305958fb403a Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 13:25:44 -0700 Subject: [PATCH 041/113] removing unused Jenkins file --- Jenkinsfile | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index e70e3dc6..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,20 +0,0 @@ -pipeline { - agent any - stages { - stage('build core') { - steps { - sh './bootstrap.sh' - sh './configure' - sh 'make' - sh 'sudo make install' - } - } - stage('test core') { - steps { - sh 'pytest daemon/tests/test_core.py' - sh 'pytest daemon/tests/test_gui.py' - sh 'pytest daemon/tests/test_emane.py' - } - } - } -} \ No newline at end of file From fbfc4929f1b3f5086db451a347406bc41cac3224 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 13:31:22 -0700 Subject: [PATCH 042/113] updated pipenv run core to default to turning on grpc --- daemon/Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/Pipfile b/daemon/Pipfile index a689f4dc..a1e33a29 100644 --- a/daemon/Pipfile +++ b/daemon/Pipfile @@ -4,7 +4,7 @@ url = "https://pypi.org/simple" verify_ssl = true [scripts] -core = "python scripts/core-daemon -f data/core.conf -l data/logging.conf" +core = "python scripts/core-daemon -f data/core.conf -l data/logging.conf --grpc" test = "pytest -v tests" test_emane = "pytest -v tests/emane" From 70bb1598e1bb5d6998fc30718b9cfe1f4d3e8ea1 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 13:49:26 -0700 Subject: [PATCH 043/113] fixed some nrl services for python3, using len(filter) --- daemon/core/services/nrl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon/core/services/nrl.py b/daemon/core/services/nrl.py index 07cdbb97..23484459 100644 --- a/daemon/core/services/nrl.py +++ b/daemon/core/services/nrl.py @@ -3,6 +3,8 @@ nrl.py: defines services provided by NRL protolib tools hosted here: http://www.nrl.navy.mil/itd/ncs/products """ +from past.builtins import filter + from core import utils from core.nodes.ipaddress import Ipv4Prefix from core.services.coreservices import CoreService From 7aa013d351980918ab4908dcdff524285ae7ab66 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 3 Oct 2019 20:38:32 -0700 Subject: [PATCH 044/113] start to wrapping commands to support remote ssh --- daemon/core/nodes/base.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index ff34984d..b655c70a 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,6 +14,8 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 +from fabric import Connection + from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes @@ -82,17 +84,23 @@ class NodeBase(object): """ raise NotImplementedError - def net_cmd(self, args): + def net_cmd(self, args, env=None): """ Runs a command that is used to configure and setup the network on the host system. :param list[str]|str args: command to run + :param dict env: environment to run command with :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - return utils.check_cmd(args) + if self.server is None: + return utils.check_cmd(args, env=env) + else: + args = " ".join(args) + result = Connection(self.server, user="root").run(args, hide=True) + return result.stderr def setposition(self, x=None, y=None, z=None): """ @@ -515,7 +523,7 @@ class CoreNode(CoreNodeBase): env["NODE_NUMBER"] = str(self.id) env["NODE_NAME"] = str(self.name) - output = utils.check_cmd(vnoded, env=env) + output = self.net_cmd(vnoded, env=env) self.pid = int(output) # create vnode client @@ -660,8 +668,8 @@ class CoreNode(CoreNodeBase): """ source = os.path.abspath(source) logging.debug("node(%s) mounting: %s at %s", self.name, source, target) - self.client.check_cmd(["mkdir", "-p", target]) - self.client.check_cmd([constants.MOUNT_BIN, "-n", "--bind", source, target]) + self.node_net_cmd(["mkdir", "-p", target]) + self.node_net_cmd([constants.MOUNT_BIN, "-n", "--bind", source, target]) self._mounts.append((source, target)) def newifindex(self): From 031517ba56c9e69cf31c1d3616d78a93c7aaaedc Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 4 Oct 2019 09:29:10 -0700 Subject: [PATCH 045/113] fixed base.py imports with isort --- daemon/core/nodes/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index b655c70a..7efad49e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,14 +14,13 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 -from fabric import Connection - from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, OvsNetClient +from fabric import Connection _DEFAULT_MTU = 1500 From bb98a4a77fe6ffa8a0d12e9cf257fabe495a2e3e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 4 Oct 2019 12:36:15 -0700 Subject: [PATCH 046/113] removed circular dependency causing issues in python2 --- daemon/core/utils.py | 8 ++------ daemon/examples/python/emane80211.py | 5 ++--- daemon/examples/python/switch.py | 6 ++---- daemon/examples/python/switch_inject.py | 5 ++--- daemon/examples/python/wlan.py | 5 ++--- daemon/scripts/core-daemon | 10 ++++++---- 6 files changed, 16 insertions(+), 23 deletions(-) diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 425fc0ac..20d2384e 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -16,7 +16,6 @@ import sys from past.builtins import basestring -from core import constants from core.errors import CoreCommandError DEVNULL = open(os.devnull, "wb") @@ -451,16 +450,13 @@ def load_classes(path, clazz): return classes -def load_logging_config(config_path=None): +def load_logging_config(config_path): """ Load CORE logging configuration file. - :param str config_path: path to logging config file, - when None defaults to /etc/core/logging.conf + :param str config_path: path to logging config file :return: nothing """ - if not config_path: - config_path = os.path.join(constants.CORE_CONF_DIR, "logging.conf") with open(config_path, "r") as log_config_file: log_config = json.load(log_config_file) logging.config.dictConfig(log_config) diff --git a/daemon/examples/python/emane80211.py b/daemon/examples/python/emane80211.py index 7cbfd586..0e42be95 100644 --- a/daemon/examples/python/emane80211.py +++ b/daemon/examples/python/emane80211.py @@ -3,6 +3,7 @@ # Example CORE Python script that attaches N nodes to an EMANE 802.11abg network. import datetime +import logging import parser from builtins import range @@ -10,9 +11,6 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes from core.emulator.enumerations import EventTypes -from core.utils import load_logging_config - -load_logging_config() def example(options): @@ -52,6 +50,7 @@ def example(options): def main(): + logging.basicConfig(level=logging.INFO) options = parser.parse_options("emane80211") start = datetime.datetime.now() print( diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index 5b086d6a..d669478d 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -6,15 +6,13 @@ # nodestep import datetime +import logging import parser from builtins import range from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes from core.emulator.enumerations import EventTypes, NodeTypes -from core.utils import load_logging_config - -load_logging_config() def example(options): @@ -56,8 +54,8 @@ def example(options): def main(): + logging.basicConfig(level=logging.INFO) options = parser.parse_options("switch") - start = datetime.datetime.now() print("running switch example: nodes(%s) time(%s)" % (options.nodes, options.time)) example(options) diff --git a/daemon/examples/python/switch_inject.py b/daemon/examples/python/switch_inject.py index b8587760..ff1ff84b 100644 --- a/daemon/examples/python/switch_inject.py +++ b/daemon/examples/python/switch_inject.py @@ -4,13 +4,11 @@ # n nodes are connected to a virtual wlan; run test for testsec # and repeat for minnodes <= n <= maxnodes with a step size of # nodestep +import logging from builtins import range from core.emulator.emudata import IpPrefixes from core.emulator.enumerations import EventTypes, NodeTypes -from core.utils import load_logging_config - -load_logging_config() def example(nodes): @@ -38,4 +36,5 @@ def example(nodes): if __name__ in {"__main__", "__builtin__"}: + logging.basicConfig(level=logging.INFO) example(2) diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index f94fd528..376f34d0 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -6,6 +6,7 @@ # nodestep import datetime +import logging import parser from builtins import range @@ -13,9 +14,6 @@ from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes from core.location.mobility import BasicRangeModel -from core.utils import load_logging_config - -load_logging_config() def example(options): @@ -60,6 +58,7 @@ def example(options): def main(): + logging.basicConfig(level=logging.INFO) options = parser.parse_options("wlan") start = datetime.datetime.now() diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 4becdaa1..07c6a9a7 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -7,6 +7,7 @@ message handlers are defined and some support for sending messages. import argparse import logging +import os import sys import threading import time @@ -96,7 +97,7 @@ def get_merged_config(filename): "numthreads": "1", "grpcport": "50051", "grpcaddress": "localhost", - "logfile": "" + "logfile": os.path.join(constants.CORE_CONF_DIR, "logging.conf") } parser = argparse.ArgumentParser( @@ -125,9 +126,6 @@ def get_merged_config(filename): cfg = ConfigParser(defaults) cfg.read(filename) - # load logging configuration - load_logging_config(args.logfile) - section = "core-daemon" if not cfg.has_section(section): cfg.add_section(section) @@ -149,6 +147,10 @@ def main(): """ # get a configuration merged from config file and command-line arguments cfg = get_merged_config("%s/core.conf" % constants.CORE_CONF_DIR) + + # load logging configuration + load_logging_config(cfg["logfile"]) + banner() try: From d00658ccb255d243ac011f7b212bf94b35ffef6d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 4 Oct 2019 12:47:01 -0700 Subject: [PATCH 047/113] updated version to 5.5.1 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a4734b9e..045425c8 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. # this defines the CORE version number, must be static for AC_INIT -AC_INIT(core, 5.5.0) +AC_INIT(core, 5.5.1) # autoconf and automake initialization AC_CONFIG_SRCDIR([netns/version.h.in]) From f83f98262f3523a76b1fa76c6b1a415adfb65900 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 4 Oct 2019 17:33:44 -0700 Subject: [PATCH 048/113] some initial remote node commands using fabric --- daemon/core/emulator/session.py | 18 ++++++- daemon/core/nodes/base.py | 68 ++++++++++++++++++++++----- daemon/examples/python/distributed.py | 55 ++++++++++++++++++++++ 3 files changed, 127 insertions(+), 14 deletions(-) create mode 100644 daemon/examples/python/distributed.py diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 7b2a03b9..f5625232 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -14,6 +14,8 @@ import threading import time from multiprocessing.pool import ThreadPool +from fabric import Connection + from core import constants, utils from core.api.tlv import coreapi from core.api.tlv.broker import CoreBroker @@ -144,6 +146,9 @@ class Session(object): self.emane = EmaneManager(session=self) self.sdt = Sdt(session=self) + # distributed servers + self.servers = set() + # initialize default node services self.services.default_services = { "mdr": ("zebra", "OSPFv3MDR", "IPForward"), @@ -153,6 +158,11 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } + def init_distributed(self): + for server in self.servers: + cmd = "mkdir -p %s" % self.session_dir + Connection(server, user="root").run(cmd, hide=False) + @classmethod def get_node_class(cls, _type): """ @@ -683,7 +693,13 @@ class Session(object): image=node_options.image, ) else: - node = self.create_node(cls=node_class, _id=_id, name=name, start=start) + node = self.create_node( + cls=node_class, + _id=_id, + name=name, + start=start, + server=node_options.emulation_server, + ) # set node attributes node.icon = node_options.icon diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 7efad49e..b08c7cd4 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,13 +14,15 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 +from fabric import Connection + from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes +from core.errors import CoreCommandError from core.nodes import client, ipaddress from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, OvsNetClient -from fabric import Connection _DEFAULT_MTU = 1500 @@ -33,7 +35,7 @@ class NodeBase(object): apitype = None # TODO: appears start has no usage, verify and remove - def __init__(self, session, _id=None, name=None, start=True): + def __init__(self, session, _id=None, name=None, start=True, server=None): """ Creates a PyCoreObj instance. @@ -41,7 +43,7 @@ class NodeBase(object): :param int _id: id :param str name: object name :param bool start: start value - :return: + :param str server: remote server node will run on, default is None for localhost """ self.session = session @@ -51,8 +53,11 @@ class NodeBase(object): if name is None: name = "o%s" % self.id self.name = name + self.server = server + if self.server is not None: + self.server_conn = Connection(self.server, user="root") + self.type = None - self.server = None self.services = None # ifindex is key, CoreInterface instance is value self._netif = {} @@ -94,12 +99,23 @@ class NodeBase(object): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ + logging.info("net cmd server(%s): %s", self.server, args) if self.server is None: return utils.check_cmd(args, env=env) else: args = " ".join(args) - result = Connection(self.server, user="root").run(args, hide=True) - return result.stderr + result = self.server_conn.run(args, hide=False) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr + ) + + logging.info( + "fabric result:\n\tstdout: %s\n\tstderr: %s", + result.stdout.strip(), + result.stderr.strip(), + ) + return result.stdout.strip() def setposition(self, x=None, y=None, z=None): """ @@ -243,7 +259,7 @@ class CoreNodeBase(NodeBase): Base class for CORE nodes. """ - def __init__(self, session, _id=None, name=None, start=True): + def __init__(self, session, _id=None, name=None, start=True, server=None): """ Create a CoreNodeBase instance. @@ -251,8 +267,9 @@ class CoreNodeBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: boolean for starting + :param str server: remote server node will run on, default is None for localhost """ - super(CoreNodeBase, self).__init__(session, _id, name, start=start) + super(CoreNodeBase, self).__init__(session, _id, name, start, server) self.services = [] self.nodedir = None self.tmpnodedir = False @@ -265,7 +282,7 @@ class CoreNodeBase(NodeBase): """ if self.nodedir is None: self.nodedir = os.path.join(self.session.session_dir, self.name + ".conf") - os.makedirs(self.nodedir) + self.net_cmd(["mkdir", "-p", self.nodedir]) self.tmpnodedir = True else: self.tmpnodedir = False @@ -446,7 +463,14 @@ class CoreNode(CoreNodeBase): valid_address_types = {"inet", "inet6", "inet6link"} def __init__( - self, session, _id=None, name=None, nodedir=None, bootsh="boot.sh", start=True + self, + session, + _id=None, + name=None, + nodedir=None, + bootsh="boot.sh", + start=True, + server=None, ): """ Create a CoreNode instance. @@ -457,8 +481,9 @@ class CoreNode(CoreNodeBase): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost """ - super(CoreNode, self).__init__(session, _id, name, start) + super(CoreNode, self).__init__(session, _id, name, start, server) self.nodedir = nodedir self.ctrlchnlname = os.path.abspath( os.path.join(self.session.session_dir, self.name) @@ -619,7 +644,24 @@ class CoreNode(CoreNodeBase): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - return self.check_cmd(args) + logging.info("net cmd server(%s): %s", self.server, args) + if self.server is None: + return self.check_cmd(args) + else: + args = self.client._cmd_args() + args + args = " ".join(args) + result = self.server_conn.run(args, hide=False) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr + ) + + logging.info( + "fabric result:\n\tstdout: %s\n\tstderr: %s", + result.stdout.strip(), + result.stderr.strip(), + ) + return result.stdout.strip() def check_cmd(self, args): """ @@ -653,7 +695,7 @@ class CoreNode(CoreNodeBase): hostpath = os.path.join( self.nodedir, os.path.normpath(path).strip("/").replace("/", ".") ) - os.mkdir(hostpath) + self.net_cmd(["mkdir", "-p", hostpath]) self.mount(hostpath, path) def mount(self, source, target): diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py new file mode 100644 index 00000000..bed75a47 --- /dev/null +++ b/daemon/examples/python/distributed.py @@ -0,0 +1,55 @@ +import logging + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import NodeOptions +from core.emulator.enumerations import EventTypes + + +def main(): + # ip generator for example + # prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # initialize distributed + session.servers.add("core2") + session.init_distributed() + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create switch network node + # switch = session.add_node(_type=NodeTypes.SWITCH) + + # create nodes + options = NodeOptions() + options.emulation_server = "10.10.4.38" + options.emulation_server = "core2" + session.add_node(node_options=options) + # interface = prefixes.create_interface(node_one) + # session.add_link(node_one.id, switch.id, interface_one=interface) + + # node_two = session.add_node() + # interface = prefixes.create_interface(node_two) + # session.add_link(node_two.id, switch.id, interface_one=interface) + + # instantiate session + session.instantiate() + + # print("starting iperf server on node: %s" % node_one.name) + # node_one.cmd(["iperf", "-s", "-D"]) + # node_one_address = prefixes.ip4_address(node_one) + # + # print("node %s connecting to %s" % (node_two.name, node_one_address)) + # node_two.client.icmd(["iperf", "-t", "10", "-c", node_one_address]) + # node_one.cmd(["killall", "-9", "iperf"]) + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From 931ee65235cea91923c16d00aaa554876f22a165 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Oct 2019 09:48:30 -0700 Subject: [PATCH 049/113] added remote_cmd func for nodes to avoid duplication --- daemon/core/nodes/base.py | 44 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index b08c7cd4..8e4d33d2 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -104,18 +104,29 @@ class NodeBase(object): return utils.check_cmd(args, env=env) else: args = " ".join(args) - result = self.server_conn.run(args, hide=False) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) + return self.remote_cmd(args) - logging.info( - "fabric result:\n\tstdout: %s\n\tstderr: %s", - result.stdout.strip(), - result.stderr.strip(), + def remote_cmd(self, cmd): + """ + Run command remotely using server connection. + + :param str cmd: command to run + :return: stdout when success + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + result = self.server_conn.run(cmd, hide=False) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr ) - return result.stdout.strip() + + logging.info( + "fabric result:\n\tstdout: %s\n\tstderr: %s", + result.stdout.strip(), + result.stderr.strip(), + ) + return result.stdout.strip() def setposition(self, x=None, y=None, z=None): """ @@ -650,18 +661,7 @@ class CoreNode(CoreNodeBase): else: args = self.client._cmd_args() + args args = " ".join(args) - result = self.server_conn.run(args, hide=False) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) - - logging.info( - "fabric result:\n\tstdout: %s\n\tstderr: %s", - result.stdout.strip(), - result.stderr.strip(), - ) - return result.stdout.strip() + return self.remote_cmd(args) def check_cmd(self, args): """ From 95296988c5e92f52ed66290b1b5d2893d464cc03 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Oct 2019 11:16:57 -0700 Subject: [PATCH 050/113] updates to Pipefile.lock and for nodes to add server to constructor --- daemon/Pipfile.lock | 414 +++++++++++++++++++++++----------- daemon/core/emane/nodes.py | 4 +- daemon/core/nodes/base.py | 5 +- daemon/core/nodes/network.py | 28 ++- daemon/core/nodes/physical.py | 6 +- ns3/corens3/obj.py | 6 +- 6 files changed, 310 insertions(+), 153 deletions(-) diff --git a/daemon/Pipfile.lock b/daemon/Pipfile.lock index 1e1cf60c..73400b8b 100644 --- a/daemon/Pipfile.lock +++ b/daemon/Pipfile.lock @@ -14,6 +14,67 @@ ] }, "default": { + "asn1crypto": { + "hashes": [ + "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", + "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" + ], + "version": "==1.0.1" + }, + "bcrypt": { + "hashes": [ + "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", + "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", + "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", + "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", + "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", + "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", + "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", + "sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0", + "sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de", + "sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e", + "sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052", + "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", + "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", + "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", + "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", + "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" + ], + "version": "==3.1.7" + }, + "cffi": { + "hashes": [ + "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", + "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", + "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", + "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", + "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", + "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", + "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", + "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", + "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", + "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", + "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", + "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", + "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", + "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", + "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", + "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", + "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", + "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", + "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", + "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", + "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", + "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", + "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", + "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", + "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", + "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", + "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", + "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" + ], + "version": "==1.12.3" + }, "configparser": { "hashes": [ "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", @@ -25,14 +86,33 @@ "editable": true, "path": "." }, - "enum34": { + "cryptography": { "hashes": [ - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", + "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", + "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", + "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", + "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", + "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", + "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", + "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", + "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", + "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", + "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", + "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", + "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", + "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", + "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", + "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" ], - "version": "==1.1.6" + "version": "==2.7" + }, + "fabric": { + "hashes": [ + "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389", + "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6" + ], + "version": "==2.5.0" }, "future": { "hashes": [ @@ -42,40 +122,48 @@ }, "grpcio": { "hashes": [ - "sha256:1303578092f1f6e4bfbc354c04ac422856c393723d3ffa032fff0f7cb5cfd693", - "sha256:229c6b313cd82bec8f979b059d87f03cc1a48939b543fe170b5a9c5cf6a6bc69", - "sha256:3cd3d99a8b5568d0d186f9520c16121a0f2a4bcad8e2b9884b76fb88a85a7774", - "sha256:41cfb222db358227521f9638a6fbc397f310042a4db5539a19dea01547c621cd", - "sha256:43330501660f636fd6547d1e196e395cd1e2c2ae57d62219d6184a668ffebda0", - "sha256:45d7a2bd8b4f25a013296683f4140d636cdbb507d94a382ea5029a21e76b1648", - "sha256:47dc935658a13b25108823dabd010194ddea9610357c5c1ef1ad7b3f5157ebee", - "sha256:480aa7e2b56238badce0b9413a96d5b4c90c3bfbd79eba5a0501e92328d9669e", - "sha256:4a0934c8b0f97e1d8c18e76c45afc0d02d33ab03125258179f2ac6c7a13f3626", - "sha256:5624dab19e950f99e560400c59d87b685809e4cfcb2c724103f1ab14c06071f7", - "sha256:60515b1405bb3dadc55e6ca99429072dad3e736afcf5048db5452df5572231ff", - "sha256:610f97ebae742a57d336a69b09a9c7d7de1f62aa54aaa8adc635b38f55ba4382", - "sha256:64ea189b2b0859d1f7b411a09185028744d494ef09029630200cc892e366f169", - "sha256:686090c6c1e09e4f49585b8508d0a31d58bc3895e4049ea55b197d1381e9f70f", - "sha256:7745c365195bb0605e3d47b480a2a4d1baa8a41a5fd0a20de5fa48900e2c886a", - "sha256:79491e0d2b77a1c438116bf9e5f9e2e04e78b78524615e2ce453eff62db59a09", - "sha256:825177dd4c601c487836b7d6b4ba268db59787157911c623ba59a7c03c8d3adc", - "sha256:8a060e1f72fb94eee8a035ed29f1201ce903ad14cbe27bda56b4a22a8abda045", - "sha256:90168cc6353e2766e47b650c963f21cfff294654b10b3a14c67e26a4e3683634", - "sha256:94b7742734bceeff6d8db5edb31ac844cb68fc7f13617eca859ff1b78bb20ba1", - "sha256:962aebf2dd01bbb2cdb64580e61760f1afc470781f9ecd5fe8f3d8dcd8cf4556", - "sha256:9c8d9eacdce840b72eee7924c752c31b675f8aec74790e08cff184a4ea8aa9c1", - "sha256:af5b929debc336f6bab9b0da6915f9ee5e41444012aed6a79a3c7e80d7662fdf", - "sha256:b9cdb87fc77e9a3eabdc42a512368538d648fa0760ad30cf97788076985c790a", - "sha256:c5e6380b90b389454669dc67d0a39fb4dc166416e01308fcddd694236b8329ef", - "sha256:d60c90fe2bfbee735397bf75a2f2c4e70c5deab51cd40c6e4fa98fae018c8db6", - "sha256:d8582c8b1b1063249da1588854251d8a91df1e210a328aeb0ece39da2b2b763b", - "sha256:ddbf86ba3aa0ad8fed2867910d2913ee237d55920b55f1d619049b3399f04efc", - "sha256:e46bc0664c5c8a0545857aa7a096289f8db148e7f9cca2d0b760113e8994bddc", - "sha256:f6437f70ec7fed0ca3a0eef1146591bb754b418bb6c6b21db74f0333d624e135", - "sha256:f71693c3396530c6b00773b029ea85e59272557e9bd6077195a6593e4229892a", - "sha256:f79f7455f8fbd43e8e9d61914ecf7f48ba1c8e271801996fef8d6a8f3cc9f39f" + "sha256:0302331e014fc4bac028b6ad480b33f7abfe20b9bdcca7be417124dda8f22115", + "sha256:0aa0cce9c5eb1261b32173a20ed42b51308d55ce28ecc2021e868b3cb90d9503", + "sha256:0c83947575300499adbc308e986d754e7f629be0bdd9bea1ffdd5cf76e1f1eff", + "sha256:0ca26ff968d45efd4ef73447c4d4b34322ea8c7d06fbb6907ce9e5db78f1bbcb", + "sha256:0cf80a7955760c2498f8821880242bb657d70998065ff0d2a082de5ffce230a7", + "sha256:0d40706e57d9833fe0e023a08b468f33940e8909affa12547874216d36bba208", + "sha256:11872069156de34c6f3f9a1deb46cc88bc35dfde88262c4c73eb22b39b16fc55", + "sha256:16065227faae0ab0abf1789bfb92a2cd2ab5da87630663f93f8178026da40e0d", + "sha256:1e33778277685f6fabb22539136269c87c029e39b6321ef1a639b756a1c0a408", + "sha256:2b16be15b1ae656bc7a36642b8c7045be2dde2048bb4b67478003e9d9db8022a", + "sha256:3701dfca3ada27ceef0d17f728ce9dfef155ed20c57979c2b05083082258c6c1", + "sha256:41912ecaf482abf2de74c69f509878f99223f5dd6b2de1a09c955afd4de3cf9b", + "sha256:4332cbd20544fe7406910137590f38b5b3a1f6170258e038652cf478c639430f", + "sha256:44068ecbdc6467c2bff4d8198816c8a2701b6dd1ec16078fceb6adc7c1f577d6", + "sha256:53115960e37059420e2d16a4b04b00dd2ab3b6c3c67babd01ffbfdcd7881a69b", + "sha256:6e7027bcd4070414751e2a5e60706facb98a1fc636497c9bac5442fe37b8ae6b", + "sha256:6ff57fb2f07b7226b5bec89e8e921ea9bd220f35f11e094f2ba38f09eecd49c6", + "sha256:73240e244d7644654bbda1f309f4911748b6a1804b7a8897ddbe8a04c90f7407", + "sha256:785234bbc469bc75e26c868789a2080ffb30bd6e93930167797729889ad06b0b", + "sha256:82f9d3c7f91d2d1885631335c003c5d45ae1cd69cc0bc4893f21fef50b8151bc", + "sha256:86bdc2a965510658407a1372eb61f0c92f763fdfb2795e4d038944da4320c950", + "sha256:95e925b56676a55e6282b3de80a1cbad5774072159779c61eac02791dface049", + "sha256:96673bb4f14bd3263613526d1e7e33fdb38a9130e3ce87bf52314965706e1900", + "sha256:970014205e76920484679035b6fb4b16e02fc977e5aac4d22025da849c79dab9", + "sha256:ace5e8bf11a1571f855f5dab38a9bd34109b6c9bc2864abf24a597598c7e3695", + "sha256:ad375f03eb3b9cb75a24d91eab8609e134d34605f199efc41e20dd642bdac855", + "sha256:b819c4c7dcf0de76788ce5f95daad6d4e753d6da2b6a5f84e5bb5b5ce95fddc4", + "sha256:c17943fd340cbd906db49f3f03c7545e5a66b617e8348b2c7a0d2c759d216af1", + "sha256:d21247150dea86dabd3b628d8bc4b563036db3d332b3f4db3c5b1b0b122cb4f6", + "sha256:d4d500a7221116de9767229ff5dd10db91f789448d85befb0adf5a37b0cd83b5", + "sha256:e2a942a3cfccbbca21a90c144867112698ef36486345c285da9e98c466f22b22", + "sha256:e983273dca91cb8a5043bc88322eb48e2b8d4e4998ff441a1ee79ced89db3909" ], - "version": "==1.23.0" + "version": "==1.24.1" + }, + "invoke": { + "hashes": [ + "sha256:c52274d2e8a6d64ef0d61093e1983268ea1fc0cd13facb9448c4ef0c9a7ac7da", + "sha256:f4ec8a134c0122ea042c8912529f87652445d9f4de590b353d23f95bfa1f0efd", + "sha256:fc803a5c9052f15e63310aa81a43498d7c55542beb18564db88a9d75a176fa44" + ], + "version": "==1.3.0" }, "lxml": { "hashes": [ @@ -104,6 +192,64 @@ ], "version": "==4.4.1" }, + "paramiko": { + "hashes": [ + "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", + "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" + ], + "version": "==2.6.0" + }, + "protobuf": { + "hashes": [ + "sha256:125713564d8cfed7610e52444c9769b8dcb0b55e25cc7841f2290ee7bc86636f", + "sha256:1accdb7a47e51503be64d9a57543964ba674edac103215576399d2d0e34eac77", + "sha256:27003d12d4f68e3cbea9eb67427cab3bfddd47ff90670cb367fcd7a3a89b9657", + "sha256:3264f3c431a631b0b31e9db2ae8c927b79fc1a7b1b06b31e8e5bcf2af91fe896", + "sha256:3c5ab0f5c71ca5af27143e60613729e3488bb45f6d3f143dc918a20af8bab0bf", + "sha256:45dcf8758873e3f69feab075e5f3177270739f146255225474ee0b90429adef6", + "sha256:56a77d61a91186cc5676d8e11b36a5feb513873e4ae88d2ee5cf530d52bbcd3b", + "sha256:5984e4947bbcef5bd849d6244aec507d31786f2dd3344139adc1489fb403b300", + "sha256:6b0441da73796dd00821763bb4119674eaf252776beb50ae3883bed179a60b2a", + "sha256:6f6677c5ade94d4fe75a912926d6796d5c71a2a90c2aeefe0d6f211d75c74789", + "sha256:84a825a9418d7196e2acc48f8746cf1ee75877ed2f30433ab92a133f3eaf8fbe", + "sha256:b842c34fe043ccf78b4a6cf1019d7b80113707d68c88842d061fa2b8fb6ddedc", + "sha256:ca33d2f09dae149a1dcf942d2d825ebb06343b77b437198c9e2ef115cf5d5bc1", + "sha256:db83b5c12c0cd30150bb568e6feb2435c49ce4e68fe2d7b903113f0e221e58fe", + "sha256:f50f3b1c5c1c1334ca7ce9cad5992f098f460ffd6388a3cabad10b66c2006b09", + "sha256:f99f127909731cafb841c52f9216e447d3e4afb99b17bebfad327a75aee206de" + ], + "version": "==3.10.0" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "pynacl": { + "hashes": [ + "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", + "sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c", + "sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e", + "sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae", + "sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621", + "sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56", + "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", + "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", + "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", + "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", + "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", + "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", + "sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b", + "sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f", + "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", + "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", + "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", + "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", + "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" + ], + "version": "==1.3.0" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -136,10 +282,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", + "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" ], - "version": "==19.1.0" + "version": "==19.2.0" }, "black": { "hashes": [ @@ -180,78 +326,78 @@ }, "grpcio": { "hashes": [ - "sha256:1303578092f1f6e4bfbc354c04ac422856c393723d3ffa032fff0f7cb5cfd693", - "sha256:229c6b313cd82bec8f979b059d87f03cc1a48939b543fe170b5a9c5cf6a6bc69", - "sha256:3cd3d99a8b5568d0d186f9520c16121a0f2a4bcad8e2b9884b76fb88a85a7774", - "sha256:41cfb222db358227521f9638a6fbc397f310042a4db5539a19dea01547c621cd", - "sha256:43330501660f636fd6547d1e196e395cd1e2c2ae57d62219d6184a668ffebda0", - "sha256:45d7a2bd8b4f25a013296683f4140d636cdbb507d94a382ea5029a21e76b1648", - "sha256:47dc935658a13b25108823dabd010194ddea9610357c5c1ef1ad7b3f5157ebee", - "sha256:480aa7e2b56238badce0b9413a96d5b4c90c3bfbd79eba5a0501e92328d9669e", - "sha256:4a0934c8b0f97e1d8c18e76c45afc0d02d33ab03125258179f2ac6c7a13f3626", - "sha256:5624dab19e950f99e560400c59d87b685809e4cfcb2c724103f1ab14c06071f7", - "sha256:60515b1405bb3dadc55e6ca99429072dad3e736afcf5048db5452df5572231ff", - "sha256:610f97ebae742a57d336a69b09a9c7d7de1f62aa54aaa8adc635b38f55ba4382", - "sha256:64ea189b2b0859d1f7b411a09185028744d494ef09029630200cc892e366f169", - "sha256:686090c6c1e09e4f49585b8508d0a31d58bc3895e4049ea55b197d1381e9f70f", - "sha256:7745c365195bb0605e3d47b480a2a4d1baa8a41a5fd0a20de5fa48900e2c886a", - "sha256:79491e0d2b77a1c438116bf9e5f9e2e04e78b78524615e2ce453eff62db59a09", - "sha256:825177dd4c601c487836b7d6b4ba268db59787157911c623ba59a7c03c8d3adc", - "sha256:8a060e1f72fb94eee8a035ed29f1201ce903ad14cbe27bda56b4a22a8abda045", - "sha256:90168cc6353e2766e47b650c963f21cfff294654b10b3a14c67e26a4e3683634", - "sha256:94b7742734bceeff6d8db5edb31ac844cb68fc7f13617eca859ff1b78bb20ba1", - "sha256:962aebf2dd01bbb2cdb64580e61760f1afc470781f9ecd5fe8f3d8dcd8cf4556", - "sha256:9c8d9eacdce840b72eee7924c752c31b675f8aec74790e08cff184a4ea8aa9c1", - "sha256:af5b929debc336f6bab9b0da6915f9ee5e41444012aed6a79a3c7e80d7662fdf", - "sha256:b9cdb87fc77e9a3eabdc42a512368538d648fa0760ad30cf97788076985c790a", - "sha256:c5e6380b90b389454669dc67d0a39fb4dc166416e01308fcddd694236b8329ef", - "sha256:d60c90fe2bfbee735397bf75a2f2c4e70c5deab51cd40c6e4fa98fae018c8db6", - "sha256:d8582c8b1b1063249da1588854251d8a91df1e210a328aeb0ece39da2b2b763b", - "sha256:ddbf86ba3aa0ad8fed2867910d2913ee237d55920b55f1d619049b3399f04efc", - "sha256:e46bc0664c5c8a0545857aa7a096289f8db148e7f9cca2d0b760113e8994bddc", - "sha256:f6437f70ec7fed0ca3a0eef1146591bb754b418bb6c6b21db74f0333d624e135", - "sha256:f71693c3396530c6b00773b029ea85e59272557e9bd6077195a6593e4229892a", - "sha256:f79f7455f8fbd43e8e9d61914ecf7f48ba1c8e271801996fef8d6a8f3cc9f39f" + "sha256:0302331e014fc4bac028b6ad480b33f7abfe20b9bdcca7be417124dda8f22115", + "sha256:0aa0cce9c5eb1261b32173a20ed42b51308d55ce28ecc2021e868b3cb90d9503", + "sha256:0c83947575300499adbc308e986d754e7f629be0bdd9bea1ffdd5cf76e1f1eff", + "sha256:0ca26ff968d45efd4ef73447c4d4b34322ea8c7d06fbb6907ce9e5db78f1bbcb", + "sha256:0cf80a7955760c2498f8821880242bb657d70998065ff0d2a082de5ffce230a7", + "sha256:0d40706e57d9833fe0e023a08b468f33940e8909affa12547874216d36bba208", + "sha256:11872069156de34c6f3f9a1deb46cc88bc35dfde88262c4c73eb22b39b16fc55", + "sha256:16065227faae0ab0abf1789bfb92a2cd2ab5da87630663f93f8178026da40e0d", + "sha256:1e33778277685f6fabb22539136269c87c029e39b6321ef1a639b756a1c0a408", + "sha256:2b16be15b1ae656bc7a36642b8c7045be2dde2048bb4b67478003e9d9db8022a", + "sha256:3701dfca3ada27ceef0d17f728ce9dfef155ed20c57979c2b05083082258c6c1", + "sha256:41912ecaf482abf2de74c69f509878f99223f5dd6b2de1a09c955afd4de3cf9b", + "sha256:4332cbd20544fe7406910137590f38b5b3a1f6170258e038652cf478c639430f", + "sha256:44068ecbdc6467c2bff4d8198816c8a2701b6dd1ec16078fceb6adc7c1f577d6", + "sha256:53115960e37059420e2d16a4b04b00dd2ab3b6c3c67babd01ffbfdcd7881a69b", + "sha256:6e7027bcd4070414751e2a5e60706facb98a1fc636497c9bac5442fe37b8ae6b", + "sha256:6ff57fb2f07b7226b5bec89e8e921ea9bd220f35f11e094f2ba38f09eecd49c6", + "sha256:73240e244d7644654bbda1f309f4911748b6a1804b7a8897ddbe8a04c90f7407", + "sha256:785234bbc469bc75e26c868789a2080ffb30bd6e93930167797729889ad06b0b", + "sha256:82f9d3c7f91d2d1885631335c003c5d45ae1cd69cc0bc4893f21fef50b8151bc", + "sha256:86bdc2a965510658407a1372eb61f0c92f763fdfb2795e4d038944da4320c950", + "sha256:95e925b56676a55e6282b3de80a1cbad5774072159779c61eac02791dface049", + "sha256:96673bb4f14bd3263613526d1e7e33fdb38a9130e3ce87bf52314965706e1900", + "sha256:970014205e76920484679035b6fb4b16e02fc977e5aac4d22025da849c79dab9", + "sha256:ace5e8bf11a1571f855f5dab38a9bd34109b6c9bc2864abf24a597598c7e3695", + "sha256:ad375f03eb3b9cb75a24d91eab8609e134d34605f199efc41e20dd642bdac855", + "sha256:b819c4c7dcf0de76788ce5f95daad6d4e753d6da2b6a5f84e5bb5b5ce95fddc4", + "sha256:c17943fd340cbd906db49f3f03c7545e5a66b617e8348b2c7a0d2c759d216af1", + "sha256:d21247150dea86dabd3b628d8bc4b563036db3d332b3f4db3c5b1b0b122cb4f6", + "sha256:d4d500a7221116de9767229ff5dd10db91f789448d85befb0adf5a37b0cd83b5", + "sha256:e2a942a3cfccbbca21a90c144867112698ef36486345c285da9e98c466f22b22", + "sha256:e983273dca91cb8a5043bc88322eb48e2b8d4e4998ff441a1ee79ced89db3909" ], - "version": "==1.23.0" + "version": "==1.24.1" }, "grpcio-tools": { "hashes": [ - "sha256:056f2a274edda4315e825ac2e3a9536f5415b43aa51669196860c8de6e76d847", - "sha256:0c953251585fdcd422072e4b7f4243fce215f22e21db94ec83c5970e41db6e18", - "sha256:142a73f5769f37bf2e4a8e4a77ef60f7af5f55635f60428322b49c87bd8f9cc0", - "sha256:1b333e2a068d8ef89a01eb23a098d2a789659c3178de79da9bd3d0ffb944cc6d", - "sha256:2124f19cc51d63405a0204ae38ef355732ab0a235975ab41ff6f6f9701905432", - "sha256:24c3a04adfb6c6f1bc4a2f8498d7661ca296ae352b498e538832c22ddde7bf81", - "sha256:3a2054e9640cbdd0ce8a345afb86be52875c5a8f9f5973a5c64791a8002da2dd", - "sha256:3fd15a09eecef83440ac849dcda2ff522f8ee1603ebfcdbb0e9b320ef2012e41", - "sha256:457e7a7dfa0b6bb608a766edba6f20c9d626a790df802016b930ad242fec4470", - "sha256:49ad5661d54ff0d164e4b441ee5e05191187d497380afa16d36d72eb8ef048de", - "sha256:561078e425d21a6720c3c3828385d949e24c0765e2852a46ecc3ad3fca2706e5", - "sha256:5a4f65ab06b32dc34112ed114dee3b698c8463670474334ece5b0b531073804c", - "sha256:8883e0e34676356d219a4cd37d239c3ead655cc550836236b52068e091416fff", - "sha256:8d2b45b1faf81098780e07d6a1c207b328b07e913160b8baa7e2e8d89723e06c", - "sha256:b0ebddb6ecc4c161369f93bb3a74c6120a498d3ddc759b64679709a885dd6d4f", - "sha256:b786ba4842c50de865dd3885b5570690a743e84a327b7213dd440eb0e6b996f8", - "sha256:be8efa010f5a80f1862ead80c3b19b5eb97dc954a0f59a1e2487078576105e03", - "sha256:c29106eaff0e2e708a9a89628dc0134ef145d0d3631f0ef421c09f380c30e354", - "sha256:c3c71236a056ec961b2b8b3b7c0b3b5a826283bc69c4a1c6415d23b70fea8243", - "sha256:cbc35031ec2b29af36947d085a7fbbcd8b79b84d563adf6156103d82565f78db", - "sha256:d47307c22744918e803c1eec7263a14f36aaf34fe496bff9ccbcae67c02b40ae", - "sha256:db088c98e563c1bb070da5984c8df08b45b61e4d9c6d2a8a1ffeed2af89fd1f3", - "sha256:df4dd1cb670062abdacc1fbce41cae4e08a4a212d28dd94fdbbf90615d027f73", - "sha256:e3adcf1499ca08d1e036ff44aedf55ed78653d946f4c4426b6e72ab757cc4dec", - "sha256:e3b3e32e0cda4dc382ec5bed8599dab644e4b3fc66a9ab54eb58248e207880b9", - "sha256:ed524195b35304c670755efa1eca579e5c290a66981f97004a5b2c0d12d6897d", - "sha256:edb42432790b1f8ec9f08faf9326d7e5dfe6e1d8c8fe4db39abc0a49c1c76537", - "sha256:eff1f995e5aa4cc941b6bbc45b5b57842f8f62bbe1a99427281c2c70cf42312c", - "sha256:f2fcdc2669662d77b400f80e20315a3661466e3cb3df1730f8083f9e49465cbc", - "sha256:f52ec9926daf48f41389d39d01570967b99c7dbc12bffc134cc3a3c5b5540ba2", - "sha256:fd007d67fdfbd2a13bf8a8c8ced8353b42a92ca72dbee54e951d8ddbc6ca12bc", - "sha256:ff9045e928dbb7943ea8559bfabebee95a43a830e00bf52c16202d2d805780fb" + "sha256:0a849994d7d6411ca6147bb1db042b61ba6232eb5c90c69de5380a441bf80a75", + "sha256:0db96ed52816471ceec8807aedf5cb4fd133ca201f614464cb46ca58584edf84", + "sha256:1b98720459204e9afa33928e4fd53aeec6598afb7f704ed497f6926c67f12b9b", + "sha256:200479310cc083c41a5020f6e5e916a99ee0f7c588b6affe317b96a839120bf4", + "sha256:25543b8f2e59ddcc9929d6f6111faa5c474b21580d2996f93347bb55f2ecba84", + "sha256:2d4609996616114c155c1e697a9faf604d81f2508cd9a4168a0bafd53c799e24", + "sha256:2fdb2a1ed2b3e43514d9c29c9de415c953a46caabbc8a9b7de1439a0c1bd3b89", + "sha256:3886a7983d8ae19df0c11a54114d6546fcdf76cf18cdccf25c3b14200fd5478a", + "sha256:408d111b9341f107bdafc523e2345471547ffe8a4104e6f2ce690b7a25c4bae5", + "sha256:60b3dd5e76c1389fc836bf83675985b92d158ff9a8d3d6d3f0a670f0c227ef13", + "sha256:629be7ce8504530b4adbf0425a44dd53007ccb6212344804294888c9662cc38f", + "sha256:6af3dde07b1051e954230e650a6ef74073cf993cf473c2078580f8a73c4fe46a", + "sha256:7a1e77539d28e90517c55561f40f7872f1348d0e23f25a38d68abbfb5b0eff88", + "sha256:87917a18b3b5951b6c9badd7b5ef09f63f61611966b58427b856bdf5c1d68e91", + "sha256:8823d0ebd185a77edb506e286c88d06847f75620a033ad96ef9c0fd7efc1d859", + "sha256:8bd3e12e1969beb813b861a2a65d4f2d4faaa87de0b60bf7f848da2d8ffc4eb2", + "sha256:8f37e9acc46e75ed9786ece89afeacd86182893eacc3f0642d81531b90fbe25f", + "sha256:9b358dd2f4142e89d760a52a7a8f4ec5dbaf955e7ada09f703f3a5d05dddd12e", + "sha256:9cb43007c4a8aa7adaacf896f5109b578028f23d259615e3fa5866e38855b311", + "sha256:9cf594bfbfbf84dcd462b20a4a753362be7ed376d2b5020a083dac24400b7b6c", + "sha256:ab79940e5c5ed949e1f95e7f417dd916b0992d29f45d073dd64501a76d128e2c", + "sha256:ba8aab6c78a82755477bb8c79f3be0824b297422d1edb21b94ae5a45407bf3ba", + "sha256:bcc00b83bf39f6e60a13f0b24ec3951f4d2ae810b01e6e125b7ff238a85da1ac", + "sha256:c1fcf5cbe6a2ecdc587b469156520b9128ccdb7c5908060c7d9712cd97e76db5", + "sha256:c6e640d39b9615388b59036b29970292b15f4519043e43833e28c674f740d1f7", + "sha256:c6ea2c385da620049b17f0135cf9307a4750e9d9c9988e15bfeeaf1f209c4ada", + "sha256:cec4f37120f93fe2ab4ab9a7eab9a877163d74c232c93a275a624971f8557b81", + "sha256:d2dbb42d237bcdecb7284535ec074c85bbf880124c1cbbff362ed3bd81ed7d41", + "sha256:d5c98a41abd4f7de43b256c21bbba2a97c57e25bf6a170927a90638b18f7509c", + "sha256:dcf5965a24179aa7dcfa00b5ff70f4f2f202e663657e0c74a642307beecda053", + "sha256:e11e3aacf0200d6e00a9b74534e0174738768fe1c41e5aa2f4aab881d6b43afd", + "sha256:e550816bdb2e49bba94bcd7f342004a8adbc46e9a25c8c4ed3fd58f2435c655f" ], "index": "pypi", - "version": "==1.23.0" + "version": "==1.24.1" }, "identify": { "hashes": [ @@ -262,11 +408,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:652234b6ab8f2506ae58e528b6fbcc668831d3cc758e1bc01ef438d328b68cdb", - "sha256:6f264986fb88042bc1f0535fa9a557e6a376cfe5679dc77caac7fe8b5d43d05f" + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" ], "markers": "python_version < '3.8'", - "version": "==0.22" + "version": "==0.23" }, "importlib-resources": { "hashes": [ @@ -314,10 +460,10 @@ }, "packaging": { "hashes": [ - "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", - "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "version": "==19.1" + "version": "==19.2" }, "pluggy": { "hashes": [ @@ -336,24 +482,24 @@ }, "protobuf": { "hashes": [ - "sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295", - "sha256:01acbca2d2c8c3f7f235f1842440adbe01bbc379fa1cbdd80753801432b3fae9", - "sha256:0a795bca65987b62d6b8a2d934aa317fd1a4d06a6dd4df36312f5b0ade44a8d9", - "sha256:0ec035114213b6d6e7713987a759d762dd94e9f82284515b3b7331f34bfaec7f", - "sha256:31b18e1434b4907cb0113e7a372cd4d92c047ce7ba0fa7ea66a404d6388ed2c1", - "sha256:32a3abf79b0bef073c70656e86d5bd68a28a1fbb138429912c4fc07b9d426b07", - "sha256:55f85b7808766e5e3f526818f5e2aeb5ba2edcc45bcccede46a3ccc19b569cb0", - "sha256:64ab9bc971989cbdd648c102a96253fdf0202b0c38f15bd34759a8707bdd5f64", - "sha256:64cf847e843a465b6c1ba90fb6c7f7844d54dbe9eb731e86a60981d03f5b2e6e", - "sha256:917c8662b585470e8fd42f052661fc66d59fccaae450a60044307dcbf82a3335", - "sha256:afed9003d7f2be2c3df20f64220c30faec441073731511728a2cb4cab4cd46a6", - "sha256:bf8e05d638b585d1752c5a84247134a0350d3a8b73d3632489a014a9f6f1e758", - "sha256:d831b047bd69becaf64019a47179eb22118a50dd008340655266a906c69c6417", - "sha256:de2760583ed28749ff885789c1cbc6c9c06d6de92fc825740ab99deb2f25ea4d", - "sha256:eabc4cf1bc19689af8022ba52fd668564a8d96e0d08f3b4732d26a64255216a4", - "sha256:fcff6086c86fb1628d94ea455c7b9de898afc50378042927a59df8065a79a549" + "sha256:125713564d8cfed7610e52444c9769b8dcb0b55e25cc7841f2290ee7bc86636f", + "sha256:1accdb7a47e51503be64d9a57543964ba674edac103215576399d2d0e34eac77", + "sha256:27003d12d4f68e3cbea9eb67427cab3bfddd47ff90670cb367fcd7a3a89b9657", + "sha256:3264f3c431a631b0b31e9db2ae8c927b79fc1a7b1b06b31e8e5bcf2af91fe896", + "sha256:3c5ab0f5c71ca5af27143e60613729e3488bb45f6d3f143dc918a20af8bab0bf", + "sha256:45dcf8758873e3f69feab075e5f3177270739f146255225474ee0b90429adef6", + "sha256:56a77d61a91186cc5676d8e11b36a5feb513873e4ae88d2ee5cf530d52bbcd3b", + "sha256:5984e4947bbcef5bd849d6244aec507d31786f2dd3344139adc1489fb403b300", + "sha256:6b0441da73796dd00821763bb4119674eaf252776beb50ae3883bed179a60b2a", + "sha256:6f6677c5ade94d4fe75a912926d6796d5c71a2a90c2aeefe0d6f211d75c74789", + "sha256:84a825a9418d7196e2acc48f8746cf1ee75877ed2f30433ab92a133f3eaf8fbe", + "sha256:b842c34fe043ccf78b4a6cf1019d7b80113707d68c88842d061fa2b8fb6ddedc", + "sha256:ca33d2f09dae149a1dcf942d2d825ebb06343b77b437198c9e2ef115cf5d5bc1", + "sha256:db83b5c12c0cd30150bb568e6feb2435c49ce4e68fe2d7b903113f0e221e58fe", + "sha256:f50f3b1c5c1c1334ca7ce9cad5992f098f460ffd6388a3cabad10b66c2006b09", + "sha256:f99f127909731cafb841c52f9216e447d3e4afb99b17bebfad327a75aee206de" ], - "version": "==3.9.1" + "version": "==3.10.0" }, "py": { "hashes": [ @@ -385,11 +531,11 @@ }, "pytest": { "hashes": [ - "sha256:95d13143cc14174ca1a01ec68e84d76ba5d9d493ac02716fd9706c949a505210", - "sha256:b78fe2881323bd44fd9bd76e5317173d4316577e7b1cddebae9136a4495ec865" + "sha256:13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5", + "sha256:d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.2.0" }, "pyyaml": { "hashes": [ diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 89c97b6b..5451506f 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -29,8 +29,8 @@ class EmaneNet(CoreNetworkBase): type = "wlan" is_emane = True - def __init__(self, session, _id=None, name=None, start=True): - super(EmaneNet, self).__init__(session, _id, name, start) + def __init__(self, session, _id=None, name=None, start=True, server=None): + super(EmaneNet, self).__init__(session, _id, name, start, server) self.conf = "" self.up = False self.nemidmap = {} diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 8e4d33d2..d0f03e6e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -1045,7 +1045,7 @@ class CoreNetworkBase(NodeBase): linktype = LinkTypes.WIRED.value is_emane = False - def __init__(self, session, _id, name, start=True): + def __init__(self, session, _id, name, start=True, server=None): """ Create a CoreNetworkBase instance. @@ -1053,8 +1053,9 @@ class CoreNetworkBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: should object start + :param str server: remote server node will run on, default is None for localhost """ - super(CoreNetworkBase, self).__init__(session, _id, name, start=start) + super(CoreNetworkBase, self).__init__(session, _id, name, start, server) self._linked = {} self._linked_lock = threading.Lock() diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 6af1ed9e..444ded56 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -281,7 +281,9 @@ class CoreNetwork(CoreNetworkBase): policy = "DROP" - def __init__(self, session, _id=None, name=None, start=True, policy=None): + def __init__( + self, session, _id=None, name=None, start=True, server=None, policy=None + ): """ Creates a LxBrNet instance. @@ -289,9 +291,10 @@ class CoreNetwork(CoreNetworkBase): :param int _id: object id :param str name: object name :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :param policy: network policy """ - CoreNetworkBase.__init__(self, session, _id, name, start) + CoreNetworkBase.__init__(self, session, _id, name, start, server) if name is None: name = str(self.id) if policy is not None: @@ -649,6 +652,7 @@ class GreTapBridge(CoreNetwork): ttl=255, key=None, start=True, + server=None, ): """ Create a GreTapBridge instance. @@ -663,9 +667,7 @@ class GreTapBridge(CoreNetwork): :param key: gre tap key :param bool start: start flag """ - CoreNetwork.__init__( - self, session=session, _id=_id, name=name, policy=policy, start=False - ) + CoreNetwork.__init__(self, session, _id, name, False, server, policy) self.grekey = key if self.grekey is None: self.grekey = self.session.id ^ self.id @@ -769,6 +771,7 @@ class CtrlNet(CoreNetwork): prefix=None, hostid=None, start=True, + server=None, assign_address=True, updown_script=None, serverintf=None, @@ -782,6 +785,7 @@ class CtrlNet(CoreNetwork): :param prefix: control network ipv4 prefix :param hostid: host id :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :param str assign_address: assigned address :param str updown_script: updown script :param serverintf: server interface @@ -792,7 +796,7 @@ class CtrlNet(CoreNetwork): self.assign_address = assign_address self.updown_script = updown_script self.serverintf = serverintf - CoreNetwork.__init__(self, session, _id=_id, name=name, start=start) + CoreNetwork.__init__(self, session, _id, name, start, server) def startup(self): """ @@ -1028,7 +1032,7 @@ class HubNode(CoreNetwork): policy = "ACCEPT" type = "hub" - def __init__(self, session, _id=None, name=None, start=True): + def __init__(self, session, _id=None, name=None, start=True, server=None): """ Creates a HubNode instance. @@ -1036,9 +1040,10 @@ class HubNode(CoreNetwork): :param int _id: node id :param str name: node namee :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ - CoreNetwork.__init__(self, session, _id, name, start) + CoreNetwork.__init__(self, session, _id, name, start, server) # TODO: move to startup method if start: @@ -1055,7 +1060,9 @@ class WlanNode(CoreNetwork): policy = "DROP" type = "wlan" - def __init__(self, session, _id=None, name=None, start=True, policy=None): + def __init__( + self, session, _id=None, name=None, start=True, server=None, policy=None + ): """ Create a WlanNode instance. @@ -1063,9 +1070,10 @@ class WlanNode(CoreNetwork): :param int _id: node id :param str name: node name :param bool start: start flag + :param str server: remote server node will run on, default is None for localhost :param policy: wlan policy """ - CoreNetwork.__init__(self, session, _id, name, start, policy) + CoreNetwork.__init__(self, session, _id, name, start, server, policy) # wireless model such as basic range self.model = None # mobility model such as scripted diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 0035f97a..ba8cdc25 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -277,7 +277,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): apitype = NodeTypes.RJ45.value type = "rj45" - def __init__(self, session, _id=None, name=None, mtu=1500, start=True): + def __init__(self, session, _id=None, name=None, mtu=1500, start=True, server=None): """ Create an RJ45Node instance. @@ -286,9 +286,9 @@ class Rj45Node(CoreNodeBase, CoreInterface): :param str name: node name :param mtu: rj45 mtu :param bool start: start flag - :return: + :param str server: remote server node will run on, default is None for localhost """ - CoreNodeBase.__init__(self, session, _id, name, start=start) + CoreNodeBase.__init__(self, session, _id, name, start, server) CoreInterface.__init__(self, node=self, name=name, mtu=mtu) self.up = False self.lock = threading.RLock() diff --git a/ns3/corens3/obj.py b/ns3/corens3/obj.py index 70291d3b..c1907f03 100644 --- a/ns3/corens3/obj.py +++ b/ns3/corens3/obj.py @@ -117,8 +117,10 @@ class CoreNs3Net(CoreNetworkBase): # icon used type = "wlan" - def __init__(self, session, _id=None, name=None, start=True, policy=None): - CoreNetworkBase.__init__(self, session, _id, name) + def __init__( + self, session, _id=None, name=None, start=True, server=None, policy=None + ): + CoreNetworkBase.__init__(self, session, _id, name, start, server) self.tapbridge = ns.tap_bridge.TapBridgeHelper() self._ns3devs = {} self._tapdevs = {} From cca57bba47fc8312f85afe2774e0500aaa809f9b Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Oct 2019 16:10:01 -0700 Subject: [PATCH 051/113] updated other node system commands to be ran in such a way that should work if local or remote using shell commands --- daemon/core/nodes/base.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d0f03e6e..848fae34 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -2,12 +2,9 @@ Defines the base logic for nodes used within core. """ -import errno import logging import os import random -import shutil -import signal import socket import string import threading @@ -309,7 +306,7 @@ class CoreNodeBase(NodeBase): return if self.tmpnodedir: - shutil.rmtree(self.nodedir, ignore_errors=True) + self.net_cmd(["rm", "-rf", self.nodedir]) def addnetif(self, netif, ifindex): """ @@ -522,8 +519,8 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - os.kill(self.pid, 0) - except OSError: + self.net_cmd(["kill", "-9", str(self.pid)]) + except CoreCommandError: return False return True @@ -560,6 +557,7 @@ class CoreNode(CoreNodeBase): output = self.net_cmd(vnoded, env=env) self.pid = int(output) + logging.debug("node(%s) pid: %s", self.name, self.pid) # create vnode client self.client = client.VnodeClient(self.name, self.ctrlchnlname) @@ -599,21 +597,17 @@ class CoreNode(CoreNodeBase): for netif in self.netifs(): netif.shutdown() - # attempt to kill node process and wait for termination of children + # kill node process if present try: - os.kill(self.pid, signal.SIGTERM) - os.waitpid(self.pid, 0) - except OSError as e: - if e.errno != 10: - logging.exception("error killing process") + self.net_cmd(["kill", "-9", str(self.pid)]) + except CoreCommandError: + logging.exception("error killing process") # remove node directory if present try: - os.unlink(self.ctrlchnlname) - except OSError as e: - # no such file or directory - if e.errno != errno.ENOENT: - logging.exception("error removing node directory") + self.net_cmd(["rm", "-rf", self.ctrlchnlname]) + except CoreCommandError: + logging.exception("error removing node directory") # clear interface data, close client, and mark self and not up self._netif.clear() @@ -1029,9 +1023,9 @@ class CoreNode(CoreNodeBase): :return: nothing """ hostfilename = self.hostfilename(filename) - shutil.copy2(srcfilename, hostfilename) + self.net_cmd(["cp", "-a", srcfilename, hostfilename]) if mode is not None: - os.chmod(hostfilename, mode) + self.net_cmd(["chmod", oct(mode), hostfilename]) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) From 4eacd815d13b9bf4a93054e744d13f2026bd66fc Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sun, 6 Oct 2019 00:06:29 -0700 Subject: [PATCH 052/113] updated to use fabric scp for copying files to remote nodes --- daemon/core/nodes/base.py | 64 ++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 848fae34..901d08a6 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -5,11 +5,13 @@ Defines the base logic for nodes used within core. import logging import os import random +import shutil import socket import string import threading from builtins import range from socket import AF_INET, AF_INET6 +from tempfile import NamedTemporaryFile from fabric import Connection @@ -519,7 +521,7 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - self.net_cmd(["kill", "-9", str(self.pid)]) + self.net_cmd(["kill", "-0", str(self.pid)]) except CoreCommandError: return False @@ -961,9 +963,13 @@ class CoreNode(CoreNodeBase): """ logging.info("adding file from %s to %s", srcname, filename) directory = os.path.dirname(filename) - self.client.check_cmd(["mkdir", "-p", directory]) - self.client.check_cmd(["mv", srcname, filename]) - self.client.check_cmd(["sync"]) + if self.server is None: + self.client.check_cmd(["mkdir", "-p", directory]) + self.client.check_cmd(["mv", srcname, filename]) + self.client.check_cmd(["sync"]) + else: + self.net_cmd(["mkdir", "-p", directory]) + self.server_conn.put(srcname, filename) def hostfilename(self, filename): """ @@ -981,21 +987,6 @@ class CoreNode(CoreNodeBase): dirname = os.path.join(self.nodedir, dirname) return os.path.join(dirname, basename) - def opennodefile(self, filename, mode="w"): - """ - Open a node file, within it"s directory. - - :param str filename: file name to open - :param str mode: mode to open file in - :return: open file - :rtype: file - """ - hostfilename = self.hostfilename(filename) - dirname, _basename = os.path.split(hostfilename) - if not os.path.isdir(dirname): - os.makedirs(dirname, mode=0o755) - return open(hostfilename, mode) - def nodefile(self, filename, contents, mode=0o644): """ Create a node file with a given mode. @@ -1005,12 +996,24 @@ class CoreNode(CoreNodeBase): :param int mode: mode for file :return: nothing """ - with self.opennodefile(filename, "w") as open_file: - open_file.write(contents) - os.chmod(open_file.name, mode) - logging.debug( - "node(%s) added file: %s; mode: 0%o", self.name, open_file.name, mode - ) + hostfilename = self.hostfilename(filename) + dirname, _basename = os.path.split(hostfilename) + if self.server is None: + if not os.path.isdir(dirname): + os.makedirs(dirname, mode=0o755) + with open(hostfilename, "w") as open_file: + open_file.write(contents) + os.chmod(open_file.name, mode) + else: + temp = NamedTemporaryFile() + temp.write(contents) + temp.close() + self.net_cmd(["mkdir", "-m", oct(0o755), "-p", dirname]) + self.server_conn.put(temp.name, hostfilename) + self.net_cmd(["chmod", oct(mode), hostfilename]) + logging.debug( + "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode + ) def nodefilecopy(self, filename, srcfilename, mode=None): """ @@ -1023,9 +1026,14 @@ class CoreNode(CoreNodeBase): :return: nothing """ hostfilename = self.hostfilename(filename) - self.net_cmd(["cp", "-a", srcfilename, hostfilename]) - if mode is not None: - self.net_cmd(["chmod", oct(mode), hostfilename]) + if self.server is None: + shutil.copy2(srcfilename, hostfilename) + if mode is not None: + os.chmod(hostfilename, mode) + else: + self.server_conn.put(srcfilename, hostfilename) + if mode is not None: + self.net_cmd(["chmod", oct(mode), hostfilename]) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) From 212fec916b77a2d9e594026a5e0caec973a5c25e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 7 Oct 2019 11:58:27 -0700 Subject: [PATCH 053/113] updated how distributed servers are added and connections are created to reduce duplicate connections --- daemon/core/emulator/session.py | 29 +++++++++++++++------ daemon/core/nodes/base.py | 36 ++++++++++++++------------- daemon/examples/python/distributed.py | 8 +++--- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index f5625232..e0afc53c 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -147,7 +147,7 @@ class Session(object): self.sdt = Sdt(session=self) # distributed servers - self.servers = set() + self.servers = {} # initialize default node services self.services.default_services = { @@ -158,10 +158,21 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } + def add_distributed(self, server): + conn = Connection(server, user="root") + self.servers[server] = conn + def init_distributed(self): for server in self.servers: + conn = self.servers[server] cmd = "mkdir -p %s" % self.session_dir - Connection(server, user="root").run(cmd, hide=False) + conn.run(cmd, hide=False) + + def shutdown_distributed(self): + for server in self.servers: + conn = self.servers[server] + cmd = "rm -rf %s" % self.session_dir + conn.run(cmd, hide=False) @classmethod def get_node_class(cls, _type): @@ -676,6 +687,13 @@ class Session(object): if not name: name = "%s%s" % (node_class.__name__, _id) + # verify distributed server + server = self.servers.get(node_options.emulation_server) + if node_options.emulation_server is not None and server is None: + raise CoreError( + "invalid distributed server: %s" % node_options.emulation_server + ) + # create node logging.info( "creating node(%s) id(%s) name(%s) start(%s)", @@ -694,11 +712,7 @@ class Session(object): ) else: node = self.create_node( - cls=node_class, - _id=_id, - name=name, - start=start, - server=node_options.emulation_server, + cls=node_class, _id=_id, name=name, start=start, server=server ) # set node attributes @@ -972,6 +986,7 @@ class Session(object): preserve = self.options.get_config("preservedir") == "1" if not preserve: shutil.rmtree(self.session_dir, ignore_errors=True) + self.shutdown_distributed() # call session shutdown handlers for handler in self.shutdown_handlers: diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 901d08a6..21324c59 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -13,8 +13,6 @@ from builtins import range from socket import AF_INET, AF_INET6 from tempfile import NamedTemporaryFile -from fabric import Connection - from core import constants, utils from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes @@ -42,7 +40,8 @@ class NodeBase(object): :param int _id: id :param str name: object name :param bool start: start value - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ self.session = session @@ -53,8 +52,6 @@ class NodeBase(object): name = "o%s" % self.id self.name = name self.server = server - if self.server is not None: - self.server_conn = Connection(self.server, user="root") self.type = None self.services = None @@ -103,18 +100,23 @@ class NodeBase(object): return utils.check_cmd(args, env=env) else: args = " ".join(args) - return self.remote_cmd(args) + return self.remote_cmd(args, env=env) - def remote_cmd(self, cmd): + def remote_cmd(self, cmd, env=None): """ Run command remotely using server connection. :param str cmd: command to run + :param dict env: environment for remote command, default is None :return: stdout when success :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - result = self.server_conn.run(cmd, hide=False) + if env is None: + result = self.server.run(cmd, hide=False) + else: + logging.info("command env: %s", env) + result = self.server.run(cmd, hide=False, env=env, replace_env=True) if result.exited: raise CoreCommandError( result.exited, result.command, result.stdout, result.stderr @@ -969,7 +971,7 @@ class CoreNode(CoreNodeBase): self.client.check_cmd(["sync"]) else: self.net_cmd(["mkdir", "-p", directory]) - self.server_conn.put(srcname, filename) + self.server.put(srcname, filename) def hostfilename(self, filename): """ @@ -992,7 +994,7 @@ class CoreNode(CoreNodeBase): Create a node file with a given mode. :param str filename: name of file to create - :param contents: contents of file + :param str contents: contents of file :param int mode: mode for file :return: nothing """ @@ -1005,12 +1007,12 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - temp = NamedTemporaryFile() - temp.write(contents) + temp = NamedTemporaryFile(delete=False) + temp.write(contents.encode("utf-8")) temp.close() - self.net_cmd(["mkdir", "-m", oct(0o755), "-p", dirname]) - self.server_conn.put(temp.name, hostfilename) - self.net_cmd(["chmod", oct(mode), hostfilename]) + self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) + self.server.put(temp.name, hostfilename) + self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode ) @@ -1031,9 +1033,9 @@ class CoreNode(CoreNodeBase): if mode is not None: os.chmod(hostfilename, mode) else: - self.server_conn.put(srcfilename, hostfilename) + self.server.put(srcfilename, hostfilename) if mode is not None: - self.net_cmd(["chmod", oct(mode), hostfilename]) + self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index bed75a47..5b5174f6 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -1,4 +1,5 @@ import logging +import pdb from core.emulator.coreemu import CoreEmu from core.emulator.emudata import NodeOptions @@ -14,7 +15,7 @@ def main(): session = coreemu.create_session() # initialize distributed - session.servers.add("core2") + session.add_distributed("core2") session.init_distributed() # must be in configuration state for nodes to start, when using "node_add" below @@ -25,13 +26,12 @@ def main(): # create nodes options = NodeOptions() - options.emulation_server = "10.10.4.38" options.emulation_server = "core2" session.add_node(node_options=options) # interface = prefixes.create_interface(node_one) # session.add_link(node_one.id, switch.id, interface_one=interface) - # node_two = session.add_node() + session.add_node() # interface = prefixes.create_interface(node_two) # session.add_link(node_two.id, switch.id, interface_one=interface) @@ -46,6 +46,8 @@ def main(): # node_two.client.icmd(["iperf", "-t", "10", "-c", node_one_address]) # node_one.cmd(["killall", "-9", "iperf"]) + pdb.set_trace() + # shutdown session coreemu.shutdown() From 0bbb686bdae99576135631d228b1d4c01b63bb0d Mon Sep 17 00:00:00 2001 From: Jeff Ahrenholz Date: Tue, 8 Oct 2019 13:05:37 -0700 Subject: [PATCH 054/113] fix exception when trying to run Python script from GUI --- daemon/core/api/tlv/corehandlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index e520ce95..7096c099 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -2042,6 +2042,7 @@ class CoreUdpHandler(CoreHandler): } self.master = False self.session = None + self.coreemu = server.mainserver.coreemu socketserver.BaseRequestHandler.__init__(self, request, client_address, server) def setup(self): From b7b0e4222c5ee8312133a3d332528a9fed0dea2d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 8 Oct 2019 15:09:26 -0700 Subject: [PATCH 055/113] updates for basic working distrbuted network using fabric --- daemon/core/emulator/distributed.py | 27 ++++++ daemon/core/emulator/session.py | 83 +++++++++++++++++-- daemon/core/nodes/base.py | 75 ++++++++--------- daemon/core/nodes/interface.py | 46 ++++++---- daemon/core/nodes/network.py | 39 +++++++-- daemon/core/utils.py | 2 +- daemon/examples/python/distributed.py | 43 +++++----- .../examples/python/distributed_switches.py | 42 ++++++++++ 8 files changed, 261 insertions(+), 96 deletions(-) create mode 100644 daemon/core/emulator/distributed.py create mode 100644 daemon/examples/python/distributed_switches.py diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py new file mode 100644 index 00000000..104d939d --- /dev/null +++ b/daemon/core/emulator/distributed.py @@ -0,0 +1,27 @@ +import logging + +from core.errors import CoreCommandError + + +def remote_cmd(server, cmd, env=None): + """ + Run command remotely using server connection. + + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost + :param str cmd: command to run + :param dict env: environment for remote command, default is None + :return: stdout when success + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + logging.info("remote cmd server(%s): %s", server, cmd) + if env is None: + result = server.run(cmd, hide=False) + else: + result = server.run(cmd, hide=False, env=env, replace_env=True) + if result.exited: + raise CoreCommandError( + result.exited, result.command, result.stdout, result.stderr + ) + return result.stdout.strip() diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index e0afc53c..9eb02a07 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -37,9 +37,11 @@ from core.location.event import EventLoop from core.location.mobility import MobilityManager from core.nodes.base import CoreNetworkBase, CoreNode, CoreNodeBase from core.nodes.docker import DockerNode -from core.nodes.ipaddress import MacAddress +from core.nodes.interface import GreTap +from core.nodes.ipaddress import IpAddress, MacAddress from core.nodes.lxd import LxcNode from core.nodes.network import ( + CoreNetwork, CtrlNet, GreTapBridge, HubNode, @@ -148,6 +150,8 @@ class Session(object): # distributed servers self.servers = {} + self.tunnels = {} + self.address = None # initialize default node services self.services.default_services = { @@ -161,19 +165,81 @@ class Session(object): def add_distributed(self, server): conn = Connection(server, user="root") self.servers[server] = conn - - def init_distributed(self): - for server in self.servers: - conn = self.servers[server] - cmd = "mkdir -p %s" % self.session_dir - conn.run(cmd, hide=False) + cmd = "mkdir -p %s" % self.session_dir + conn.run(cmd, hide=False) def shutdown_distributed(self): + # shutdown all tunnels + for key in self.tunnels: + tunnels = self.tunnels[key] + for tunnel in tunnels: + tunnel.shutdown() + + # remove all remote session directories for server in self.servers: conn = self.servers[server] cmd = "rm -rf %s" % self.session_dir conn.run(cmd, hide=False) + # clear tunnels + self.tunnels.clear() + + def initialize_distributed(self): + for node_id in self.nodes: + node = self.nodes[node_id] + + if not isinstance(node, CoreNetwork): + continue + + if isinstance(node, CtrlNet) and node.serverintf is not None: + continue + + for server in self.servers: + conn = self.servers[server] + key = self.tunnelkey(node_id, IpAddress.to_int(server)) + + # local to server + logging.info( + "local tunnel node(%s) to remote(%s) key(%s)", + node.name, + server, + key, + ) + local_tap = GreTap(session=self, remoteip=server, key=key) + local_tap.net_client.create_interface(node.brname, local_tap.localname) + + # server to local + logging.info( + "remote tunnel node(%s) to local(%s) key(%s)", + node.name, + self.address, + key, + ) + remote_tap = GreTap( + session=self, remoteip=self.address, key=key, server=conn + ) + remote_tap.net_client.create_interface( + node.brname, remote_tap.localname + ) + + # save tunnels for shutdown + self.tunnels[key] = [local_tap, remote_tap] + + def tunnelkey(self, n1num, n2num): + """ + Compute a 32-bit key used to uniquely identify a GRE tunnel. + The hash(n1num), hash(n2num) values are used, so node numbers may be + None or string values (used for e.g. "ctrlnet"). + + :param int n1num: node one id + :param int n2num: node two id + :return: tunnel key for the node pair + :rtype: int + """ + logging.debug("creating tunnel key for: %s, %s", n1num, n2num) + key = (self.id << 16) ^ utils.hashkey(n1num) ^ (utils.hashkey(n2num) << 8) + return key & 0xFFFFFFFF + @classmethod def get_node_class(cls, _type): """ @@ -1493,6 +1559,9 @@ class Session(object): self.add_remove_control_interface(node=None, remove=False) self.broker.startup() + # initialize distributed tunnels + self.initialize_distributed() + # instantiate will be invoked again upon Emane configure if self.emane.startup() == self.emane.NOT_READY: return diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 21324c59..82915b38 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,6 +14,7 @@ from socket import AF_INET, AF_INET6 from tempfile import NamedTemporaryFile from core import constants, utils +from core.emulator import distributed from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.errors import CoreCommandError @@ -95,39 +96,7 @@ class NodeBase(object): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - logging.info("net cmd server(%s): %s", self.server, args) - if self.server is None: - return utils.check_cmd(args, env=env) - else: - args = " ".join(args) - return self.remote_cmd(args, env=env) - - def remote_cmd(self, cmd, env=None): - """ - Run command remotely using server connection. - - :param str cmd: command to run - :param dict env: environment for remote command, default is None - :return: stdout when success - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - if env is None: - result = self.server.run(cmd, hide=False) - else: - logging.info("command env: %s", env) - result = self.server.run(cmd, hide=False, env=env, replace_env=True) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) - - logging.info( - "fabric result:\n\tstdout: %s\n\tstderr: %s", - result.stdout.strip(), - result.stderr.strip(), - ) - return result.stdout.strip() + raise NotImplementedError def setposition(self, x=None, y=None, z=None): """ @@ -279,7 +248,8 @@ class CoreNodeBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: boolean for starting - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ super(CoreNodeBase, self).__init__(session, _id, name, start, server) self.services = [] @@ -412,6 +382,23 @@ class CoreNodeBase(NodeBase): return common + def net_cmd(self, args, env=None): + """ + Runs a command that is used to configure and setup the network on the host + system. + + :param list[str]|str args: command to run + :param dict env: environment to run command with + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + if self.server is None: + return utils.check_cmd(args, env=env) + else: + args = " ".join(args) + return distributed.remote_cmd(self.server, args, env=env) + def node_net_cmd(self, args): """ Runs a command that is used to configure and setup the network within a @@ -493,7 +480,8 @@ class CoreNode(CoreNodeBase): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ super(CoreNode, self).__init__(session, _id, name, start, server) self.nodedir = nodedir @@ -653,13 +641,13 @@ class CoreNode(CoreNodeBase): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - logging.info("net cmd server(%s): %s", self.server, args) if self.server is None: + logging.info("node(%s) cmd: %s", self.name, args) return self.check_cmd(args) else: args = self.client._cmd_args() + args args = " ".join(args) - return self.remote_cmd(args) + return distributed.remote_cmd(self.server, args) def check_cmd(self, args): """ @@ -753,7 +741,11 @@ class CoreNode(CoreNodeBase): raise ValueError("interface name (%s) too long" % name) veth = Veth( - node=self, name=name, localname=localname, net=net, start=self.up + node=self, + name=name, + localname=localname, + start=self.up, + server=self.server, ) if self.up: @@ -806,9 +798,7 @@ class CoreNode(CoreNodeBase): sessionid = self.session.short_session_id() localname = "tap%s.%s.%s" % (self.id, ifindex, sessionid) name = ifname - tuntap = TunTap( - node=self, name=name, localname=localname, net=net, start=self.up - ) + tuntap = TunTap(node=self, name=name, localname=localname, start=self.up) try: self.addnetif(tuntap, ifindex) @@ -1057,7 +1047,8 @@ class CoreNetworkBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: should object start - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ super(CoreNetworkBase, self).__init__(session, _id, name, start, server) self._linked = {} diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 51859e3a..8b73b1b7 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -7,6 +7,7 @@ import time from builtins import int, range from core import utils +from core.emulator import distributed from core.errors import CoreCommandError from core.nodes.netclient import LinuxNetClient @@ -16,13 +17,15 @@ class CoreInterface(object): Base class for network interfaces. """ - def __init__(self, node, name, mtu): + def __init__(self, node, name, mtu, server=None): """ Creates a PyCoreNetIf instance. :param core.nodes.base.CoreNode node: node for interface :param str name: interface name :param mtu: mtu value + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ self.node = node @@ -42,7 +45,15 @@ class CoreInterface(object): self.netindex = None # index used to find flow data self.flow_id = None - self.net_client = LinuxNetClient(utils.check_cmd) + self.server = server + self.net_client = LinuxNetClient(self.net_cmd) + + def net_cmd(self, args): + if self.server is None: + return utils.check_cmd(args) + else: + args = " ".join(args) + return distributed.remote_cmd(self.server, args) def startup(self): """ @@ -191,8 +202,7 @@ class Veth(CoreInterface): Provides virtual ethernet functionality for core nodes. """ - # TODO: network is not used, why was it needed? - def __init__(self, node, name, localname, mtu=1500, net=None, start=True): + def __init__(self, node, name, localname, mtu=1500, server=None, start=True): """ Creates a VEth instance. @@ -200,12 +210,13 @@ class Veth(CoreInterface): :param str name: interface name :param str localname: interface local name :param mtu: interface mtu - :param net: network + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param bool start: start flag :raises CoreCommandError: when there is a command exception """ # note that net arg is ignored - CoreInterface.__init__(self, node=node, name=name, mtu=mtu) + CoreInterface.__init__(self, node, name, mtu, server) self.localname = localname self.up = False if start: @@ -251,8 +262,7 @@ class TunTap(CoreInterface): TUN/TAP virtual device in TAP mode """ - # TODO: network is not used, why was it needed? - def __init__(self, node, name, localname, mtu=1500, net=None, start=True): + def __init__(self, node, name, localname, mtu=1500, server=None, start=True): """ Create a TunTap instance. @@ -260,10 +270,11 @@ class TunTap(CoreInterface): :param str name: interface name :param str localname: local interface name :param mtu: interface mtu - :param core.nodes.base.CoreNetworkBase net: related network + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param bool start: start flag """ - CoreInterface.__init__(self, node=node, name=name, mtu=mtu) + CoreInterface.__init__(self, node, name, mtu, server) self.localname = localname self.up = False self.transport_type = "virtual" @@ -427,6 +438,7 @@ class GreTap(CoreInterface): ttl=255, key=None, start=True, + server=None, ): """ Creates a GreTap instance. @@ -441,9 +453,11 @@ class GreTap(CoreInterface): :param ttl: ttl value :param key: gre tap key :param bool start: start flag + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :raises CoreCommandError: when there is a command exception """ - CoreInterface.__init__(self, node=node, name=name, mtu=mtu) + CoreInterface.__init__(self, node, name, mtu, server) self.session = session if _id is None: # from PyCoreObj @@ -460,9 +474,13 @@ class GreTap(CoreInterface): if remoteip is None: raise ValueError("missing remote IP required for GRE TAP device") - self.net_client.create_gretap( - self.localname, str(remoteip), str(localip), str(ttl), str(key) - ) + if localip is not None: + localip = str(localip) + if ttl is not None: + ttl = str(ttl) + if key is not None: + key = str(key) + self.net_client.create_gretap(self.localname, str(remoteip), localip, ttl, key) self.net_client.device_up(self.localname) self.up = True diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 444ded56..3ec84282 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -10,6 +10,7 @@ import time from socket import AF_INET, AF_INET6 from core import constants, utils +from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs from core.errors import CoreCommandError, CoreError @@ -291,7 +292,8 @@ class CoreNetwork(CoreNetworkBase): :param int _id: object id :param str name: object name :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param policy: network policy """ CoreNetworkBase.__init__(self, session, _id, name, start, server) @@ -307,6 +309,27 @@ class CoreNetwork(CoreNetworkBase): self.startup() ebq.startupdateloop(self) + def net_cmd(self, args, env=None): + """ + Runs a command that is used to configure and setup the network on the host + system. + + :param list[str]|str args: command to run + :param dict env: environment to run command with + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + logging.info("network node(%s) cmd", self.name) + output = utils.check_cmd(args, env=env) + + args = " ".join(args) + for server in self.session.servers: + conn = self.session.servers[server] + distributed.remote_cmd(conn, args, env=env) + + return output + def startup(self): """ Linux bridge starup logic. @@ -381,11 +404,11 @@ class CoreNetwork(CoreNetworkBase): """ Attach a network interface. - :param core.netns.vnode.VEth netif: network interface to attach + :param core.nodes.interface.Veth netif: network interface to attach :return: nothing """ if self.up: - self.net_client.create_interface(self.brname, netif.localname) + netif.net_client.create_interface(self.brname, netif.localname) CoreNetworkBase.attach(self, netif) @@ -397,7 +420,7 @@ class CoreNetwork(CoreNetworkBase): :return: nothing """ if self.up: - self.net_client.delete_interface(self.brname, netif.localname) + netif.net_client.delete_interface(self.brname, netif.localname) CoreNetworkBase.detach(self, netif) @@ -591,13 +614,11 @@ class CoreNetwork(CoreNetworkBase): if len(name) >= 16: raise ValueError("interface name %s too long" % name) - netif = Veth( - node=None, name=name, localname=localname, mtu=1500, net=self, start=self.up - ) + netif = Veth(node=None, name=name, localname=localname, mtu=1500, start=self.up) self.attach(netif) if net.up: # this is similar to net.attach() but uses netif.name instead of localname - self.net_client.create_interface(net.brname, netif.name) + netif.net_client.create_interface(net.brname, netif.name) i = net.newifindex() net._netif[i] = netif with net._linked_lock: @@ -666,6 +687,8 @@ class GreTapBridge(CoreNetwork): :param ttl: ttl value :param key: gre tap key :param bool start: start flag + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ CoreNetwork.__init__(self, session, _id, name, False, server, policy) self.grekey = key diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 20d2384e..8e59a050 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -263,7 +263,7 @@ def check_cmd(args, **kwargs): kwargs["stdout"] = subprocess.PIPE kwargs["stderr"] = subprocess.STDOUT args = split_args(args) - logging.debug("command: %s", args) + logging.info("command: %s", args) try: p = subprocess.Popen(args, **kwargs) stdout, _ = p.communicate() diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index 5b5174f6..feb5e8bb 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -1,51 +1,46 @@ import logging import pdb +import sys from core.emulator.coreemu import CoreEmu -from core.emulator.emudata import NodeOptions -from core.emulator.enumerations import EventTypes +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes def main(): # ip generator for example - # prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods coreemu = CoreEmu() session = coreemu.create_session() # initialize distributed - session.add_distributed("core2") - session.init_distributed() + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) - # create switch network node - # switch = session.add_node(_type=NodeTypes.SWITCH) - - # create nodes + # create local node, switch, and remote nodes + node_one = session.add_node() + switch = session.add_node(_type=NodeTypes.SWITCH) options = NodeOptions() - options.emulation_server = "core2" - session.add_node(node_options=options) - # interface = prefixes.create_interface(node_one) - # session.add_link(node_one.id, switch.id, interface_one=interface) + options.emulation_server = remote + node_two = session.add_node(node_options=options) - session.add_node() - # interface = prefixes.create_interface(node_two) - # session.add_link(node_two.id, switch.id, interface_one=interface) + # create not interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, switch.id, interface_one=interface_one) + session.add_link(node_two.id, switch.id, interface_one=interface_two) # instantiate session session.instantiate() - # print("starting iperf server on node: %s" % node_one.name) - # node_one.cmd(["iperf", "-s", "-D"]) - # node_one_address = prefixes.ip4_address(node_one) - # - # print("node %s connecting to %s" % (node_two.name, node_one_address)) - # node_two.client.icmd(["iperf", "-t", "10", "-c", node_one_address]) - # node_one.cmd(["killall", "-9", "iperf"]) - + # pause script for verification pdb.set_trace() # shutdown session diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py new file mode 100644 index 00000000..c6366d5d --- /dev/null +++ b/daemon/examples/python/distributed_switches.py @@ -0,0 +1,42 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.enumerations import EventTypes, NodeTypes + + +def main(): + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + switch_one = session.add_node(_type=NodeTypes.SWITCH) + switch_two = session.add_node(_type=NodeTypes.SWITCH) + + # create not interfaces and link + session.add_link(switch_one.id, switch_two.id) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From c8d68c332a65704097ae0fd251eb910402b9c808 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 8 Oct 2019 21:06:22 -0700 Subject: [PATCH 056/113] updates for testing using examples --- daemon/core/emulator/session.py | 9 ++++++++- daemon/core/nodes/network.py | 3 ++- daemon/examples/python/distributed.py | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 9eb02a07..31e75de2 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -1869,7 +1869,14 @@ class Session(object): assign_address = self.master prefix = prefixes[0] - logging.info("controlnet prefix: %s - %s", type(prefix), prefix) + logging.info( + "controlnet(%s) prefix(%s) assign(%s) updown(%s) serverintf(%s)", + _id, + prefix, + assign_address, + updown_script, + server_interface, + ) control_net = self.create_node( cls=CtrlNet, _id=_id, diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 3ec84282..f7d6af69 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -808,7 +808,8 @@ class CtrlNet(CoreNetwork): :param prefix: control network ipv4 prefix :param hostid: host id :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param str assign_address: assigned address :param str updown_script: updown script :param serverintf: server interface diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index feb5e8bb..3cb3debd 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -15,6 +15,9 @@ def main(): coreemu = CoreEmu() session = coreemu.create_session() + # set controlnet + session.options.set_config("controlnet", "172.16.0.0/24") + # initialize distributed address = sys.argv[1] remote = sys.argv[2] From 7e45168e777a01835589f249b7ca2647ae29b244 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 8 Oct 2019 21:17:15 -0700 Subject: [PATCH 057/113] distributed example for ptp --- daemon/core/nodes/network.py | 4 +- daemon/examples/python/distributed.py | 2 +- daemon/examples/python/distributed_ptp.py | 50 +++++++++++++++++++ .../examples/python/distributed_switches.py | 2 +- 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 daemon/examples/python/distributed_ptp.py diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index f7d6af69..c404db5a 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -852,7 +852,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - utils.check_cmd([self.updown_script, self.brname, "startup"]) + self.net_cmd([self.updown_script, self.brname, "startup"]) if self.serverintf: self.net_client.create_interface(self.brname, self.serverintf) @@ -880,7 +880,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - utils.check_cmd([self.updown_script, self.brname, "shutdown"]) + self.net_cmd([self.updown_script, self.brname, "shutdown"]) except CoreCommandError: logging.exception("error issuing shutdown script shutdown") diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index 3cb3debd..ca9ca928 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -34,7 +34,7 @@ def main(): options.emulation_server = remote node_two = session.add_node(node_options=options) - # create not interfaces and link + # create node interfaces and link interface_one = prefixes.create_interface(node_one) interface_two = prefixes.create_interface(node_two) session.add_link(node_one.id, switch.id, interface_one=interface_one) diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py new file mode 100644 index 00000000..6ab5b9dc --- /dev/null +++ b/daemon/examples/python/distributed_ptp.py @@ -0,0 +1,50 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes + + +def main(): + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + node_one = session.add_node() + options = NodeOptions() + options.emulation_server = remote + node_two = session.add_node(node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, node_two.id, interface_one, interface_two) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py index c6366d5d..b7ed166b 100644 --- a/daemon/examples/python/distributed_switches.py +++ b/daemon/examples/python/distributed_switches.py @@ -24,7 +24,7 @@ def main(): switch_one = session.add_node(_type=NodeTypes.SWITCH) switch_two = session.add_node(_type=NodeTypes.SWITCH) - # create not interfaces and link + # create node interfaces and link session.add_link(switch_one.id, switch_two.id) # instantiate session From 859f473ba9f50b5207a09494f7c7c7fd1f8ad422 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Oct 2019 12:13:26 -0700 Subject: [PATCH 058/113] updated ebtables to use net_cmd --- daemon/core/nodes/network.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index c404db5a..2fab566c 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -3,7 +3,6 @@ Defines network nodes used within core. """ import logging -import os import socket import threading import time @@ -165,20 +164,20 @@ class EbtablesQueue(object): """ # save kernel ebtables snapshot to a file args = self.ebatomiccmd(["--atomic-save"]) - utils.check_cmd(args) + wlan.net_cmd(args) # modify the table file using queued ebtables commands for c in self.cmds: args = self.ebatomiccmd(c) - utils.check_cmd(args) + wlan.net_cmd(args) self.cmds = [] # commit the table file to the kernel args = self.ebatomiccmd(["--atomic-commit"]) - utils.check_cmd(args) + wlan.net_cmd(args) try: - os.unlink(self.atomic_file) + wlan.net_cmd(["rm", "-f", self.atomic_file]) except OSError: logging.exception("error removing atomic file: %s", self.atomic_file) @@ -312,7 +311,7 @@ class CoreNetwork(CoreNetworkBase): def net_cmd(self, args, env=None): """ Runs a command that is used to configure and setup the network on the host - system. + system and all configured distributed servers. :param list[str]|str args: command to run :param dict env: environment to run command with @@ -341,7 +340,7 @@ class CoreNetwork(CoreNetworkBase): # create a new ebtables chain for this bridge ebtablescmds( - utils.check_cmd, + self.net_cmd, [ [constants.EBTABLES_BIN, "-N", self.brname, "-P", self.policy], [ @@ -372,7 +371,7 @@ class CoreNetwork(CoreNetworkBase): try: self.net_client.delete_bridge(self.brname) ebtablescmds( - utils.check_cmd, + self.net_cmd, [ [ constants.EBTABLES_BIN, @@ -844,7 +843,6 @@ class CtrlNet(CoreNetwork): if self.assign_address: addrlist = ["%s/%s" % (addr, self.prefix.prefixlen)] self.addrconfig(addrlist=addrlist) - logging.info("address %s", addr) if self.updown_script: logging.info( From a4b6b8be510e2a0ea04e365abbaf5c52bbe446ed Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Oct 2019 15:44:45 -0700 Subject: [PATCH 059/113] updated link config to work distributed, added crude locking for fabric --- daemon/core/emulator/distributed.py | 17 +++++-- daemon/core/nodes/base.py | 6 +-- daemon/core/nodes/network.py | 8 +-- daemon/examples/python/distributed_ptp.py | 2 +- daemon/examples/python/distributed_wlan.py | 58 ++++++++++++++++++++++ 5 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 daemon/examples/python/distributed_wlan.py diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 104d939d..2c7d7bbb 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -1,7 +1,10 @@ import logging +import threading from core.errors import CoreCommandError +LOCK = threading.Lock() + def remote_cmd(server, cmd, env=None): """ @@ -16,12 +19,18 @@ def remote_cmd(server, cmd, env=None): :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("remote cmd server(%s): %s", server, cmd) - if env is None: - result = server.run(cmd, hide=False) - else: - result = server.run(cmd, hide=False, env=env, replace_env=True) + with LOCK: + if env is None: + result = server.run(cmd, hide=False) + else: + result = server.run(cmd, hide=False, env=env, replace_env=True) if result.exited: raise CoreCommandError( result.exited, result.command, result.stdout, result.stderr ) return result.stdout.strip() + + +def remote_put(server, source, destination): + with LOCK: + server.put(source, destination) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 82915b38..4f95c56e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -961,7 +961,7 @@ class CoreNode(CoreNodeBase): self.client.check_cmd(["sync"]) else: self.net_cmd(["mkdir", "-p", directory]) - self.server.put(srcname, filename) + distributed.remote_put(self.server, srcname, filename) def hostfilename(self, filename): """ @@ -1001,7 +1001,7 @@ class CoreNode(CoreNodeBase): temp.write(contents.encode("utf-8")) temp.close() self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) - self.server.put(temp.name, hostfilename) + distributed.remote_put(self.server, temp.name, hostfilename) self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode @@ -1023,7 +1023,7 @@ class CoreNode(CoreNodeBase): if mode is not None: os.chmod(hostfilename, mode) else: - self.server.put(srcfilename, hostfilename) + distributed.remote_put(self.server, srcfilename, hostfilename) if mode is not None: self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.info( diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 2fab566c..81dfc34b 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -525,14 +525,14 @@ class CoreNetwork(CoreNetworkBase): logging.debug( "linkconfig: %s" % ([tc + parent + ["handle", "1:"] + tbf],) ) - utils.check_cmd(tc + parent + ["handle", "1:"] + tbf) + netif.net_cmd(tc + parent + ["handle", "1:"] + tbf) netif.setparam("has_tbf", True) changed = True elif netif.getparam("has_tbf") and bw <= 0: tcd = [] + tc tcd[2] = "delete" if self.up: - utils.check_cmd(tcd + parent) + netif.net_cmd(tcd + parent) netif.setparam("has_tbf", False) # removing the parent removes the child netif.setparam("has_netem", False) @@ -575,14 +575,14 @@ class CoreNetwork(CoreNetworkBase): tc[2] = "delete" if self.up: logging.debug("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) - utils.check_cmd(tc + parent + ["handle", "10:"]) + netif.net_cmd(tc + parent + ["handle", "10:"]) netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: logging.debug( "linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],) ) - utils.check_cmd(tc + parent + ["handle", "10:"] + netem) + netif.net_cmd(tc + parent + ["handle", "10:"] + netem) netif.setparam("has_netem", True) def linknet(self, net): diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index 6ab5b9dc..2b611816 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -25,8 +25,8 @@ def main(): session.set_state(EventTypes.CONFIGURATION_STATE) # create local node, switch, and remote nodes - node_one = session.add_node() options = NodeOptions() + node_one = session.add_node(node_options=options) options.emulation_server = remote node_two = session.add_node(node_options=options) diff --git a/daemon/examples/python/distributed_wlan.py b/daemon/examples/python/distributed_wlan.py new file mode 100644 index 00000000..ca64ee01 --- /dev/null +++ b/daemon/examples/python/distributed_wlan.py @@ -0,0 +1,58 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes +from core.location.mobility import BasicRangeModel + + +def main(): + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # set controlnet + # session.options.set_config("controlnet", "172.16.0.0/24") + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + options = NodeOptions() + options.set_position(0, 0) + options.emulation_server = remote + node_one = session.add_node(node_options=options) + wlan = session.add_node(_type=NodeTypes.WIRELESS_LAN) + session.mobility.set_model(wlan, BasicRangeModel) + node_two = session.add_node(node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, wlan.id, interface_one=interface_one) + session.add_link(node_two.id, wlan.id, interface_one=interface_two) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From bc586933399fe7b227222a26f87cfa496d760fe7 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 10 Oct 2019 11:53:52 -0700 Subject: [PATCH 060/113] updated emane config files to be generated for remote servers, fixed services not using node remote server compatible commands --- daemon/core/emane/emanemanager.py | 13 +++- daemon/core/emane/emanemodel.py | 11 ++- daemon/core/emulator/distributed.py | 41 ++++++++--- daemon/core/nodes/base.py | 6 +- daemon/core/services/coreservices.py | 10 +-- daemon/core/xml/emanexml.py | 77 +++++++++++++++++---- daemon/examples/python/distributed_emane.py | 65 +++++++++++++++++ 7 files changed, 184 insertions(+), 39 deletions(-) create mode 100644 daemon/examples/python/distributed_emane.py diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 746016f9..2902c47c 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -18,6 +18,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel +from core.emulator import distributed from core.emulator.enumerations import ( ConfigDataTypes, ConfigFlags, @@ -679,8 +680,12 @@ class EmaneManager(ModelManager): return dev = self.get_config("eventservicedevice") - emanexml.create_event_service_xml(group, port, dev, self.session.session_dir) + for server in self.session.servers: + conn = self.session.servers[server] + emanexml.create_event_service_xml( + group, port, dev, self.session.session_dir, conn + ) def startdaemons(self): """ @@ -745,7 +750,7 @@ class EmaneManager(ModelManager): os.path.join(path, "emane%d.log" % n), os.path.join(path, "platform%d.xml" % n), ] - output = node.check_cmd(args) + output = node.node_net_cmd(args) logging.info("node(%s) emane daemon running: %s", node.name, args) logging.info("node(%s) emane daemon output: %s", node.name, output) @@ -756,6 +761,10 @@ class EmaneManager(ModelManager): emanecmd += ["-f", os.path.join(path, "emane.log")] args = emanecmd + [os.path.join(path, "platform.xml")] utils.check_cmd(args, cwd=path) + args = " ".join(args) + for server in self.session.servers: + conn = self.session.servers[server] + distributed.remote_cmd(conn, args, cwd=path) logging.info("host emane daemon running: %s", args) def stopdaemons(self): diff --git a/daemon/core/emane/emanemodel.py b/daemon/core/emane/emanemodel.py index 56eee289..3ca2a18f 100644 --- a/daemon/core/emane/emanemodel.py +++ b/daemon/core/emane/emanemodel.py @@ -102,6 +102,11 @@ class EmaneModel(WirelessModel): mac_name = emanexml.mac_file_name(self, interface) phy_name = emanexml.phy_file_name(self, interface) + # remote server for file + server = None + if interface is not None: + server = interface.node.server + # check if this is external transport_type = "virtual" if interface and interface.transport_type == "raw": @@ -111,16 +116,16 @@ class EmaneModel(WirelessModel): # create nem xml file nem_file = os.path.join(self.session.session_dir, nem_name) emanexml.create_nem_xml( - self, config, nem_file, transport_name, mac_name, phy_name + self, config, nem_file, transport_name, mac_name, phy_name, server ) # create mac xml file mac_file = os.path.join(self.session.session_dir, mac_name) - emanexml.create_mac_xml(self, config, mac_file) + emanexml.create_mac_xml(self, config, mac_file, server) # create phy xml file phy_file = os.path.join(self.session.session_dir, phy_name) - emanexml.create_phy_xml(self, config, phy_file) + emanexml.create_phy_xml(self, config, phy_file, server) def post_startup(self): """ diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 2c7d7bbb..35cbf208 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -1,12 +1,16 @@ import logging +import os import threading +from tempfile import NamedTemporaryFile + +from invoke import UnexpectedExit from core.errors import CoreCommandError LOCK = threading.Lock() -def remote_cmd(server, cmd, env=None): +def remote_cmd(server, cmd, env=None, cwd=None): """ Run command remotely using server connection. @@ -14,23 +18,38 @@ def remote_cmd(server, cmd, env=None): default is None for localhost :param str cmd: command to run :param dict env: environment for remote command, default is None + :param str cwd: directory to run command in, defaults to None, which is the user's + home directory :return: stdout when success :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("remote cmd server(%s): %s", server, cmd) - with LOCK: - if env is None: - result = server.run(cmd, hide=False) - else: - result = server.run(cmd, hide=False, env=env, replace_env=True) - if result.exited: - raise CoreCommandError( - result.exited, result.command, result.stdout, result.stderr - ) - return result.stdout.strip() + replace_env = env is not None + try: + with LOCK: + if cwd is None: + result = server.run(cmd, hide=False, env=env, replace_env=replace_env) + else: + with server.cd(cwd): + result = server.run( + cmd, hide=False, env=env, replace_env=replace_env + ) + return result.stdout.strip() + except UnexpectedExit as e: + stdout, stderr = e.streams_for_display() + raise CoreCommandError(e.result.exited, cmd, stdout, stderr) def remote_put(server, source, destination): with LOCK: server.put(source, destination) + + +def remote_put_temp(server, destination, data): + with LOCK: + temp = NamedTemporaryFile(delete=False) + temp.write(data.encode("utf-8")) + temp.close() + server.put(temp.name, destination) + os.unlink(temp.name) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 4f95c56e..d8cac8c5 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -11,7 +11,6 @@ import string import threading from builtins import range from socket import AF_INET, AF_INET6 -from tempfile import NamedTemporaryFile from core import constants, utils from core.emulator import distributed @@ -997,11 +996,8 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - temp = NamedTemporaryFile(delete=False) - temp.write(contents.encode("utf-8")) - temp.close() self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) - distributed.remote_put(self.server, temp.name, hostfilename) + distributed.remote_put_temp(self.server, hostfilename, contents) self.net_cmd(["chmod", "%o" % mode, hostfilename]) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index 4563d4b7..b34daa73 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -9,6 +9,7 @@ services. import enum import logging +import shlex import time from multiprocessing.pool import ThreadPool @@ -597,8 +598,9 @@ class CoreServices(object): status = 0 for cmd in cmds: logging.debug("validating service(%s) using: %s", service.name, cmd) + cmd = shlex.split(cmd) try: - node.check_cmd(cmd) + node.node_net_cmd(cmd) except CoreCommandError as e: logging.debug( "node(%s) service(%s) validate failed", node.name, service.name @@ -728,11 +730,11 @@ class CoreServices(object): status = 0 for cmd in cmds: + cmd = shlex.split(cmd) try: if wait: - node.check_cmd(cmd) - else: - node.cmd(cmd, wait=False) + cmd.append("&") + node.node_net_cmd(cmd) except CoreCommandError: logging.exception("error starting command") status = -1 diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index d73f3d5b..3b4fafef 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -1,9 +1,11 @@ import logging import os +from tempfile import NamedTemporaryFile from lxml import etree from core import utils +from core.emulator import distributed from core.nodes.ipaddress import MacAddress from core.xml import corexml @@ -44,20 +46,29 @@ def _value_to_params(value): return None -def create_file(xml_element, doc_name, file_path): +def create_file(xml_element, doc_name, file_path, server=None): """ Create xml file. :param lxml.etree.Element xml_element: root element to write to file :param str doc_name: name to use in the emane doctype :param str file_path: file path to write xml file to + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ doctype = ( '' % {"doc_name": doc_name} ) - corexml.write_xml_file(xml_element, file_path, doctype=doctype) + if server is not None: + temp = NamedTemporaryFile(delete=False) + create_file(xml_element, doc_name, temp.name) + temp.close() + distributed.remote_put(server, temp.name, file_path) + os.unlink(temp.name) + else: + corexml.write_xml_file(xml_element, file_path, doctype=doctype) def add_param(xml_element, name, value): @@ -204,17 +215,18 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x # increment nem id nem_id += 1 + doc_name = "platform" for key in sorted(platform_xmls.keys()): + platform_element = platform_xmls[key] if key == "host": file_name = "platform.xml" + file_path = os.path.join(emane_manager.session.session_dir, file_name) + create_file(platform_element, doc_name, file_path) else: file_name = "platform%d.xml" % key - - platform_element = platform_xmls[key] - - doc_name = "platform" - file_path = os.path.join(emane_manager.session.session_dir, file_name) - create_file(platform_element, doc_name, file_path) + file_path = os.path.join(emane_manager.session.session_dir, file_name) + linked_node = emane_manager.session.nodes[key] + create_file(platform_element, doc_name, file_path, linked_node.server) return nem_id @@ -303,15 +315,20 @@ def build_transport_xml(emane_manager, node, transport_type): file_name = transport_file_name(node.id, transport_type) file_path = os.path.join(emane_manager.session.session_dir, file_name) create_file(transport_element, doc_name, file_path) + for server in emane_manager.session.servers: + conn = emane_manager.session.servers[server] + create_file(transport_element, doc_name, file_path, conn) -def create_phy_xml(emane_model, config, file_path): +def create_phy_xml(emane_model, config, file_path, server): """ Create the phy xml document. :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ phy_element = etree.Element("phy", name="%s PHY" % emane_model.name) @@ -322,15 +339,24 @@ def create_phy_xml(emane_model, config, file_path): phy_element, emane_model.phy_config, config, emane_model.config_ignore ) create_file(phy_element, "phy", file_path) + if server is not None: + create_file(phy_element, "phy", file_path, server) + else: + create_file(phy_element, "phy", file_path) + for server in emane_model.session.servers: + conn = emane_model.session.servers[server] + create_file(phy_element, "phy", file_path, conn) -def create_mac_xml(emane_model, config, file_path): +def create_mac_xml(emane_model, config, file_path, server): """ Create the mac xml document. :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ if not emane_model.mac_library: @@ -343,10 +369,23 @@ def create_mac_xml(emane_model, config, file_path): mac_element, emane_model.mac_config, config, emane_model.config_ignore ) create_file(mac_element, "mac", file_path) + if server is not None: + create_file(mac_element, "mac", file_path, server) + else: + create_file(mac_element, "mac", file_path) + for server in emane_model.session.servers: + conn = emane_model.session.servers[server] + create_file(mac_element, "mac", file_path, conn) def create_nem_xml( - emane_model, config, nem_file, transport_definition, mac_definition, phy_definition + emane_model, + config, + nem_file, + transport_definition, + mac_definition, + phy_definition, + server, ): """ Create the nem xml document. @@ -357,6 +396,8 @@ def create_nem_xml( :param str transport_definition: transport file definition path :param str mac_definition: mac file definition path :param str phy_definition: phy file definition path + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) @@ -366,10 +407,16 @@ def create_nem_xml( etree.SubElement(nem_element, "transport", definition=transport_definition) etree.SubElement(nem_element, "mac", definition=mac_definition) etree.SubElement(nem_element, "phy", definition=phy_definition) - create_file(nem_element, "nem", nem_file) + if server is not None: + create_file(nem_element, "nem", nem_file, server) + else: + create_file(nem_element, "nem", nem_file) + for server in emane_model.session.servers: + conn = emane_model.session.servers[server] + create_file(nem_element, "nem", nem_file, conn) -def create_event_service_xml(group, port, device, file_directory): +def create_event_service_xml(group, port, device, file_directory, server=None): """ Create a emane event service xml file. @@ -377,6 +424,8 @@ def create_event_service_xml(group, port, device, file_directory): :param str port: event port :param str device: event device :param str file_directory: directory to create file in + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :return: nothing """ event_element = etree.Element("emaneeventmsgsvc") @@ -391,7 +440,7 @@ def create_event_service_xml(group, port, device, file_directory): sub_element.text = value file_name = "libemaneeventservice.xml" file_path = os.path.join(file_directory, file_name) - create_file(event_element, "emaneeventmsgsvc", file_path) + create_file(event_element, "emaneeventmsgsvc", file_path, server) def transport_file_name(node_id, transport_type): diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py new file mode 100644 index 00000000..1ffe5795 --- /dev/null +++ b/daemon/examples/python/distributed_emane.py @@ -0,0 +1,65 @@ +import logging +import pdb +import sys + +from core.emane.ieee80211abg import EmaneIeee80211abgModel +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes + + +def main(): + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu() + session = coreemu.create_session() + + # set controlnet + session.options.set_config( + "controlnet", + "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 " + "core4:172.16.4.0/24 core5:172.16.5.0/24", + ) + + # initialize distributed + address = sys.argv[1] + remote = sys.argv[2] + session.address = address + session.add_distributed(remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + options = NodeOptions(model="mdr") + options.set_position(0, 0) + node_one = session.add_node(node_options=options) + emane_net = session.add_node(_type=NodeTypes.EMANE) + session.emane.set_model(emane_net, EmaneIeee80211abgModel) + options.emulation_server = remote + node_two = session.add_node(node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, emane_net.id, interface_one=interface_one) + session.add_link(node_two.id, emane_net.id, interface_one=interface_two) + + # instantiate session + try: + session.instantiate() + except Exception: + logging.exception("error during instantiate") + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From f6cdeb23de01aba78089c7c8192883ab9ece365b Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 10 Oct 2019 15:25:12 -0700 Subject: [PATCH 061/113] changes to update commands to leverage either node_net_cmd/net_cmd --- daemon/core/api/tlv/corehandlers.py | 13 +++++++++---- daemon/core/emane/emanemanager.py | 24 ++++++++++++++++++------ daemon/core/emulator/distributed.py | 10 ++++++++-- daemon/core/emulator/session.py | 3 ++- daemon/core/errors.py | 7 ++++++- daemon/core/nodes/base.py | 10 ++++++---- daemon/core/nodes/client.py | 5 +++-- daemon/core/nodes/physical.py | 2 +- daemon/core/services/coreservices.py | 9 +++------ daemon/examples/python/switch.py | 9 ++++++--- daemon/examples/python/wlan.py | 6 +++--- daemon/tests/emane/test_emane.py | 2 +- daemon/tests/test_core.py | 4 +--- 13 files changed, 67 insertions(+), 37 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index e520ce95..a531efe2 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -37,7 +37,7 @@ from core.emulator.enumerations import ( RegisterTlvs, SessionTlvs, ) -from core.errors import CoreError +from core.errors import CoreCommandError, CoreError from core.location.mobility import BasicRangeModel from core.nodes.network import WlanNode from core.services.coreservices import ServiceManager, ServiceShim @@ -882,16 +882,21 @@ class CoreHandler(socketserver.BaseRequestHandler): return (reply,) else: logging.info("execute message with cmd=%s", command) + command = utils.split_args(command) # execute command and send a response if ( message.flags & MessageFlags.STRING.value or message.flags & MessageFlags.TEXT.value ): - # shlex.split() handles quotes within the string if message.flags & MessageFlags.LOCAL.value: status, res = utils.cmd_output(command) else: - status, res = node.cmd_output(command) + try: + res = node.node_net_cmd(command) + status = 0 + except CoreCommandError as e: + res = e.stderr + status = e.returncode logging.info( "done exec cmd=%s with status=%d res=(%d bytes)", command, @@ -913,7 +918,7 @@ class CoreHandler(socketserver.BaseRequestHandler): if message.flags & MessageFlags.LOCAL.value: utils.mute_detach(command) else: - node.cmd(command, wait=False) + node.node_net_cmd(command, wait=False) except CoreError: logging.exception("error getting object: %s", node_num) # XXX wait and queue this message to try again later diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 2902c47c..49cf4f24 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -772,7 +772,8 @@ class EmaneManager(ModelManager): Kill the appropriate EMANE daemons. """ # TODO: we may want to improve this if we had the PIDs from the specific EMANE daemons that we"ve started - args = ["killall", "-q", "emane"] + kill_emaned = ["killall", "-q", "emane"] + kill_transortd = ["killall", "-q", "emanetransportd"] stop_emane_on_host = False for node in self.getnodes(): if hasattr(node, "transport_type") and node.transport_type == "raw": @@ -780,13 +781,19 @@ class EmaneManager(ModelManager): continue if node.up: - node.cmd(args, wait=False) + node.node_net_cmd(kill_emaned, wait=False) # TODO: RJ45 node if stop_emane_on_host: try: - utils.check_cmd(args) - utils.check_cmd(["killall", "-q", "emanetransportd"]) + utils.check_cmd(kill_emaned) + utils.check_cmd(kill_transortd) + kill_emaned = " ".join(kill_emaned) + kill_transortd = " ".join(kill_transortd) + for server in self.session.servers: + conn = self.session[server] + distributed.remote_cmd(conn, kill_emaned) + distributed.remote_cmd(conn, kill_transortd) except CoreCommandError: logging.exception("error shutting down emane daemons") @@ -976,8 +983,13 @@ class EmaneManager(ModelManager): Return True if an EMANE process associated with the given node is running, False otherwise. """ args = ["pkill", "-0", "-x", "emane"] - status = node.cmd(args) - return status == 0 + try: + node.node_net_cmd(args) + result = True + except CoreCommandError: + result = False + + return result class EmaneGlobalModel(EmaneModel): diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 35cbf208..abec0a57 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -10,7 +10,7 @@ from core.errors import CoreCommandError LOCK = threading.Lock() -def remote_cmd(server, cmd, env=None, cwd=None): +def remote_cmd(server, cmd, env=None, cwd=None, wait=True): """ Run command remotely using server connection. @@ -20,12 +20,18 @@ def remote_cmd(server, cmd, env=None, cwd=None): :param dict env: environment for remote command, default is None :param str cwd: directory to run command in, defaults to None, which is the user's home directory + :param bool wait: True to wait for status, False to background process :return: stdout when success :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - logging.info("remote cmd server(%s): %s", server, cmd) + replace_env = env is not None + if not wait: + cmd += " &" + logging.info( + "remote cmd server(%s) cwd(%s) wait(%s): %s", server.host, cwd, wait, cmd + ) try: with LOCK: if cwd is None: diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 31e75de2..9bc2ab80 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -2044,4 +2044,5 @@ class Session(object): utils.mute_detach(data) else: node = self.get_node(node_id) - node.cmd(data, wait=False) + data = utils.split_args(data) + node.node_net_cmd(data, wait=False) diff --git a/daemon/core/errors.py b/daemon/core/errors.py index bb124434..5b76abb3 100644 --- a/daemon/core/errors.py +++ b/daemon/core/errors.py @@ -10,7 +10,12 @@ class CoreCommandError(subprocess.CalledProcessError): """ def __str__(self): - return "Command(%s), Status(%s):\n%s" % (self.cmd, self.returncode, self.output) + return "Command(%s), Status(%s):\nstdout: %s\nstderr: %s" % ( + self.cmd, + self.returncode, + self.output, + self.stderr, + ) class CoreError(Exception): diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d8cac8c5..147989d7 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -630,29 +630,31 @@ class CoreNode(CoreNodeBase): """ return self.client.cmd_output(args) - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): """ Runs a command that is used to configure and setup the network within a node. - :param list[str]|str args: command to run + :param list[str] args: command to run + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: logging.info("node(%s) cmd: %s", self.name, args) - return self.check_cmd(args) + return self.client.check_cmd(args, wait=wait) else: args = self.client._cmd_args() + args args = " ".join(args) - return distributed.remote_cmd(self.server, args) + return distributed.remote_cmd(self.server, args, wait=wait) def check_cmd(self, args): """ Runs shell command on node. :param list[str]|str args: command to run + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 6b7fa44c..6c72547b 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -97,17 +97,18 @@ class VnodeClient(object): status = p.wait() return status, output.decode("utf-8").strip() - def check_cmd(self, args): + def check_cmd(self, args, wait=True): """ Run command and return exit status and combined stdout and stderr. :param list[str]|str args: command to run + :param bool wait: True to wait for command status, False otherwise :return: combined stdout and stderr :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ status, output = self.cmd_output(args) - if status != 0: + if wait and status != 0: raise CoreCommandError(status, args, output) return output.strip() diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index ba8cdc25..48bd5a5a 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -94,7 +94,7 @@ class PhysicalNode(CoreNodeBase): return output.strip() def shcmd(self, cmdstr, sh="/bin/sh"): - return self.cmd([sh, "-c", cmdstr]) + return self.node_net_cmd([sh, "-c", cmdstr]) def sethwaddr(self, ifindex, addr): """ diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index b34daa73..d6eeb1b5 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -9,7 +9,6 @@ services. import enum import logging -import shlex import time from multiprocessing.pool import ThreadPool @@ -598,7 +597,7 @@ class CoreServices(object): status = 0 for cmd in cmds: logging.debug("validating service(%s) using: %s", service.name, cmd) - cmd = shlex.split(cmd) + cmd = utils.split_args(cmd) try: node.node_net_cmd(cmd) except CoreCommandError as e: @@ -730,11 +729,9 @@ class CoreServices(object): status = 0 for cmd in cmds: - cmd = shlex.split(cmd) + cmd = utils.split_args(cmd) try: - if wait: - cmd.append("&") - node.node_net_cmd(cmd) + node.node_net_cmd(cmd, wait) except CoreCommandError: logging.exception("error starting command") status = -1 diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index d669478d..3c6ec383 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -43,11 +43,14 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd(["iperf", "-s", "-D"]) first_node_address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, first_node_address)) - last_node.client.icmd(["iperf", "-t", str(options.time), "-c", first_node_address]) - first_node.cmd(["killall", "-9", "iperf"]) + output = last_node.node_net_cmd( + ["iperf", "-t", str(options.time), "-c", first_node_address] + ) + print(output) + first_node.node_net_cmd(["killall", "-9", "iperf"]) # shutdown session coreemu.shutdown() diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index 376f34d0..3d5171c2 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -47,11 +47,11 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd(["iperf", "-s", "-D"]) address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, address)) - last_node.client.icmd(["iperf", "-t", str(options.time), "-c", address]) - first_node.cmd(["killall", "-9", "iperf"]) + last_node.node_net_cmd(["iperf", "-t", str(options.time), "-c", address]) + first_node.node_net_cmd(["killall", "-9", "iperf"]) # shutdown session coreemu.shutdown() diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 85836605..d9001065 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -26,7 +26,7 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) - return from_node.cmd(["ping", "-c", str(count), address]) + return from_node.node_net_cmd(["ping", "-c", str(count), address]) class TestEmane: diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index e120dcc8..7d64ae69 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -38,7 +38,7 @@ def createclients(sessiondir, clientcls=VnodeClient, cmdchnlfilterfunc=None): def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) - return from_node.cmd(["ping", "-c", "3", address]) + return from_node.node_net_cmd(["ping", "-c", "3", address]) class TestCore: @@ -102,7 +102,6 @@ class TestCore: # check various command using vcmd module command = ["ls"] - assert not client.cmd(command) status, output = client.cmd_output(command) assert not status p, stdin, stdout, stderr = client.popen(command) @@ -110,7 +109,6 @@ class TestCore: assert not client.icmd(command) # check various command using command line - assert not client.cmd(command) status, output = client.cmd_output(command) assert not status p, stdin, stdout, stderr = client.popen(command) From c3d27eb8a53b380d19e7656e6b865aef4cb8adde Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 10 Oct 2019 23:01:16 -0700 Subject: [PATCH 062/113] removed utils.cmd and related node functions --- daemon/core/nodes/base.py | 22 ---------------------- daemon/core/nodes/docker.py | 20 -------------------- daemon/core/nodes/lxd.py | 25 ------------------------- daemon/core/nodes/physical.py | 13 ------------- daemon/core/utils.py | 20 -------------------- daemon/tests/emane/test_emane.py | 9 +++++++-- daemon/tests/test_core.py | 8 +++++++- 7 files changed, 14 insertions(+), 103 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 147989d7..d2d49fb4 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -421,17 +421,6 @@ class CoreNodeBase(NodeBase): """ raise NotImplementedError - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - raise NotImplementedError - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. @@ -609,17 +598,6 @@ class CoreNode(CoreNodeBase): finally: self.rmnodedir() - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - return self.client.cmd(args, wait) - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index ad7deff2..69bf376f 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -47,15 +47,6 @@ class DockerClient(object): name=self.name )) - def cmd(self, cmd, wait=True): - if isinstance(cmd, list): - cmd = " ".join(cmd) - logging.info("docker cmd wait(%s): %s", wait, cmd) - return utils.cmd("docker exec {name} {cmd}".format( - name=self.name, - cmd=cmd - ), wait) - def cmd_output(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) @@ -155,17 +146,6 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - return self.client.cmd(args, wait) - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 86ca192e..54a66182 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -55,13 +55,6 @@ class LxdClient(object): logging.info("lxc cmd output: %s", args) return utils.cmd_output(args) - def cmd(self, cmd, wait=True): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._cmd_args(cmd) - logging.info("lxc cmd: %s", args) - return utils.cmd(args, wait) - def _ns_args(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) @@ -72,13 +65,6 @@ class LxdClient(object): logging.info("ns cmd: %s", args) return utils.cmd_output(args) - def ns_cmd(self, cmd, wait=True): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._ns_args(cmd) - logging.info("ns cmd: %s", args) - return utils.cmd(args, wait) - def copy_file(self, source, destination): if destination[0] != "/": destination = os.path.join("/root/", destination) @@ -158,17 +144,6 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - return self.client.cmd(args, wait) - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 48bd5a5a..159c746d 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -52,19 +52,6 @@ class PhysicalNode(CoreNodeBase): """ return sh - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - os.chdir(self.nodedir) - status = utils.cmd(args, wait) - return status - def cmd_output(self, args): """ Runs shell command on node and get exit status and output. diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 8e59a050..83a18c6a 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -207,26 +207,6 @@ def mute_detach(args, **kwargs): return subprocess.Popen(args, **kwargs).pid -def cmd(args, wait=True): - """ - Runs a command on and returns the exit status. - - :param list[str]|str args: command arguments - :param bool wait: wait for command to end or not - :return: command status - :rtype: int - """ - args = split_args(args) - logging.debug("command: %s", args) - try: - p = subprocess.Popen(args) - if not wait: - return 0 - return p.wait() - except OSError: - raise CoreCommandError(-1, args) - - def cmd_output(args): """ Execute a command on the host and return a tuple containing the exit status and diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index d9001065..3d9b9eb2 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -12,7 +12,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel from core.emulator.emudata import NodeOptions -from core.errors import CoreError +from core.errors import CoreCommandError, CoreError _EMANE_MODELS = [ EmaneIeee80211abgModel, @@ -26,7 +26,12 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) - return from_node.node_net_cmd(["ping", "-c", str(count), address]) + try: + from_node.node_net_cmd(["ping", "-c", str(count), address]) + status = 0 + except CoreCommandError as e: + status = e.returncode + return status class TestEmane: diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 7d64ae69..9a59b4ce 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -10,6 +10,7 @@ import pytest from core.emulator.emudata import NodeOptions from core.emulator.enumerations import MessageFlags, NodeTypes +from core.errors import CoreCommandError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility from core.nodes.client import VnodeClient @@ -38,7 +39,12 @@ def createclients(sessiondir, clientcls=VnodeClient, cmdchnlfilterfunc=None): def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) - return from_node.node_net_cmd(["ping", "-c", "3", address]) + try: + from_node.node_net_cmd(["ping", "-c", "3", address]) + status = 0 + except CoreCommandError as e: + status = e.returncode + return status class TestCore: From 4a6d69bb09eb27724e7b10342a666f1fd4725487 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 09:34:49 -0700 Subject: [PATCH 063/113] removing cmd_output function from utils and nodes --- daemon/core/api/grpc/server.py | 9 ++++-- daemon/core/api/tlv/corehandlers.py | 7 +++- daemon/core/nodes/base.py | 20 ------------ daemon/core/nodes/client.py | 50 +++++------------------------ daemon/core/nodes/docker.py | 46 ++++++++------------------ daemon/core/nodes/lxd.py | 37 ++++++--------------- daemon/core/nodes/physical.py | 42 ++---------------------- daemon/core/services/utility.py | 8 +++-- daemon/core/utils.py | 21 ------------ daemon/tests/test_core.py | 4 --- 10 files changed, 51 insertions(+), 193 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 7fb6ed31..3d589fab 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,6 +10,7 @@ from queue import Empty, Queue import grpc +from core import utils from core.api.grpc import core_pb2, core_pb2_grpc from core.emane.nodes import EmaneNet from core.emulator.data import ( @@ -22,7 +23,7 @@ from core.emulator.data import ( ) from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions from core.emulator.enumerations import EventTypes, LinkTypes, NodeTypes -from core.errors import CoreError +from core.errors import CoreCommandError, CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility from core.nodes.base import CoreNetworkBase from core.nodes.docker import DockerNode @@ -882,7 +883,11 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): logging.debug("sending node command: %s", request) session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) - _, output = node.cmd_output(request.command) + try: + args = utils.split_args(request.command) + output = node.node_net_cmd(args) + except CoreCommandError as e: + output = e.stderr return core_pb2.NodeCommandResponse(output=output) def GetNodeTerminal(self, request, context): diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index a531efe2..e1a32e5f 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -889,7 +889,12 @@ class CoreHandler(socketserver.BaseRequestHandler): or message.flags & MessageFlags.TEXT.value ): if message.flags & MessageFlags.LOCAL.value: - status, res = utils.cmd_output(command) + try: + res = utils.check_cmd(command) + status = 0 + except CoreCommandError as e: + res = e.stderr + status = e.returncode else: try: res = node.node_net_cmd(command) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d2d49fb4..78279b5e 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -421,16 +421,6 @@ class CoreNodeBase(NodeBase): """ raise NotImplementedError - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - raise NotImplementedError - def termcmdstring(self, sh): """ Create a terminal command string. @@ -598,16 +588,6 @@ class CoreNode(CoreNodeBase): finally: self.rmnodedir() - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - return self.client.cmd_output(args) - def node_net_cmd(self, args, wait=True): """ Runs a command that is used to configure and setup the network within a diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 6c72547b..3b0b2843 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -57,46 +57,6 @@ class VnodeClient(object): def _cmd_args(self): return [constants.VCMD_BIN, "-c", self.ctrlchnlname, "--"] - def cmd(self, args, wait=True): - """ - Execute a command on a node and return the status (return code). - - :param list[str]|str args: command arguments - :param bool wait: wait for command to end or not - :return: command status - :rtype: int - """ - self._verify_connection() - args = utils.split_args(args) - - # run command, return process when not waiting - cmd = self._cmd_args() + args - logging.debug("cmd wait(%s): %s", wait, cmd) - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - if not wait: - return 0 - - # wait for and return exit status - return p.wait() - - def cmd_output(self, args): - """ - Execute a command on a node and return a tuple containing the - exit status and result string. stderr output - is folded into the stdout result string. - - :param list[str]|str args: command to run - :return: command status and combined stdout and stderr output - :rtype: tuple[int, str] - """ - p, stdin, stdout, stderr = self.popen(args) - stdin.close() - output = stdout.read() + stderr.read() - stdout.close() - stderr.close() - status = p.wait() - return status, output.decode("utf-8").strip() - def check_cmd(self, args, wait=True): """ Run command and return exit status and combined stdout and stderr. @@ -107,10 +67,16 @@ class VnodeClient(object): :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ - status, output = self.cmd_output(args) + p, stdin, stdout, stderr = self.popen(args) + stdin.close() + output = stdout.read() + stderr.read() + output = output.decode("utf-8").strip() + stdout.close() + stderr.close() + status = p.wait() if wait and status != 0: raise CoreCommandError(status, args, output) - return output.strip() + return output def popen(self, args): """ diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 69bf376f..028b3e0b 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -27,12 +27,12 @@ class DockerClient(object): def get_info(self): args = "docker inspect {name}".format(name=self.name) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + output = utils.check_cmd(args) data = json.loads(output) if not data: - raise CoreCommandError(status, args, "docker({name}) not present".format(name=self.name)) + raise CoreCommandError( + -1, args, "docker({name}) not present".format(name=self.name) + ) return data[0] def is_alive(self): @@ -47,11 +47,11 @@ class DockerClient(object): name=self.name )) - def cmd_output(self, cmd): + def check_cmd(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) logging.info("docker cmd output: %s", cmd) - return utils.cmd_output("docker exec {name} {cmd}".format( + return utils.check_cmd("docker exec {name} {cmd}".format( name=self.name, cmd=cmd )) @@ -64,13 +64,11 @@ class DockerClient(object): cmd=cmd ) logging.info("ns cmd: %s", args) - return utils.cmd_output(args) + return utils.check_cmd(args) def get_pid(self): args = "docker inspect -f '{{{{.State.Pid}}}}' {name}".format(name=self.name) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + output = utils.check_cmd(args) self.pid = output logging.debug("node(%s) pid: %s", self.name, self.pid) return output @@ -81,9 +79,7 @@ class DockerClient(object): name=self.name, destination=destination ) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + return utils.check_cmd(args) class DockerNode(CoreNode): @@ -146,16 +142,6 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - return self.client.cmd_output(args) - def check_cmd(self, args): """ Runs shell command on node. @@ -165,20 +151,14 @@ class DockerNode(CoreNode): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - status, output = self.client.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) - return output + return self.client.check_cmd(args) - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) - return 0 + return "" - status, output = self.client.ns_cmd(args) - if status: - raise CoreCommandError(status, args, output) - return output + return self.client.ns_cmd(args) def termcmdstring(self, sh="/bin/sh"): """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 54a66182..c28306b6 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -25,13 +25,11 @@ class LxdClient(object): def get_info(self): args = "lxc list {name} --format json".format(name=self.name) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + output = utils.check_cmd(args) data = json.loads(output) if not data: raise CoreCommandError( - status, args, "LXC({name}) not present".format(name=self.name) + -1, args, "LXC({name}) not present".format(name=self.name) ) return data[0] @@ -48,22 +46,22 @@ class LxdClient(object): def _cmd_args(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def cmd_output(self, cmd): + def check_cmd(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) args = self._cmd_args(cmd) logging.info("lxc cmd output: %s", args) - return utils.cmd_output(args) + return utils.check_cmd(args) def _ns_args(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) - def ns_cmd_output(self, cmd): + def ns_check_cmd(self, cmd): if isinstance(cmd, list): cmd = " ".join(cmd) args = self._ns_args(cmd) logging.info("ns cmd: %s", args) - return utils.cmd_output(args) + return utils.check_cmd(args) def copy_file(self, source, destination): if destination[0] != "/": @@ -72,9 +70,7 @@ class LxdClient(object): args = "lxc file push {source} {name}/{destination}".format( source=source, name=self.name, destination=destination ) - status, output = utils.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) + utils.check_cmd(args) class LxcNode(CoreNode): @@ -144,16 +140,6 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - return self.client.cmd_output(args) - def check_cmd(self, args): """ Runs shell command on node. @@ -163,15 +149,12 @@ class LxcNode(CoreNode): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - status, output = self.client.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) - return output + return self.client.check_cmd(args) - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) - return 0 + return "" return self.check_cmd(args) def termcmdstring(self, sh="/bin/sh"): diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 159c746d..60e445ea 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -4,7 +4,6 @@ PhysicalNode class for including real systems in the emulated network. import logging import os -import subprocess import threading from core import constants, utils @@ -52,20 +51,6 @@ class PhysicalNode(CoreNodeBase): """ return sh - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - os.chdir(self.nodedir) - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout, _ = p.communicate() - status = p.wait() - return status, stdout.strip() - def check_cmd(self, args): """ Runs shell command on node. @@ -75,10 +60,8 @@ class PhysicalNode(CoreNodeBase): :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - status, output = self.cmd_output(args) - if status: - raise CoreCommandError(status, args, output) - return output.strip() + os.chdir(self.nodedir) + return utils.check_cmd(args) def shcmd(self, cmdstr, sh="/bin/sh"): return self.node_net_cmd([sh, "-c", cmdstr]) @@ -526,27 +509,6 @@ class Rj45Node(CoreNodeBase, CoreInterface): """ raise NotImplementedError - def cmd(self, args, wait=True): - """ - Runs shell command on node, with option to not wait for a result. - - :param list[str]|str args: command to run - :param bool wait: wait for command to exit, defaults to True - :return: exit status for command - :rtype: int - """ - raise NotImplementedError - - def cmd_output(self, args): - """ - Runs shell command on node and get exit status and output. - - :param list[str]|str args: command to run - :return: exit status and combined stdout and stderr - :rtype: tuple[int, str] - """ - raise NotImplementedError - def termcmdstring(self, sh): """ Create a terminal command string. diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 8088b149..66a84dd6 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -415,9 +415,11 @@ class HttpService(UtilService): Detect the apache2 version using the 'a2query' command. """ try: - status, result = utils.cmd_output(["a2query", "-v"]) - except CoreCommandError: - status = -1 + result = utils.check_cmd(["a2query", "-v"]) + status = 0 + except CoreCommandError as e: + status = e.returncode + result = e.stderr if status == 0 and result[:3] == "2.4": return cls.APACHEVER24 diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 83a18c6a..ead65eeb 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -207,27 +207,6 @@ def mute_detach(args, **kwargs): return subprocess.Popen(args, **kwargs).pid -def cmd_output(args): - """ - Execute a command on the host and return a tuple containing the exit status and - result string. stderr output is folded into the stdout result string. - - :param list[str]|str args: command arguments - :return: command status and stdout - :rtype: tuple[int, str] - :raises CoreCommandError: when the file to execute is not found - """ - args = split_args(args) - logging.debug("command: %s", args) - try: - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout, _ = p.communicate() - status = p.wait() - return status, stdout.decode("utf-8").strip() - except OSError: - raise CoreCommandError(-1, args) - - def check_cmd(args, **kwargs): """ Execute a command on the host and return a tuple containing the exit status and diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 9a59b4ce..7432ee58 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -108,15 +108,11 @@ class TestCore: # check various command using vcmd module command = ["ls"] - status, output = client.cmd_output(command) - assert not status p, stdin, stdout, stderr = client.popen(command) assert not p.wait() assert not client.icmd(command) # check various command using command line - status, output = client.cmd_output(command) - assert not status p, stdin, stdout, stderr = client.popen(command) assert not p.wait() assert not client.icmd(command) From d326f246a7adbc2cb49a5ded4f568f5e8f326902 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 12:57:37 -0700 Subject: [PATCH 064/113] removed node based check_cmd, updated to use appropriate function --- daemon/core/emane/emanemanager.py | 7 +++- daemon/core/emulator/session.py | 10 +++--- daemon/core/nodes/base.py | 53 ++++++---------------------- daemon/core/nodes/docker.py | 13 +------ daemon/core/nodes/interface.py | 17 +++++++-- daemon/core/nodes/lxd.py | 15 ++------ daemon/core/nodes/netclient.py | 3 +- daemon/core/nodes/network.py | 8 +++-- daemon/core/nodes/physical.py | 33 +++-------------- daemon/core/services/coreservices.py | 3 +- daemon/tests/test_nodes.py | 2 +- 11 files changed, 51 insertions(+), 113 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 49cf4f24..90ca9dfe 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -152,8 +152,13 @@ class EmaneManager(ModelManager): """ try: # check for emane - emane_version = utils.check_cmd(["emane", "--version"]) + args = ["emane", "--version"] + emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) + args = " ".join(args) + for server in self.session.servers: + conn = self.session.servers[server] + distributed.remote_cmd(conn, args) # load default emane models self.load_models(EMANE_MODELS) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 9bc2ab80..e3a2389b 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -140,6 +140,11 @@ class Session(object): self.options.set_config(key, value) self.metadata = SessionMetaData() + # distributed servers + self.servers = {} + self.tunnels = {} + self.address = None + # initialize session feature helpers self.broker = CoreBroker(session=self) self.location = CoreLocation() @@ -148,11 +153,6 @@ class Session(object): self.emane = EmaneManager(session=self) self.sdt = Sdt(session=self) - # distributed servers - self.servers = {} - self.tunnels = {} - self.address = None - # initialize default node services self.services.default_services = { "mdr": ("zebra", "OSPFv3MDR", "IPForward"), diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 78279b5e..2b9e08eb 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -84,18 +84,24 @@ class NodeBase(object): """ raise NotImplementedError - def net_cmd(self, args, env=None): + def net_cmd(self, args, env=None, cwd=None, wait=True): """ Runs a command that is used to configure and setup the network on the host system. :param list[str]|str args: command to run :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ - raise NotImplementedError + if self.server is None: + return utils.check_cmd(args, env=env, cwd=cwd) + else: + args = " ".join(args) + return distributed.remote_cmd(self.server, args, env, cwd, wait) def setposition(self, x=None, y=None, z=None): """ @@ -381,40 +387,13 @@ class CoreNodeBase(NodeBase): return common - def net_cmd(self, args, env=None): - """ - Runs a command that is used to configure and setup the network on the host - system. - - :param list[str]|str args: command to run - :param dict env: environment to run command with - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - if self.server is None: - return utils.check_cmd(args, env=env) - else: - args = " ".join(args) - return distributed.remote_cmd(self.server, args, env=env) - - def node_net_cmd(self, args): + def node_net_cmd(self, args, wait=True): """ Runs a command that is used to configure and setup the network within a node. :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - raise NotImplementedError - - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs @@ -607,18 +586,6 @@ class CoreNode(CoreNodeBase): args = " ".join(args) return distributed.remote_cmd(self.server, args, wait=wait) - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :param bool wait: True to wait for status, False otherwise - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - return self.client.check_cmd(args) - def termcmdstring(self, sh="/bin/sh"): """ Create a terminal command string. diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 028b3e0b..28a92607 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -142,17 +142,6 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - return self.client.check_cmd(args) - def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) @@ -178,7 +167,7 @@ class DockerNode(CoreNode): """ logging.debug("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - self.check_cmd(args) + self.client.check_cmd(args) def mount(self, source, target): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 8b73b1b7..08f1d662 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -48,12 +48,23 @@ class CoreInterface(object): self.server = server self.net_client = LinuxNetClient(self.net_cmd) - def net_cmd(self, args): + def net_cmd(self, args, env=None, cwd=None, wait=True): + """ + Runs a command on the host system or distributed servers. + + :param list[str]|str args: command to run + :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise + :return: combined stdout and stderr + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ if self.server is None: - return utils.check_cmd(args) + return utils.check_cmd(args, env=env, cwd=cwd) else: args = " ".join(args) - return distributed.remote_cmd(self.server, args) + return distributed.remote_cmd(self.server, args, env, cwd, wait) def startup(self): """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index c28306b6..f16df715 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -140,22 +140,11 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - return self.client.check_cmd(args) - def node_net_cmd(self, args, wait=True): if not self.up: logging.debug("node down, not running network command: %s", args) return "" - return self.check_cmd(args) + return self.client.check_cmd(args) def termcmdstring(self, sh="/bin/sh"): """ @@ -175,7 +164,7 @@ class LxcNode(CoreNode): """ logging.info("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - self.check_cmd(args) + return self.client.check_cmd(args) def mount(self, source, target): """ diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 34fee343..a689dde9 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -5,7 +5,6 @@ Clients for dealing with bridge/interface commands. import os from core.constants import BRCTL_BIN, ETHTOOL_BIN, IP_BIN, OVS_BIN, TC_BIN -from core.utils import check_cmd class LinuxNetClient(object): @@ -275,7 +274,7 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - check_cmd([BRCTL_BIN, "setageing", name, "0"]) + self.run([BRCTL_BIN, "setageing", name, "0"]) class OvsNetClient(LinuxNetClient): diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 81dfc34b..b236b96b 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -308,24 +308,26 @@ class CoreNetwork(CoreNetworkBase): self.startup() ebq.startupdateloop(self) - def net_cmd(self, args, env=None): + def net_cmd(self, args, env=None, cwd=None, wait=True): """ Runs a command that is used to configure and setup the network on the host system and all configured distributed servers. :param list[str]|str args: command to run :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("network node(%s) cmd", self.name) - output = utils.check_cmd(args, env=env) + output = utils.check_cmd(args, env=env, cwd=cwd) args = " ".join(args) for server in self.session.servers: conn = self.session.servers[server] - distributed.remote_cmd(conn, args, env=env) + distributed.remote_cmd(conn, args, env, cwd, wait) return output diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 60e445ea..4c219258 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -51,21 +51,6 @@ class PhysicalNode(CoreNodeBase): """ return sh - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: combined stdout and stderr - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs - """ - os.chdir(self.nodedir) - return utils.check_cmd(args) - - def shcmd(self, cmdstr, sh="/bin/sh"): - return self.node_net_cmd([sh, "-c", cmdstr]) - def sethwaddr(self, ifindex, addr): """ Set hardware address for an interface. @@ -205,13 +190,13 @@ class PhysicalNode(CoreNodeBase): source = os.path.abspath(source) logging.info("mounting %s at %s", source, target) os.makedirs(target) - self.check_cmd([constants.MOUNT_BIN, "--bind", source, target]) + self.net_cmd([constants.MOUNT_BIN, "--bind", source, target], cwd=self.nodedir) self._mounts.append((source, target)) def umount(self, target): logging.info("unmounting '%s'" % target) try: - self.check_cmd([constants.UMOUNT_BIN, "-l", target]) + self.net_cmd([constants.UMOUNT_BIN, "-l", target], cwd=self.nodedir) except CoreCommandError: logging.exception("unmounting failed for %s", target) @@ -256,7 +241,8 @@ class Rj45Node(CoreNodeBase, CoreInterface): :param str name: node name :param mtu: rj45 mtu :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost """ CoreNodeBase.__init__(self, session, _id, name, start, server) CoreInterface.__init__(self, node=self, name=name, mtu=mtu) @@ -498,17 +484,6 @@ class Rj45Node(CoreNodeBase, CoreInterface): CoreInterface.setposition(self, x, y, z) return result - def check_cmd(self, args): - """ - Runs shell command on node. - - :param list[str]|str args: command to run - :return: exist status and combined stdout and stderr - :rtype: tuple[int, str] - :raises CoreCommandError: when a non-zero exit status occurs - """ - raise NotImplementedError - def termcmdstring(self, sh): """ Create a terminal command string. diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index d6eeb1b5..dc45fa33 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -631,8 +631,9 @@ class CoreServices(object): """ status = 0 for args in service.shutdown: + args = utils.split_args(args) try: - node.check_cmd(args) + node.node_net_cmd(args) except CoreCommandError: logging.exception("error running stop command %s", args) status = -1 diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index baf0c20c..ad5476d4 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -30,7 +30,7 @@ class TestNodes: assert os.path.exists(node.nodedir) assert node.alive() assert node.up - assert node.check_cmd(["ip", "addr", "show", "lo"]) + assert node.node_net_cmd(["ip", "addr", "show", "lo"]) def test_node_update(self, session): # given From fc7a161221b44fffcebba3bca2a6189efa7661b8 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 13:15:57 -0700 Subject: [PATCH 065/113] updated utils.check_cmd to accept the same parameters as other commands and be leveraged for node cmds --- daemon/core/nodes/base.py | 2 +- daemon/core/nodes/docker.py | 6 +++--- daemon/core/nodes/interface.py | 2 +- daemon/core/nodes/lxd.py | 6 +++--- daemon/core/nodes/network.py | 2 +- daemon/core/utils.py | 29 ++++++++++++++++------------- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 2b9e08eb..6a3bebf0 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -98,7 +98,7 @@ class NodeBase(object): :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - return utils.check_cmd(args, env=env, cwd=cwd) + return utils.check_cmd(args, env, cwd, wait) else: args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 28a92607..1b8322ae 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -56,7 +56,7 @@ class DockerClient(object): cmd=cmd )) - def ns_cmd(self, cmd): + def ns_cmd(self, cmd, wait): if isinstance(cmd, list): cmd = " ".join(cmd) args = "nsenter -t {pid} -u -i -p -n {cmd}".format( @@ -64,7 +64,7 @@ class DockerClient(object): cmd=cmd ) logging.info("ns cmd: %s", args) - return utils.check_cmd(args) + return utils.check_cmd(args, wait=wait) def get_pid(self): args = "docker inspect -f '{{{{.State.Pid}}}}' {name}".format(name=self.name) @@ -147,7 +147,7 @@ class DockerNode(CoreNode): logging.debug("node down, not running network command: %s", args) return "" - return self.client.ns_cmd(args) + return self.client.ns_cmd(args, wait) def termcmdstring(self, sh="/bin/sh"): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 08f1d662..d749bbde 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -61,7 +61,7 @@ class CoreInterface(object): :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - return utils.check_cmd(args, env=env, cwd=cwd) + return utils.check_cmd(args, env, cwd, wait) else: args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index f16df715..df01f4ad 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -46,12 +46,12 @@ class LxdClient(object): def _cmd_args(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def check_cmd(self, cmd): + def check_cmd(self, cmd, wait): if isinstance(cmd, list): cmd = " ".join(cmd) args = self._cmd_args(cmd) logging.info("lxc cmd output: %s", args) - return utils.check_cmd(args) + return utils.check_cmd(args, wait=wait) def _ns_args(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) @@ -144,7 +144,7 @@ class LxcNode(CoreNode): if not self.up: logging.debug("node down, not running network command: %s", args) return "" - return self.client.check_cmd(args) + return self.client.check_cmd(args, wait) def termcmdstring(self, sh="/bin/sh"): """ diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index b236b96b..7197c77f 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -322,7 +322,7 @@ class CoreNetwork(CoreNetworkBase): :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("network node(%s) cmd", self.name) - output = utils.check_cmd(args, env=env, cwd=cwd) + output = utils.check_cmd(args, env, cwd, wait) args = " ".join(args) for server in self.session.servers: diff --git a/daemon/core/utils.py b/daemon/core/utils.py index ead65eeb..df59f9ea 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -11,8 +11,8 @@ import logging import logging.config import os import shlex -import subprocess import sys +from subprocess import PIPE, STDOUT, Popen from past.builtins import basestring @@ -203,33 +203,36 @@ def mute_detach(args, **kwargs): args = split_args(args) kwargs["preexec_fn"] = _detach_init kwargs["stdout"] = DEVNULL - kwargs["stderr"] = subprocess.STDOUT - return subprocess.Popen(args, **kwargs).pid + kwargs["stderr"] = STDOUT + return Popen(args, **kwargs).pid -def check_cmd(args, **kwargs): +def check_cmd(args, env=None, cwd=None, wait=True): """ Execute a command on the host and return a tuple containing the exit status and result string. stderr output is folded into the stdout result string. :param list[str]|str args: command arguments - :param dict kwargs: keyword arguments to pass to subprocess.Popen + :param dict env: environment to run command with + :param str cwd: directory to run command in + :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when there is a non-zero exit status or the file to execute is not found """ - kwargs["stdout"] = subprocess.PIPE - kwargs["stderr"] = subprocess.STDOUT args = split_args(args) logging.info("command: %s", args) try: - p = subprocess.Popen(args, **kwargs) - stdout, _ = p.communicate() - status = p.wait() - if status != 0: - raise CoreCommandError(status, args, stdout) - return stdout.decode("utf-8").strip() + p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd) + if wait: + stdout, stderr = p.communicate() + status = p.wait() + if status != 0: + raise CoreCommandError(status, args, stdout, stderr) + return stdout.decode("utf-8").strip() + else: + return "" except OSError: raise CoreCommandError(-1, args) From b5d71bab8243ad8700b397530cf2d889cf0158ca Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 13:36:00 -0700 Subject: [PATCH 066/113] removed VnodeClient.popen --- daemon/core/nodes/base.py | 1 - daemon/core/nodes/client.py | 28 ++-------------------------- daemon/core/utils.py | 2 +- daemon/tests/test_core.py | 4 ---- 4 files changed, 3 insertions(+), 32 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 6a3bebf0..120c53ca 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -579,7 +579,6 @@ class CoreNode(CoreNodeBase): :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - logging.info("node(%s) cmd: %s", self.name, args) return self.client.check_cmd(args, wait=wait) else: args = self.client._cmd_args() + args diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 3b0b2843..13072c05 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -4,12 +4,9 @@ over a control channel to the vnoded process running in a network namespace. The control channel can be accessed via calls using the vcmd shell. """ -import logging import os -from subprocess import PIPE, Popen from core import constants, utils -from core.errors import CoreCommandError class VnodeClient(object): @@ -67,31 +64,10 @@ class VnodeClient(object): :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ - p, stdin, stdout, stderr = self.popen(args) - stdin.close() - output = stdout.read() + stderr.read() - output = output.decode("utf-8").strip() - stdout.close() - stderr.close() - status = p.wait() - if wait and status != 0: - raise CoreCommandError(status, args, output) - return output - - def popen(self, args): - """ - Execute a popen command against the node. - - :param list[str]|str args: command arguments - :return: popen object, stdin, stdout, and stderr - :rtype: tuple - """ self._verify_connection() args = utils.split_args(args) - cmd = self._cmd_args() + args - logging.debug("popen: %s", cmd) - p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) - return p, p.stdin, p.stdout, p.stderr + args = self._cmd_args() + args + return utils.check_cmd(args, wait=wait) def icmd(self, args): """ diff --git a/daemon/core/utils.py b/daemon/core/utils.py index df59f9ea..003c7134 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -222,7 +222,7 @@ def check_cmd(args, env=None, cwd=None, wait=True): execute is not found """ args = split_args(args) - logging.info("command: %s", args) + logging.info("command cwd(%s) wait(%s): %s", cwd, wait, args) try: p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd) if wait: diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 7432ee58..4360ba06 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -108,13 +108,9 @@ class TestCore: # check various command using vcmd module command = ["ls"] - p, stdin, stdout, stderr = client.popen(command) - assert not p.wait() assert not client.icmd(command) # check various command using command line - p, stdin, stdout, stderr = client.popen(command) - assert not p.wait() assert not client.icmd(command) # check module methods From 69772f993c47b90357f93c7c8a1a5157ac864600 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 13:55:06 -0700 Subject: [PATCH 067/113] removed VnodeClient.icmd and VnodeClient.term --- daemon/core/nodes/client.py | 52 ---------------------------- daemon/examples/python/emane80211.py | 4 --- daemon/tests/test_core.py | 31 ++--------------- 3 files changed, 2 insertions(+), 85 deletions(-) diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 13072c05..81297cf5 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -4,8 +4,6 @@ over a control channel to the vnoded process running in a network namespace. The control channel can be accessed via calls using the vcmd shell. """ -import os - from core import constants, utils @@ -69,56 +67,6 @@ class VnodeClient(object): args = self._cmd_args() + args return utils.check_cmd(args, wait=wait) - def icmd(self, args): - """ - Execute an icmd against a node. - - :param list[str]|str args: command arguments - :return: command result - :rtype: int - """ - args = utils.split_args(args) - return os.spawnlp( - os.P_WAIT, - constants.VCMD_BIN, - constants.VCMD_BIN, - "-c", - self.ctrlchnlname, - "--", - *args - ) - - def term(self, sh="/bin/sh"): - """ - Open a terminal on a node. - - :param str sh: shell to open terminal with - :return: terminal command result - :rtype: int - """ - args = ( - "xterm", - "-ut", - "-title", - self.name, - "-e", - constants.VCMD_BIN, - "-c", - self.ctrlchnlname, - "--", - sh, - ) - if "SUDO_USER" in os.environ: - args = ( - "su", - "-s", - "/bin/sh", - "-c", - "exec " + " ".join(map(lambda x: "'%s'" % x, args)), - os.environ["SUDO_USER"], - ) - return os.spawnvp(os.P_NOWAIT, args[0], args) - def termcmdstring(self, sh="/bin/sh"): """ Create a terminal command string. diff --git a/daemon/examples/python/emane80211.py b/daemon/examples/python/emane80211.py index 0e42be95..adf6959b 100644 --- a/daemon/examples/python/emane80211.py +++ b/daemon/examples/python/emane80211.py @@ -40,10 +40,6 @@ def example(options): # instantiate session session.instantiate() - # start a shell on the first node - node = session.get_node(2) - node.client.term("bash") - # shutdown session input("press enter to exit...") coreemu.shutdown() diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 4360ba06..392794d0 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -3,7 +3,6 @@ Unit tests for testing basic CORE networks. """ import os -import stat import threading import pytest @@ -12,31 +11,12 @@ from core.emulator.emudata import NodeOptions from core.emulator.enumerations import MessageFlags, NodeTypes from core.errors import CoreCommandError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility -from core.nodes.client import VnodeClient _PATH = os.path.abspath(os.path.dirname(__file__)) _MOBILITY_FILE = os.path.join(_PATH, "mobility.scen") _WIRED = [NodeTypes.PEER_TO_PEER, NodeTypes.HUB, NodeTypes.SWITCH] -def createclients(sessiondir, clientcls=VnodeClient, cmdchnlfilterfunc=None): - """ - Create clients - - :param str sessiondir: session directory to create clients - :param class clientcls: class to create clients from - :param func cmdchnlfilterfunc: command channel filter function - :return: list of created clients - :rtype: list - """ - direntries = map(lambda x: os.path.join(sessiondir, x), os.listdir(sessiondir)) - cmdchnls = list(filter(lambda x: stat.S_ISSOCK(os.stat(x).st_mode), direntries)) - if cmdchnlfilterfunc: - cmdchnls = list(filter(cmdchnlfilterfunc, cmdchnls)) - cmdchnls.sort() - return map(lambda x: clientcls(os.path.basename(x), x), cmdchnls) - - def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) try: @@ -106,15 +86,8 @@ class TestCore: # check we are connected assert client.connected() - # check various command using vcmd module - command = ["ls"] - assert not client.icmd(command) - - # check various command using command line - assert not client.icmd(command) - - # check module methods - assert createclients(session.session_dir) + # validate command + assert client.check_cmd("echo hello") == "hello" def test_netif(self, session, ip_prefixes): """ From 02ef91242eb3c9b33354021bfed01983854e4889 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 16:36:57 -0700 Subject: [PATCH 068/113] initial changes to convert all commands to be string based for consistency --- daemon/core/api/grpc/server.py | 4 +- daemon/core/api/tlv/corehandlers.py | 1 - daemon/core/emane/emanemanager.py | 41 ++++----- daemon/core/emane/tdma.py | 3 +- daemon/core/emulator/session.py | 1 - daemon/core/location/mobility.py | 2 +- daemon/core/nodes/base.py | 60 ++++++------- daemon/core/nodes/client.py | 10 +-- daemon/core/nodes/docker.py | 1 - daemon/core/nodes/interface.py | 3 +- daemon/core/nodes/ipaddress.py | 6 +- daemon/core/nodes/lxd.py | 20 ++--- daemon/core/nodes/netclient.py | 86 +++++++++---------- daemon/core/nodes/network.py | 123 ++++++++------------------- daemon/core/services/coreservices.py | 3 - daemon/core/services/utility.py | 2 +- daemon/core/utils.py | 22 +---- daemon/core/xml/corexmldeployment.py | 5 +- daemon/tests/emane/test_emane.py | 2 +- daemon/tests/test_core.py | 2 +- daemon/tests/test_nodes.py | 4 +- 21 files changed, 145 insertions(+), 256 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 3d589fab..84d019e3 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,7 +10,6 @@ from queue import Empty, Queue import grpc -from core import utils from core.api.grpc import core_pb2, core_pb2_grpc from core.emane.nodes import EmaneNet from core.emulator.data import ( @@ -884,8 +883,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) try: - args = utils.split_args(request.command) - output = node.node_net_cmd(args) + output = node.node_net_cmd(request.command) except CoreCommandError as e: output = e.stderr return core_pb2.NodeCommandResponse(output=output) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index e1a32e5f..444ae5a5 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -882,7 +882,6 @@ class CoreHandler(socketserver.BaseRequestHandler): return (reply,) else: logging.info("execute message with cmd=%s", command) - command = utils.split_args(command) # execute command and send a response if ( message.flags & MessageFlags.STRING.value diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 90ca9dfe..65fed8bd 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -152,10 +152,9 @@ class EmaneManager(ModelManager): """ try: # check for emane - args = ["emane", "--version"] + args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - args = " ".join(args) for server in self.session.servers: conn = self.session.servers[server] distributed.remote_cmd(conn, args) @@ -652,14 +651,6 @@ class EmaneManager(ModelManager): emane_net = self._emane_nets[key] emanexml.build_xml_files(self, emane_net) - def buildtransportxml(self): - """ - Calls emanegentransportxml using a platform.xml file to build the transportdaemon*.xml. - """ - utils.check_cmd( - ["emanegentransportxml", "platform.xml"], cwd=self.session.session_dir - ) - def buildeventservicexml(self): """ Build the libemaneeventservice.xml file if event service options @@ -705,9 +696,9 @@ class EmaneManager(ModelManager): logging.info("setting user-defined EMANE log level: %d", cfgloglevel) loglevel = str(cfgloglevel) - emanecmd = ["emane", "-d", "-l", loglevel] + emanecmd = "emane -d -l %s" % loglevel if realtime: - emanecmd += ("-r",) + emanecmd += " -r" otagroup, _otaport = self.get_config("otamanagergroup").split(":") otadev = self.get_config("otamanagerdevice") @@ -750,11 +741,11 @@ class EmaneManager(ModelManager): node.node_net_client.create_route(eventgroup, eventdev) # start emane - args = emanecmd + [ - "-f", + args = "%s -f %s %s" % ( + emanecmd, os.path.join(path, "emane%d.log" % n), os.path.join(path, "platform%d.xml" % n), - ] + ) output = node.node_net_cmd(args) logging.info("node(%s) emane daemon running: %s", node.name, args) logging.info("node(%s) emane daemon output: %s", node.name, output) @@ -763,22 +754,22 @@ class EmaneManager(ModelManager): return path = self.session.session_dir - emanecmd += ["-f", os.path.join(path, "emane.log")] - args = emanecmd + [os.path.join(path, "platform.xml")] - utils.check_cmd(args, cwd=path) - args = " ".join(args) + emanecmd += " -f %s" % os.path.join(path, "emane.log") + emanecmd += " %s" % os.path.join(path, "platform.xml") + utils.check_cmd(emanecmd, cwd=path) for server in self.session.servers: conn = self.session.servers[server] - distributed.remote_cmd(conn, args, cwd=path) - logging.info("host emane daemon running: %s", args) + distributed.remote_cmd(conn, emanecmd, cwd=path) + logging.info("host emane daemon running: %s", emanecmd) def stopdaemons(self): """ Kill the appropriate EMANE daemons. """ - # TODO: we may want to improve this if we had the PIDs from the specific EMANE daemons that we"ve started - kill_emaned = ["killall", "-q", "emane"] - kill_transortd = ["killall", "-q", "emanetransportd"] + # TODO: we may want to improve this if we had the PIDs from the specific EMANE + # daemons that we"ve started + kill_emaned = "killall -q emane" + kill_transortd = "killall -q emanetransportd" stop_emane_on_host = False for node in self.getnodes(): if hasattr(node, "transport_type") and node.transport_type == "raw": @@ -793,8 +784,6 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - kill_emaned = " ".join(kill_emaned) - kill_transortd = " ".join(kill_transortd) for server in self.session.servers: conn = self.session[server] distributed.remote_cmd(conn, kill_emaned) diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index dfd36b5d..249e81b6 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -62,4 +62,5 @@ class EmaneTdmaModel(emanemodel.EmaneModel): logging.info( "setting up tdma schedule: schedule(%s) device(%s)", schedule, event_device ) - utils.check_cmd(["emaneevent-tdmaschedule", "-i", event_device, schedule]) + args = "emaneevent-tdmaschedule -i %s %s" % (event_device, schedule) + utils.check_cmd(args) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index e3a2389b..7c4cd651 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -2044,5 +2044,4 @@ class Session(object): utils.mute_detach(data) else: node = self.get_node(node_id) - data = utils.split_args(data) node.node_net_cmd(data, wait=False) diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index f6ce60ca..3522a3f7 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -1281,7 +1281,7 @@ class Ns2ScriptedMobility(WayPointMobility): if filename is None or filename == "": return filename = self.findfile(filename) - args = ["/bin/sh", filename, typestr] + args = "/bin/sh %s %s" % (filename, typestr) utils.check_cmd( args, cwd=self.session.session_dir, env=self.session.get_environment() ) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 120c53ca..85e25074 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -12,7 +12,8 @@ import threading from builtins import range from socket import AF_INET, AF_INET6 -from core import constants, utils +from core import utils +from core.constants import MOUNT_BIN, VNODED_BIN from core.emulator import distributed from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes @@ -89,7 +90,7 @@ class NodeBase(object): Runs a command that is used to configure and setup the network on the host system. - :param list[str]|str args: command to run + :param str args: command to run :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -100,7 +101,6 @@ class NodeBase(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) def setposition(self, x=None, y=None, z=None): @@ -269,7 +269,7 @@ class CoreNodeBase(NodeBase): """ if self.nodedir is None: self.nodedir = os.path.join(self.session.session_dir, self.name + ".conf") - self.net_cmd(["mkdir", "-p", self.nodedir]) + self.net_cmd("mkdir -p %s" % self.nodedir) self.tmpnodedir = True else: self.tmpnodedir = False @@ -285,7 +285,7 @@ class CoreNodeBase(NodeBase): return if self.tmpnodedir: - self.net_cmd(["rm", "-rf", self.nodedir]) + self.net_cmd("rm -rf %s" % self.nodedir) def addnetif(self, netif, ifindex): """ @@ -334,7 +334,7 @@ class CoreNodeBase(NodeBase): Attach a network. :param int ifindex: interface of index to attach - :param core.nodes.interface.CoreInterface net: network to attach + :param core.nodes.base.CoreNetworkBase net: network to attach :return: nothing """ if ifindex not in self._netif: @@ -392,7 +392,7 @@ class CoreNodeBase(NodeBase): Runs a command that is used to configure and setup the network within a node. - :param list[str]|str args: command to run + :param str args: command to run :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str @@ -468,7 +468,7 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - self.net_cmd(["kill", "-0", str(self.pid)]) + self.net_cmd("kill -0 %s" % self.pid) except CoreCommandError: return False @@ -488,18 +488,11 @@ class CoreNode(CoreNodeBase): raise ValueError("starting a node that is already up") # create a new namespace for this node using vnoded - vnoded = [ - constants.VNODED_BIN, - "-v", - "-c", - self.ctrlchnlname, - "-l", - self.ctrlchnlname + ".log", - "-p", - self.ctrlchnlname + ".pid", - ] + vnoded = "{cmd} -v -c {name} -l {name}.log -p {name}.pid".format( + cmd=VNODED_BIN, name=self.ctrlchnlname + ) if self.nodedir: - vnoded += ["-C", self.nodedir] + vnoded += " -C %s" % self.nodedir env = self.session.get_environment(state=False) env["NODE_NUMBER"] = str(self.id) env["NODE_NAME"] = str(self.name) @@ -548,13 +541,13 @@ class CoreNode(CoreNodeBase): # kill node process if present try: - self.net_cmd(["kill", "-9", str(self.pid)]) + self.net_cmd("kill -9 %s" % self.pid) except CoreCommandError: logging.exception("error killing process") # remove node directory if present try: - self.net_cmd(["rm", "-rf", self.ctrlchnlname]) + self.net_cmd("rm -rf %s" % self.ctrlchnlname) except CoreCommandError: logging.exception("error removing node directory") @@ -572,7 +565,7 @@ class CoreNode(CoreNodeBase): Runs a command that is used to configure and setup the network within a node. - :param list[str] args: command to run + :param str args: command to run :param bool wait: True to wait for status, False otherwise :return: combined stdout and stderr :rtype: str @@ -581,8 +574,7 @@ class CoreNode(CoreNodeBase): if self.server is None: return self.client.check_cmd(args, wait=wait) else: - args = self.client._cmd_args() + args - args = " ".join(args) + args = self.client.create_cmd(args) return distributed.remote_cmd(self.server, args, wait=wait) def termcmdstring(self, sh="/bin/sh"): @@ -606,7 +598,7 @@ class CoreNode(CoreNodeBase): hostpath = os.path.join( self.nodedir, os.path.normpath(path).strip("/").replace("/", ".") ) - self.net_cmd(["mkdir", "-p", hostpath]) + self.net_cmd("mkdir -p %s" % hostpath) self.mount(hostpath, path) def mount(self, source, target): @@ -620,8 +612,8 @@ class CoreNode(CoreNodeBase): """ source = os.path.abspath(source) logging.debug("node(%s) mounting: %s at %s", self.name, source, target) - self.node_net_cmd(["mkdir", "-p", target]) - self.node_net_cmd([constants.MOUNT_BIN, "-n", "--bind", source, target]) + self.node_net_cmd("mkdir -p %s" % target) + self.node_net_cmd("%s -n --bind %s %s" % (MOUNT_BIN, source, target)) self._mounts.append((source, target)) def newifindex(self): @@ -881,11 +873,11 @@ class CoreNode(CoreNodeBase): logging.info("adding file from %s to %s", srcname, filename) directory = os.path.dirname(filename) if self.server is None: - self.client.check_cmd(["mkdir", "-p", directory]) - self.client.check_cmd(["mv", srcname, filename]) - self.client.check_cmd(["sync"]) + self.client.check_cmd("mkdir -p %s" % directory) + self.client.check_cmd("mv %s %s" % (srcname, filename)) + self.client.check_cmd("sync") else: - self.net_cmd(["mkdir", "-p", directory]) + self.net_cmd("mkdir -p %s" % directory) distributed.remote_put(self.server, srcname, filename) def hostfilename(self, filename): @@ -922,9 +914,9 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - self.net_cmd(["mkdir", "-m", "%o" % 0o755, "-p", dirname]) + self.net_cmd("mkdir -m %o -p %s" % (0o755, dirname)) distributed.remote_put_temp(self.server, hostfilename, contents) - self.net_cmd(["chmod", "%o" % mode, hostfilename]) + self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode ) @@ -947,7 +939,7 @@ class CoreNode(CoreNodeBase): else: distributed.remote_put(self.server, srcfilename, hostfilename) if mode is not None: - self.net_cmd(["chmod", "%o" % mode, hostfilename]) + self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 81297cf5..e09c72fa 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -5,6 +5,7 @@ The control channel can be accessed via calls using the vcmd shell. """ from core import constants, utils +from core.constants import VCMD_BIN class VnodeClient(object): @@ -49,22 +50,21 @@ class VnodeClient(object): """ pass - def _cmd_args(self): - return [constants.VCMD_BIN, "-c", self.ctrlchnlname, "--"] + def create_cmd(self, args): + return "%s -c %s -- %s" % (VCMD_BIN, self.ctrlchnlname, args) def check_cmd(self, args, wait=True): """ Run command and return exit status and combined stdout and stderr. - :param list[str]|str args: command to run + :param str args: command to run :param bool wait: True to wait for command status, False otherwise :return: combined stdout and stderr :rtype: str :raises core.CoreCommandError: when there is a non-zero exit status """ self._verify_connection() - args = utils.split_args(args) - args = self._cmd_args() + args + args = self.create_cmd(args) return utils.check_cmd(args, wait=wait) def termcmdstring(self, sh="/bin/sh"): diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 1b8322ae..416b31d1 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -63,7 +63,6 @@ class DockerClient(object): pid=self.pid, cmd=cmd ) - logging.info("ns cmd: %s", args) return utils.check_cmd(args, wait=wait) def get_pid(self): diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index d749bbde..4e834fd3 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -52,7 +52,7 @@ class CoreInterface(object): """ Runs a command on the host system or distributed servers. - :param list[str]|str args: command to run + :param str args: command to run :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -63,7 +63,6 @@ class CoreInterface(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - args = " ".join(args) return distributed.remote_cmd(self.server, args, env, cwd, wait) def startup(self): diff --git a/daemon/core/nodes/ipaddress.py b/daemon/core/nodes/ipaddress.py index 00aed74c..c7860dbc 100644 --- a/daemon/core/nodes/ipaddress.py +++ b/daemon/core/nodes/ipaddress.py @@ -19,7 +19,7 @@ class MacAddress(object): """ Creates a MacAddress instance. - :param str address: mac address + :param bytes address: mac address """ self.addr = address @@ -42,7 +42,7 @@ class MacAddress(object): """ if not self.addr: return IpAddress.from_string("::") - tmp = struct.unpack("!Q", "\x00\x00" + self.addr)[0] + tmp = struct.unpack("!Q", b"\x00\x00" + self.addr)[0] nic = int(tmp) & 0x000000FFFFFF oui = int(tmp) & 0xFFFFFF000000 # toggle U/L bit @@ -88,7 +88,7 @@ class IpAddress(object): Create a IpAddress instance. :param int af: address family - :param str address: ip address + :param bytes address: ip address :return: """ # check if (af, addr) is valid diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index df01f4ad..96b107f8 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -43,25 +43,19 @@ class LxdClient(object): def stop_container(self): utils.check_cmd("lxc delete --force {name}".format(name=self.name)) - def _cmd_args(self, cmd): + def create_cmd(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def check_cmd(self, cmd, wait): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._cmd_args(cmd) - logging.info("lxc cmd output: %s", args) + def check_cmd(self, cmd, wait=True): + args = self.create_cmd(cmd) return utils.check_cmd(args, wait=wait) - def _ns_args(self, cmd): + def create_ns_cmd(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) - def ns_check_cmd(self, cmd): - if isinstance(cmd, list): - cmd = " ".join(cmd) - args = self._ns_args(cmd) - logging.info("ns cmd: %s", args) - return utils.check_cmd(args) + def ns_check_cmd(self, cmd, wait=True): + args = self.create_ns_cmd(cmd) + return utils.check_cmd(args, wait=wait) def copy_file(self, source, destination): if destination[0] != "/": diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index a689dde9..930e13b4 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -27,7 +27,7 @@ class LinuxNetClient(object): :param str name: name for hostname :return: nothing """ - self.run(["hostname", name]) + self.run("hostname %s" % name) def create_route(self, route, device): """ @@ -37,7 +37,7 @@ class LinuxNetClient(object): :param str device: device to add route to :return: nothing """ - self.run([IP_BIN, "route", "add", route, "dev", device]) + self.run("%s route add %s dev %s" % (IP_BIN, route, device)) def device_up(self, device): """ @@ -46,7 +46,7 @@ class LinuxNetClient(object): :param str device: device to bring up :return: nothing """ - self.run([IP_BIN, "link", "set", device, "up"]) + self.run("%s link set %s up" % (IP_BIN, device)) def device_down(self, device): """ @@ -55,7 +55,7 @@ class LinuxNetClient(object): :param str device: device to bring down :return: nothing """ - self.run([IP_BIN, "link", "set", device, "down"]) + self.run("%s link set %s down" % (IP_BIN, device)) def device_name(self, device, name): """ @@ -65,7 +65,7 @@ class LinuxNetClient(object): :param str name: name to set :return: nothing """ - self.run([IP_BIN, "link", "set", device, "name", name]) + self.run("%s link set %s name %s" % (IP_BIN, device, name)) def device_show(self, device): """ @@ -75,7 +75,7 @@ class LinuxNetClient(object): :return: device information :rtype: str """ - return self.run([IP_BIN, "link", "show", device]) + return self.run("%s link show %s" % (IP_BIN, device)) def device_ns(self, device, namespace): """ @@ -85,7 +85,7 @@ class LinuxNetClient(object): :param str namespace: namespace to set device to :return: nothing """ - self.run([IP_BIN, "link", "set", device, "netns", namespace]) + self.run("%s link set %s netns %s" % (IP_BIN, device, namespace)) def device_flush(self, device): """ @@ -94,7 +94,7 @@ class LinuxNetClient(object): :param str device: device to flush :return: nothing """ - self.run([IP_BIN, "-6", "address", "flush", "dev", device]) + self.run("%s -6 address flush dev %s" % (IP_BIN, device)) def device_mac(self, device, mac): """ @@ -104,7 +104,7 @@ class LinuxNetClient(object): :param str mac: mac to set :return: nothing """ - self.run([IP_BIN, "link", "set", "dev", device, "address", mac]) + self.run("%s link set dev %s address %s" % (IP_BIN, device, mac)) def delete_device(self, device): """ @@ -113,7 +113,7 @@ class LinuxNetClient(object): :param str device: device to delete :return: nothing """ - self.run([IP_BIN, "link", "delete", device]) + self.run("%s link delete %s" % (IP_BIN, device)) def delete_tc(self, device): """ @@ -122,7 +122,7 @@ class LinuxNetClient(object): :param str device: device to remove tc :return: nothing """ - self.run([TC_BIN, "qdisc", "del", "dev", device, "root"]) + self.run("%s qdisc del dev %s root" % (TC_BIN, device)) def checksums_off(self, interface_name): """ @@ -131,7 +131,7 @@ class LinuxNetClient(object): :param str interface_name: interface to update :return: nothing """ - self.run([ETHTOOL_BIN, "-K", interface_name, "rx", "off", "tx", "off"]) + self.run("%s -K %s rx off tx off" % (ETHTOOL_BIN, interface_name)) def create_address(self, device, address, broadcast=None): """ @@ -144,19 +144,11 @@ class LinuxNetClient(object): """ if broadcast is not None: self.run( - [ - IP_BIN, - "address", - "add", - address, - "broadcast", - broadcast, - "dev", - device, - ] + "%s address add %s broadcast %s dev %s" + % (IP_BIN, address, broadcast, device) ) else: - self.run([IP_BIN, "address", "add", address, "dev", device]) + self.run("%s address add %s dev %s" % (IP_BIN, address, device)) def delete_address(self, device, address): """ @@ -166,7 +158,7 @@ class LinuxNetClient(object): :param str address: address to remove :return: nothing """ - self.run([IP_BIN, "address", "delete", address, "dev", device]) + self.run("%s address delete %s dev %s" % (IP_BIN, address, device)) def create_veth(self, name, peer): """ @@ -176,9 +168,7 @@ class LinuxNetClient(object): :param str peer: peer name :return: nothing """ - self.run( - [IP_BIN, "link", "add", "name", name, "type", "veth", "peer", "name", peer] - ) + self.run("%s link add name %s type veth peer name %s" % (IP_BIN, name, peer)) def create_gretap(self, device, address, local, ttl, key): """ @@ -191,13 +181,13 @@ class LinuxNetClient(object): :param str key: key for tap :return: nothing """ - cmd = [IP_BIN, "link", "add", device, "type", "gretap", "remote", address] + cmd = "%s link add %s type gretap remote %s" % (IP_BIN, device, address) if local is not None: - cmd.extend(["local", local]) + cmd += " local %s" % local if ttl is not None: - cmd.extend(["ttl", ttl]) + cmd += " ttl %s" % ttl if key is not None: - cmd.extend(["key", key]) + cmd += " key %s" % key self.run(cmd) def create_bridge(self, name): @@ -207,9 +197,9 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - self.run([BRCTL_BIN, "addbr", name]) - self.run([BRCTL_BIN, "stp", name, "off"]) - self.run([BRCTL_BIN, "setfd", name, "0"]) + self.run("%s addbr %s" % (BRCTL_BIN, name)) + self.run("%s stp %s off" % (BRCTL_BIN, name)) + self.run("%s setfd %s 0" % (BRCTL_BIN, name)) self.device_up(name) # turn off multicast snooping so forwarding occurs w/o IGMP joins @@ -226,7 +216,7 @@ class LinuxNetClient(object): :return: nothing """ self.device_down(name) - self.run([BRCTL_BIN, "delbr", name]) + self.run("%s delbr %s" % (BRCTL_BIN, name)) def create_interface(self, bridge_name, interface_name): """ @@ -236,7 +226,7 @@ class LinuxNetClient(object): :param str interface_name: interface name :return: nothing """ - self.run([BRCTL_BIN, "addif", bridge_name, interface_name]) + self.run("%s addif %s %s" % (BRCTL_BIN, bridge_name, interface_name)) self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): @@ -247,7 +237,7 @@ class LinuxNetClient(object): :param str interface_name: interface name :return: nothing """ - self.run([BRCTL_BIN, "delif", bridge_name, interface_name]) + self.run("%s delif %s %s" % (BRCTL_BIN, bridge_name, interface_name)) def existing_bridges(self, _id): """ @@ -255,7 +245,7 @@ class LinuxNetClient(object): :param _id: node id to check bridges for """ - output = self.run([BRCTL_BIN, "show"]) + output = self.run("%s show" % BRCTL_BIN) lines = output.split("\n") for line in lines[1:]: columns = line.split() @@ -274,7 +264,7 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - self.run([BRCTL_BIN, "setageing", name, "0"]) + self.run("%s setageing %s 0" % (BRCTL_BIN, name)) class OvsNetClient(LinuxNetClient): @@ -289,10 +279,10 @@ class OvsNetClient(LinuxNetClient): :param str name: bridge name :return: nothing """ - self.run([OVS_BIN, "add-br", name]) - self.run([OVS_BIN, "set", "bridge", name, "stp_enable=false"]) - self.run([OVS_BIN, "set", "bridge", name, "other_config:stp-max-age=6"]) - self.run([OVS_BIN, "set", "bridge", name, "other_config:stp-forward-delay=4"]) + self.run("%s add-br %s" % (OVS_BIN, name)) + self.run("%s set bridge %s stp_enable=false" % (OVS_BIN, name)) + self.run("%s set bridge %s other_config:stp-max-age=6" % (OVS_BIN, name)) + self.run("%s set bridge %s other_config:stp-forward-delay=4" % (OVS_BIN, name)) self.device_up(name) def delete_bridge(self, name): @@ -303,7 +293,7 @@ class OvsNetClient(LinuxNetClient): :return: nothing """ self.device_down(name) - self.run([OVS_BIN, "del-br", name]) + self.run("%s del-br %s" % (OVS_BIN, name)) def create_interface(self, bridge_name, interface_name): """ @@ -313,7 +303,7 @@ class OvsNetClient(LinuxNetClient): :param str interface_name: interface name :return: nothing """ - self.run([OVS_BIN, "add-port", bridge_name, interface_name]) + self.run("%s add-port %s %s" % (OVS_BIN, bridge_name, interface_name)) self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): @@ -324,7 +314,7 @@ class OvsNetClient(LinuxNetClient): :param str interface_name: interface name :return: nothing """ - self.run([OVS_BIN, "del-port", bridge_name, interface_name]) + self.run("%s del-port %s %s" % (OVS_BIN, bridge_name, interface_name)) def existing_bridges(self, _id): """ @@ -332,7 +322,7 @@ class OvsNetClient(LinuxNetClient): :param _id: node id to check bridges for """ - output = self.run([OVS_BIN, "list-br"]) + output = self.run("%s list-br" % OVS_BIN) if output: for line in output.split("\n"): fields = line.split(".") @@ -347,4 +337,4 @@ class OvsNetClient(LinuxNetClient): :param str name: bridge name :return: nothing """ - self.run([OVS_BIN, "set", "bridge", name, "other_config:mac-aging-time=0"]) + self.run("%s set bridge %s other_config:mac-aging-time=0" % (OVS_BIN, name)) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 7197c77f..e0fcd839 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -9,6 +9,7 @@ import time from socket import AF_INET, AF_INET6 from core import constants, utils +from core.constants import EBTABLES_BIN from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs @@ -92,14 +93,11 @@ class EbtablesQueue(object): """ Helper for building ebtables atomic file command list. - :param list[str] cmd: ebtable command + :param str cmd: ebtable command :return: ebtable atomic command :rtype: list[str] """ - r = [constants.EBTABLES_BIN, "--atomic-file", self.atomic_file] - if cmd: - r.extend(cmd) - return r + return "%s --atomic-file %s %s" % (EBTABLES_BIN, self.atomic_file, cmd) def lastupdate(self, wlan): """ @@ -163,7 +161,7 @@ class EbtablesQueue(object): :return: nothing """ # save kernel ebtables snapshot to a file - args = self.ebatomiccmd(["--atomic-save"]) + args = self.ebatomiccmd("--atomic-save") wlan.net_cmd(args) # modify the table file using queued ebtables commands @@ -173,12 +171,12 @@ class EbtablesQueue(object): self.cmds = [] # commit the table file to the kernel - args = self.ebatomiccmd(["--atomic-commit"]) + args = self.ebatomiccmd("--atomic-commit") wlan.net_cmd(args) try: - wlan.net_cmd(["rm", "-f", self.atomic_file]) - except OSError: + wlan.net_cmd("rm -f %s" % self.atomic_file) + except CoreCommandError: logging.exception("error removing atomic file: %s", self.atomic_file) def ebchange(self, wlan): @@ -200,58 +198,26 @@ class EbtablesQueue(object): """ with wlan._linked_lock: # flush the chain - self.cmds.extend([["-F", wlan.brname]]) + self.cmds.append("-F %s" % wlan.brname) # rebuild the chain for netif1, v in wlan._linked.items(): for netif2, linked in v.items(): if wlan.policy == "DROP" and linked: self.cmds.extend( [ - [ - "-A", - wlan.brname, - "-i", - netif1.localname, - "-o", - netif2.localname, - "-j", - "ACCEPT", - ], - [ - "-A", - wlan.brname, - "-o", - netif1.localname, - "-i", - netif2.localname, - "-j", - "ACCEPT", - ], + "-A %s -i %s -o %s -j ACCEPT" + % (wlan.brname, netif1.localname, netif2.localname), + "-A %s -o %s -i %s -j ACCEPT" + % (wlan.brname, netif1.localname, netif2.localname), ] ) elif wlan.policy == "ACCEPT" and not linked: self.cmds.extend( [ - [ - "-A", - wlan.brname, - "-i", - netif1.localname, - "-o", - netif2.localname, - "-j", - "DROP", - ], - [ - "-A", - wlan.brname, - "-o", - netif1.localname, - "-i", - netif2.localname, - "-j", - "DROP", - ], + "-A %s -i %s -o %s -j DROP" + % (wlan.brname, netif1.localname, netif2.localname), + "-A %s -o %s -i %s -j DROP" + % (wlan.brname, netif1.localname, netif2.localname), ] ) @@ -313,7 +279,7 @@ class CoreNetwork(CoreNetworkBase): Runs a command that is used to configure and setup the network on the host system and all configured distributed servers. - :param list[str]|str args: command to run + :param str args: command to run :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -323,12 +289,9 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - - args = " ".join(args) for server in self.session.servers: conn = self.session.servers[server] distributed.remote_cmd(conn, args, env, cwd, wait) - return output def startup(self): @@ -341,21 +304,12 @@ class CoreNetwork(CoreNetworkBase): self.net_client.create_bridge(self.brname) # create a new ebtables chain for this bridge - ebtablescmds( - self.net_cmd, - [ - [constants.EBTABLES_BIN, "-N", self.brname, "-P", self.policy], - [ - constants.EBTABLES_BIN, - "-A", - "FORWARD", - "--logical-in", - self.brname, - "-j", - self.brname, - ], - ], - ) + cmds = [ + "%s -N %s -P %s" % (EBTABLES_BIN, self.brname, self.policy), + "%s -A FORWARD --logical-in %s -j %s" + % (EBTABLES_BIN, self.brname, self.brname), + ] + ebtablescmds(self.net_cmd, cmds) self.up = True @@ -372,21 +326,12 @@ class CoreNetwork(CoreNetworkBase): try: self.net_client.delete_bridge(self.brname) - ebtablescmds( - self.net_cmd, - [ - [ - constants.EBTABLES_BIN, - "-D", - "FORWARD", - "--logical-in", - self.brname, - "-j", - self.brname, - ], - [constants.EBTABLES_BIN, "-X", self.brname], - ], - ) + cmds = [ + "%s -D FORWARD --logical-in %s -j %s" + % (EBTABLES_BIN, self.brname, self.brname), + "%s -X %s" % (EBTABLES_BIN, self.brname), + ] + ebtablescmds(self.net_cmd, cmds) except CoreCommandError: logging.exception("error during shutdown") @@ -852,7 +797,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd([self.updown_script, self.brname, "startup"]) + self.net_cmd("%s %s startup" % (self.updown_script, self.brname)) if self.serverintf: self.net_client.create_interface(self.brname, self.serverintf) @@ -880,7 +825,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd([self.updown_script, self.brname, "shutdown"]) + self.net_cmd("%s %s shutdown" % (self.updown_script, self.brname)) except CoreCommandError: logging.exception("error issuing shutdown script shutdown") @@ -1064,7 +1009,8 @@ class HubNode(CoreNetwork): :param int _id: node id :param str name: node namee :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :raises CoreCommandError: when there is a command exception """ CoreNetwork.__init__(self, session, _id, name, start, server) @@ -1094,7 +1040,8 @@ class WlanNode(CoreNetwork): :param int _id: node id :param str name: node name :param bool start: start flag - :param str server: remote server node will run on, default is None for localhost + :param fabric.connection.Connection server: remote server node will run on, + default is None for localhost :param policy: wlan policy """ CoreNetwork.__init__(self, session, _id, name, start, server, policy) diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index dc45fa33..6db2d8ed 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -597,7 +597,6 @@ class CoreServices(object): status = 0 for cmd in cmds: logging.debug("validating service(%s) using: %s", service.name, cmd) - cmd = utils.split_args(cmd) try: node.node_net_cmd(cmd) except CoreCommandError as e: @@ -631,7 +630,6 @@ class CoreServices(object): """ status = 0 for args in service.shutdown: - args = utils.split_args(args) try: node.node_net_cmd(args) except CoreCommandError: @@ -730,7 +728,6 @@ class CoreServices(object): status = 0 for cmd in cmds: - cmd = utils.split_args(cmd) try: node.node_net_cmd(cmd, wait) except CoreCommandError: diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 66a84dd6..14bd5a90 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -415,7 +415,7 @@ class HttpService(UtilService): Detect the apache2 version using the 'a2query' command. """ try: - result = utils.check_cmd(["a2query", "-v"]) + result = utils.check_cmd("a2query -v") status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 003c7134..8f8da19c 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -14,8 +14,6 @@ import shlex import sys from subprocess import PIPE, STDOUT, Popen -from past.builtins import basestring - from core.errors import CoreCommandError DEVNULL = open(os.devnull, "wb") @@ -177,20 +175,6 @@ def make_tuple_fromstr(s, value_type): return tuple(value_type(i) for i in values) -def split_args(args): - """ - Convenience method for splitting potential string commands into a shell-like - syntax list. - - :param list/str args: command list or string - :return: shell-like syntax list - :rtype: list - """ - if isinstance(args, basestring): - args = shlex.split(args) - return args - - def mute_detach(args, **kwargs): """ Run a muted detached process by forking it. @@ -200,7 +184,7 @@ def mute_detach(args, **kwargs): :return: process id of the command :rtype: int """ - args = split_args(args) + args = shlex.split(args) kwargs["preexec_fn"] = _detach_init kwargs["stdout"] = DEVNULL kwargs["stderr"] = STDOUT @@ -212,7 +196,7 @@ def check_cmd(args, env=None, cwd=None, wait=True): Execute a command on the host and return a tuple containing the exit status and result string. stderr output is folded into the stdout result string. - :param list[str]|str args: command arguments + :param str args: command arguments :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise @@ -221,8 +205,8 @@ def check_cmd(args, env=None, cwd=None, wait=True): :raises CoreCommandError: when there is a non-zero exit status or the file to execute is not found """ - args = split_args(args) logging.info("command cwd(%s) wait(%s): %s", cwd, wait, args) + args = shlex.split(args) try: p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd) if wait: diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index c410ef5f..ee316ffc 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -3,7 +3,8 @@ import socket from lxml import etree -from core import constants, utils +from core import utils +from core.constants import IP_BIN from core.emane.nodes import EmaneNet from core.nodes import ipaddress from core.nodes.base import CoreNodeBase @@ -67,7 +68,7 @@ def get_address_type(address): def get_ipv4_addresses(hostname): if hostname == "localhost": addresses = [] - args = [constants.IP_BIN, "-o", "-f", "inet", "addr", "show"] + args = "%s -o -f inet address show" % IP_BIN output = utils.check_cmd(args) for line in output.split(os.linesep): split = line.split() diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 3d9b9eb2..65065665 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -27,7 +27,7 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd(["ping", "-c", str(count), address]) + from_node.node_net_cmd("ping -c %s %s" % (count, address)) status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 392794d0..3fc90da8 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -20,7 +20,7 @@ _WIRED = [NodeTypes.PEER_TO_PEER, NodeTypes.HUB, NodeTypes.SWITCH] def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd(["ping", "-c", "3", address]) + from_node.node_net_cmd("ping -c 3 %s" % address) status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index ad5476d4..1f18c87e 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -30,7 +30,7 @@ class TestNodes: assert os.path.exists(node.nodedir) assert node.alive() assert node.up - assert node.node_net_cmd(["ip", "addr", "show", "lo"]) + assert node.node_net_cmd("ip address show lo") def test_node_update(self, session): # given @@ -67,4 +67,4 @@ class TestNodes: # then assert node assert node.up - assert utils.check_cmd(["brctl", "show", node.brname]) + assert utils.check_cmd("brctl show %s" % node.brname) From 5b3308a23159553621b51e5b79c48277afe5d44d Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 22:27:04 -0700 Subject: [PATCH 069/113] updated linkconfig to use string commands, fixed issues for wlan configuration --- daemon/core/api/tlv/corehandlers.py | 2 +- daemon/core/location/mobility.py | 2 +- daemon/core/nodes/netclient.py | 2 +- daemon/core/nodes/network.py | 49 ++++++++++++++--------------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 444ae5a5..6ff7f55b 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1425,7 +1425,7 @@ class CoreHandler(socketserver.BaseRequestHandler): parsed_config = ConfigShim.str_to_dict(values_str) self.session.mobility.set_model_config(node_id, object_name, parsed_config) - if self.session.state == EventTypes.RUNTIME_STATE.value: + if self.session.state == EventTypes.RUNTIME_STATE.value and parsed_config: try: node = self.session.get_node(node_id) if object_name == BasicRangeModel.name: diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index 3522a3f7..2f323783 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -426,7 +426,7 @@ class BasicRangeModel(WirelessModel): self.delay = int(config["delay"]) if self.delay == 0: self.delay = None - self.loss = int(config["error"]) + self.loss = int(float(config["error"])) if self.loss == 0: self.loss = None self.jitter = int(config["jitter"]) diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 930e13b4..43d21ead 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -122,7 +122,7 @@ class LinuxNetClient(object): :param str device: device to remove tc :return: nothing """ - self.run("%s qdisc del dev %s root" % (TC_BIN, device)) + self.run("%s qdisc delete dev %s root" % (TC_BIN, device)) def checksums_off(self, interface_name): """ diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index e0fcd839..4df7b28a 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -8,8 +8,8 @@ import threading import time from socket import AF_INET, AF_INET6 -from core import constants, utils -from core.constants import EBTABLES_BIN +from core import utils +from core.constants import EBTABLES_BIN, TC_BIN from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs @@ -457,8 +457,8 @@ class CoreNetwork(CoreNetworkBase): """ if devname is None: devname = netif.localname - tc = [constants.TC_BIN, "qdisc", "replace", "dev", devname] - parent = ["root"] + tc = "%s qdisc replace dev %s" % (TC_BIN, devname) + parent = "root" changed = False if netif.setparam("bw", bw): # from tc-tbf(8): minimum value for burst is rate / kernel_hz @@ -466,27 +466,24 @@ class CoreNetwork(CoreNetworkBase): burst = max(2 * netif.mtu, bw / 1000) # max IP payload limit = 0xFFFF - tbf = ["tbf", "rate", str(bw), "burst", str(burst), "limit", str(limit)] + tbf = "tbf rate %s burst %s limit %s" % (bw, burst, limit) if bw > 0: if self.up: - logging.debug( - "linkconfig: %s" % ([tc + parent + ["handle", "1:"] + tbf],) - ) - netif.net_cmd(tc + parent + ["handle", "1:"] + tbf) + cmd = "%s %s handle 1: %s" % (tc, parent, tbf) + netif.net_cmd(cmd) netif.setparam("has_tbf", True) changed = True elif netif.getparam("has_tbf") and bw <= 0: - tcd = [] + tc - tcd[2] = "delete" if self.up: - netif.net_cmd(tcd + parent) + cmd = "%s qdisc delete dev %s %s" % (TC_BIN, devname, parent) + netif.net_cmd(cmd) netif.setparam("has_tbf", False) # removing the parent removes the child netif.setparam("has_netem", False) changed = True if netif.getparam("has_tbf"): - parent = ["parent", "1:1"] - netem = ["netem"] + parent = "parent 1:1" + netem = "netem" changed = max(changed, netif.setparam("delay", delay)) if loss is not None: loss = float(loss) @@ -499,17 +496,17 @@ class CoreNetwork(CoreNetworkBase): return # jitter and delay use the same delay statement if delay is not None: - netem += ["delay", "%sus" % delay] + netem += " delay %sus" % delay if jitter is not None: if delay is None: - netem += ["delay", "0us", "%sus" % jitter, "25%"] + netem += " delay 0us %sus 25%%" % jitter else: - netem += ["%sus" % jitter, "25%"] + netem += " %sus 25%%" % jitter if loss is not None and loss > 0: - netem += ["loss", "%s%%" % min(loss, 100)] + netem += " loss %s%%" % min(loss, 100) if duplicate is not None and duplicate > 0: - netem += ["duplicate", "%s%%" % min(duplicate, 100)] + netem += " duplicate %s%%" % min(duplicate, 100) delay_check = delay is None or delay <= 0 jitter_check = jitter is None or jitter <= 0 @@ -519,17 +516,19 @@ class CoreNetwork(CoreNetworkBase): # possibly remove netem if it exists and parent queue wasn't removed if not netif.getparam("has_netem"): return - tc[2] = "delete" if self.up: - logging.debug("linkconfig: %s" % ([tc + parent + ["handle", "10:"]],)) - netif.net_cmd(tc + parent + ["handle", "10:"]) + cmd = "%s qdisc delete dev %s %s handle 10:" % (TC_BIN, devname, parent) + netif.net_cmd(cmd) netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: - logging.debug( - "linkconfig: %s" % ([tc + parent + ["handle", "10:"] + netem],) + cmd = "%s qdisc replace dev %s %s handle 10: %s" % ( + TC_BIN, + devname, + parent, + netem, ) - netif.net_cmd(tc + parent + ["handle", "10:"] + netem) + netif.net_cmd(cmd) netif.setparam("has_netem", True) def linknet(self, net): From 2bfd05088094ed8d93d6f9737843793e4f66e0a4 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Oct 2019 22:37:33 -0700 Subject: [PATCH 070/113] updated missed commands to be string based --- daemon/core/emane/emanemanager.py | 5 +++-- daemon/core/emulator/session.py | 3 ++- daemon/core/nodes/physical.py | 7 ++++--- daemon/examples/python/switch.py | 6 +++--- daemon/examples/python/wlan.py | 6 +++--- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 65fed8bd..35cc7ca9 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -974,9 +974,10 @@ class EmaneManager(ModelManager): def emanerunning(self, node): """ - Return True if an EMANE process associated with the given node is running, False otherwise. + Return True if an EMANE process associated with the given node is running, + False otherwise. """ - args = ["pkill", "-0", "-x", "emane"] + args = "pkill -0 -x emane" try: node.node_net_cmd(args) result = True diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 7c4cd651..7d3928b2 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -2025,7 +2025,8 @@ class Session(object): data, ) - # TODO: if data is None, this blows up, but this ties into how event functions are ran, need to clean that up + # TODO: if data is None, this blows up, but this ties into how event functions + # are ran, need to clean that up def run_event(self, node_id=None, name=None, data=None): """ Run a scheduled event, executing commands in the data string. diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 4c219258..9bd04c53 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -6,7 +6,8 @@ import logging import os import threading -from core import constants, utils +from core import utils +from core.constants import MOUNT_BIN, UMOUNT_BIN from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNodeBase @@ -190,13 +191,13 @@ class PhysicalNode(CoreNodeBase): source = os.path.abspath(source) logging.info("mounting %s at %s", source, target) os.makedirs(target) - self.net_cmd([constants.MOUNT_BIN, "--bind", source, target], cwd=self.nodedir) + self.net_cmd("%s --bind %s %s" % (MOUNT_BIN, source, target), cwd=self.nodedir) self._mounts.append((source, target)) def umount(self, target): logging.info("unmounting '%s'" % target) try: - self.net_cmd([constants.UMOUNT_BIN, "-l", target], cwd=self.nodedir) + self.net_cmd("%s -l %s" % (UMOUNT_BIN, target), cwd=self.nodedir) except CoreCommandError: logging.exception("unmounting failed for %s", target) diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index 3c6ec383..e4d0fd02 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -43,14 +43,14 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.node_net_cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd("iperf -s -D") first_node_address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, first_node_address)) output = last_node.node_net_cmd( - ["iperf", "-t", str(options.time), "-c", first_node_address] + "iperf -t %s -c %s" % (options.time, first_node_address) ) print(output) - first_node.node_net_cmd(["killall", "-9", "iperf"]) + first_node.node_net_cmd("killall -9 iperf") # shutdown session coreemu.shutdown() diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index 3d5171c2..b16af7cd 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -47,11 +47,11 @@ def example(options): last_node = session.get_node(options.nodes + 1) print("starting iperf server on node: %s" % first_node.name) - first_node.node_net_cmd(["iperf", "-s", "-D"]) + first_node.node_net_cmd("iperf -s -D") address = prefixes.ip4_address(first_node) print("node %s connecting to %s" % (last_node.name, address)) - last_node.node_net_cmd(["iperf", "-t", str(options.time), "-c", address]) - first_node.node_net_cmd(["killall", "-9", "iperf"]) + last_node.node_net_cmd("iperf -t %s -c %s" % (options.time, address)) + first_node.node_net_cmd("killall -9 iperf") # shutdown session coreemu.shutdown() From fee56302a1477269f95fb8384e992549cf3799a0 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 12 Oct 2019 08:03:20 -0700 Subject: [PATCH 071/113] fix for wlan configuration issues --- daemon/core/api/tlv/corehandlers.py | 2 +- daemon/core/location/mobility.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 7096c099..1264af84 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1416,7 +1416,7 @@ class CoreHandler(socketserver.BaseRequestHandler): parsed_config = ConfigShim.str_to_dict(values_str) self.session.mobility.set_model_config(node_id, object_name, parsed_config) - if self.session.state == EventTypes.RUNTIME_STATE.value: + if self.session.state == EventTypes.RUNTIME_STATE.value and parsed_config: try: node = self.session.get_node(node_id) if object_name == BasicRangeModel.name: diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index f6ce60ca..aaaf45d2 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -426,7 +426,7 @@ class BasicRangeModel(WirelessModel): self.delay = int(config["delay"]) if self.delay == 0: self.delay = None - self.loss = int(config["error"]) + self.loss = int(float(config["error"])) if self.loss == 0: self.loss = None self.jitter = int(config["jitter"]) From eb248291a59337740c3fabbd8ecd81888bcef5e9 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 12 Oct 2019 08:10:30 -0700 Subject: [PATCH 072/113] fix for gretap interfaces giving bad values to netclient --- daemon/core/nodes/interface.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 51859e3a..a33c0be2 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -460,9 +460,13 @@ class GreTap(CoreInterface): if remoteip is None: raise ValueError("missing remote IP required for GRE TAP device") - self.net_client.create_gretap( - self.localname, str(remoteip), str(localip), str(ttl), str(key) - ) + if localip is not None: + localip = str(localip) + if ttl is not None: + ttl = str(ttl) + if key is not None: + key = str(key) + self.net_client.create_gretap(self.localname, str(remoteip), localip, ttl, key) self.net_client.device_up(self.localname) self.up = True From 4e4fcddd00322d9d0e23cd9c8d027edb0935ff6f Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 12 Oct 2019 08:56:39 -0700 Subject: [PATCH 073/113] added grpc api for creating emane link/unlink messages, which will be sent to the gui for being added/removed --- daemon/core/api/grpc/client.py | 15 +++++++++ daemon/core/api/grpc/server.py | 46 ++++++++++++++++++++++++++- daemon/proto/core/api/grpc/core.proto | 13 ++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index dfb96d21..2d767993 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -864,6 +864,21 @@ class CoreGrpcClient(object): request = core_pb2.OpenXmlRequest(data=data) return self.stub.OpenXml(request) + def emane_link(self, session_id, nem_one, nem_two, linked): + """ + Helps broadcast wireless link/unlink between EMANE nodes. + + :param int session_id: session id + :param int nem_one: + :param int nem_two: + :param bool linked: True to link, False to unlink + :return: core_pb2.EmaneLinkResponse + """ + request = core_pb2.EmaneLinkRequest( + session_id=session_id, nem_one=nem_one, nem_two=nem_two, linked=linked + ) + return self.stub.EmaneLink(request) + def connect(self): """ Open connection to server, must be closed manually. diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 7fb6ed31..c438a414 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -21,7 +21,7 @@ from core.emulator.data import ( NodeData, ) from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions -from core.emulator.enumerations import EventTypes, LinkTypes, NodeTypes +from core.emulator.enumerations import EventTypes, LinkTypes, MessageFlags, NodeTypes from core.errors import CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility from core.nodes.base import CoreNetworkBase @@ -1557,3 +1557,47 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): continue interfaces.append(interface) return core_pb2.GetInterfacesResponse(interfaces=interfaces) + + def EmaneLink(self, request, context): + """ + Helps broadcast wireless link/unlink between EMANE nodes. + + :param core.api.grpc.core_pb2.EmaneLinkRequest request: get-interfaces request + :param grpc.ServicerContext context: context object + :return: emane link response with success status + :rtype: core.api.grpc.core_pb2.EmaneLinkResponse + """ + logging.debug("emane link: %s", request) + session = self.get_session(request.session_id, context) + nem_one = request.nem_one + emane_one, netif = session.emane.nemlookup(nem_one) + if not emane_one or not netif: + context.abort( + grpc.StatusCode.NOT_FOUND, "nem one {} not found".format(nem_one) + ) + node_one = netif.node + + nem_two = request.nem_two + emane_two, netif = session.emane.nemlookup(nem_two) + if not emane_two or not netif: + context.abort( + grpc.StatusCode.NOT_FOUND, "nem two {} not found".format(nem_two) + ) + node_two = netif.node + + if emane_one.id == emane_two.id: + if request.linked: + flag = MessageFlags.ADD.value + else: + flag = MessageFlags.DELETE.value + link = LinkData( + message_type=flag, + link_type=LinkTypes.WIRELESS.value, + node1_id=node_one.id, + node2_id=node_two.id, + network_id=emane_one.id, + ) + session.broadcast_link(link) + return core_pb2.EmaneLinkResponse(result=True) + else: + return core_pb2.EmaneLinkResponse(result=False) diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 91049f40..68c83ad2 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -119,6 +119,8 @@ service CoreApi { // utilities rpc GetInterfaces (GetInterfacesRequest) returns (GetInterfacesResponse) { } + rpc EmaneLink (EmaneLinkRequest) returns (EmaneLinkResponse) { + } } // rpc request/response messages @@ -633,6 +635,17 @@ message GetInterfacesResponse { repeated string interfaces = 1; } +message EmaneLinkRequest { + int32 session_id = 1; + int32 nem_one = 2; + int32 nem_two = 3; + bool linked = 4; +} + +message EmaneLinkResponse { + bool result = 1; +} + // data structures for messages below message MessageType { enum Enum { From a93763f4a7593a5435ee2e157e6863a3095a36d5 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 12 Oct 2019 08:57:14 -0700 Subject: [PATCH 074/113] bump version for release --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 045425c8..fec902a3 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. # this defines the CORE version number, must be static for AC_INIT -AC_INIT(core, 5.5.1) +AC_INIT(core, 5.5.2) # autoconf and automake initialization AC_CONFIG_SRCDIR([netns/version.h.in]) From 82bdbd776b3d36b88f45c9397097735cf5ad1f93 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 14 Oct 2019 12:31:41 -0700 Subject: [PATCH 075/113] removed parameter conversion for creating GreTap commands --- daemon/core/nodes/interface.py | 14 ++++---------- daemon/core/nodes/netclient.py | 4 ++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 4e834fd3..2f42fdfe 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -456,12 +456,12 @@ class GreTap(CoreInterface): :param core.nodes.base.CoreNode node: related core node :param str name: interface name :param core.emulator.session.Session session: core session instance - :param mtu: interface mtu + :param int mtu: interface mtu :param str remoteip: remote address :param int _id: object id :param str localip: local address - :param ttl: ttl value - :param key: gre tap key + :param int ttl: ttl value + :param int key: gre tap key :param bool start: start flag :param fabric.connection.Connection server: remote server node will run on, default is None for localhost @@ -484,13 +484,7 @@ class GreTap(CoreInterface): if remoteip is None: raise ValueError("missing remote IP required for GRE TAP device") - if localip is not None: - localip = str(localip) - if ttl is not None: - ttl = str(ttl) - if key is not None: - key = str(key) - self.net_client.create_gretap(self.localname, str(remoteip), localip, ttl, key) + self.net_client.create_gretap(self.localname, remoteip, localip, ttl, key) self.net_client.device_up(self.localname) self.up = True diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 43d21ead..6de5d698 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -177,8 +177,8 @@ class LinuxNetClient(object): :param str device: device to add tap to :param str address: address to add tap for :param str local: local address to tie to - :param str ttl: time to live value - :param str key: key for tap + :param int ttl: time to live value + :param int key: key for tap :return: nothing """ cmd = "%s link add %s type gretap remote %s" % (IP_BIN, device, address) From 5f282bb6950e575c8d6c9f1c920c0072dbf376a1 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 14 Oct 2019 14:28:18 -0700 Subject: [PATCH 076/113] updates to lxd/docker to work with net_cmd/node_net_cmd --- daemon/core/nodes/base.py | 20 ++++++-- daemon/core/nodes/client.py | 11 +---- daemon/core/nodes/docker.py | 89 ++++++++++++++++++++++++---------- daemon/core/nodes/lxd.py | 70 +++++++++++++++----------- daemon/examples/lxd/lxd2lxd.py | 4 +- 5 files changed, 122 insertions(+), 72 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 85e25074..f263d0f3 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -452,14 +452,24 @@ class CoreNode(CoreNodeBase): self._mounts = [] self.bootsh = bootsh - if session.options.get_config("ovs") == "True": - self.node_net_client = OvsNetClient(self.node_net_cmd) - else: - self.node_net_client = LinuxNetClient(self.node_net_cmd) + use_ovs = session.options.get_config("ovs") == "True" + self.node_net_client = self.create_node_net_client(use_ovs) if start: self.startup() + def create_node_net_client(self, use_ovs): + """ + Create a client for running network orchestration commands. + + :param bool use_ovs: True to use OVS bridges, False for Linux bridge + :return: network client + """ + if use_ovs: + return OvsNetClient(self.node_net_cmd) + else: + return LinuxNetClient(self.node_net_cmd) + def alive(self): """ Check if the node is alive. @@ -584,7 +594,7 @@ class CoreNode(CoreNodeBase): :param str sh: shell to execute command in :return: str """ - return self.client.termcmdstring(sh) + return self.client.create_cmd(sh) def privatedir(self, path): """ diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index e09c72fa..632e12bc 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -4,7 +4,7 @@ over a control channel to the vnoded process running in a network namespace. The control channel can be accessed via calls using the vcmd shell. """ -from core import constants, utils +from core import utils from core.constants import VCMD_BIN @@ -66,12 +66,3 @@ class VnodeClient(object): self._verify_connection() args = self.create_cmd(args) return utils.check_cmd(args, wait=wait) - - def termcmdstring(self, sh="/bin/sh"): - """ - Create a terminal command string. - - :param str sh: shell to execute command in - :return: str - """ - return "%s -c %s -- %s" % (constants.VCMD_BIN, self.ctrlchnlname, sh) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 416b31d1..e465a768 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -1,21 +1,25 @@ import json import logging import os +from tempfile import NamedTemporaryFile from core import utils +from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode +from core.nodes.netclient import LinuxNetClient, OvsNetClient class DockerClient(object): - def __init__(self, name, image): + def __init__(self, name, image, run): self.name = name self.image = image + self.run = run self.pid = None def create_container(self): - utils.check_cmd( + self.run( "docker run -td --init --net=none --hostname {name} --name {name} " "--sysctl net.ipv6.conf.all.disable_ipv6=0 " "{image} /bin/bash".format( @@ -27,7 +31,7 @@ class DockerClient(object): def get_info(self): args = "docker inspect {name}".format(name=self.name) - output = utils.check_cmd(args) + output = self.run(args) data = json.loads(output) if not data: raise CoreCommandError( @@ -43,22 +47,24 @@ class DockerClient(object): return False def stop_container(self): - utils.check_cmd("docker rm -f {name}".format( + self.run("docker rm -f {name}".format( name=self.name )) def check_cmd(self, cmd): - if isinstance(cmd, list): - cmd = " ".join(cmd) logging.info("docker cmd output: %s", cmd) return utils.check_cmd("docker exec {name} {cmd}".format( name=self.name, cmd=cmd )) + def create_ns_cmd(self, cmd): + return "nsenter -t {pid} -u -i -p -n {cmd}".format( + pid=self.pid, + cmd=cmd + ) + def ns_cmd(self, cmd, wait): - if isinstance(cmd, list): - cmd = " ".join(cmd) args = "nsenter -t {pid} -u -i -p -n {cmd}".format( pid=self.pid, cmd=cmd @@ -67,7 +73,7 @@ class DockerClient(object): def get_pid(self): args = "docker inspect -f '{{{{.State.Pid}}}}' {name}".format(name=self.name) - output = utils.check_cmd(args) + output = self.run(args) self.pid = output logging.debug("node(%s) pid: %s", self.name, self.pid) return output @@ -78,7 +84,7 @@ class DockerClient(object): name=self.name, destination=destination ) - return utils.check_cmd(args) + return self.run(args) class DockerNode(CoreNode): @@ -101,6 +107,12 @@ class DockerNode(CoreNode): self.image = image super(DockerNode, self).__init__(session, _id, name, nodedir, bootsh, start) + def create_node_net_client(self, use_ovs): + if use_ovs: + return OvsNetClient(self.nsenter_cmd) + else: + return LinuxNetClient(self.nsenter_cmd) + def alive(self): """ Check if the node is alive. @@ -122,7 +134,7 @@ class DockerNode(CoreNode): if self.up: raise ValueError("starting a node that is already up") self.makenodedir() - self.client = DockerClient(self.name, self.image) + self.client = DockerClient(self.name, self.image, self.net_cmd) self.pid = self.client.create_container() self.up = True @@ -141,12 +153,13 @@ class DockerNode(CoreNode): self.client.stop_container() self.up = False - def node_net_cmd(self, args, wait=True): - if not self.up: - logging.debug("node down, not running network command: %s", args) - return "" - - return self.client.ns_cmd(args, wait) + def nsenter_cmd(self, args, wait=True): + if self.server is None: + args = self.client.create_ns_cmd(args) + return utils.check_cmd(args, wait=wait) + else: + args = self.client.create_ns_cmd(args) + return distributed.remote_cmd(self.server, args, wait=wait) def termcmdstring(self, sh="/bin/sh"): """ @@ -166,7 +179,7 @@ class DockerNode(CoreNode): """ logging.debug("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - self.client.check_cmd(args) + self.node_net_cmd(args) def mount(self, source, target): """ @@ -189,13 +202,24 @@ class DockerNode(CoreNode): :param int mode: mode for file :return: nothing """ - logging.debug("node dir(%s) ctrlchannel(%s)", self.nodedir, self.ctrlchnlname) logging.debug("nodefile filename(%s) mode(%s)", filename, mode) - file_path = os.path.join(self.nodedir, filename) - with open(file_path, "w") as f: - os.chmod(f.name, mode) - f.write(contents) - self.client.copy_file(file_path, filename) + directory = os.path.dirname(filename) + temp = NamedTemporaryFile(delete=False) + temp.write(contents.encode("utf-8")) + temp.close() + + if directory: + self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) + if self.server is not None: + distributed.remote_put(self.server, temp.name, temp.name) + self.client.copy_file(temp.name, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) + if self.server is not None: + self.net_cmd("rm -f %s" % temp.name) + os.unlink(temp.name) + logging.debug( + "node(%s) added file: %s; mode: 0%o", self.name, filename, mode + ) def nodefilecopy(self, filename, srcfilename, mode=None): """ @@ -207,5 +231,18 @@ class DockerNode(CoreNode): :param int mode: mode to copy to :return: nothing """ - logging.info("node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode) - raise Exception("not supported") + logging.info( + "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode + ) + directory = os.path.dirname(filename) + self.node_net_cmd("mkdir -p %s" % directory) + + if self.server is None: + source = srcfilename + else: + temp = NamedTemporaryFile(delete=False) + source = temp.name + distributed.remote_put(self.server, source, temp.name) + + self.client.copy_file(source, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 96b107f8..afd36db2 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -2,30 +2,31 @@ import json import logging import os import time +from tempfile import NamedTemporaryFile from core import utils +from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode class LxdClient(object): - def __init__(self, name, image): + def __init__(self, name, image, run): self.name = name self.image = image + self.run = run self.pid = None def create_container(self): - utils.check_cmd( - "lxc launch {image} {name}".format(name=self.name, image=self.image) - ) + self.run("lxc launch {image} {name}".format(name=self.name, image=self.image)) data = self.get_info() self.pid = data["state"]["pid"] return self.pid def get_info(self): args = "lxc list {name} --format json".format(name=self.name) - output = utils.check_cmd(args) + output = self.run(args) data = json.loads(output) if not data: raise CoreCommandError( @@ -41,20 +42,16 @@ class LxdClient(object): return False def stop_container(self): - utils.check_cmd("lxc delete --force {name}".format(name=self.name)) + self.run("lxc delete --force {name}".format(name=self.name)) def create_cmd(self, cmd): return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) - def check_cmd(self, cmd, wait=True): - args = self.create_cmd(cmd) - return utils.check_cmd(args, wait=wait) - def create_ns_cmd(self, cmd): return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) - def ns_check_cmd(self, cmd, wait=True): - args = self.create_ns_cmd(cmd) + def check_cmd(self, cmd, wait=True): + args = self.create_cmd(cmd) return utils.check_cmd(args, wait=wait) def copy_file(self, source, destination): @@ -64,7 +61,7 @@ class LxdClient(object): args = "lxc file push {source} {name}/{destination}".format( source=source, name=self.name, destination=destination ) - utils.check_cmd(args) + self.run(args) class LxcNode(CoreNode): @@ -115,7 +112,7 @@ class LxcNode(CoreNode): if self.up: raise ValueError("starting a node that is already up") self.makenodedir() - self.client = LxdClient(self.name, self.image) + self.client = LxdClient(self.name, self.image, self.net_cmd) self.pid = self.client.create_container() self.up = True @@ -134,12 +131,6 @@ class LxcNode(CoreNode): self.client.stop_container() self.up = False - def node_net_cmd(self, args, wait=True): - if not self.up: - logging.debug("node down, not running network command: %s", args) - return "" - return self.client.check_cmd(args, wait) - def termcmdstring(self, sh="/bin/sh"): """ Create a terminal command string. @@ -147,7 +138,7 @@ class LxcNode(CoreNode): :param str sh: shell to execute command in :return: str """ - return "lxc exec {name} -- bash".format(name=self.name) + return "lxc exec {name} -- {sh}".format(name=self.name, sh=sh) def privatedir(self, path): """ @@ -158,7 +149,7 @@ class LxcNode(CoreNode): """ logging.info("creating node dir: %s", path) args = "mkdir -p {path}".format(path=path) - return self.client.check_cmd(args) + return self.node_net_cmd(args) def mount(self, source, target): """ @@ -181,13 +172,23 @@ class LxcNode(CoreNode): :param int mode: mode for file :return: nothing """ - logging.debug("node dir(%s) ctrlchannel(%s)", self.nodedir, self.ctrlchnlname) logging.debug("nodefile filename(%s) mode(%s)", filename, mode) - file_path = os.path.join(self.nodedir, filename) - with open(file_path, "w") as f: - os.chmod(f.name, mode) - f.write(contents) - self.client.copy_file(file_path, filename) + + directory = os.path.dirname(filename) + temp = NamedTemporaryFile(delete=False) + temp.write(contents.encode("utf-8")) + temp.close() + + if directory: + self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) + if self.server is not None: + distributed.remote_put(self.server, temp.name, temp.name) + self.client.copy_file(temp.name, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) + if self.server is not None: + self.net_cmd("rm -f %s" % temp.name) + os.unlink(temp.name) + logging.debug("node(%s) added file: %s; mode: 0%o", self.name, filename, mode) def nodefilecopy(self, filename, srcfilename, mode=None): """ @@ -202,7 +203,18 @@ class LxcNode(CoreNode): logging.info( "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode ) - raise Exception("not supported") + directory = os.path.dirname(filename) + self.node_net_cmd("mkdir -p %s" % directory) + + if self.server is None: + source = srcfilename + else: + temp = NamedTemporaryFile(delete=False) + source = temp.name + distributed.remote_put(self.server, source, temp.name) + + self.client.copy_file(source, filename) + self.node_net_cmd("chmod %o %s" % (mode, filename)) def addnetif(self, netif, ifindex): super(LxcNode, self).addnetif(netif, ifindex) diff --git a/daemon/examples/lxd/lxd2lxd.py b/daemon/examples/lxd/lxd2lxd.py index e0ff13a3..4f27de95 100644 --- a/daemon/examples/lxd/lxd2lxd.py +++ b/daemon/examples/lxd/lxd2lxd.py @@ -5,7 +5,7 @@ from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes if __name__ == "__main__": - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.INFO) coreemu = CoreEmu() session = coreemu.create_session() @@ -14,7 +14,7 @@ if __name__ == "__main__": # create nodes and interfaces try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(image="ubuntu") + options = NodeOptions(image="ubuntu:18.04") # create node one node_one = session.add_node(_type=NodeTypes.LXC, node_options=options) From 6570f22ccf1151df3120f87428805f0edd3d060a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 14 Oct 2019 15:43:57 -0700 Subject: [PATCH 077/113] refactor fabric distributed to use a class and update sessions to create and provide these to nodes --- daemon/core/emane/emanemanager.py | 21 +++--- daemon/core/emulator/distributed.py | 113 ++++++++++++++++++---------- daemon/core/emulator/session.py | 32 ++++---- daemon/core/nodes/base.py | 35 ++++----- daemon/core/nodes/docker.py | 7 +- daemon/core/nodes/interface.py | 19 +++-- daemon/core/nodes/lxd.py | 5 +- daemon/core/nodes/network.py | 27 ++++--- daemon/core/nodes/physical.py | 4 +- daemon/core/xml/emanexml.py | 23 +++--- 10 files changed, 153 insertions(+), 133 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 35cc7ca9..f48d2e2e 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -18,7 +18,6 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel -from core.emulator import distributed from core.emulator.enumerations import ( ConfigDataTypes, ConfigFlags, @@ -155,9 +154,9 @@ class EmaneManager(ModelManager): args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - for server in self.session.servers: - conn = self.session.servers[server] - distributed.remote_cmd(conn, args) + for host in self.session.servers: + server = self.session.servers[host] + server.remote_cmd(args) # load default emane models self.load_models(EMANE_MODELS) @@ -757,9 +756,9 @@ class EmaneManager(ModelManager): emanecmd += " -f %s" % os.path.join(path, "emane.log") emanecmd += " %s" % os.path.join(path, "platform.xml") utils.check_cmd(emanecmd, cwd=path) - for server in self.session.servers: - conn = self.session.servers[server] - distributed.remote_cmd(conn, emanecmd, cwd=path) + for host in self.session.servers: + server = self.session.servers[host] + server.remote_cmd(emanecmd, cwd=path) logging.info("host emane daemon running: %s", emanecmd) def stopdaemons(self): @@ -784,10 +783,10 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - for server in self.session.servers: - conn = self.session[server] - distributed.remote_cmd(conn, kill_emaned) - distributed.remote_cmd(conn, kill_transortd) + for host in self.session.servers: + server = self.session[host] + server.remote_cmd(kill_emaned) + server.remote_cmd(kill_transortd) except CoreCommandError: logging.exception("error shutting down emane daemons") diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index abec0a57..19594ae1 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -1,8 +1,13 @@ +""" +Defines distributed server functionality. +""" + import logging import os import threading from tempfile import NamedTemporaryFile +from fabric import Connection from invoke import UnexpectedExit from core.errors import CoreCommandError @@ -10,52 +15,80 @@ from core.errors import CoreCommandError LOCK = threading.Lock() -def remote_cmd(server, cmd, env=None, cwd=None, wait=True): +class DistributedServer(object): """ - Run command remotely using server connection. - - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost - :param str cmd: command to run - :param dict env: environment for remote command, default is None - :param str cwd: directory to run command in, defaults to None, which is the user's - home directory - :param bool wait: True to wait for status, False to background process - :return: stdout when success - :rtype: str - :raises CoreCommandError: when a non-zero exit status occurs + Provides distributed server interactions. """ - replace_env = env is not None - if not wait: - cmd += " &" - logging.info( - "remote cmd server(%s) cwd(%s) wait(%s): %s", server.host, cwd, wait, cmd - ) - try: - with LOCK: - if cwd is None: - result = server.run(cmd, hide=False, env=env, replace_env=replace_env) - else: - with server.cd(cwd): - result = server.run( + def __init__(self, host): + """ + Create a DistributedServer instance. + + :param str host: host to connect to + """ + self.host = host + self.conn = Connection(host, user="root") + self.lock = threading.Lock() + + def remote_cmd(self, cmd, env=None, cwd=None, wait=True): + """ + Run command remotely using server connection. + + :param str cmd: command to run + :param dict env: environment for remote command, default is None + :param str cwd: directory to run command in, defaults to None, which is the user's + home directory + :param bool wait: True to wait for status, False to background process + :return: stdout when success + :rtype: str + :raises CoreCommandError: when a non-zero exit status occurs + """ + + replace_env = env is not None + if not wait: + cmd += " &" + logging.info( + "remote cmd server(%s) cwd(%s) wait(%s): %s", self.host, cwd, wait, cmd + ) + try: + with self.lock: + if cwd is None: + result = self.conn.run( cmd, hide=False, env=env, replace_env=replace_env ) - return result.stdout.strip() - except UnexpectedExit as e: - stdout, stderr = e.streams_for_display() - raise CoreCommandError(e.result.exited, cmd, stdout, stderr) + else: + with self.conn.cd(cwd): + result = self.conn.run( + cmd, hide=False, env=env, replace_env=replace_env + ) + return result.stdout.strip() + except UnexpectedExit as e: + stdout, stderr = e.streams_for_display() + raise CoreCommandError(e.result.exited, cmd, stdout, stderr) + def remote_put(self, source, destination): + """ + Push file to remote server. -def remote_put(server, source, destination): - with LOCK: - server.put(source, destination) + :param str source: source file to push + :param str destination: destination file location + :return: nothing + """ + with self.lock: + self.conn.put(source, destination) + def remote_put_temp(self, destination, data): + """ + Remote push file contents to a remote server, using a temp file as an + intermediate step. -def remote_put_temp(server, destination, data): - with LOCK: - temp = NamedTemporaryFile(delete=False) - temp.write(data.encode("utf-8")) - temp.close() - server.put(temp.name, destination) - os.unlink(temp.name) + :param str destination: file destination for data + :param str data: data to store in remote file + :return: nothing + """ + with self.lock: + temp = NamedTemporaryFile(delete=False) + temp.write(data.encode("utf-8")) + temp.close() + self.conn.put(temp.name, destination) + os.unlink(temp.name) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 7d3928b2..a0b4ec38 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -14,14 +14,13 @@ import threading import time from multiprocessing.pool import ThreadPool -from fabric import Connection - from core import constants, utils from core.api.tlv import coreapi from core.api.tlv.broker import CoreBroker from core.emane.emanemanager import EmaneManager from core.emane.nodes import EmaneNet from core.emulator.data import EventData, ExceptionData, NodeData +from core.emulator.distributed import DistributedServer from core.emulator.emudata import ( IdGen, LinkOptions, @@ -162,11 +161,11 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } - def add_distributed(self, server): - conn = Connection(server, user="root") - self.servers[server] = conn + def add_distributed(self, host): + server = DistributedServer(host) + self.servers[host] = server cmd = "mkdir -p %s" % self.session_dir - conn.run(cmd, hide=False) + server.remote_cmd(cmd) def shutdown_distributed(self): # shutdown all tunnels @@ -176,10 +175,10 @@ class Session(object): tunnel.shutdown() # remove all remote session directories - for server in self.servers: - conn = self.servers[server] + for host in self.servers: + server = self.servers[host] cmd = "rm -rf %s" % self.session_dir - conn.run(cmd, hide=False) + server.remote_cmd(cmd) # clear tunnels self.tunnels.clear() @@ -194,18 +193,15 @@ class Session(object): if isinstance(node, CtrlNet) and node.serverintf is not None: continue - for server in self.servers: - conn = self.servers[server] - key = self.tunnelkey(node_id, IpAddress.to_int(server)) + for host in self.servers: + server = self.servers[host] + key = self.tunnelkey(node_id, IpAddress.to_int(host)) # local to server logging.info( - "local tunnel node(%s) to remote(%s) key(%s)", - node.name, - server, - key, + "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key ) - local_tap = GreTap(session=self, remoteip=server, key=key) + local_tap = GreTap(session=self, remoteip=host, key=key) local_tap.net_client.create_interface(node.brname, local_tap.localname) # server to local @@ -216,7 +212,7 @@ class Session(object): key, ) remote_tap = GreTap( - session=self, remoteip=self.address, key=key, server=conn + session=self, remoteip=self.address, key=key, server=server ) remote_tap.net_client.create_interface( node.brname, remote_tap.localname diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index f263d0f3..7758e4af 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -14,7 +14,6 @@ from socket import AF_INET, AF_INET6 from core import utils from core.constants import MOUNT_BIN, VNODED_BIN -from core.emulator import distributed from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.errors import CoreCommandError @@ -41,8 +40,8 @@ class NodeBase(object): :param int _id: id :param str name: object name :param bool start: start value - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ self.session = session @@ -101,7 +100,7 @@ class NodeBase(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - return distributed.remote_cmd(self.server, args, env, cwd, wait) + return self.server.remote_cmd(args, env, cwd, wait) def setposition(self, x=None, y=None, z=None): """ @@ -200,7 +199,7 @@ class NodeBase(object): x, y, _ = self.getposition() model = self.type - emulation_server = self.server + emulation_server = self.server.host services = self.services if services is not None: @@ -253,8 +252,8 @@ class CoreNodeBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: boolean for starting - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ super(CoreNodeBase, self).__init__(session, _id, name, start, server) self.services = [] @@ -437,8 +436,8 @@ class CoreNode(CoreNodeBase): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ super(CoreNode, self).__init__(session, _id, name, start, server) self.nodedir = nodedir @@ -585,7 +584,7 @@ class CoreNode(CoreNodeBase): return self.client.check_cmd(args, wait=wait) else: args = self.client.create_cmd(args) - return distributed.remote_cmd(self.server, args, wait=wait) + return self.server.remote_cmd(args, wait=wait) def termcmdstring(self, sh="/bin/sh"): """ @@ -888,7 +887,7 @@ class CoreNode(CoreNodeBase): self.client.check_cmd("sync") else: self.net_cmd("mkdir -p %s" % directory) - distributed.remote_put(self.server, srcname, filename) + self.server.remote_put(srcname, filename) def hostfilename(self, filename): """ @@ -925,7 +924,7 @@ class CoreNode(CoreNodeBase): os.chmod(open_file.name, mode) else: self.net_cmd("mkdir -m %o -p %s" % (0o755, dirname)) - distributed.remote_put_temp(self.server, hostfilename, contents) + self.server.remote_put_temp(hostfilename, contents) self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode @@ -944,12 +943,10 @@ class CoreNode(CoreNodeBase): hostfilename = self.hostfilename(filename) if self.server is None: shutil.copy2(srcfilename, hostfilename) - if mode is not None: - os.chmod(hostfilename, mode) else: - distributed.remote_put(self.server, srcfilename, hostfilename) - if mode is not None: - self.net_cmd("chmod %o %s" % (mode, hostfilename)) + self.server.remote_put(srcfilename, hostfilename) + if mode is not None: + self.net_cmd("chmod %o %s" % (mode, hostfilename)) logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) @@ -971,8 +968,8 @@ class CoreNetworkBase(NodeBase): :param int _id: object id :param str name: object name :param bool start: should object start - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ super(CoreNetworkBase, self).__init__(session, _id, name, start, server) self._linked = {} diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index e465a768..2679704f 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -4,7 +4,6 @@ import os from tempfile import NamedTemporaryFile from core import utils -from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode @@ -159,7 +158,7 @@ class DockerNode(CoreNode): return utils.check_cmd(args, wait=wait) else: args = self.client.create_ns_cmd(args) - return distributed.remote_cmd(self.server, args, wait=wait) + return self.server.remote_cmd(args, wait=wait) def termcmdstring(self, sh="/bin/sh"): """ @@ -211,7 +210,7 @@ class DockerNode(CoreNode): if directory: self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) if self.server is not None: - distributed.remote_put(self.server, temp.name, temp.name) + self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) if self.server is not None: @@ -242,7 +241,7 @@ class DockerNode(CoreNode): else: temp = NamedTemporaryFile(delete=False) source = temp.name - distributed.remote_put(self.server, source, temp.name) + self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 2f42fdfe..bfcb8583 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -7,7 +7,6 @@ import time from builtins import int, range from core import utils -from core.emulator import distributed from core.errors import CoreCommandError from core.nodes.netclient import LinuxNetClient @@ -24,8 +23,8 @@ class CoreInterface(object): :param core.nodes.base.CoreNode node: node for interface :param str name: interface name :param mtu: mtu value - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ self.node = node @@ -63,7 +62,7 @@ class CoreInterface(object): if self.server is None: return utils.check_cmd(args, env, cwd, wait) else: - return distributed.remote_cmd(self.server, args, env, cwd, wait) + return self.server.remote_cmd(args, env, cwd, wait) def startup(self): """ @@ -220,8 +219,8 @@ class Veth(CoreInterface): :param str name: interface name :param str localname: interface local name :param mtu: interface mtu - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param bool start: start flag :raises CoreCommandError: when there is a command exception """ @@ -280,8 +279,8 @@ class TunTap(CoreInterface): :param str name: interface name :param str localname: local interface name :param mtu: interface mtu - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param bool start: start flag """ CoreInterface.__init__(self, node, name, mtu, server) @@ -463,8 +462,8 @@ class GreTap(CoreInterface): :param int ttl: ttl value :param int key: gre tap key :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ CoreInterface.__init__(self, node, name, mtu, server) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index afd36db2..eef3dc8f 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -5,7 +5,6 @@ import time from tempfile import NamedTemporaryFile from core import utils -from core.emulator import distributed from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode @@ -182,7 +181,7 @@ class LxcNode(CoreNode): if directory: self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) if self.server is not None: - distributed.remote_put(self.server, temp.name, temp.name) + self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) if self.server is not None: @@ -211,7 +210,7 @@ class LxcNode(CoreNode): else: temp = NamedTemporaryFile(delete=False) source = temp.name - distributed.remote_put(self.server, source, temp.name) + self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) self.node_net_cmd("chmod %o %s" % (mode, filename)) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 4df7b28a..4d68ccaf 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -10,7 +10,6 @@ from socket import AF_INET, AF_INET6 from core import utils from core.constants import EBTABLES_BIN, TC_BIN -from core.emulator import distributed from core.emulator.data import LinkData from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs from core.errors import CoreCommandError, CoreError @@ -257,8 +256,8 @@ class CoreNetwork(CoreNetworkBase): :param int _id: object id :param str name: object name :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param policy: network policy """ CoreNetworkBase.__init__(self, session, _id, name, start, server) @@ -289,9 +288,9 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - for server in self.session.servers: - conn = self.session.servers[server] - distributed.remote_cmd(conn, args, env, cwd, wait) + for host in self.session.servers: + server = self.session.servers[host] + server.remote_cmd(args, env, cwd, wait) return output def startup(self): @@ -632,8 +631,8 @@ class GreTapBridge(CoreNetwork): :param ttl: ttl value :param key: gre tap key :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ CoreNetwork.__init__(self, session, _id, name, False, server, policy) self.grekey = key @@ -753,8 +752,8 @@ class CtrlNet(CoreNetwork): :param prefix: control network ipv4 prefix :param hostid: host id :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param str assign_address: assigned address :param str updown_script: updown script :param serverintf: server interface @@ -1008,8 +1007,8 @@ class HubNode(CoreNetwork): :param int _id: node id :param str name: node namee :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ CoreNetwork.__init__(self, session, _id, name, start, server) @@ -1039,8 +1038,8 @@ class WlanNode(CoreNetwork): :param int _id: node id :param str name: node name :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param policy: wlan policy """ CoreNetwork.__init__(self, session, _id, name, start, server, policy) diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 9bd04c53..9daf4f35 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -242,8 +242,8 @@ class Rj45Node(CoreNodeBase, CoreInterface): :param str name: node name :param mtu: rj45 mtu :param bool start: start flag - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost """ CoreNodeBase.__init__(self, session, _id, name, start, server) CoreInterface.__init__(self, node=self, name=name, mtu=mtu) diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 3b4fafef..0005c378 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -5,7 +5,6 @@ from tempfile import NamedTemporaryFile from lxml import etree from core import utils -from core.emulator import distributed from core.nodes.ipaddress import MacAddress from core.xml import corexml @@ -53,8 +52,8 @@ def create_file(xml_element, doc_name, file_path, server=None): :param lxml.etree.Element xml_element: root element to write to file :param str doc_name: name to use in the emane doctype :param str file_path: file path to write xml file to - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ doctype = ( @@ -65,7 +64,7 @@ def create_file(xml_element, doc_name, file_path, server=None): temp = NamedTemporaryFile(delete=False) create_file(xml_element, doc_name, temp.name) temp.close() - distributed.remote_put(server, temp.name, file_path) + server.remote_put(temp.name, file_path) os.unlink(temp.name) else: corexml.write_xml_file(xml_element, file_path, doctype=doctype) @@ -327,8 +326,8 @@ def create_phy_xml(emane_model, config, file_path, server): :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ phy_element = etree.Element("phy", name="%s PHY" % emane_model.name) @@ -355,8 +354,8 @@ def create_mac_xml(emane_model, config, file_path, server): :param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml :param dict config: all current configuration values :param str file_path: path to write file to - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ if not emane_model.mac_library: @@ -396,8 +395,8 @@ def create_nem_xml( :param str transport_definition: transport file definition path :param str mac_definition: mac file definition path :param str phy_definition: phy file definition path - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) @@ -424,8 +423,8 @@ def create_event_service_xml(group, port, device, file_directory, server=None): :param str port: event port :param str device: event device :param str file_directory: directory to create file in - :param fabric.connection.Connection server: remote server node will run on, - default is None for localhost + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :return: nothing """ event_element = etree.Element("emaneeventmsgsvc") From b2d2705849cb2f547586214790495ccb4bc19996 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 15 Oct 2019 14:13:42 -0700 Subject: [PATCH 078/113] removed broker from session, updated most places using broker to use alternative logic to compensate where needed --- daemon/core/__init__.py | 3 + daemon/core/api/tlv/coreapi.py | 18 -- daemon/core/api/tlv/corehandlers.py | 72 +++---- daemon/core/emane/emanemanager.py | 182 ++---------------- daemon/core/emulator/distributed.py | 8 +- daemon/core/emulator/session.py | 111 ++++------- daemon/core/location/mobility.py | 93 --------- daemon/core/nodes/base.py | 12 +- daemon/core/nodes/network.py | 4 +- daemon/core/plugins/sdt.py | 2 +- daemon/core/xml/corexmldeployment.py | 4 - daemon/core/xml/emanexml.py | 24 +-- daemon/examples/python/distributed.py | 16 +- daemon/examples/python/distributed_emane.py | 26 +-- daemon/examples/python/distributed_ptp.py | 13 +- .../examples/python/distributed_switches.py | 11 +- daemon/examples/python/distributed_wlan.py | 16 +- daemon/tests/conftest.py | 1 - daemon/tests/test_gui.py | 6 +- 19 files changed, 151 insertions(+), 471 deletions(-) diff --git a/daemon/core/__init__.py b/daemon/core/__init__.py index c847c8dc..40ca3604 100644 --- a/daemon/core/__init__.py +++ b/daemon/core/__init__.py @@ -2,3 +2,6 @@ import logging.config # setup default null handler logging.getLogger(__name__).addHandler(logging.NullHandler()) + +# disable paramiko logging +logging.getLogger("paramiko").setLevel(logging.WARNING) diff --git a/daemon/core/api/tlv/coreapi.py b/daemon/core/api/tlv/coreapi.py index 63747642..1e1de8be 100644 --- a/daemon/core/api/tlv/coreapi.py +++ b/daemon/core/api/tlv/coreapi.py @@ -15,7 +15,6 @@ from core.api.tlv import structutils from core.emulator.enumerations import ( ConfigTlvs, EventTlvs, - EventTypes, ExceptionTlvs, ExecuteTlvs, FileTlvs, @@ -1017,20 +1016,3 @@ def str_to_list(value): return None return value.split("|") - - -def state_name(value): - """ - Helper to convert state number into state name using event types. - - :param int value: state value to derive name from - :return: state name - :rtype: str - """ - - try: - value = EventTypes(value).name - except ValueError: - value = "unknown" - - return value diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 6ff7f55b..4b4e7c1e 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -86,6 +86,7 @@ class CoreHandler(socketserver.BaseRequestHandler): self.master = False self.session = None + self.session_clients = {} # core emulator self.coreemu = server.coreemu @@ -138,8 +139,9 @@ class CoreHandler(socketserver.BaseRequestHandler): if self.session: # remove client from session broker and shutdown if there are no clients self.remove_session_handlers() - self.session.broker.session_clients.remove(self) - if not self.session.broker.session_clients and not self.session.is_active(): + clients = self.session_clients[self.session.id] + clients.remove(self) + if not clients and not self.session.is_active(): logging.info( "no session clients left and not active, initiating shutdown" ) @@ -407,9 +409,7 @@ class CoreHandler(socketserver.BaseRequestHandler): tlv_data += coreapi.CoreRegisterTlv.pack( RegisterTlvs.EMULATION_SERVER.value, "core-daemon" ) - tlv_data += coreapi.CoreRegisterTlv.pack( - self.session.broker.config_type, self.session.broker.name - ) + tlv_data += coreapi.CoreRegisterTlv.pack(RegisterTlvs.UTILITY.value, "broker") tlv_data += coreapi.CoreRegisterTlv.pack( self.session.location.config_type, self.session.location.name ) @@ -533,10 +533,6 @@ class CoreHandler(socketserver.BaseRequestHandler): :param message: message to handle :return: nothing """ - if self.session and self.session.broker.handle_message(message): - logging.debug("message not being handled locally") - return - logging.debug( "%s handling message:\n%s", threading.currentThread().getName(), message ) @@ -606,12 +602,11 @@ class CoreHandler(socketserver.BaseRequestHandler): self.session = self.coreemu.create_session(port, master=False) logging.debug("created new session for client: %s", self.session.id) - # TODO: hack to associate this handler with this sessions broker for broadcasting - # TODO: broker needs to be pulled out of session to the server/handler level if self.master: logging.debug("session set to master") self.session.master = True - self.session.broker.session_clients.append(self) + clients = self.session_clients.setdefault(self.session.id, []) + clients.append(self) # add handlers for various data self.add_session_handlers() @@ -643,7 +638,8 @@ class CoreHandler(socketserver.BaseRequestHandler): ]: continue - for client in self.session.broker.session_clients: + clients = self.session_clients[self.session.id] + for client in clients: if client == self: continue @@ -734,6 +730,7 @@ class CoreHandler(socketserver.BaseRequestHandler): node_options.icon = message.get_tlv(NodeTlvs.ICON.value) node_options.canvas = message.get_tlv(NodeTlvs.CANVAS.value) node_options.opaque = message.get_tlv(NodeTlvs.OPAQUE.value) + node_options.emulation_server = message.get_tlv(NodeTlvs.EMULATION_SERVER.value) services = message.get_tlv(NodeTlvs.SERVICES.value) if services: @@ -1027,8 +1024,9 @@ class CoreHandler(socketserver.BaseRequestHandler): # find the session containing this client and set the session to master for _id in self.coreemu.sessions: - session = self.coreemu.sessions[_id] - if self in session.broker.session_clients: + clients = self.session_clients[_id] + if self in clients: + session = self.coreemu.sessions[_id] logging.debug("setting session to master: %s", session.id) session.master = True break @@ -1077,7 +1075,7 @@ class CoreHandler(socketserver.BaseRequestHandler): self.handle_config_location(message_type, config_data) elif config_data.object == self.session.metadata.name: replies = self.handle_config_metadata(message_type, config_data) - elif config_data.object == self.session.broker.name: + elif config_data.object == "broker": self.handle_config_broker(message_type, config_data) elif config_data.object == self.session.services.name: replies = self.handle_config_services(message_type, config_data) @@ -1182,7 +1180,6 @@ class CoreHandler(socketserver.BaseRequestHandler): def handle_config_broker(self, message_type, config_data): if message_type not in [ConfigFlags.REQUEST, ConfigFlags.RESET]: - session_id = config_data.session if not config_data.data_values: logging.info("emulation server data missing") else: @@ -1194,29 +1191,10 @@ class CoreHandler(socketserver.BaseRequestHandler): for server in server_list: server_items = server.split(":") - name, host, port = server_items[:3] - - if host == "": - host = None - - if port == "": - port = None - else: - port = int(port) - - if session_id is not None: - # receive session ID and my IP from master - self.session.broker.session_id_master = int( - session_id.split("|")[0] - ) - self.session.broker.myip = host - host = None - port = None - - # this connects to the server immediately; maybe we should wait - # or spin off a new "client" thread here - self.session.broker.addserver(name, host, port) - self.session.broker.setupserver(name) + name, host, _ = server_items[:3] + self.session.add_distributed(name, host) + elif message_type == ConfigFlags.RESET: + self.session.shutdown_distributed() def handle_config_services(self, message_type, config_data): replies = [] @@ -1842,11 +1820,9 @@ class CoreHandler(socketserver.BaseRequestHandler): # remove client from session broker and shutdown if needed self.remove_session_handlers() - self.session.broker.session_clients.remove(self) - if ( - not self.session.broker.session_clients - and not self.session.is_active() - ): + clients = self.session_clients[self.session.id] + clients.remove(self) + if not clients and not self.session.is_active(): self.coreemu.delete_session(self.session.id) # set session to join @@ -1855,7 +1831,8 @@ class CoreHandler(socketserver.BaseRequestHandler): # add client to session broker and set master if needed if self.master: self.session.master = True - self.session.broker.session_clients.append(self) + clients = self.session_clients.setdefault(self.session.id, []) + clients.append(self) # add broadcast handlers logging.info("adding session broadcast handlers") @@ -2139,7 +2116,8 @@ class CoreUdpHandler(CoreHandler): if not isinstance(message, (coreapi.CoreNodeMessage, coreapi.CoreLinkMessage)): return - for client in self.session.broker.session_clients: + clients = self.session_clients[self.session.id] + for client in clients: try: client.sendall(message.raw_message) except IOError: diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index f48d2e2e..e4208189 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -2,14 +2,12 @@ emane.py: definition of an Emane class for implementing configuration control of an EMANE emulation. """ -import copy import logging import os import threading from core import utils -from core.api.tlv import coreapi, dataconversion -from core.config import ConfigGroup, ConfigShim, Configuration, ModelManager +from core.config import ConfigGroup, Configuration, ModelManager from core.emane import emanemanifest from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel @@ -18,14 +16,7 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emane.rfpipe import EmaneRfPipeModel from core.emane.tdma import EmaneTdmaModel -from core.emulator.enumerations import ( - ConfigDataTypes, - ConfigFlags, - ConfigTlvs, - MessageFlags, - MessageTypes, - RegisterTlvs, -) +from core.emulator.enumerations import ConfigDataTypes, RegisterTlvs from core.errors import CoreCommandError, CoreError from core.xml import emanexml @@ -75,8 +66,6 @@ class EmaneManager(ModelManager): self.session = session self._emane_nets = {} self._emane_node_lock = threading.Lock() - self._ifccounts = {} - self._ifccountslock = threading.Lock() # port numbers are allocated from these counters self.platformport = self.session.options.get_config_int( "emane_platform_port", 8100 @@ -91,7 +80,6 @@ class EmaneManager(ModelManager): self.emane_config = EmaneGlobalModel(session) self.set_configs(self.emane_config.default_values()) - session.broker.handlers.add(self.handledistributed) self.service = None self.event_device = None self.emane_check() @@ -154,8 +142,8 @@ class EmaneManager(ModelManager): args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - for host in self.session.servers: - server = self.session.servers[host] + for name in self.session.servers: + server = self.session.servers[name] server.remote_cmd(args) # load default emane models @@ -282,7 +270,6 @@ class EmaneManager(ModelManager): return EmaneManager.NOT_NEEDED # control network bridge required for EMANE 0.9.2 - # - needs to be configured before checkdistributed() for distributed # - needs to exist when eventservice binds to it (initeventservice) if self.session.master: otadev = self.get_config("otamanagerdevice") @@ -297,10 +284,9 @@ class EmaneManager(ModelManager): ) return EmaneManager.NOT_READY - ctrlnet = self.session.add_remove_control_net( + self.session.add_remove_control_net( net_index=netidx, remove=False, conf_required=False ) - self.distributedctrlnet(ctrlnet) eventdev = self.get_config("eventservicedevice") logging.debug("emane event service device: eventdev(%s)", eventdev) if eventdev != otadev: @@ -313,18 +299,9 @@ class EmaneManager(ModelManager): ) return EmaneManager.NOT_READY - ctrlnet = self.session.add_remove_control_net( + self.session.add_remove_control_net( net_index=netidx, remove=False, conf_required=False ) - self.distributedctrlnet(ctrlnet) - - if self.checkdistributed(): - # we are slave, but haven't received a platformid yet - platform_id_start = "platform_id_start" - default_values = self.emane_config.default_values() - value = self.get_config(platform_id_start) - if value == default_values[platform_id_start]: - return EmaneManager.NOT_READY self.check_node_models() return EmaneManager.SUCCESS @@ -413,9 +390,6 @@ class EmaneManager(ModelManager): """ stop all EMANE daemons """ - with self._ifccountslock: - self._ifccounts.clear() - with self._emane_node_lock: if not self._emane_nets: return @@ -424,92 +398,6 @@ class EmaneManager(ModelManager): self.stopdaemons() self.stopeventmonitor() - def handledistributed(self, message): - """ - Broker handler for processing CORE API messages as they are - received. This is used to snoop the Link add messages to get NEM - counts of NEMs that exist on other servers. - """ - if ( - message.message_type == MessageTypes.LINK.value - and message.flags & MessageFlags.ADD.value - ): - nn = message.node_numbers() - # first node is always link layer node in Link add message - if nn[0] in self.session.broker.network_nodes: - serverlist = self.session.broker.getserversbynode(nn[1]) - for server in serverlist: - with self._ifccountslock: - if server not in self._ifccounts: - self._ifccounts[server] = 1 - else: - self._ifccounts[server] += 1 - - def checkdistributed(self): - """ - Check for EMANE nodes that exist on multiple emulation servers and - coordinate the NEM id and port number space. - If we are the master EMANE node, return False so initialization will - proceed as normal; otherwise slaves return True here and - initialization is deferred. - """ - # check with the session if we are the "master" Emane object? - master = False - - with self._emane_node_lock: - if self._emane_nets: - master = self.session.master - logging.info("emane check distributed as master: %s.", master) - - # we are not the master Emane object, wait for nem id and ports - if not master: - return True - - nemcount = 0 - with self._emane_node_lock: - for key in self._emane_nets: - emane_node = self._emane_nets[key] - nemcount += emane_node.numnetif() - - nemid = int(self.get_config("nem_id_start")) - nemid += nemcount - - platformid = int(self.get_config("platform_id_start")) - - # build an ordered list of servers so platform ID is deterministic - servers = [] - for key in sorted(self._emane_nets): - for server in self.session.broker.getserversbynode(key): - if server not in servers: - servers.append(server) - - servers.sort(key=lambda x: x.name) - for server in servers: - if server.name == "localhost": - continue - - if server.sock is None: - continue - - platformid += 1 - - # create temporary config for updating distributed nodes - typeflags = ConfigFlags.UPDATE.value - config = copy.deepcopy(self.get_configs()) - config["platform_id_start"] = str(platformid) - config["nem_id_start"] = str(nemid) - config_data = ConfigShim.config_data( - 0, None, typeflags, self.emane_config, config - ) - message = dataconversion.convert_config(config_data) - server.sock.send(message) - # increment nemid for next server by number of interfaces - with self._ifccountslock: - if server in self._ifccounts: - nemid += self._ifccounts[server] - - return False - def buildxml(self): """ Build XML files required to run EMANE on each node. @@ -526,52 +414,6 @@ class EmaneManager(ModelManager): self.buildnemxml() self.buildeventservicexml() - # TODO: remove need for tlv messaging - def distributedctrlnet(self, ctrlnet): - """ - Distributed EMANE requires multiple control network prefixes to - be configured. This generates configuration for slave control nets - using the default list of prefixes. - """ - # slave server - session = self.session - if not session.master: - return - - # not distributed - servers = session.broker.getservernames() - if len(servers) < 2: - return - - # normal Config messaging will distribute controlnets - prefix = session.options.get_config("controlnet", default="") - prefixes = prefix.split() - if len(prefixes) < len(servers): - logging.info( - "setting up default controlnet prefixes for distributed (%d configured)", - len(prefixes), - ) - prefix = ctrlnet.DEFAULT_PREFIX_LIST[0] - prefixes = prefix.split() - servers.remove("localhost") - servers.insert(0, "localhost") - prefix = " ".join("%s:%s" % (s, prefixes[i]) for i, s in enumerate(servers)) - - # this generates a config message having controlnet prefix assignments - logging.info("setting up controlnet prefixes for distributed: %s", prefix) - vals = "controlnet=%s" % prefix - tlvdata = b"" - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.OBJECT.value, "session") - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.TYPE.value, 0) - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.VALUES.value, vals) - rawmsg = coreapi.CoreConfMessage.pack(0, tlvdata) - msghdr = rawmsg[: coreapi.CoreMessage.header_len] - msg = coreapi.CoreConfMessage( - flags=0, hdr=msghdr, data=rawmsg[coreapi.CoreMessage.header_len :] - ) - logging.debug("sending controlnet message:\n%s", msg) - self.session.broker.handle_message(msg) - def check_node_models(self): """ Associate EMANE model classes with EMANE network nodes. @@ -676,8 +518,8 @@ class EmaneManager(ModelManager): dev = self.get_config("eventservicedevice") emanexml.create_event_service_xml(group, port, dev, self.session.session_dir) - for server in self.session.servers: - conn = self.session.servers[server] + for name in self.session.servers: + conn = self.session.servers[name] emanexml.create_event_service_xml( group, port, dev, self.session.session_dir, conn ) @@ -756,8 +598,8 @@ class EmaneManager(ModelManager): emanecmd += " -f %s" % os.path.join(path, "emane.log") emanecmd += " %s" % os.path.join(path, "platform.xml") utils.check_cmd(emanecmd, cwd=path) - for host in self.session.servers: - server = self.session.servers[host] + for name in self.session.servers: + server = self.session.servers[name] server.remote_cmd(emanecmd, cwd=path) logging.info("host emane daemon running: %s", emanecmd) @@ -783,8 +625,8 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - for host in self.session.servers: - server = self.session[host] + for name in self.session.servers: + server = self.session[name] server.remote_cmd(kill_emaned) server.remote_cmd(kill_transortd) except CoreCommandError: diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 19594ae1..4e258937 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -20,12 +20,14 @@ class DistributedServer(object): Provides distributed server interactions. """ - def __init__(self, host): + def __init__(self, name, host): """ Create a DistributedServer instance. + :param str name: convenience name to associate with host :param str host: host to connect to """ + self.name = name self.host = host self.conn = Connection(host, user="root") self.lock = threading.Lock() @@ -36,8 +38,8 @@ class DistributedServer(object): :param str cmd: command to run :param dict env: environment for remote command, default is None - :param str cwd: directory to run command in, defaults to None, which is the user's - home directory + :param str cwd: directory to run command in, defaults to None, which is the + user's home directory :param bool wait: True to wait for status, False to background process :return: stdout when success :rtype: str diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index a0b4ec38..1445cb8f 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -15,8 +15,6 @@ import time from multiprocessing.pool import ThreadPool from core import constants, utils -from core.api.tlv import coreapi -from core.api.tlv.broker import CoreBroker from core.emane.emanemanager import EmaneManager from core.emane.nodes import EmaneNet from core.emulator.data import EventData, ExceptionData, NodeData @@ -142,10 +140,9 @@ class Session(object): # distributed servers self.servers = {} self.tunnels = {} - self.address = None + self.address = self.options.get_config("distributed_address", default=None) # initialize session feature helpers - self.broker = CoreBroker(session=self) self.location = CoreLocation() self.mobility = MobilityManager(session=self) self.services = CoreServices(session=self) @@ -161,9 +158,9 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } - def add_distributed(self, host): - server = DistributedServer(host) - self.servers[host] = server + def add_distributed(self, name, host): + server = DistributedServer(name, host) + self.servers[name] = server cmd = "mkdir -p %s" % self.session_dir server.remote_cmd(cmd) @@ -175,8 +172,8 @@ class Session(object): tunnel.shutdown() # remove all remote session directories - for host in self.servers: - server = self.servers[host] + for name in self.servers: + server = self.servers[name] cmd = "rm -rf %s" % self.session_dir server.remote_cmd(cmd) @@ -193,8 +190,9 @@ class Session(object): if isinstance(node, CtrlNet) and node.serverintf is not None: continue - for host in self.servers: - server = self.servers[host] + for name in self.servers: + server = self.servers[name] + host = server.host key = self.tunnelkey(node_id, IpAddress.to_int(host)) # local to server @@ -219,23 +217,35 @@ class Session(object): ) # save tunnels for shutdown - self.tunnels[key] = [local_tap, remote_tap] + self.tunnels[key] = (local_tap, remote_tap) - def tunnelkey(self, n1num, n2num): + def tunnelkey(self, n1_id, n2_id): """ Compute a 32-bit key used to uniquely identify a GRE tunnel. The hash(n1num), hash(n2num) values are used, so node numbers may be None or string values (used for e.g. "ctrlnet"). - :param int n1num: node one id - :param int n2num: node two id + :param int n1_id: node one id + :param int n2_id: node two id :return: tunnel key for the node pair :rtype: int """ - logging.debug("creating tunnel key for: %s, %s", n1num, n2num) - key = (self.id << 16) ^ utils.hashkey(n1num) ^ (utils.hashkey(n2num) << 8) + logging.debug("creating tunnel key for: %s, %s", n1_id, n2_id) + key = (self.id << 16) ^ utils.hashkey(n1_id) ^ (utils.hashkey(n2_id) << 8) return key & 0xFFFFFFFF + def gettunnel(self, n1_id, n2_id): + """ + Return the GreTap between two nodes if it exists. + + :param int n1_id: node one id + :param int n2_id: node two id + :return: gre tap between nodes or None + """ + key = self.tunnelkey(n1_id, n2_id) + logging.debug("checking for tunnel key(%s) in: %s", key, self.tunnels) + return self.tunnels.get(key) + @classmethod def get_node_class(cls, _type): """ @@ -285,7 +295,7 @@ class Session(object): node_two = self.get_node(node_two_id) # both node ids are provided - tunnel = self.broker.gettunnel(node_one_id, node_two_id) + tunnel = self.gettunnel(node_one_id, node_two_id) logging.debug("tunnel between nodes: %s", tunnel) if isinstance(tunnel, GreTapBridge): net_one = tunnel @@ -958,13 +968,13 @@ class Session(object): def clear(self): """ - Clear all CORE session data. (objects, hooks, broker) + Clear all CORE session data. (nodes, hooks, etc) :return: nothing """ self.delete_nodes() + self.shutdown_distributed() self.del_hooks() - self.broker.reset() self.emane.reset() def start_events(self): @@ -1038,17 +1048,16 @@ class Session(object): # shutdown/cleanup feature helpers self.emane.shutdown() - self.broker.shutdown() self.sdt.shutdown() - # delete all current nodes + # remove and shutdown all nodes and tunnels self.delete_nodes() + self.shutdown_distributed() # remove this sessions working directory preserve = self.options.get_config("preservedir") == "1" if not preserve: shutil.rmtree(self.session_dir, ignore_errors=True) - self.shutdown_distributed() # call session shutdown handlers for handler in self.shutdown_handlers: @@ -1160,7 +1169,7 @@ class Session(object): """ try: state_file = open(self._state_file, "w") - state_file.write("%d %s\n" % (state, coreapi.state_name(state))) + state_file.write("%d %s\n" % (state, EventTypes(self.state).name)) state_file.close() except IOError: logging.exception("error writing state file: %s", state) @@ -1278,7 +1287,7 @@ class Session(object): hook(state) except Exception: message = "exception occured when running %s state hook: %s" % ( - coreapi.state_name(state), + EventTypes(self.state).name, hook, ) logging.exception(message) @@ -1549,11 +1558,10 @@ class Session(object): # write current nodes out to session directory file self.write_nodes() - # create control net interfaces and broker network tunnels + # create control net interfaces and network tunnels # which need to exist for emane to sync on location events # in distributed scenarios self.add_remove_control_interface(node=None, remove=False) - self.broker.startup() # initialize distributed tunnels self.initialize_distributed() @@ -1566,9 +1574,6 @@ class Session(object): self.boot_nodes() self.mobility.startup() - # set broker local instantiation to complete - self.broker.local_instantiation_complete() - # notify listeners that instantiation is complete event = EventData(event_type=EventTypes.INSTANTIATION_COMPLETE.value) self.broadcast_event(event) @@ -1606,21 +1611,16 @@ class Session(object): have entered runtime (time=0). """ # this is called from instantiate() after receiving an event message - # for the instantiation state, and from the broker when distributed - # nodes have been started + # for the instantiation state logging.debug( "session(%s) checking if not in runtime state, current state: %s", self.id, - coreapi.state_name(self.state), + EventTypes(self.state).name, ) if self.state == EventTypes.RUNTIME_STATE.value: logging.info("valid runtime state found, returning") return - # check to verify that all nodes and networks are running - if not self.broker.instantiation_complete(): - return - # start event loop and set to runtime self.event_loop.run() self.set_state(EventTypes.RUNTIME_STATE, send_event=True) @@ -1830,37 +1830,11 @@ class Session(object): except IndexError: # no server name. possibly only one server prefix = prefixes[0] - else: - # slave servers have their name and localhost in the serverlist - servers = self.broker.getservernames() - servers.remove("localhost") - prefix = None - for server_prefix in prefixes: - try: - # split each entry into server and prefix - server, p = server_prefix.split(":") - except ValueError: - server = "" - p = None - - if server == servers[0]: - # the server name in the list matches this server - prefix = p - break - - if not prefix: - logging.error( - "control network prefix not found for server: %s", servers[0] - ) - assign_address = False - try: - prefix = prefixes[0].split(":", 1)[1] - except IndexError: - prefix = prefixes[0] # len(prefixes) == 1 else: - # TODO: can we get the server name from the servers.conf or from the node assignments? + # TODO: can we get the server name from the servers.conf or from the node + # assignments?o # with one prefix, only master gets a ctrlnet address assign_address = self.master prefix = prefixes[0] @@ -1882,13 +1856,6 @@ class Session(object): serverintf=server_interface, ) - # tunnels between controlnets will be built with Broker.addnettunnels() - # TODO: potentially remove documentation saying node ids are ints - # TODO: need to move broker code out of the session object - self.broker.addnet(_id) - for server in self.broker.getservers(): - self.broker.addnodemap(server, _id) - return control_net def add_remove_control_interface( diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index 2f323783..eae46ce4 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -19,13 +19,9 @@ from core.emulator.enumerations import ( EventTypes, LinkTypes, MessageFlags, - MessageTypes, - NodeTlvs, RegisterTlvs, ) from core.errors import CoreError -from core.nodes.base import CoreNodeBase -from core.nodes.ipaddress import IpAddress class MobilityManager(ModelManager): @@ -48,11 +44,6 @@ class MobilityManager(ModelManager): self.models[BasicRangeModel.name] = BasicRangeModel self.models[Ns2ScriptedMobility.name] = Ns2ScriptedMobility - # dummy node objects for tracking position of nodes on other servers - self.phys = {} - self.physnets = {} - self.session.broker.handlers.add(self.physnodehandlelink) - def reset(self): """ Clear out all current configurations. @@ -93,9 +84,6 @@ class MobilityManager(ModelManager): model_class = self.models[model_name] self.set_model(node, model_class, config) - if self.session.master: - self.installphysnodes(node) - if node.mobility: self.session.event_loop.add_event(0.0, node.mobility.startup) @@ -209,87 +197,6 @@ class MobilityManager(ModelManager): if node.model: node.model.update(moved, moved_netifs) - def addphys(self, netnum, node): - """ - Keep track of PhysicalNodes and which network they belong to. - - :param int netnum: network number - :param core.coreobj.PyCoreNode node: node to add physical network to - :return: nothing - """ - node_id = node.id - self.phys[node_id] = node - if netnum not in self.physnets: - self.physnets[netnum] = [node_id] - else: - self.physnets[netnum].append(node_id) - - # TODO: remove need for handling old style message - - def physnodehandlelink(self, message): - """ - Broker handler. Snoop Link add messages to get - node numbers of PhyiscalNodes and their nets. - Physical nodes exist only on other servers, but a shadow object is - created here for tracking node position. - - :param message: link message to handle - :return: nothing - """ - if ( - message.message_type == MessageTypes.LINK.value - and message.flags & MessageFlags.ADD.value - ): - nn = message.node_numbers() - # first node is always link layer node in Link add message - if nn[0] not in self.session.broker.network_nodes: - return - if nn[1] in self.session.broker.physical_nodes: - # record the fact that this PhysicalNode is linked to a net - dummy = CoreNodeBase( - session=self.session, _id=nn[1], name="n%d" % nn[1], start=False - ) - self.addphys(nn[0], dummy) - - # TODO: remove need to handling old style messages - def physnodeupdateposition(self, message): - """ - Snoop node messages belonging to physical nodes. The dummy object - in self.phys[] records the node position. - - :param message: message to handle - :return: nothing - """ - nodenum = message.node_numbers()[0] - try: - dummy = self.phys[nodenum] - nodexpos = message.get_tlv(NodeTlvs.X_POSITION.value) - nodeypos = message.get_tlv(NodeTlvs.Y_POSITION.value) - dummy.setposition(nodexpos, nodeypos, None) - except KeyError: - logging.exception("error retrieving physical node: %s", nodenum) - - def installphysnodes(self, net): - """ - After installing a mobility model on a net, include any physical - nodes that we have recorded. Use the GreTap tunnel to the physical node - as the node's interface. - - :param net: network to install - :return: nothing - """ - node_ids = self.physnets.get(net.id, []) - for node_id in node_ids: - node = self.phys[node_id] - # TODO: fix this bad logic, relating to depending on a break to get a valid server - for server in self.session.broker.getserversbynode(node_id): - break - netif = self.session.broker.gettunnel(net.id, IpAddress.to_int(server.host)) - node.addnetif(netif, 0) - netif.node = node - x, y, z = netif.node.position.get() - netif.poshook(netif, x, y, z) - class WirelessModel(ConfigurableOptions): """ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 7758e4af..a9e8dc43 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -199,7 +199,9 @@ class NodeBase(object): x, y, _ = self.getposition() model = self.type - emulation_server = self.server.host + emulation_server = None + if self.server is not None: + emulation_server = self.server.host services = self.services if services is not None: @@ -593,7 +595,13 @@ class CoreNode(CoreNodeBase): :param str sh: shell to execute command in :return: str """ - return self.client.create_cmd(sh) + terminal = self.client.create_cmd(sh) + if self.server is None: + return terminal + else: + return "ssh -X -f {host} xterm -e {terminal}".format( + host=self.server.host, terminal=terminal + ) def privatedir(self, path): """ diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 4d68ccaf..f25c800c 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -288,8 +288,8 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - for host in self.session.servers: - server = self.session.servers[host] + for name in self.session.servers: + server = self.session.servers[name] server.remote_cmd(args, env, cwd, wait) return output diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index 32800eea..fd674b45 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -76,7 +76,7 @@ class Sdt(object): # node information for remote nodes not in session._objs # local nodes also appear here since their obj may not exist yet self.remotes = {} - session.broker.handlers.add(self.handle_distributed) + # session.broker.handlers.add(self.handle_distributed) # add handler for node updates self.session.node_handlers.append(self.handle_node_update) diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index ee316ffc..0a81b75e 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -107,10 +107,6 @@ class CoreXmlDeployment(object): def add_deployment(self): physical_host = self.add_physical_host(socket.gethostname()) - # TODO: handle other servers - # servers = self.session.broker.getservernames() - # servers.remove("localhost") - for node_id in self.session.nodes: node = self.session.nodes[node_id] if isinstance(node, CoreNodeBase): diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 0005c378..881ff373 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -314,9 +314,9 @@ def build_transport_xml(emane_manager, node, transport_type): file_name = transport_file_name(node.id, transport_type) file_path = os.path.join(emane_manager.session.session_dir, file_name) create_file(transport_element, doc_name, file_path) - for server in emane_manager.session.servers: - conn = emane_manager.session.servers[server] - create_file(transport_element, doc_name, file_path, conn) + for name in emane_manager.session.servers: + server = emane_manager.session.servers[name] + create_file(transport_element, doc_name, file_path, server) def create_phy_xml(emane_model, config, file_path, server): @@ -342,9 +342,9 @@ def create_phy_xml(emane_model, config, file_path, server): create_file(phy_element, "phy", file_path, server) else: create_file(phy_element, "phy", file_path) - for server in emane_model.session.servers: - conn = emane_model.session.servers[server] - create_file(phy_element, "phy", file_path, conn) + for name in emane_model.session.servers: + server = emane_model.session.servers[name] + create_file(phy_element, "phy", file_path, server) def create_mac_xml(emane_model, config, file_path, server): @@ -372,9 +372,9 @@ def create_mac_xml(emane_model, config, file_path, server): create_file(mac_element, "mac", file_path, server) else: create_file(mac_element, "mac", file_path) - for server in emane_model.session.servers: - conn = emane_model.session.servers[server] - create_file(mac_element, "mac", file_path, conn) + for name in emane_model.session.servers: + server = emane_model.session.servers[name] + create_file(mac_element, "mac", file_path, server) def create_nem_xml( @@ -410,9 +410,9 @@ def create_nem_xml( create_file(nem_element, "nem", nem_file, server) else: create_file(nem_element, "nem", nem_file) - for server in emane_model.session.servers: - conn = emane_model.session.servers[server] - create_file(nem_element, "nem", nem_file, conn) + for name in emane_model.session.servers: + server = emane_model.session.servers[name] + create_file(nem_element, "nem", nem_file, server) def create_event_service_xml(group, port, device, file_directory, server=None): diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index ca9ca928..8bcf2972 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -8,21 +8,19 @@ from core.emulator.enumerations import EventTypes, NodeTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"controlnet": "172.16.0.0/24", "distributed_address": address}) session = coreemu.create_session() - # set controlnet - session.options.set_config("controlnet", "172.16.0.0/24") - # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -31,7 +29,7 @@ def main(): node_one = session.add_node() switch = session.add_node(_type=NodeTypes.SWITCH) options = NodeOptions() - options.emulation_server = remote + options.emulation_server = server_name node_two = session.add_node(node_options=options) # create node interfaces and link diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py index 1ffe5795..c64d1f0c 100644 --- a/daemon/examples/python/distributed_emane.py +++ b/daemon/examples/python/distributed_emane.py @@ -9,25 +9,25 @@ from core.emulator.enumerations import EventTypes, NodeTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu( + { + "controlnet": "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 " + "core4:172.16.4.0/24 core5:172.16.5.0/24", + "distributed_address": address, + } + ) session = coreemu.create_session() - # set controlnet - session.options.set_config( - "controlnet", - "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 " - "core4:172.16.4.0/24 core5:172.16.5.0/24", - ) - # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -38,7 +38,7 @@ def main(): node_one = session.add_node(node_options=options) emane_net = session.add_node(_type=NodeTypes.EMANE) session.emane.set_model(emane_net, EmaneIeee80211abgModel) - options.emulation_server = remote + options.emulation_server = server_name node_two = session.add_node(node_options=options) # create node interfaces and link diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index 2b611816..b0f27c28 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -8,18 +8,19 @@ from core.emulator.enumerations import EventTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"distributed_address": address}) session = coreemu.create_session() # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -27,7 +28,7 @@ def main(): # create local node, switch, and remote nodes options = NodeOptions() node_one = session.add_node(node_options=options) - options.emulation_server = remote + options.emulation_server = server_name node_two = session.add_node(node_options=options) # create node interfaces and link diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py index b7ed166b..bc13bf2c 100644 --- a/daemon/examples/python/distributed_switches.py +++ b/daemon/examples/python/distributed_switches.py @@ -7,15 +7,16 @@ from core.emulator.enumerations import EventTypes, NodeTypes def main(): + address = sys.argv[1] + remote = sys.argv[2] + # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"distributed_address": address}) session = coreemu.create_session() # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_wlan.py b/daemon/examples/python/distributed_wlan.py index ca64ee01..f8af1f5f 100644 --- a/daemon/examples/python/distributed_wlan.py +++ b/daemon/examples/python/distributed_wlan.py @@ -9,21 +9,19 @@ from core.location.mobility import BasicRangeModel def main(): + address = sys.argv[1] + remote = sys.argv[2] + # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu() + coreemu = CoreEmu({"distributed_address": address}) session = coreemu.create_session() - # set controlnet - # session.options.set_config("controlnet", "172.16.0.0/24") - # initialize distributed - address = sys.argv[1] - remote = sys.argv[2] - session.address = address - session.add_distributed(remote) + server_name = "core2" + session.add_distributed(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -31,7 +29,7 @@ def main(): # create local node, switch, and remote nodes options = NodeOptions() options.set_position(0, 0) - options.emulation_server = remote + options.emulation_server = server_name node_one = session.add_node(node_options=options) wlan = session.add_node(_type=NodeTypes.WIRELESS_LAN) session.mobility.set_model(wlan, BasicRangeModel) diff --git a/daemon/tests/conftest.py b/daemon/tests/conftest.py index 001233bb..ead3c2b4 100644 --- a/daemon/tests/conftest.py +++ b/daemon/tests/conftest.py @@ -58,7 +58,6 @@ class CoreServerTest(object): self.request_handler = CoreHandler(request_mock, "", self.server) self.request_handler.session = self.session self.request_handler.add_session_handlers() - self.session.broker.session_clients.append(self.request_handler) # have broker handle a configuration state change self.session.set_state(EventTypes.DEFINITION_STATE) diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index caff15fe..02e634be 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -763,13 +763,11 @@ class TestGui: (ConfigTlvs.VALUES, "%s:%s:%s" % (server, host, port)), ], ) - coreserver.session.broker.addserver = mock.MagicMock() - coreserver.session.broker.setupserver = mock.MagicMock() + coreserver.session.add_distributed = mock.MagicMock() coreserver.request_handler.handle_message(message) - coreserver.session.broker.addserver.assert_called_once_with(server, host, port) - coreserver.session.broker.setupserver.assert_called_once_with(server) + coreserver.session.add_distributed.assert_called_once_with(server, host) def test_config_services_request_all(self, coreserver): message = coreapi.CoreConfMessage.create( From 0b8bc7bd1362e96d3089eb28b239afb2d0198b7e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 15 Oct 2019 15:02:38 -0700 Subject: [PATCH 079/113] updated corehandlers to allow sdt snooping to help mimic previous behavior --- daemon/core/api/tlv/corehandlers.py | 2 ++ daemon/core/plugins/sdt.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 4b4e7c1e..60ddfcca 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -2082,6 +2082,7 @@ class CoreUdpHandler(CoreHandler): logging.debug("session handling message: %s", session.session_id) self.session = session self.handle_message(message) + self.session.sdt.handle_distributed(message) self.broadcast(message) else: logging.error( @@ -2106,6 +2107,7 @@ class CoreUdpHandler(CoreHandler): if session or message.message_type == MessageTypes.REGISTER.value: self.session = session self.handle_message(message) + self.session.sdt.handle_distributed(message) self.broadcast(message) else: logging.error( diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index fd674b45..52635da3 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -76,7 +76,6 @@ class Sdt(object): # node information for remote nodes not in session._objs # local nodes also appear here since their obj may not exist yet self.remotes = {} - # session.broker.handlers.add(self.handle_distributed) # add handler for node updates self.session.node_handlers.append(self.handle_node_update) From 61a4e228a158ec9ffa5b4ce1b5df111f985c93e9 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 10:14:36 -0700 Subject: [PATCH 080/113] updated ctrlnets to assign unique addresses per server, fixed ovs command issue for interface specific commands --- daemon/core/nodes/base.py | 62 ++++++---------------------------- daemon/core/nodes/docker.py | 14 +++++--- daemon/core/nodes/interface.py | 33 +++++++++++------- daemon/core/nodes/netclient.py | 14 ++++++++ daemon/core/nodes/network.py | 35 ++++++++++++++----- daemon/core/nodes/physical.py | 2 +- 6 files changed, 81 insertions(+), 79 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index a9e8dc43..4e9de039 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -4,12 +4,9 @@ Defines the base logic for nodes used within core. import logging import os -import random import shutil import socket -import string import threading -from builtins import range from socket import AF_INET, AF_INET6 from core import utils @@ -18,8 +15,8 @@ from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes from core.errors import CoreCommandError from core.nodes import client, ipaddress -from core.nodes.interface import CoreInterface, TunTap, Veth -from core.nodes.netclient import LinuxNetClient, OvsNetClient +from core.nodes.interface import TunTap, Veth +from core.nodes.netclient import get_net_client _DEFAULT_MTU = 1500 @@ -63,10 +60,8 @@ class NodeBase(object): self.opaque = None self.position = Position() - if session.options.get_config("ovs") == "True": - self.net_client = OvsNetClient(self.net_cmd) - else: - self.net_client = LinuxNetClient(self.net_cmd) + use_ovs = session.options.get_config("ovs") == "True" + self.net_client = get_net_client(use_ovs, self.net_cmd) def startup(self): """ @@ -461,15 +456,13 @@ class CoreNode(CoreNodeBase): def create_node_net_client(self, use_ovs): """ - Create a client for running network orchestration commands. + Create node network client for running network commands within the nodes + container. - :param bool use_ovs: True to use OVS bridges, False for Linux bridge - :return: network client + :param bool use_ovs: True for OVS bridges, False for Linux bridges + :return:node network client """ - if use_ovs: - return OvsNetClient(self.node_net_cmd) - else: - return LinuxNetClient(self.node_net_cmd) + return get_net_client(use_ovs, self.node_net_cmd) def alive(self): """ @@ -675,11 +668,7 @@ class CoreNode(CoreNodeBase): raise ValueError("interface name (%s) too long" % name) veth = Veth( - node=self, - name=name, - localname=localname, - start=self.up, - server=self.server, + self.session, self, name, localname, start=self.up, server=self.server ) if self.up: @@ -732,7 +721,7 @@ class CoreNode(CoreNodeBase): sessionid = self.session.short_session_id() localname = "tap%s.%s.%s" % (self.id, ifindex, sessionid) name = ifname - tuntap = TunTap(node=self, name=name, localname=localname, start=self.up) + tuntap = TunTap(self.session, self, name, localname, start=self.up) try: self.addnetif(tuntap, ifindex) @@ -849,35 +838,6 @@ class CoreNode(CoreNodeBase): self.ifup(ifindex) return ifindex - def connectnode(self, ifname, othernode, otherifname): - """ - Connect a node. - - :param str ifname: name of interface to connect - :param core.nodes.base.CoreNode othernode: node to connect to - :param str otherifname: interface name to connect to - :return: nothing - """ - tmplen = 8 - tmp1 = "tmp." + "".join( - [random.choice(string.ascii_lowercase) for _ in range(tmplen)] - ) - tmp2 = "tmp." + "".join( - [random.choice(string.ascii_lowercase) for _ in range(tmplen)] - ) - self.net_client.create_veth(tmp1, tmp2) - self.net_client.device_ns(tmp1, str(self.pid)) - self.node_net_client.device_name(tmp1, ifname) - interface = CoreInterface(node=self, name=ifname, mtu=_DEFAULT_MTU) - self.addnetif(interface, self.newifindex()) - - self.net_client.device_ns(tmp2, str(othernode.pid)) - othernode.node_net_client.device_name(tmp2, otherifname) - other_interface = CoreInterface( - node=othernode, name=otherifname, mtu=_DEFAULT_MTU - ) - othernode.addnetif(other_interface, othernode.newifindex()) - def addfile(self, srcname, filename): """ Add a file. diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 2679704f..b91e987e 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -7,7 +7,7 @@ from core import utils from core.emulator.enumerations import NodeTypes from core.errors import CoreCommandError from core.nodes.base import CoreNode -from core.nodes.netclient import LinuxNetClient, OvsNetClient +from core.nodes.netclient import get_net_client class DockerClient(object): @@ -107,10 +107,14 @@ class DockerNode(CoreNode): super(DockerNode, self).__init__(session, _id, name, nodedir, bootsh, start) def create_node_net_client(self, use_ovs): - if use_ovs: - return OvsNetClient(self.nsenter_cmd) - else: - return LinuxNetClient(self.nsenter_cmd) + """ + Create node network client for running network commands within the nodes + container. + + :param bool use_ovs: True for OVS bridges, False for Linux bridges + :return:node network client + """ + return get_net_client(use_ovs, self.nsenter_cmd) def alive(self): """ diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index bfcb8583..a6e04eb5 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -8,7 +8,7 @@ from builtins import int, range from core import utils from core.errors import CoreCommandError -from core.nodes.netclient import LinuxNetClient +from core.nodes.netclient import get_net_client class CoreInterface(object): @@ -16,17 +16,18 @@ class CoreInterface(object): Base class for network interfaces. """ - def __init__(self, node, name, mtu, server=None): + def __init__(self, session, node, name, mtu, server=None): """ Creates a PyCoreNetIf instance. + :param core.emulator.session.Session session: core session instance :param core.nodes.base.CoreNode node: node for interface :param str name: interface name - :param mtu: mtu value + :param int mtu: mtu value :param core.emulator.distributed.DistributedServer server: remote server node will run on, default is None for localhost """ - + self.session = session self.node = node self.name = name if not isinstance(mtu, int): @@ -45,7 +46,8 @@ class CoreInterface(object): # index used to find flow data self.flow_id = None self.server = server - self.net_client = LinuxNetClient(self.net_cmd) + use_ovs = session.options.get_config("ovs") == "True" + self.net_client = get_net_client(use_ovs, self.net_cmd) def net_cmd(self, args, env=None, cwd=None, wait=True): """ @@ -211,21 +213,24 @@ class Veth(CoreInterface): Provides virtual ethernet functionality for core nodes. """ - def __init__(self, node, name, localname, mtu=1500, server=None, start=True): + def __init__( + self, session, node, name, localname, mtu=1500, server=None, start=True + ): """ Creates a VEth instance. + :param core.emulator.session.Session session: core session instance :param core.nodes.base.CoreNode node: related core node :param str name: interface name :param str localname: interface local name - :param mtu: interface mtu + :param int mtu: interface mtu :param core.emulator.distributed.DistributedServer server: remote server node will run on, default is None for localhost :param bool start: start flag :raises CoreCommandError: when there is a command exception """ # note that net arg is ignored - CoreInterface.__init__(self, node, name, mtu, server) + CoreInterface.__init__(self, session, node, name, mtu, server) self.localname = localname self.up = False if start: @@ -271,19 +276,22 @@ class TunTap(CoreInterface): TUN/TAP virtual device in TAP mode """ - def __init__(self, node, name, localname, mtu=1500, server=None, start=True): + def __init__( + self, session, node, name, localname, mtu=1500, server=None, start=True + ): """ Create a TunTap instance. + :param core.emulator.session.Session session: core session instance :param core.nodes.base.CoreNode node: related core node :param str name: interface name :param str localname: local interface name - :param mtu: interface mtu + :param int mtu: interface mtu :param core.emulator.distributed.DistributedServer server: remote server node will run on, default is None for localhost :param bool start: start flag """ - CoreInterface.__init__(self, node, name, mtu, server) + CoreInterface.__init__(self, session, node, name, mtu, server) self.localname = localname self.up = False self.transport_type = "virtual" @@ -466,8 +474,7 @@ class GreTap(CoreInterface): will run on, default is None for localhost :raises CoreCommandError: when there is a command exception """ - CoreInterface.__init__(self, node, name, mtu, server) - self.session = session + CoreInterface.__init__(self, session, node, name, mtu, server) if _id is None: # from PyCoreObj _id = ((id(self) >> 16) ^ (id(self) & 0xFFFF)) & 0xFFFF diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 6de5d698..9234bef5 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -7,6 +7,20 @@ import os from core.constants import BRCTL_BIN, ETHTOOL_BIN, IP_BIN, OVS_BIN, TC_BIN +def get_net_client(use_ovs, run): + """ + Retrieve desired net client for running network commands. + + :param bool use_ovs: True for OVS bridges, False for Linux bridges + :param func run: function used to run net client commands + :return: net client class + """ + if use_ovs: + return OvsNetClient(run) + else: + return LinuxNetClient(run) + + class LinuxNetClient(object): """ Client for creating Linux bridges and ip interfaces for nodes. diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index f25c800c..931622bb 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -16,6 +16,7 @@ from core.errors import CoreCommandError, CoreError from core.nodes import ipaddress from core.nodes.base import CoreNetworkBase from core.nodes.interface import GreTap, Veth +from core.nodes.netclient import get_net_client ebtables_lock = threading.Lock() @@ -558,7 +559,7 @@ class CoreNetwork(CoreNetworkBase): if len(name) >= 16: raise ValueError("interface name %s too long" % name) - netif = Veth(node=None, name=name, localname=localname, mtu=1500, start=self.up) + netif = Veth(self.session, None, name, localname, start=self.up) self.attach(netif) if net.up: # this is similar to net.attach() but uses netif.name instead of localname @@ -766,6 +767,24 @@ class CtrlNet(CoreNetwork): self.serverintf = serverintf CoreNetwork.__init__(self, session, _id, name, start, server) + def add_addresses(self, address): + """ + Add addresses used for created control networks, + + :param core.nodes.interfaces.IpAddress address: starting address to use + :return: + """ + use_ovs = self.session.options.get_config("ovs") == "True" + current = "%s/%s" % (address, self.prefix.prefixlen) + net_client = get_net_client(use_ovs, utils.check_cmd) + net_client.create_address(self.brname, current) + for name in self.session.servers: + server = self.session.servers[name] + address -= 1 + current = "%s/%s" % (address, self.prefix.prefixlen) + net_client = get_net_client(use_ovs, server.remote_cmd) + net_client.create_address(self.brname, current) + def startup(self): """ Startup functionality for the control network. @@ -778,16 +797,14 @@ class CtrlNet(CoreNetwork): CoreNetwork.startup(self) - if self.hostid: - addr = self.prefix.addr(self.hostid) - else: - addr = self.prefix.max_addr() - logging.info("added control network bridge: %s %s", self.brname, self.prefix) - if self.assign_address: - addrlist = ["%s/%s" % (addr, self.prefix.prefixlen)] - self.addrconfig(addrlist=addrlist) + if self.hostid and self.assign_address: + address = self.prefix.addr(self.hostid) + self.add_addresses(address) + elif self.assign_address: + address = self.prefix.max_addr() + self.add_addresses(address) if self.updown_script: logging.info( diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 9daf4f35..93e04c5e 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -246,7 +246,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): will run on, default is None for localhost """ CoreNodeBase.__init__(self, session, _id, name, start, server) - CoreInterface.__init__(self, node=self, name=name, mtu=mtu) + CoreInterface.__init__(self, session, self, name, mtu, server) self.up = False self.lock = threading.RLock() self.ifindex = None From 8aef9f273feb57da549b2246b7cb880676079e63 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 17:11:21 -0700 Subject: [PATCH 081/113] updates to clear broker from physical node --- daemon/core/emulator/session.py | 49 +++++++++++++++++---------------- daemon/core/nodes/physical.py | 43 +++++++++++++++-------------- gui/nodes.tcl | 2 +- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 1445cb8f..e864ec8b 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -153,7 +153,7 @@ class Session(object): self.services.default_services = { "mdr": ("zebra", "OSPFv3MDR", "IPForward"), "PC": ("DefaultRoute",), - "prouter": ("zebra", "OSPFv2", "OSPFv3", "IPForward"), + "prouter": (), "router": ("zebra", "OSPFv2", "OSPFv3", "IPForward"), "host": ("DefaultRoute", "SSH"), } @@ -192,32 +192,33 @@ class Session(object): for name in self.servers: server = self.servers[name] - host = server.host - key = self.tunnelkey(node_id, IpAddress.to_int(host)) + self.create_gre_tunnel(node, server) - # local to server - logging.info( - "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key - ) - local_tap = GreTap(session=self, remoteip=host, key=key) - local_tap.net_client.create_interface(node.brname, local_tap.localname) + def create_gre_tunnel(self, node, server): + host = server.host + key = self.tunnelkey(node.id, IpAddress.to_int(host)) + tunnel = self.tunnels.get(key) + if tunnel is not None: + return tunnel - # server to local - logging.info( - "remote tunnel node(%s) to local(%s) key(%s)", - node.name, - self.address, - key, - ) - remote_tap = GreTap( - session=self, remoteip=self.address, key=key, server=server - ) - remote_tap.net_client.create_interface( - node.brname, remote_tap.localname - ) + # local to server + logging.info( + "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key + ) + local_tap = GreTap(session=self, remoteip=host, key=key) + local_tap.net_client.create_interface(node.brname, local_tap.localname) - # save tunnels for shutdown - self.tunnels[key] = (local_tap, remote_tap) + # server to local + logging.info( + "remote tunnel node(%s) to local(%s) key(%s)", node.name, self.address, key + ) + remote_tap = GreTap(session=self, remoteip=self.address, key=key, server=server) + remote_tap.net_client.create_interface(node.brname, remote_tap.localname) + + # save tunnels for shutdown + tunnel = (local_tap, remote_tap) + self.tunnels[key] = tunnel + return tunnel def tunnelkey(self, n1_id, n2_id): """ diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 93e04c5e..ecbcf368 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -9,15 +9,19 @@ import threading from core import utils from core.constants import MOUNT_BIN, UMOUNT_BIN from core.emulator.enumerations import NodeTypes -from core.errors import CoreCommandError +from core.errors import CoreCommandError, CoreError from core.nodes.base import CoreNodeBase from core.nodes.interface import CoreInterface from core.nodes.network import CoreNetwork, GreTap class PhysicalNode(CoreNodeBase): - def __init__(self, session, _id=None, name=None, nodedir=None, start=True): - CoreNodeBase.__init__(self, session, _id, name, start=start) + def __init__( + self, session, _id=None, name=None, nodedir=None, start=True, server=None + ): + CoreNodeBase.__init__(self, session, _id, name, start, server) + if not self.server: + raise CoreError("physical nodes must be assigned to a remote server") self.nodedir = nodedir self.up = start self.lock = threading.RLock() @@ -86,7 +90,6 @@ class PhysicalNode(CoreNodeBase): def adoptnetif(self, netif, ifindex, hwaddr, addrlist): """ - The broker builds a GreTap tunnel device to this physical node. When a link message is received linking this node to another part of the emulation, no new interface is created; instead, adopt the GreTap netif as the node interface. @@ -157,26 +160,21 @@ class PhysicalNode(CoreNodeBase): if ifindex is None: ifindex = self.newifindex() - if self.up: - # this is reached when this node is linked to a network node - # tunnel to net not built yet, so build it now and adopt it - gt = self.session.broker.addnettunnel(net.id) - if gt is None or len(gt) != 1: - raise ValueError( - "error building tunnel from adding a new network interface: %s" % gt - ) - gt = gt[0] - net.detach(gt) - self.adoptnetif(gt, ifindex, hwaddr, addrlist) - return ifindex - - # this is reached when configuring services (self.up=False) if ifname is None: ifname = "gt%d" % ifindex - netif = GreTap(node=self, name=ifname, session=self.session, start=False) - self.adoptnetif(netif, ifindex, hwaddr, addrlist) - return ifindex + if self.up: + # this is reached when this node is linked to a network node + # tunnel to net not built yet, so build it now and adopt it + _, remote_tap = self.session.create_gre_tunnel(net, self.server) + # net.detach(remote_tap) + self.adoptnetif(remote_tap, ifindex, hwaddr, addrlist) + return ifindex + else: + # this is reached when configuring services (self.up=False) + netif = GreTap(node=self, name=ifname, session=self.session, start=False) + self.adoptnetif(netif, ifindex, hwaddr, addrlist) + return ifindex def privatedir(self, path): if path[0] != "/": @@ -223,6 +221,9 @@ class PhysicalNode(CoreNodeBase): os.chmod(node_file.name, mode) logging.info("created nodefile: '%s'; mode: 0%o", node_file.name, mode) + def node_net_cmd(self, args, wait=True): + return self.net_cmd(args, wait=wait) + class Rj45Node(CoreNodeBase, CoreInterface): """ diff --git a/gui/nodes.tcl b/gui/nodes.tcl index 00e52c5d..c8645f03 100644 --- a/gui/nodes.tcl +++ b/gui/nodes.tcl @@ -19,7 +19,7 @@ array set g_node_types_default { 4 {mdr mdr.gif mdr.gif {zebra OSPFv3MDR IPForward} \ netns {built-in type for wireless routers}} 5 {prouter router_green.gif router_green.gif \ - {zebra OSPFv2 OSPFv3 IPForward} \ + {} \ physical {built-in type for physical nodes}} } From 009ce8143efec337cb6afd22b27afd5217d12a11 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 20:19:51 -0700 Subject: [PATCH 082/113] removed lock for distributed commands and limited usage to uploads --- daemon/core/emulator/distributed.py | 13 ++++++------- daemon/core/nodes/physical.py | 1 - 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 4e258937..2df33541 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -53,16 +53,15 @@ class DistributedServer(object): "remote cmd server(%s) cwd(%s) wait(%s): %s", self.host, cwd, wait, cmd ) try: - with self.lock: - if cwd is None: + if cwd is None: + result = self.conn.run( + cmd, hide=False, env=env, replace_env=replace_env + ) + else: + with self.conn.cd(cwd): result = self.conn.run( cmd, hide=False, env=env, replace_env=replace_env ) - else: - with self.conn.cd(cwd): - result = self.conn.run( - cmd, hide=False, env=env, replace_env=replace_env - ) return result.stdout.strip() except UnexpectedExit as e: stdout, stderr = e.streams_for_display() diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index ecbcf368..37a2eb54 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -167,7 +167,6 @@ class PhysicalNode(CoreNodeBase): # this is reached when this node is linked to a network node # tunnel to net not built yet, so build it now and adopt it _, remote_tap = self.session.create_gre_tunnel(net, self.server) - # net.detach(remote_tap) self.adoptnetif(remote_tap, ifindex, hwaddr, addrlist) return ifindex else: From 774dd8330cd81c0a72ecdc7cbafddfe70427b1a6 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 16 Oct 2019 20:26:14 -0700 Subject: [PATCH 083/113] removed broker.py --- daemon/core/api/tlv/broker.py | 1147 --------------------------------- 1 file changed, 1147 deletions(-) delete mode 100644 daemon/core/api/tlv/broker.py diff --git a/daemon/core/api/tlv/broker.py b/daemon/core/api/tlv/broker.py deleted file mode 100644 index d71d050c..00000000 --- a/daemon/core/api/tlv/broker.py +++ /dev/null @@ -1,1147 +0,0 @@ -""" -Broker class that is part of the session object. Handles distributing parts of the emulation out to -other emulation servers. The broker is consulted when handling messages to determine if messages -should be handled locally or forwarded on to another emulation server. -""" - -import logging -import os -import select -import socket -import threading - -from core import utils -from core.api.tlv import coreapi -from core.emane.nodes import EmaneNet -from core.emulator.enumerations import ( - ConfigDataTypes, - ConfigFlags, - ConfigTlvs, - EventTlvs, - EventTypes, - ExecuteTlvs, - FileTlvs, - LinkTlvs, - MessageFlags, - MessageTypes, - NodeTlvs, - NodeTypes, - RegisterTlvs, -) -from core.nodes.base import CoreNetworkBase, CoreNodeBase -from core.nodes.interface import GreTap -from core.nodes.ipaddress import IpAddress -from core.nodes.network import CtrlNet, GreTapBridge -from core.nodes.physical import PhysicalNode - - -class CoreDistributedServer(object): - """ - Represents CORE daemon servers for communication. - """ - - def __init__(self, name, host, port): - """ - Creates a CoreServer instance. - - :param str name: name of the CORE server - :param str host: server address - :param int port: server port - """ - self.name = name - self.host = host - self.port = port - self.sock = None - self.instantiation_complete = False - - def connect(self): - """ - Connect to CORE server and save connection. - - :return: nothing - """ - if self.sock: - raise ValueError("socket already connected") - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - try: - sock.connect((self.host, self.port)) - except IOError as e: - sock.close() - raise e - - self.sock = sock - - def close(self): - """ - Close connection with CORE server. - - :return: nothing - """ - if self.sock is not None: - self.sock.close() - self.sock = None - - -class CoreBroker(object): - """ - Helps with brokering messages between CORE daemon servers. - """ - - # configurable manager name - name = "broker" - - # configurable manager type - config_type = RegisterTlvs.UTILITY.value - - def __init__(self, session): - """ - Creates a CoreBroker instance. - - :param core.emulator.session.Session session: session this manager is tied to - :return: nothing - """ - - # ConfigurableManager.__init__(self) - self.session = session - self.session_clients = [] - self.session_id_master = None - self.myip = None - # dict containing tuples of (host, port, sock) - self.servers = {} - self.servers_lock = threading.Lock() - self.addserver("localhost", None, None) - # dict containing node number to server name mapping - self.nodemap = {} - # this lock also protects self.nodecounts - self.nodemap_lock = threading.Lock() - # reference counts of nodes on servers - self.nodecounts = {} - # set of node numbers that are link-layer nodes (networks) - self.network_nodes = set() - # set of node numbers that are PhysicalNode nodes - self.physical_nodes = set() - # allows for other message handlers to process API messages (e.g. EMANE) - self.handlers = set() - # dict with tunnel key to tunnel device mapping - self.tunnels = {} - self.dorecvloop = False - self.recvthread = None - self.bootcount = 0 - - def startup(self): - """ - Build tunnels between network-layer nodes now that all node - and link information has been received; called when session - enters the instantation state. - """ - self.addnettunnels() - self.writeservers() - - def shutdown(self): - """ - Close all active sockets; called when the session enters the - data collect state - """ - self.reset() - with self.servers_lock: - while len(self.servers) > 0: - name, server = self.servers.popitem() - if server.sock is not None: - logging.info( - "closing connection with %s: %s:%s", - name, - server.host, - server.port, - ) - server.close() - self.dorecvloop = False - if self.recvthread is not None: - self.recvthread.join() - - def reset(self): - """ - Reset to initial state. - """ - logging.debug("broker reset") - self.nodemap_lock.acquire() - self.nodemap.clear() - for server in self.nodecounts: - count = self.nodecounts[server] - if count < 1: - self.delserver(server) - self.nodecounts.clear() - self.bootcount = 0 - self.nodemap_lock.release() - self.network_nodes.clear() - self.physical_nodes.clear() - while len(self.tunnels) > 0: - _key, gt = self.tunnels.popitem() - gt.shutdown() - - def startrecvloop(self): - """ - Spawn the receive loop for receiving messages. - """ - if self.recvthread is not None: - logging.info("server receive loop already started") - if self.recvthread.isAlive(): - return - else: - self.recvthread.join() - # start reading data from connected sockets - logging.info("starting server receive loop") - self.dorecvloop = True - self.recvthread = threading.Thread(target=self.recvloop) - self.recvthread.daemon = True - self.recvthread.start() - - def recvloop(self): - """ - Receive loop for receiving messages from server sockets. - """ - self.dorecvloop = True - # note: this loop continues after emulation is stopped, - # even with 0 servers - while self.dorecvloop: - rlist = [] - with self.servers_lock: - # build a socket list for select call - for name in self.servers: - server = self.servers[name] - if server.sock is not None: - rlist.append(server.sock) - r, _w, _x = select.select(rlist, [], [], 1.0) - for sock in r: - server = self.getserverbysock(sock) - logging.info( - "attempting to receive from server: peer:%s remote:%s", - server.sock.getpeername(), - server.sock.getsockname(), - ) - if server is None: - # servers may have changed; loop again - continue - rcvlen = self.recv(server) - if rcvlen == 0: - logging.info( - "connection with server(%s) closed: %s:%s", - server.name, - server.host, - server.port, - ) - - def recv(self, server): - """ - Receive data on an emulation server socket and broadcast it to - all connected session handlers. Returns the length of data recevied - and forwarded. Return value of zero indicates the socket has closed - and should be removed from the self.servers dict. - - :param CoreDistributedServer server: server to receive from - :return: message length - :rtype: int - """ - msghdr = server.sock.recv(coreapi.CoreMessage.header_len) - if len(msghdr) == 0: - # server disconnected - logging.info("server disconnected, closing server") - server.close() - return 0 - - if len(msghdr) != coreapi.CoreMessage.header_len: - logging.warning( - "warning: broker received not enough data len=%s", len(msghdr) - ) - return len(msghdr) - - msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(msghdr) - msgdata = server.sock.recv(msglen) - data = msghdr + msgdata - count = None - logging.debug("received message type: %s", MessageTypes(msgtype)) - # snoop exec response for remote interactive TTYs - if msgtype == MessageTypes.EXECUTE.value and msgflags & MessageFlags.TTY.value: - data = self.fixupremotetty(msghdr, msgdata, server.host) - logging.debug("created remote tty message: %s", data) - elif msgtype == MessageTypes.NODE.value: - # snoop node delete response to decrement node counts - if msgflags & MessageFlags.DELETE.value: - msg = coreapi.CoreNodeMessage(msgflags, msghdr, msgdata) - nodenum = msg.get_tlv(NodeTlvs.NUMBER.value) - if nodenum is not None: - count = self.delnodemap(server, nodenum) - elif msgtype == MessageTypes.LINK.value: - # this allows green link lines for remote WLANs - msg = coreapi.CoreLinkMessage(msgflags, msghdr, msgdata) - self.session.sdt.handle_distributed(msg) - elif msgtype == MessageTypes.EVENT.value: - msg = coreapi.CoreEventMessage(msgflags, msghdr, msgdata) - eventtype = msg.get_tlv(EventTlvs.TYPE.value) - if eventtype == EventTypes.INSTANTIATION_COMPLETE.value: - server.instantiation_complete = True - if self.instantiation_complete(): - self.session.check_runtime() - else: - logging.error("unknown message type received: %s", msgtype) - - try: - for session_client in self.session_clients: - session_client.sendall(data) - except IOError: - logging.exception("error sending message") - - if count is not None and count < 1: - return 0 - else: - return len(data) - - def addserver(self, name, host, port): - """ - Add a new server, and try to connect to it. If we"re already connected to this - (host, port), then leave it alone. When host,port is None, do not try to connect. - - :param str name: name of server - :param str host: server address - :param int port: server port - :return: nothing - """ - with self.servers_lock: - server = self.servers.get(name) - if server is not None: - if ( - host == server.host - and port == server.port - and server.sock is not None - ): - # leave this socket connected - return - - logging.debug( - "closing connection with %s @ %s:%s", name, server.host, server.port - ) - server.close() - del self.servers[name] - - logging.debug("adding broker server(%s): %s:%s", name, host, port) - server = CoreDistributedServer(name, host, port) - if host is not None and port is not None: - try: - server.connect() - except IOError: - logging.exception( - "error connecting to server(%s): %s:%s", name, host, port - ) - if server.sock is not None: - self.startrecvloop() - self.servers[name] = server - - def delserver(self, server): - """ - Remove a server and hang up any connection. - - :param CoreDistributedServer server: server to delete - :return: nothing - """ - with self.servers_lock: - try: - s = self.servers.pop(server.name) - if s != server: - raise ValueError("server removed was not the server provided") - except KeyError: - logging.exception("error deleting server") - - if server.sock is not None: - logging.info( - "closing connection with %s @ %s:%s", - server.name, - server.host, - server.port, - ) - server.close() - - def getserverbyname(self, name): - """ - Return the server object having the given name, or None. - - :param str name: name of server to retrieve - :return: server for given name - :rtype: CoreDistributedServer - """ - with self.servers_lock: - return self.servers.get(name) - - def getserverbysock(self, sock): - """ - Return the server object corresponding to the given socket, or None. - - :param sock: socket associated with a server - :return: core server associated wit the socket - :rtype: CoreDistributedServer - """ - with self.servers_lock: - for name in self.servers: - server = self.servers[name] - if server.sock == sock: - return server - return None - - def getservers(self): - """ - Return a list of servers sorted by name. - - :return: sorted server list - :rtype: list - """ - with self.servers_lock: - return sorted(self.servers.values(), key=lambda x: x.name) - - def getservernames(self): - """ - Return a sorted list of server names (keys from self.servers). - - :return: sorted server names - :rtype: list - """ - with self.servers_lock: - return sorted(self.servers.keys()) - - def tunnelkey(self, n1num, n2num): - """ - Compute a 32-bit key used to uniquely identify a GRE tunnel. - The hash(n1num), hash(n2num) values are used, so node numbers may be - None or string values (used for e.g. "ctrlnet"). - - :param int n1num: node one id - :param int n2num: node two id - :return: tunnel key for the node pair - :rtype: int - """ - logging.debug("creating tunnel key for: %s, %s", n1num, n2num) - sid = self.session_id_master - if sid is None: - # this is the master session - sid = self.session.id - - key = (sid << 16) ^ utils.hashkey(n1num) ^ (utils.hashkey(n2num) << 8) - return key & 0xFFFFFFFF - - def addtunnel(self, remoteip, n1num, n2num, localnum): - """ - Adds a new GreTapBridge between nodes on two different machines. - - :param str remoteip: remote address for tunnel - :param int n1num: node one id - :param int n2num: node two id - :param int localnum: local id - :return: nothing - """ - key = self.tunnelkey(n1num, n2num) - if localnum == n2num: - remotenum = n1num - else: - remotenum = n2num - - if key in self.tunnels.keys(): - logging.warning( - "tunnel with key %s (%s-%s) already exists!", key, n1num, n2num - ) - else: - _id = key & ((1 << 16) - 1) - logging.info( - "adding tunnel for %s-%s to %s with key %s", n1num, n2num, remoteip, key - ) - if localnum in self.physical_nodes: - # no bridge is needed on physical nodes; use the GreTap directly - gt = GreTap( - node=None, - name=None, - session=self.session, - remoteip=remoteip, - key=key, - ) - else: - gt = self.session.create_node( - cls=GreTapBridge, - _id=_id, - policy="ACCEPT", - remoteip=remoteip, - key=key, - ) - gt.localnum = localnum - gt.remotenum = remotenum - self.tunnels[key] = gt - - def addnettunnels(self): - """ - Add GreTaps between network devices on different machines. - The GreTapBridge is not used since that would add an extra bridge. - """ - logging.debug("adding network tunnels for nodes: %s", self.network_nodes) - for n in self.network_nodes: - self.addnettunnel(n) - - def addnettunnel(self, node_id): - """ - Add network tunnel between node and broker. - - :param int node_id: node id of network to add tunnel to - :return: list of gre taps - :rtype: list - :raises core.CoreError: when node to add net tunnel to does not exist - """ - net = self.session.get_node(node_id) - logging.debug("adding net tunnel for: id(%s) %s", node_id, net.name) - - # add other nets here that do not require tunnels - if isinstance(net, EmaneNet): - logging.debug("emane network does not require a tunnel") - return None - - server_interface = getattr(net, "serverintf", None) - if isinstance(net, CtrlNet) and server_interface is not None: - logging.debug( - "control networks with server interfaces do not need a tunnel" - ) - return None - - servers = self.getserversbynode(node_id) - if len(servers) < 2: - logging.debug("not enough servers to create a tunnel for node: %s", node_id) - return None - - hosts = [] - for server in servers: - if server.host is None: - continue - logging.debug("adding server host for net tunnel: %s", server.host) - hosts.append(server.host) - - if len(hosts) == 0: - for session_client in self.session_clients: - # get IP address from API message sender (master) - if session_client.client_address != "": - address = session_client.client_address[0] - logging.debug("adding session_client host: %s", address) - hosts.append(address) - - r = [] - for host in hosts: - if self.myip: - # we are the remote emulation server - myip = self.myip - else: - # we are the session master - myip = host - key = self.tunnelkey(node_id, IpAddress.to_int(myip)) - if key in self.tunnels.keys(): - logging.debug( - "tunnel already exists, returning existing tunnel: %s", key - ) - gt = self.tunnels[key] - r.append(gt) - continue - logging.info( - "adding tunnel for net %s to %s with key %s", node_id, host, key - ) - gt = GreTap( - node=None, name=None, session=self.session, remoteip=host, key=key - ) - self.tunnels[key] = gt - r.append(gt) - # attaching to net will later allow gt to be destroyed - # during net.shutdown() - net.attach(gt) - - return r - - def deltunnel(self, n1num, n2num): - """ - Delete tunnel between nodes. - - :param int n1num: node one id - :param int n2num: node two id - :return: nothing - """ - key = self.tunnelkey(n1num, n2num) - try: - logging.info( - "deleting tunnel between %s - %s with key: %s", n1num, n2num, key - ) - gt = self.tunnels.pop(key) - except KeyError: - gt = None - if gt: - self.session.delete_node(gt.id) - del gt - - def gettunnel(self, n1num, n2num): - """ - Return the GreTap between two nodes if it exists. - - :param int n1num: node one id - :param int n2num: node two id - :return: gre tap between nodes or none - """ - key = self.tunnelkey(n1num, n2num) - logging.debug("checking for tunnel(%s) in: %s", key, self.tunnels.keys()) - if key in self.tunnels.keys(): - return self.tunnels[key] - else: - return None - - def addnodemap(self, server, nodenum): - """ - Record a node number to emulation server mapping. - - :param CoreDistributedServer server: core server to associate node with - :param int nodenum: node id - :return: nothing - """ - with self.nodemap_lock: - if nodenum in self.nodemap: - if server in self.nodemap[nodenum]: - return - self.nodemap[nodenum].add(server) - else: - self.nodemap[nodenum] = {server} - - if server in self.nodecounts: - self.nodecounts[server] += 1 - else: - self.nodecounts[server] = 1 - - def delnodemap(self, server, nodenum): - """ - Remove a node number to emulation server mapping. - Return the number of nodes left on this server. - - :param CoreDistributedServer server: server to remove from node map - :param int nodenum: node id - :return: number of nodes left on server - :rtype: int - """ - count = None - with self.nodemap_lock: - if nodenum not in self.nodemap: - return count - - self.nodemap[nodenum].remove(server) - if server in self.nodecounts: - count = self.nodecounts[server] - count -= 1 - self.nodecounts[server] = count - - return count - - def getserversbynode(self, nodenum): - """ - Retrieve a set of emulation servers given a node number. - - :param int nodenum: node id - :return: core server associated with node - :rtype: set - """ - with self.nodemap_lock: - if nodenum not in self.nodemap: - return set() - return self.nodemap[nodenum] - - def addnet(self, nodenum): - """ - Add a node number to the list of link-layer nodes. - - :param int nodenum: node id to add - :return: nothing - """ - logging.debug("adding net to broker: %s", nodenum) - self.network_nodes.add(nodenum) - logging.debug("broker network nodes: %s", self.network_nodes) - - def addphys(self, nodenum): - """ - Add a node number to the list of physical nodes. - - :param int nodenum: node id to add - :return: nothing - """ - self.physical_nodes.add(nodenum) - - def handle_message(self, message): - """ - Handle an API message. Determine whether this needs to be handled - by the local server or forwarded on to another one. - Returns True when message does not need to be handled locally, - and performs forwarding if required. - Returning False indicates this message should be handled locally. - - :param core.api.coreapi.CoreMessage message: message to handle - :return: true or false for handling locally - :rtype: bool - """ - servers = set() - handle_locally = False - # Do not forward messages when in definition state - # (for e.g. configuring services) - if self.session.state == EventTypes.DEFINITION_STATE.value: - return False - - # Decide whether message should be handled locally or forwarded, or both - if message.message_type == MessageTypes.NODE.value: - handle_locally, servers = self.handlenodemsg(message) - elif message.message_type == MessageTypes.EVENT.value: - # broadcast events everywhere - servers = self.getservers() - elif message.message_type == MessageTypes.CONFIG.value: - # broadcast location and services configuration everywhere - confobj = message.get_tlv(ConfigTlvs.OBJECT.value) - if ( - confobj == "location" - or confobj == "services" - or confobj == "session" - or confobj == "all" - ): - servers = self.getservers() - elif message.message_type == MessageTypes.FILE.value: - # broadcast hook scripts and custom service files everywhere - filetype = message.get_tlv(FileTlvs.TYPE.value) - if filetype is not None and ( - filetype[:5] == "hook:" or filetype[:8] == "service:" - ): - servers = self.getservers() - if message.message_type == MessageTypes.LINK.value: - # prepare a server list from two node numbers in link message - handle_locally, servers, message = self.handlelinkmsg(message) - elif len(servers) == 0: - # check for servers based on node numbers in all messages but link - nn = message.node_numbers() - if len(nn) == 0: - return False - servers = self.getserversbynode(nn[0]) - - # allow other handlers to process this message (this is used - # by e.g. EMANE to use the link add message to keep counts of - # interfaces on other servers) - for handler in self.handlers: - handler(message) - - # perform any message forwarding - handle_locally |= self.forwardmsg(message, servers) - return not handle_locally - - def setupserver(self, servername): - """ - Send the appropriate API messages for configuring the specified emulation server. - - :param str servername: name of server to configure - :return: nothing - """ - server = self.getserverbyname(servername) - if server is None: - logging.warning("ignoring unknown server: %s", servername) - return - - if server.sock is None or server.host is None or server.port is None: - logging.info("ignoring disconnected server: %s", servername) - return - - # communicate this session"s current state to the server - tlvdata = coreapi.CoreEventTlv.pack(EventTlvs.TYPE.value, self.session.state) - msg = coreapi.CoreEventMessage.pack(0, tlvdata) - server.sock.send(msg) - - # send a Configuration message for the broker object and inform the - # server of its local name - tlvdata = b"" - tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.OBJECT.value, "broker") - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.TYPE.value, ConfigFlags.UPDATE.value - ) - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.DATA_TYPES.value, (ConfigDataTypes.STRING.value,) - ) - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.VALUES.value, - "%s:%s:%s" % (server.name, server.host, server.port), - ) - tlvdata += coreapi.CoreConfigTlv.pack( - ConfigTlvs.SESSION.value, "%s" % self.session.id - ) - msg = coreapi.CoreConfMessage.pack(0, tlvdata) - server.sock.send(msg) - - @staticmethod - def fixupremotetty(msghdr, msgdata, host): - """ - When an interactive TTY request comes from the GUI, snoop the reply - and add an SSH command to the appropriate remote server. - - :param msghdr: message header - :param msgdata: message data - :param str host: host address - :return: packed core execute tlv data - """ - msgtype, msgflags, _msglen = coreapi.CoreMessage.unpack_header(msghdr) - msgcls = coreapi.CLASS_MAP[msgtype] - msg = msgcls(msgflags, msghdr, msgdata) - - nodenum = msg.get_tlv(ExecuteTlvs.NODE.value) - execnum = msg.get_tlv(ExecuteTlvs.NUMBER.value) - cmd = msg.get_tlv(ExecuteTlvs.COMMAND.value) - res = msg.get_tlv(ExecuteTlvs.RESULT.value) - - tlvdata = b"" - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.NODE.value, nodenum) - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.NUMBER.value, execnum) - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.COMMAND.value, cmd) - res = "ssh -X -f " + host + " xterm -e " + res - tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.RESULT.value, res) - - return coreapi.CoreExecMessage.pack(msgflags, tlvdata) - - def handlenodemsg(self, message): - """ - Determine and return the servers to which this node message should - be forwarded. Also keep track of link-layer nodes and the mapping of - nodes to servers. - - :param core.api.coreapi.CoreMessage message: message to handle - :return: boolean for handling locally and set of servers - :rtype: tuple - """ - servers = set() - handle_locally = False - serverfiletxt = None - - # snoop Node Message for emulation server TLV and record mapping - n = message.tlv_data[NodeTlvs.NUMBER.value] - - # replicate link-layer nodes on all servers - nodetype = message.get_tlv(NodeTlvs.TYPE.value) - if nodetype is not None: - try: - nodetype = NodeTypes(nodetype) - nodecls = self.session.get_node_class(nodetype) - except KeyError: - logging.warning("broker invalid node type %s", nodetype) - return handle_locally, servers - if nodecls is None: - logging.warning("broker unimplemented node type %s", nodetype) - return handle_locally, servers - if ( - issubclass(nodecls, CoreNetworkBase) - and nodetype != NodeTypes.WIRELESS_LAN.value - ): - # network node replicated on all servers; could be optimized - # don"t replicate WLANs, because ebtables rules won"t work - servers = self.getservers() - handle_locally = True - self.addnet(n) - for server in servers: - self.addnodemap(server, n) - # do not record server name for networks since network - # nodes are replicated across all server - return handle_locally, servers - elif issubclass(nodecls, CoreNodeBase): - name = message.get_tlv(NodeTlvs.NAME.value) - if name: - serverfiletxt = "%s %s %s" % (n, name, nodecls) - if issubclass(nodecls, PhysicalNode): - # remember physical nodes - self.addphys(n) - - # emulation server TLV specifies server - servername = message.get_tlv(NodeTlvs.EMULATION_SERVER.value) - server = self.getserverbyname(servername) - if server is not None: - self.addnodemap(server, n) - if server not in servers: - servers.add(server) - if serverfiletxt and self.session.master: - self.writenodeserver(serverfiletxt, server) - - # hook to update coordinates of physical nodes - if n in self.physical_nodes: - self.session.mobility.physnodeupdateposition(message) - - return handle_locally, servers - - def handlelinkmsg(self, message): - """ - Determine and return the servers to which this link message should - be forwarded. Also build tunnels between different servers or add - opaque data to the link message before forwarding. - - :param core.api.coreapi.CoreMessage message: message to handle - :return: boolean to handle locally, a set of server, and message - :rtype: tuple - """ - servers = set() - handle_locally = False - - # determine link message destination using non-network nodes - nn = message.node_numbers() - logging.debug( - "checking link nodes (%s) with network nodes (%s)", nn, self.network_nodes - ) - if nn[0] in self.network_nodes: - if nn[1] in self.network_nodes: - # two network nodes linked together - prevent loops caused by - # the automatic tunnelling - handle_locally = True - else: - servers = self.getserversbynode(nn[1]) - elif nn[1] in self.network_nodes: - servers = self.getserversbynode(nn[0]) - else: - logging.debug("link nodes are not network nodes") - servers1 = self.getserversbynode(nn[0]) - logging.debug("servers for node(%s): %s", nn[0], servers1) - servers2 = self.getserversbynode(nn[1]) - logging.debug("servers for node(%s): %s", nn[1], servers2) - # nodes are on two different servers, build tunnels as needed - if servers1 != servers2: - localn = None - if len(servers1) == 0 or len(servers2) == 0: - handle_locally = True - servers = servers1.union(servers2) - host = None - # get the IP of remote server and decide which node number - # is for a local node - for server in servers: - host = server.host - if host is None: - # server is local - handle_locally = True - if server in servers1: - localn = nn[0] - else: - localn = nn[1] - if handle_locally and localn is None: - # having no local node at this point indicates local node is - # the one with the empty server set - if len(servers1) == 0: - localn = nn[0] - elif len(servers2) == 0: - localn = nn[1] - if host is None: - host = self.getlinkendpoint(message, localn == nn[0]) - - logging.debug( - "handle locally(%s) and local node(%s)", handle_locally, localn - ) - if localn is None: - message = self.addlinkendpoints(message, servers1, servers2) - elif message.flags & MessageFlags.ADD.value: - self.addtunnel(host, nn[0], nn[1], localn) - elif message.flags & MessageFlags.DELETE.value: - self.deltunnel(nn[0], nn[1]) - handle_locally = False - else: - servers = servers1.union(servers2) - - return handle_locally, servers, message - - def addlinkendpoints(self, message, servers1, servers2): - """ - For a link message that is not handled locally, inform the remote - servers of the IP addresses used as tunnel endpoints by adding - opaque data to the link message. - - :param core.api.coreapi.CoreMessage message: message to link end points - :param servers1: - :param servers2: - :return: core link message - :rtype: coreapi.CoreLinkMessage - """ - ip1 = "" - for server in servers1: - if server.host is not None: - ip1 = server.host - break - ip2 = "" - for server in servers2: - if server.host is not None: - ip2 = server.host - break - tlvdata = message.raw_message[coreapi.CoreMessage.header_len :] - tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.OPAQUE.value, "%s:%s" % (ip1, ip2)) - newraw = coreapi.CoreLinkMessage.pack(message.flags, tlvdata) - msghdr = newraw[: coreapi.CoreMessage.header_len] - return coreapi.CoreLinkMessage(message.flags, msghdr, tlvdata) - - def getlinkendpoint(self, msg, first_is_local): - """ - A link message between two different servers has been received, - and we need to determine the tunnel endpoint. First look for - opaque data in the link message, otherwise use the IP of the message - sender (the master server). - - :param core.api.tlv.coreapi.CoreLinkMessage msg: link message - :param bool first_is_local: is first local - :return: host address - :rtype: str - """ - host = None - opaque = msg.get_tlv(LinkTlvs.OPAQUE.value) - if opaque is not None: - if first_is_local: - host = opaque.split(":")[1] - else: - host = opaque.split(":")[0] - if host == "": - host = None - - if host is None: - for session_client in self.session_clients: - # get IP address from API message sender (master) - if session_client.client_address != "": - host = session_client.client_address[0] - break - - return host - - def handlerawmsg(self, msg): - """ - Helper to invoke message handler, using raw (packed) message bytes. - - :param msg: raw message butes - :return: should handle locally or not - :rtype: bool - """ - hdr = msg[: coreapi.CoreMessage.header_len] - msgtype, flags, _msglen = coreapi.CoreMessage.unpack_header(hdr) - msgcls = coreapi.CLASS_MAP[msgtype] - return self.handle_message( - msgcls(flags, hdr, msg[coreapi.CoreMessage.header_len :]) - ) - - def forwardmsg(self, message, servers): - """ - Forward API message to all given servers. - - Return True if an empty host/port is encountered, indicating - the message should be handled locally. - - :param core.api.coreapi.CoreMessage message: message to forward - :param list servers: server to forward message to - :return: handle locally value - :rtype: bool - """ - handle_locally = len(servers) == 0 - for server in servers: - if server.host is None and server.port is None: - # local emulation server, handle this locally - handle_locally = True - elif server.sock is None: - logging.info( - "server %s @ %s:%s is disconnected", - server.name, - server.host, - server.port, - ) - else: - logging.info( - "forwarding message to server(%s): %s:%s", - server.name, - server.host, - server.port, - ) - logging.debug("message being forwarded:\n%s", message) - server.sock.send(message.raw_message) - return handle_locally - - def writeservers(self): - """ - Write the server list to a text file in the session directory upon - startup: /tmp/pycore.nnnnn/servers - - :return: nothing - """ - servers = self.getservers() - filename = os.path.join(self.session.session_dir, "servers") - master = self.session_id_master - if master is None: - master = self.session.id - try: - with open(filename, "w") as f: - f.write("master=%s\n" % master) - for server in servers: - if server.name == "localhost": - continue - - lhost, lport = None, None - if server.sock: - lhost, lport = server.sock.getsockname() - f.write( - "%s %s %s %s %s\n" - % (server.name, server.host, server.port, lhost, lport) - ) - except IOError: - logging.exception("error writing server list to the file: %s", filename) - - def writenodeserver(self, nodestr, server): - """ - Creates a /tmp/pycore.nnnnn/nX.conf/server file having the node - and server info. This may be used by scripts for accessing nodes on - other machines, much like local nodes may be accessed via the - VnodeClient class. - - :param str nodestr: node string - :param CoreDistributedServer server: core server - :return: nothing - """ - serverstr = "%s %s %s" % (server.name, server.host, server.port) - name = nodestr.split()[1] - dirname = os.path.join(self.session.session_dir, name + ".conf") - filename = os.path.join(dirname, "server") - try: - os.makedirs(dirname) - except OSError: - # directory may already exist from previous distributed run - logging.exception("error creating directory: %s", dirname) - - try: - with open(filename, "w") as f: - f.write("%s\n%s\n" % (serverstr, nodestr)) - except IOError: - logging.exception( - "error writing server file %s for node %s", filename, name - ) - - def local_instantiation_complete(self): - """ - Set the local server"s instantiation-complete status to True. - - :return: nothing - """ - # TODO: do we really want to allow a localhost to not exist? - with self.servers_lock: - server = self.servers.get("localhost") - if server is not None: - server.instantiation_complete = True - - # broadcast out instantiate complete - tlvdata = b"" - tlvdata += coreapi.CoreEventTlv.pack( - EventTlvs.TYPE.value, EventTypes.INSTANTIATION_COMPLETE.value - ) - message = coreapi.CoreEventMessage.pack(0, tlvdata) - for session_client in self.session_clients: - session_client.sendall(message) - - def instantiation_complete(self): - """ - Return True if all servers have completed instantiation, False - otherwise. - - :return: have all server completed instantiation - :rtype: bool - """ - with self.servers_lock: - for name in self.servers: - server = self.servers[name] - if not server.instantiation_complete: - return False - return True From 7afaff8cbb28c9d7eb7b59bee780ee4af26b3d41 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 08:41:48 -0700 Subject: [PATCH 084/113] updated requirements and setup.py to include fabric/invoke --- daemon/requirements.txt | 2 ++ daemon/setup.py.in | 2 ++ 2 files changed, 4 insertions(+) diff --git a/daemon/requirements.txt b/daemon/requirements.txt index a32f12de..d9029923 100644 --- a/daemon/requirements.txt +++ b/daemon/requirements.txt @@ -1,7 +1,9 @@ configparser==4.0.2 +fabric==2.5.0 future==0.17.1 grpcio==1.23.0 grpcio-tools==1.21.1 +invoke==1.3.0 lxml==4.4.1 protobuf==3.9.1 six==1.12.0 diff --git a/daemon/setup.py.in b/daemon/setup.py.in index 49af9cfe..3a451fb4 100644 --- a/daemon/setup.py.in +++ b/daemon/setup.py.in @@ -35,8 +35,10 @@ setup( packages=find_packages(), install_requires=[ "configparser", + "fabric", "future", "grpcio", + "invoke", "lxml", "protobuf", ], From b7dd8ddb6670933bd9f3374ffc2a3353ca8602aa Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 09:09:03 -0700 Subject: [PATCH 085/113] fix for docker/lxd based nodes to use remote servers and example for lxd --- daemon/core/emulator/session.py | 1 + daemon/core/nodes/docker.py | 18 +++++++- daemon/core/nodes/lxd.py | 7 +++- daemon/examples/python/distributed_lxd.py | 51 +++++++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 daemon/examples/python/distributed_lxd.py diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index e864ec8b..5e3997eb 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -782,6 +782,7 @@ class Session(object): name=name, start=start, image=node_options.image, + server=server, ) else: node = self.create_node( diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index b91e987e..17d7578a 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -89,7 +89,17 @@ class DockerClient(object): class DockerNode(CoreNode): apitype = NodeTypes.DOCKER.value - def __init__(self, session, _id=None, name=None, nodedir=None, bootsh="boot.sh", start=True, image=None): + def __init__( + self, + session, + _id=None, + name=None, + nodedir=None, + bootsh="boot.sh", + start=True, + server=None, + image=None + ): """ Create a DockerNode instance. @@ -99,12 +109,16 @@ class DockerNode(CoreNode): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param str image: image to start container with """ if image is None: image = "ubuntu" self.image = image - super(DockerNode, self).__init__(session, _id, name, nodedir, bootsh, start) + super(DockerNode, self).__init__( + session, _id, name, nodedir, bootsh, start, server + ) def create_node_net_client(self, use_ovs): """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index eef3dc8f..b11086e7 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -74,6 +74,7 @@ class LxcNode(CoreNode): nodedir=None, bootsh="boot.sh", start=True, + server=None, image=None, ): """ @@ -85,12 +86,16 @@ class LxcNode(CoreNode): :param str nodedir: node directory :param str bootsh: boot shell to use :param bool start: start flag + :param core.emulator.distributed.DistributedServer server: remote server node + will run on, default is None for localhost :param str image: image to start container with """ if image is None: image = "ubuntu" self.image = image - super(LxcNode, self).__init__(session, _id, name, nodedir, bootsh, start) + super(LxcNode, self).__init__( + session, _id, name, nodedir, bootsh, start, server + ) def alive(self): """ diff --git a/daemon/examples/python/distributed_lxd.py b/daemon/examples/python/distributed_lxd.py new file mode 100644 index 00000000..8bafeb7a --- /dev/null +++ b/daemon/examples/python/distributed_lxd.py @@ -0,0 +1,51 @@ +import logging +import pdb +import sys + +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes + + +def main(): + address = sys.argv[1] + remote = sys.argv[2] + + # ip generator for example + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + + # create emulator instance for creating sessions and utility methods + coreemu = CoreEmu({"distributed_address": address}) + session = coreemu.create_session() + + # initialize distributed + server_name = "core2" + session.add_distributed(server_name, remote) + + # must be in configuration state for nodes to start, when using "node_add" below + session.set_state(EventTypes.CONFIGURATION_STATE) + + # create local node, switch, and remote nodes + options = NodeOptions(image="ubuntu:18.04") + node_one = session.add_node(_type=NodeTypes.LXC, node_options=options) + options.emulation_server = server_name + node_two = session.add_node(_type=NodeTypes.LXC, node_options=options) + + # create node interfaces and link + interface_one = prefixes.create_interface(node_one) + interface_two = prefixes.create_interface(node_two) + session.add_link(node_one.id, node_two.id, interface_one, interface_two) + + # instantiate session + session.instantiate() + + # pause script for verification + pdb.set_trace() + + # shutdown session + coreemu.shutdown() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From 0ef06a0167a43092ae25090b871eb1ad9f162a31 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 09:32:32 -0700 Subject: [PATCH 086/113] added docs for session distributed commands --- daemon/core/emulator/session.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 5e3997eb..fe371c44 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -159,12 +159,25 @@ class Session(object): } def add_distributed(self, name, host): + """ + Add distributed server configuration. + + :param str name: distributed server name + :param str host: distributed server host address + :return: nothing + """ server = DistributedServer(name, host) self.servers[name] = server cmd = "mkdir -p %s" % self.session_dir server.remote_cmd(cmd) def shutdown_distributed(self): + """ + Shutdown logic for dealing with distributed tunnels and server session + directories. + + :return: nothing + """ # shutdown all tunnels for key in self.tunnels: tunnels = self.tunnels[key] @@ -180,7 +193,12 @@ class Session(object): # clear tunnels self.tunnels.clear() - def initialize_distributed(self): + def start_distributed(self): + """ + Start distributed network tunnels. + + :return: nothing + """ for node_id in self.nodes: node = self.nodes[node_id] @@ -195,6 +213,16 @@ class Session(object): self.create_gre_tunnel(node, server) def create_gre_tunnel(self, node, server): + """ + Create gre tunnel using a pair of gre taps between the local and remote server. + + + :param core.nodes.network.CoreNetwork node: node to create gre tunnel for + :param core.emulator.distributed.DistributedServer server: server to create + tunnel for + :return: local and remote gre taps created for tunnel + :rtype: tuple + """ host = server.host key = self.tunnelkey(node.id, IpAddress.to_int(host)) tunnel = self.tunnels.get(key) @@ -1566,7 +1594,7 @@ class Session(object): self.add_remove_control_interface(node=None, remove=False) # initialize distributed tunnels - self.initialize_distributed() + self.start_distributed() # instantiate will be invoked again upon Emane configure if self.emane.startup() == self.emane.NOT_READY: From e94a6d1afa7ec4014faa2287f79f0f2437374795 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 11:10:59 -0700 Subject: [PATCH 087/113] separated distributed session logic into its own class to help reduce session.py size as it is already too big --- daemon/core/api/tlv/corehandlers.py | 4 +- daemon/core/emane/emanemanager.py | 22 +-- daemon/core/emulator/distributed.py | 152 ++++++++++++++++++ daemon/core/emulator/session.py | 139 ++-------------- daemon/core/nodes/network.py | 9 +- daemon/core/nodes/physical.py | 2 +- daemon/core/xml/emanexml.py | 24 +-- daemon/examples/python/distributed.py | 2 +- daemon/examples/python/distributed_emane.py | 2 +- daemon/examples/python/distributed_lxd.py | 2 +- daemon/examples/python/distributed_ptp.py | 2 +- .../examples/python/distributed_switches.py | 2 +- daemon/examples/python/distributed_wlan.py | 2 +- daemon/tests/test_gui.py | 4 +- 14 files changed, 196 insertions(+), 172 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 60ddfcca..8f995920 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1192,9 +1192,9 @@ class CoreHandler(socketserver.BaseRequestHandler): for server in server_list: server_items = server.split(":") name, host, _ = server_items[:3] - self.session.add_distributed(name, host) + self.session.distributed.add_server(name, host) elif message_type == ConfigFlags.RESET: - self.session.shutdown_distributed() + self.session.distributed.shutdown() def handle_config_services(self, message_type, config_data): replies = [] diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index e4208189..91553b5a 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -142,9 +142,7 @@ class EmaneManager(ModelManager): args = "emane --version" emane_version = utils.check_cmd(args) logging.info("using EMANE: %s", emane_version) - for name in self.session.servers: - server = self.session.servers[name] - server.remote_cmd(args) + self.session.distributed.execute(lambda x: x.remote_cmd(args)) # load default emane models self.load_models(EMANE_MODELS) @@ -518,11 +516,11 @@ class EmaneManager(ModelManager): dev = self.get_config("eventservicedevice") emanexml.create_event_service_xml(group, port, dev, self.session.session_dir) - for name in self.session.servers: - conn = self.session.servers[name] - emanexml.create_event_service_xml( - group, port, dev, self.session.session_dir, conn + self.session.distributed.execute( + lambda x: emanexml.create_event_service_xml( + group, port, dev, self.session.session_dir, x ) + ) def startdaemons(self): """ @@ -598,9 +596,7 @@ class EmaneManager(ModelManager): emanecmd += " -f %s" % os.path.join(path, "emane.log") emanecmd += " %s" % os.path.join(path, "platform.xml") utils.check_cmd(emanecmd, cwd=path) - for name in self.session.servers: - server = self.session.servers[name] - server.remote_cmd(emanecmd, cwd=path) + self.session.distributed.execute(lambda x: x.remote_cmd(emanecmd, cwd=path)) logging.info("host emane daemon running: %s", emanecmd) def stopdaemons(self): @@ -625,10 +621,8 @@ class EmaneManager(ModelManager): try: utils.check_cmd(kill_emaned) utils.check_cmd(kill_transortd) - for name in self.session.servers: - server = self.session[name] - server.remote_cmd(kill_emaned) - server.remote_cmd(kill_transortd) + self.session.distributed.execute(lambda x: x.remote_cmd(kill_emaned)) + self.session.distributed.execute(lambda x: x.remote_cmd(kill_transortd)) except CoreCommandError: logging.exception("error shutting down emane daemons") diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 2df33541..c6218441 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -5,12 +5,17 @@ Defines distributed server functionality. import logging import os import threading +from collections import OrderedDict from tempfile import NamedTemporaryFile from fabric import Connection from invoke import UnexpectedExit +from core import utils from core.errors import CoreCommandError +from core.nodes.interface import GreTap +from core.nodes.ipaddress import IpAddress +from core.nodes.network import CoreNetwork, CtrlNet LOCK = threading.Lock() @@ -93,3 +98,150 @@ class DistributedServer(object): temp.close() self.conn.put(temp.name, destination) os.unlink(temp.name) + + +class DistributedController(object): + def __init__(self, session): + """ + Create + + :param session: + """ + self.session = session + self.servers = OrderedDict() + self.tunnels = {} + self.address = self.session.options.get_config( + "distributed_address", default=None + ) + + def add_server(self, name, host): + """ + Add distributed server configuration. + + :param str name: distributed server name + :param str host: distributed server host address + :return: nothing + """ + server = DistributedServer(name, host) + self.servers[name] = server + cmd = "mkdir -p %s" % self.session.session_dir + server.remote_cmd(cmd) + + def execute(self, func): + """ + Convenience for executing logic against all distributed servers. + + :param func: function to run, that takes a DistributedServer as a parameter + :return: nothing + """ + for name in self.servers: + server = self.servers[name] + func(server) + + def shutdown(self): + """ + Shutdown logic for dealing with distributed tunnels and server session + directories. + + :return: nothing + """ + # shutdown all tunnels + for key in self.tunnels: + tunnels = self.tunnels[key] + for tunnel in tunnels: + tunnel.shutdown() + + # remove all remote session directories + for name in self.servers: + server = self.servers[name] + cmd = "rm -rf %s" % self.session.session_dir + server.remote_cmd(cmd) + + # clear tunnels + self.tunnels.clear() + + def start(self): + """ + Start distributed network tunnels. + + :return: nothing + """ + for node_id in self.session.nodes: + node = self.session.nodes[node_id] + + if not isinstance(node, CoreNetwork): + continue + + if isinstance(node, CtrlNet) and node.serverintf is not None: + continue + + for name in self.servers: + server = self.servers[name] + self.create_gre_tunnel(node, server) + + def create_gre_tunnel(self, node, server): + """ + Create gre tunnel using a pair of gre taps between the local and remote server. + + + :param core.nodes.network.CoreNetwork node: node to create gre tunnel for + :param core.emulator.distributed.DistributedServer server: server to create + tunnel for + :return: local and remote gre taps created for tunnel + :rtype: tuple + """ + host = server.host + key = self.tunnel_key(node.id, IpAddress.to_int(host)) + tunnel = self.tunnels.get(key) + if tunnel is not None: + return tunnel + + # local to server + logging.info( + "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key + ) + local_tap = GreTap(session=self.session, remoteip=host, key=key) + local_tap.net_client.create_interface(node.brname, local_tap.localname) + + # server to local + logging.info( + "remote tunnel node(%s) to local(%s) key(%s)", node.name, self.address, key + ) + remote_tap = GreTap( + session=self.session, remoteip=self.address, key=key, server=server + ) + remote_tap.net_client.create_interface(node.brname, remote_tap.localname) + + # save tunnels for shutdown + tunnel = (local_tap, remote_tap) + self.tunnels[key] = tunnel + return tunnel + + def tunnel_key(self, n1_id, n2_id): + """ + Compute a 32-bit key used to uniquely identify a GRE tunnel. + The hash(n1num), hash(n2num) values are used, so node numbers may be + None or string values (used for e.g. "ctrlnet"). + + :param int n1_id: node one id + :param int n2_id: node two id + :return: tunnel key for the node pair + :rtype: int + """ + logging.debug("creating tunnel key for: %s, %s", n1_id, n2_id) + key = ( + (self.session.id << 16) ^ utils.hashkey(n1_id) ^ (utils.hashkey(n2_id) << 8) + ) + return key & 0xFFFFFFFF + + def get_tunnel(self, n1_id, n2_id): + """ + Return the GreTap between two nodes if it exists. + + :param int n1_id: node one id + :param int n2_id: node two id + :return: gre tap between nodes or None + """ + key = self.tunnel_key(n1_id, n2_id) + logging.debug("checking for tunnel key(%s) in: %s", key, self.tunnels) + return self.tunnels.get(key) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index fe371c44..d962da28 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -18,7 +18,7 @@ from core import constants, utils from core.emane.emanemanager import EmaneManager from core.emane.nodes import EmaneNet from core.emulator.data import EventData, ExceptionData, NodeData -from core.emulator.distributed import DistributedServer +from core.emulator.distributed import DistributedController from core.emulator.emudata import ( IdGen, LinkOptions, @@ -34,11 +34,9 @@ from core.location.event import EventLoop from core.location.mobility import MobilityManager from core.nodes.base import CoreNetworkBase, CoreNode, CoreNodeBase from core.nodes.docker import DockerNode -from core.nodes.interface import GreTap -from core.nodes.ipaddress import IpAddress, MacAddress +from core.nodes.ipaddress import MacAddress from core.nodes.lxd import LxcNode from core.nodes.network import ( - CoreNetwork, CtrlNet, GreTapBridge, HubNode, @@ -137,10 +135,8 @@ class Session(object): self.options.set_config(key, value) self.metadata = SessionMetaData() - # distributed servers - self.servers = {} - self.tunnels = {} - self.address = self.options.get_config("distributed_address", default=None) + # distributed support and logic + self.distributed = DistributedController(self) # initialize session feature helpers self.location = CoreLocation() @@ -158,123 +154,6 @@ class Session(object): "host": ("DefaultRoute", "SSH"), } - def add_distributed(self, name, host): - """ - Add distributed server configuration. - - :param str name: distributed server name - :param str host: distributed server host address - :return: nothing - """ - server = DistributedServer(name, host) - self.servers[name] = server - cmd = "mkdir -p %s" % self.session_dir - server.remote_cmd(cmd) - - def shutdown_distributed(self): - """ - Shutdown logic for dealing with distributed tunnels and server session - directories. - - :return: nothing - """ - # shutdown all tunnels - for key in self.tunnels: - tunnels = self.tunnels[key] - for tunnel in tunnels: - tunnel.shutdown() - - # remove all remote session directories - for name in self.servers: - server = self.servers[name] - cmd = "rm -rf %s" % self.session_dir - server.remote_cmd(cmd) - - # clear tunnels - self.tunnels.clear() - - def start_distributed(self): - """ - Start distributed network tunnels. - - :return: nothing - """ - for node_id in self.nodes: - node = self.nodes[node_id] - - if not isinstance(node, CoreNetwork): - continue - - if isinstance(node, CtrlNet) and node.serverintf is not None: - continue - - for name in self.servers: - server = self.servers[name] - self.create_gre_tunnel(node, server) - - def create_gre_tunnel(self, node, server): - """ - Create gre tunnel using a pair of gre taps between the local and remote server. - - - :param core.nodes.network.CoreNetwork node: node to create gre tunnel for - :param core.emulator.distributed.DistributedServer server: server to create - tunnel for - :return: local and remote gre taps created for tunnel - :rtype: tuple - """ - host = server.host - key = self.tunnelkey(node.id, IpAddress.to_int(host)) - tunnel = self.tunnels.get(key) - if tunnel is not None: - return tunnel - - # local to server - logging.info( - "local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key - ) - local_tap = GreTap(session=self, remoteip=host, key=key) - local_tap.net_client.create_interface(node.brname, local_tap.localname) - - # server to local - logging.info( - "remote tunnel node(%s) to local(%s) key(%s)", node.name, self.address, key - ) - remote_tap = GreTap(session=self, remoteip=self.address, key=key, server=server) - remote_tap.net_client.create_interface(node.brname, remote_tap.localname) - - # save tunnels for shutdown - tunnel = (local_tap, remote_tap) - self.tunnels[key] = tunnel - return tunnel - - def tunnelkey(self, n1_id, n2_id): - """ - Compute a 32-bit key used to uniquely identify a GRE tunnel. - The hash(n1num), hash(n2num) values are used, so node numbers may be - None or string values (used for e.g. "ctrlnet"). - - :param int n1_id: node one id - :param int n2_id: node two id - :return: tunnel key for the node pair - :rtype: int - """ - logging.debug("creating tunnel key for: %s, %s", n1_id, n2_id) - key = (self.id << 16) ^ utils.hashkey(n1_id) ^ (utils.hashkey(n2_id) << 8) - return key & 0xFFFFFFFF - - def gettunnel(self, n1_id, n2_id): - """ - Return the GreTap between two nodes if it exists. - - :param int n1_id: node one id - :param int n2_id: node two id - :return: gre tap between nodes or None - """ - key = self.tunnelkey(n1_id, n2_id) - logging.debug("checking for tunnel key(%s) in: %s", key, self.tunnels) - return self.tunnels.get(key) - @classmethod def get_node_class(cls, _type): """ @@ -324,7 +203,7 @@ class Session(object): node_two = self.get_node(node_two_id) # both node ids are provided - tunnel = self.gettunnel(node_one_id, node_two_id) + tunnel = self.distributed.get_tunnel(node_one_id, node_two_id) logging.debug("tunnel between nodes: %s", tunnel) if isinstance(tunnel, GreTapBridge): net_one = tunnel @@ -789,7 +668,7 @@ class Session(object): name = "%s%s" % (node_class.__name__, _id) # verify distributed server - server = self.servers.get(node_options.emulation_server) + server = self.distributed.servers.get(node_options.emulation_server) if node_options.emulation_server is not None and server is None: raise CoreError( "invalid distributed server: %s" % node_options.emulation_server @@ -1003,7 +882,7 @@ class Session(object): :return: nothing """ self.delete_nodes() - self.shutdown_distributed() + self.distributed.shutdown() self.del_hooks() self.emane.reset() @@ -1082,7 +961,7 @@ class Session(object): # remove and shutdown all nodes and tunnels self.delete_nodes() - self.shutdown_distributed() + self.distributed.shutdown() # remove this sessions working directory preserve = self.options.get_config("preservedir") == "1" @@ -1594,7 +1473,7 @@ class Session(object): self.add_remove_control_interface(node=None, remove=False) # initialize distributed tunnels - self.start_distributed() + self.distributed.start() # instantiate will be invoked again upon Emane configure if self.emane.startup() == self.emane.NOT_READY: diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 931622bb..98bec198 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -289,9 +289,7 @@ class CoreNetwork(CoreNetworkBase): """ logging.info("network node(%s) cmd", self.name) output = utils.check_cmd(args, env, cwd, wait) - for name in self.session.servers: - server = self.session.servers[name] - server.remote_cmd(args, env, cwd, wait) + self.session.distributed.execute(lambda x: x.remote_cmd(args, env, cwd, wait)) return output def startup(self): @@ -778,8 +776,9 @@ class CtrlNet(CoreNetwork): current = "%s/%s" % (address, self.prefix.prefixlen) net_client = get_net_client(use_ovs, utils.check_cmd) net_client.create_address(self.brname, current) - for name in self.session.servers: - server = self.session.servers[name] + servers = self.session.distributed.servers + for name in servers: + server = servers[name] address -= 1 current = "%s/%s" % (address, self.prefix.prefixlen) net_client = get_net_client(use_ovs, server.remote_cmd) diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 37a2eb54..0f9e0217 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -166,7 +166,7 @@ class PhysicalNode(CoreNodeBase): if self.up: # this is reached when this node is linked to a network node # tunnel to net not built yet, so build it now and adopt it - _, remote_tap = self.session.create_gre_tunnel(net, self.server) + _, remote_tap = self.session.distributed.create_gre_tunnel(net, self.server) self.adoptnetif(remote_tap, ifindex, hwaddr, addrlist) return ifindex else: diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 881ff373..41319ea4 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -314,9 +314,9 @@ def build_transport_xml(emane_manager, node, transport_type): file_name = transport_file_name(node.id, transport_type) file_path = os.path.join(emane_manager.session.session_dir, file_name) create_file(transport_element, doc_name, file_path) - for name in emane_manager.session.servers: - server = emane_manager.session.servers[name] - create_file(transport_element, doc_name, file_path, server) + emane_manager.session.distributed.execute( + lambda x: create_file(transport_element, doc_name, file_path, x) + ) def create_phy_xml(emane_model, config, file_path, server): @@ -342,9 +342,9 @@ def create_phy_xml(emane_model, config, file_path, server): create_file(phy_element, "phy", file_path, server) else: create_file(phy_element, "phy", file_path) - for name in emane_model.session.servers: - server = emane_model.session.servers[name] - create_file(phy_element, "phy", file_path, server) + emane_model.session.distributed.execute( + lambda x: create_file(phy_element, "phy", file_path, x) + ) def create_mac_xml(emane_model, config, file_path, server): @@ -372,9 +372,9 @@ def create_mac_xml(emane_model, config, file_path, server): create_file(mac_element, "mac", file_path, server) else: create_file(mac_element, "mac", file_path) - for name in emane_model.session.servers: - server = emane_model.session.servers[name] - create_file(mac_element, "mac", file_path, server) + emane_model.session.distributed.execute( + lambda x: create_file(mac_element, "mac", file_path, x) + ) def create_nem_xml( @@ -410,9 +410,9 @@ def create_nem_xml( create_file(nem_element, "nem", nem_file, server) else: create_file(nem_element, "nem", nem_file) - for name in emane_model.session.servers: - server = emane_model.session.servers[name] - create_file(nem_element, "nem", nem_file, server) + emane_model.session.distributed.execute( + lambda x: create_file(nem_element, "nem", nem_file, x) + ) def create_event_service_xml(group, port, device, file_directory, server=None): diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed.py index 8bcf2972..8eb23b2c 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed.py @@ -20,7 +20,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py index c64d1f0c..4ef50ccb 100644 --- a/daemon/examples/python/distributed_emane.py +++ b/daemon/examples/python/distributed_emane.py @@ -27,7 +27,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_lxd.py b/daemon/examples/python/distributed_lxd.py index 8bafeb7a..130942ea 100644 --- a/daemon/examples/python/distributed_lxd.py +++ b/daemon/examples/python/distributed_lxd.py @@ -20,7 +20,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index b0f27c28..62e7df64 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -20,7 +20,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py index bc13bf2c..f9b69757 100644 --- a/daemon/examples/python/distributed_switches.py +++ b/daemon/examples/python/distributed_switches.py @@ -16,7 +16,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/examples/python/distributed_wlan.py b/daemon/examples/python/distributed_wlan.py index f8af1f5f..10f25aa8 100644 --- a/daemon/examples/python/distributed_wlan.py +++ b/daemon/examples/python/distributed_wlan.py @@ -21,7 +21,7 @@ def main(): # initialize distributed server_name = "core2" - session.add_distributed(server_name, remote) + session.distributed.add_server(server_name, remote) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index 02e634be..c07e2bd3 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -763,11 +763,11 @@ class TestGui: (ConfigTlvs.VALUES, "%s:%s:%s" % (server, host, port)), ], ) - coreserver.session.add_distributed = mock.MagicMock() + coreserver.session.distributed.add_server = mock.MagicMock() coreserver.request_handler.handle_message(message) - coreserver.session.add_distributed.assert_called_once_with(server, host) + coreserver.session.distributed.add_server.assert_called_once_with(server, host) def test_config_services_request_all(self, coreserver): message = coreapi.CoreConfMessage.create( From 4746fe67ef78adde410ff00e0c07f7af63e783eb Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 11:35:48 -0700 Subject: [PATCH 088/113] added docs for distributed.py --- daemon/core/emulator/distributed.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index c6218441..83576434 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -101,6 +101,10 @@ class DistributedServer(object): class DistributedController(object): + """ + Provides logic for dealing with remote tunnels and distributed servers. + """ + def __init__(self, session): """ Create From 5d5ffb70c2be8fd4d0e20d257034cf6a37c0d616 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 13:00:20 -0700 Subject: [PATCH 089/113] update to grpc edit_node to allow editing icon and broadcasting a node update for all to listen to a change --- daemon/core/api/grpc/client.py | 5 +++-- daemon/core/api/grpc/server.py | 7 +++++-- daemon/proto/core/api/grpc/core.proto | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index 2d767993..54f77fc6 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -326,19 +326,20 @@ class CoreGrpcClient(object): request = core_pb2.GetNodeRequest(session_id=session_id, node_id=node_id) return self.stub.GetNode(request) - def edit_node(self, session_id, node_id, position): + def edit_node(self, session_id, node_id, position, icon=None): """ Edit a node, currently only changes position. :param int session_id: session id :param int node_id: node id :param core_pb2.Position position: position to set node to + :param str icon: path to icon for gui to use for node :return: response with result of success or failure :rtype: core_pb2.EditNodeResponse :raises grpc.RpcError: when session or node doesn't exist """ request = core_pb2.EditNodeRequest( - session_id=session_id, node_id=node_id, position=position + session_id=session_id, node_id=node_id, position=position, icon=icon ) return self.stub.EditNode(request) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index f61b6204..3c70ed09 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -842,8 +842,9 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): """ logging.debug("edit node: %s", request) session = self.get_session(request.session_id, context) - node_id = request.node_id + node = self.get_node(session, request.node_id, context) node_options = NodeOptions() + node_options.icon = request.icon x = request.position.x y = request.position.y node_options.set_position(x, y) @@ -853,7 +854,9 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): node_options.set_location(lat, lon, alt) result = True try: - session.update_node(node_id, node_options) + session.update_node(node.id, node_options) + node_data = node.data(0) + session.broadcast_node(node_data) except CoreError: result = False return core_pb2.EditNodeResponse(result=result) diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 68c83ad2..1e17b327 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -316,6 +316,7 @@ message EditNodeRequest { int32 session_id = 1; int32 node_id = 2; Position position = 3; + string icon = 4; } message EditNodeResponse { From 6edd6a7fdb43d424a23c6d91ba5c079fd57a63a2 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 14:52:31 -0700 Subject: [PATCH 090/113] first pass at removing all python2 specific dependencies, updating python requirements.txt/setup.py/Pipfiles/Makefiles, and removing python2 compat imports --- Makefile.am | 13 +-- configure.ac | 8 +- daemon/Makefile.am | 8 +- daemon/Pipfile.lock | 141 +++++++++++++---------------- daemon/core/api/tlv/coreapi.py | 4 +- daemon/core/api/tlv/structutils.py | 4 +- daemon/core/emane/commeffect.py | 3 +- daemon/core/location/event.py | 21 ++--- daemon/core/plugins/sdt.py | 3 +- daemon/core/services/nrl.py | 10 +- daemon/requirements.txt | 3 - daemon/setup.py.in | 6 -- ns3/Makefile.am | 8 +- 13 files changed, 89 insertions(+), 143 deletions(-) diff --git a/Makefile.am b/Makefile.am index ff31a5a4..c51c3707 100644 --- a/Makefile.am +++ b/Makefile.am @@ -44,15 +44,6 @@ DISTCLEANFILES = aclocal.m4 \ MAINTAINERCLEANFILES = .version \ .version.date - -if PYTHON3 -PYTHON_DEB_DEP = python3 >= 3.6 -PYTHON_RPM_DEP = python3 >= 3.6 -else -PYTHON_DEB_DEP = python (>= 2.7), python (<< 3.0) -PYTHON_RPM_DEP = python >= 2.7, python < 3.0 -endif - define fpm-rpm = fpm -s dir -t rpm -n core \ -m "$(PACKAGE_MAINTAINERS)" \ @@ -74,7 +65,7 @@ fpm -s dir -t rpm -n core \ -d "iproute" \ -d "libev" \ -d "net-tools" \ - -d "$(PYTHON_RPM_DEP)" \ + -d "python3 >= 3.6" \ -C $(DESTDIR) endef @@ -101,7 +92,7 @@ fpm -s dir -t deb -n core \ -d "ebtables" \ -d "iproute2" \ -d "libev4" \ - -d "$(PYTHON_DEB_DEP)" \ + -d "python3 >= 3.6" \ -C $(DESTDIR) endef diff --git a/configure.ac b/configure.ac index fec902a3..5d04356e 100644 --- a/configure.ac +++ b/configure.ac @@ -55,12 +55,6 @@ else want_python=no fi -AC_ARG_ENABLE([python3], - [AS_HELP_STRING([--enable-python3], - [sets python3 flag for building packages])], - [enable_python3=yes], [enable_python3=no]) -AM_CONDITIONAL([PYTHON3], [test "x$enable_python3" == "xyes"]) - AC_ARG_ENABLE([daemon], [AS_HELP_STRING([--enable-daemon[=ARG]], [build and install the daemon with Python modules @@ -116,7 +110,7 @@ if test "x$enable_daemon" = "xyes"; then AC_FUNC_REALLOC AC_CHECK_FUNCS([atexit dup2 gettimeofday memset socket strerror uname]) - AM_PATH_PYTHON(2.7) + AM_PATH_PYTHON(3.6) AS_IF([$PYTHON -m grpc_tools.protoc -h &> /dev/null], [], [AC_MSG_ERROR([please install python grpcio-tools])]) AC_CHECK_PROG(brctl_path, brctl, $as_dir, no, $SEARCHPATH) diff --git a/daemon/Makefile.am b/daemon/Makefile.am index a6503cc0..80483387 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -14,8 +14,6 @@ if WANT_DOCS DOCS = doc endif -PYTHONLIBDIR=$(subst site-packages,dist-packages,$(pythondir)) - SUBDIRS = proto $(DOCS) SCRIPT_FILES := $(notdir $(wildcard scripts/*)) @@ -31,7 +29,7 @@ install-exec-hook: $(PYTHON) $(SETUPPY) $(SETUPPYFLAGS) install \ --root=/$(DESTDIR) \ --prefix=$(prefix) \ - --install-lib=$(PYTHONLIBDIR) \ + --install-lib=$(pythondir) \ --single-version-externally-managed # Python package uninstall @@ -40,8 +38,8 @@ uninstall-hook: rm -rf $(DESTDIR)/$(datadir)/core rm -f $(addprefix $(DESTDIR)/$(datarootdir)/man/man1/, $(MAN_FILES)) rm -f $(addprefix $(DESTDIR)/$(bindir)/,$(SCRIPT_FILES)) - rm -rf $(DESTDIR)/$(PYTHONLIBDIR)/core-$(PACKAGE_VERSION)-py$(PYTHON_VERSION).egg-info - rm -rf $(DESTDIR)/$(PYTHONLIBDIR)/core + rm -rf $(DESTDIR)/$(pythondir)/core-$(PACKAGE_VERSION)-py$(PYTHON_VERSION).egg-info + rm -rf $(DESTDIR)/$(pythondir)/core # Python package cleanup clean-local: diff --git a/daemon/Pipfile.lock b/daemon/Pipfile.lock index 73400b8b..4bdaea3f 100644 --- a/daemon/Pipfile.lock +++ b/daemon/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d702e6eed5a1362bf261543572bbffd2e8a87140b8d8cb07b99fb0d25220a2b5" + "sha256": "6195c89ec6e2e449fcbd7f3fa41cbab79c02d952984a913e0f80114e1904bf11" }, "pipfile-spec": 6, "requires": {}, @@ -14,13 +14,6 @@ ] }, "default": { - "asn1crypto": { - "hashes": [ - "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", - "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" - ], - "version": "==1.0.1" - }, "bcrypt": { "hashes": [ "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", @@ -44,43 +37,40 @@ }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" + "sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8", + "sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d", + "sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36", + "sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45", + "sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b", + "sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2", + "sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b", + "sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5", + "sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a", + "sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4", + "sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2", + "sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5", + "sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f", + "sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa", + "sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a", + "sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c", + "sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1", + "sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24", + "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226", + "sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895", + "sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529", + "sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1", + "sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27", + "sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165", + "sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5", + "sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa", + "sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2", + "sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89", + "sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59", + "sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df", + "sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f", + "sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1" ], - "version": "==1.12.3" - }, - "configparser": { - "hashes": [ - "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", - "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df" - ], - "version": "==4.0.2" + "version": "==1.13.0" }, "core": { "editable": true, @@ -88,24 +78,29 @@ }, "cryptography": { "hashes": [ - "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", - "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", - "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", - "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", - "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", - "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", - "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", - "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", - "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", - "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", - "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", - "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", - "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", - "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", - "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", - "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" ], - "version": "==2.7" + "version": "==2.8" }, "fabric": { "hashes": [ @@ -114,12 +109,6 @@ ], "version": "==2.5.0" }, - "future": { - "hashes": [ - "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" - ], - "version": "==0.17.1" - }, "grpcio": { "hashes": [ "sha256:0302331e014fc4bac028b6ad480b33f7abfe20b9bdcca7be417124dda8f22115", @@ -282,10 +271,10 @@ }, "attrs": { "hashes": [ - "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", - "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.2.0" + "version": "==19.3.0" }, "black": { "hashes": [ @@ -531,11 +520,11 @@ }, "pytest": { "hashes": [ - "sha256:13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5", - "sha256:d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5" + "sha256:7e4800063ccfc306a53c461442526c5571e1462f61583506ce97e4da6a1d88c8", + "sha256:ca563435f4941d0cb34767301c27bc65c510cb82e90b9ecf9cb52dc2c63caaa0" ], "index": "pypi", - "version": "==5.2.0" + "version": "==5.2.1" }, "pyyaml": { "hashes": [ @@ -571,10 +560,10 @@ }, "virtualenv": { "hashes": [ - "sha256:680af46846662bb38c5504b78bad9ed9e4f3ba2d54f54ba42494fdf94337fe30", - "sha256:f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2" + "sha256:3e3597e89c73df9313f5566e8fc582bd7037938d15b05329c232ec57a11a7ad5", + "sha256:5d370508bf32e522d79096e8cbea3499d47e624ac7e11e9089f9397a0b3318df" ], - "version": "==16.7.5" + "version": "==16.7.6" }, "wcwidth": { "hashes": [ diff --git a/daemon/core/api/tlv/coreapi.py b/daemon/core/api/tlv/coreapi.py index 1e1de8be..ba737fd4 100644 --- a/daemon/core/api/tlv/coreapi.py +++ b/daemon/core/api/tlv/coreapi.py @@ -9,8 +9,6 @@ import socket import struct from enum import Enum -from past.builtins import basestring - from core.api.tlv import structutils from core.emulator.enumerations import ( ConfigTlvs, @@ -181,7 +179,7 @@ class CoreTlvDataString(CoreTlvData): :return: length of data packed and the packed data :rtype: tuple """ - if not isinstance(value, basestring): + if not isinstance(value, str): raise ValueError("value not a string: %s" % type(value)) value = value.encode("utf-8") diff --git a/daemon/core/api/tlv/structutils.py b/daemon/core/api/tlv/structutils.py index 28e22a27..41358848 100644 --- a/daemon/core/api/tlv/structutils.py +++ b/daemon/core/api/tlv/structutils.py @@ -4,8 +4,6 @@ Utilities for working with python struct data. import logging -from past.builtins import basestring - def pack_values(clazz, packers): """ @@ -31,7 +29,7 @@ def pack_values(clazz, packers): # only pack actual values and avoid packing empty strings # protobuf defaults to empty strings and does no imply a value to set - if value is None or (isinstance(value, basestring) and not value): + if value is None or (isinstance(value, str) and not value): continue # transform values as needed diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index 1f829b18..4ae20107 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -7,7 +7,6 @@ import os from builtins import int from lxml import etree -from past.builtins import basestring from core.config import ConfigGroup from core.emane import emanemanifest, emanemodel @@ -26,7 +25,7 @@ def convert_none(x): """ Helper to use 0 for None values. """ - if isinstance(x, basestring): + if isinstance(x, str): x = float(x) if x is None: return 0 diff --git a/daemon/core/location/event.py b/daemon/core/location/event.py index b3170083..1872ac18 100644 --- a/daemon/core/location/event.py +++ b/daemon/core/location/event.py @@ -5,8 +5,7 @@ event.py: event loop implementation using a heap queue and threads. import heapq import threading import time - -from past.builtins import cmp +from functools import total_ordering class Timer(threading.Thread): @@ -70,6 +69,7 @@ class Timer(threading.Thread): self.finished.set() +@total_ordering class Event(object): """ Provides event objects that can be used within the EventLoop class. @@ -92,18 +92,11 @@ class Event(object): self.kwds = kwds self.canceled = False - def __cmp__(self, other): - """ - Comparison function. - - :param Event other: event to compare with - :return: comparison result - :rtype: int - """ - tmp = cmp(self.time, other.time) - if tmp == 0: - tmp = cmp(self.eventnum, other.eventnum) - return tmp + def __lt__(self, other): + result = self.time < other.time + if result: + result = self.eventnum < other.eventnum + return result def run(self): """ diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index 52635da3..934233e9 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -4,8 +4,7 @@ sdt.py: Scripted Display Tool (SDT3D) helper import logging import socket - -from future.moves.urllib.parse import urlparse +from urllib.parse import urlparse from core import constants from core.emane.nodes import EmaneNet diff --git a/daemon/core/services/nrl.py b/daemon/core/services/nrl.py index 23484459..a610f1cc 100644 --- a/daemon/core/services/nrl.py +++ b/daemon/core/services/nrl.py @@ -3,8 +3,6 @@ nrl.py: defines services provided by NRL protolib tools hosted here: http://www.nrl.navy.mil/itd/ncs/products """ -from past.builtins import filter - from core import utils from core.nodes.ipaddress import Ipv4Prefix from core.services.coreservices import CoreService @@ -94,7 +92,7 @@ class NrlNhdp(NrlService): cmd += " -flooding ecds" cmd += " -smfClient %s_smf" % node.name - netifs = filter(lambda x: not getattr(x, "control", False), node.netifs()) + netifs = list(filter(lambda x: not getattr(x, "control", False), node.netifs())) if len(netifs) > 0: interfacenames = map(lambda x: x.name, netifs) cmd += " -i " @@ -128,7 +126,7 @@ class NrlSmf(NrlService): cmd = "nrlsmf instance %s_smf" % node.name servicenames = map(lambda x: x.name, node.services) - netifs = filter(lambda x: not getattr(x, "control", False), node.netifs()) + netifs = list(filter(lambda x: not getattr(x, "control", False), node.netifs())) if len(netifs) == 0: return "" @@ -218,7 +216,7 @@ class NrlOlsrv2(NrlService): cmd += " -p olsr" - netifs = filter(lambda x: not getattr(x, "control", False), node.netifs()) + netifs = list(filter(lambda x: not getattr(x, "control", False), node.netifs())) if len(netifs) > 0: interfacenames = map(lambda x: x.name, netifs) cmd += " -i " @@ -246,7 +244,7 @@ class OlsrOrg(NrlService): Generate the appropriate command-line based on node interfaces. """ cmd = cls.startup[0] - netifs = filter(lambda x: not getattr(x, "control", False), node.netifs()) + netifs = list(filter(lambda x: not getattr(x, "control", False), node.netifs())) if len(netifs) > 0: interfacenames = map(lambda x: x.name, netifs) cmd += " -i " diff --git a/daemon/requirements.txt b/daemon/requirements.txt index d9029923..96fe83ca 100644 --- a/daemon/requirements.txt +++ b/daemon/requirements.txt @@ -1,9 +1,6 @@ -configparser==4.0.2 fabric==2.5.0 -future==0.17.1 grpcio==1.23.0 grpcio-tools==1.21.1 invoke==1.3.0 lxml==4.4.1 protobuf==3.9.1 -six==1.12.0 diff --git a/daemon/setup.py.in b/daemon/setup.py.in index 3a451fb4..378912d3 100644 --- a/daemon/setup.py.in +++ b/daemon/setup.py.in @@ -34,18 +34,12 @@ setup( version="@PACKAGE_VERSION@", packages=find_packages(), install_requires=[ - "configparser", "fabric", - "future", "grpcio", "invoke", "lxml", "protobuf", ], - extra_require={ - ":python_version<'3.2'": ["futures"], - ":python_version<'3.4'": ["enum34"], - }, tests_require=[ "pytest", "mock", diff --git a/ns3/Makefile.am b/ns3/Makefile.am index 115a7008..62086e82 100644 --- a/ns3/Makefile.am +++ b/ns3/Makefile.am @@ -9,8 +9,6 @@ if WANT_PYTHON -PYTHONLIBDIR=$(subst site-packages,dist-packages,$(pythondir)) - SETUPPY = setup.py SETUPPYFLAGS = -v @@ -24,15 +22,15 @@ install-exec-hook: $(PYTHON) $(SETUPPY) $(SETUPPYFLAGS) install \ --root=/$(DESTDIR) \ --prefix=$(prefix) \ - --install-lib=$(PYTHONLIBDIR) \ + --install-lib=$(pythondir) \ --single-version-externally-managed \ --no-compile # Python package uninstall uninstall-hook: -rm -rf core_ns3.egg-info - -rm -rf $(DESTDIR)/$(PYTHONLIBDIR)/core_ns3-$(PACKAGE_VERSION)-py$(PYTHON_VERSION).egg-info - -rm -rf $(DESTDIR)/$(PYTHONLIBDIR)/corens3 + -rm -rf $(DESTDIR)/$(pythondir)/core_ns3-$(PACKAGE_VERSION)-py$(PYTHON_VERSION).egg-info + -rm -rf $(DESTDIR)/$(pythondir)/corens3 -rm -rf $(DESTDIR)/$(datadir)/corens3 # Python package cleanup From da946f1f56cebb81eadcf1852290af33afd24d20 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 15:02:24 -0700 Subject: [PATCH 091/113] removing builtins imports --- daemon/core/api/grpc/server.py | 1 - daemon/core/api/tlv/corehandlers.py | 1 - daemon/core/emane/commeffect.py | 1 - daemon/core/location/mobility.py | 1 - daemon/core/nodes/interface.py | 1 - daemon/core/nodes/ipaddress.py | 1 - daemon/examples/grpc/switch.py | 1 - daemon/examples/python/emane80211.py | 1 - daemon/examples/python/switch.py | 1 - daemon/examples/python/switch_inject.py | 1 - daemon/examples/python/wlan.py | 1 - daemon/tests/test_grpc.py | 1 - 12 files changed, 12 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 3c70ed09..dcea5fb5 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -4,7 +4,6 @@ import os import re import tempfile import time -from builtins import int from concurrent import futures from queue import Empty, Queue diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 9a5e487a..f3966ba1 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -10,7 +10,6 @@ import socketserver import sys import threading import time -from builtins import range from itertools import repeat from queue import Empty, Queue diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index 4ae20107..13831291 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -4,7 +4,6 @@ commeffect.py: EMANE CommEffect model for CORE import logging import os -from builtins import int from lxml import etree diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index eae46ce4..eb2f244f 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -8,7 +8,6 @@ import math import os import threading import time -from builtins import int from functools import total_ordering from core import utils diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index a6e04eb5..3e4f73ef 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -4,7 +4,6 @@ virtual ethernet classes that implement the interfaces available under Linux. import logging import time -from builtins import int, range from core import utils from core.errors import CoreCommandError diff --git a/daemon/core/nodes/ipaddress.py b/daemon/core/nodes/ipaddress.py index c7860dbc..be2ec36d 100644 --- a/daemon/core/nodes/ipaddress.py +++ b/daemon/core/nodes/ipaddress.py @@ -6,7 +6,6 @@ import logging import random import socket import struct -from builtins import bytes, int, range from socket import AF_INET, AF_INET6 diff --git a/daemon/examples/grpc/switch.py b/daemon/examples/grpc/switch.py index 89dc371d..48aa63bc 100644 --- a/daemon/examples/grpc/switch.py +++ b/daemon/examples/grpc/switch.py @@ -1,5 +1,4 @@ import logging -from builtins import range from core.api.grpc import client, core_pb2 diff --git a/daemon/examples/python/emane80211.py b/daemon/examples/python/emane80211.py index adf6959b..93222f9e 100644 --- a/daemon/examples/python/emane80211.py +++ b/daemon/examples/python/emane80211.py @@ -5,7 +5,6 @@ import datetime import logging import parser -from builtins import range from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emulator.coreemu import CoreEmu diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index e4d0fd02..6702802e 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -8,7 +8,6 @@ import datetime import logging import parser -from builtins import range from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes diff --git a/daemon/examples/python/switch_inject.py b/daemon/examples/python/switch_inject.py index ff1ff84b..1b7b634c 100644 --- a/daemon/examples/python/switch_inject.py +++ b/daemon/examples/python/switch_inject.py @@ -5,7 +5,6 @@ # and repeat for minnodes <= n <= maxnodes with a step size of # nodestep import logging -from builtins import range from core.emulator.emudata import IpPrefixes from core.emulator.enumerations import EventTypes, NodeTypes diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index b16af7cd..b3b4544e 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -8,7 +8,6 @@ import datetime import logging import parser -from builtins import range from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions diff --git a/daemon/tests/test_grpc.py b/daemon/tests/test_grpc.py index f9604229..37d8e7ae 100644 --- a/daemon/tests/test_grpc.py +++ b/daemon/tests/test_grpc.py @@ -1,5 +1,4 @@ import time -from builtins import int from queue import Queue import grpc From c9326b6a97a1f9fbb04a2d9903d8bba0b5391c7e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 18:59:50 -0700 Subject: [PATCH 092/113] initial changes to use f strings --- daemon/core/errors.py | 8 ++-- daemon/core/nodes/client.py | 2 +- daemon/core/nodes/docker.py | 10 ++--- daemon/core/nodes/interface.py | 4 +- daemon/core/nodes/ipaddress.py | 15 +++---- daemon/core/nodes/lxd.py | 10 ++--- daemon/core/nodes/netclient.py | 77 +++++++++++++++++----------------- daemon/core/nodes/physical.py | 16 +++---- 8 files changed, 70 insertions(+), 72 deletions(-) diff --git a/daemon/core/errors.py b/daemon/core/errors.py index 5b76abb3..f5c38b5b 100644 --- a/daemon/core/errors.py +++ b/daemon/core/errors.py @@ -10,11 +10,9 @@ class CoreCommandError(subprocess.CalledProcessError): """ def __str__(self): - return "Command(%s), Status(%s):\nstdout: %s\nstderr: %s" % ( - self.cmd, - self.returncode, - self.output, - self.stderr, + return ( + f"Command({self.cmd}), Status({self.returncode}):\n" + f"stdout: {self.output}\nstderr: {self.stderr}" ) diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 632e12bc..299b8135 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -51,7 +51,7 @@ class VnodeClient(object): pass def create_cmd(self, args): - return "%s -c %s -- %s" % (VCMD_BIN, self.ctrlchnlname, args) + return f"{VCMD_BIN} -c {self.ctrlchnlname} -- {args}" def check_cmd(self, args, wait=True): """ diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 17d7578a..df8422af 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -226,13 +226,13 @@ class DockerNode(CoreNode): temp.close() if directory: - self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) + self.node_net_cmd(f"mkdir -m {0o755:o} -p {directory}") if self.server is not None: self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) - self.node_net_cmd("chmod %o %s" % (mode, filename)) + self.node_net_cmd(f"chmod {mode:o} {filename}") if self.server is not None: - self.net_cmd("rm -f %s" % temp.name) + self.net_cmd(f"rm -f {temp.name}") os.unlink(temp.name) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, filename, mode @@ -252,7 +252,7 @@ class DockerNode(CoreNode): "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode ) directory = os.path.dirname(filename) - self.node_net_cmd("mkdir -p %s" % directory) + self.node_net_cmd(f"mkdir -p {directory}") if self.server is None: source = srcfilename @@ -262,4 +262,4 @@ class DockerNode(CoreNode): self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) - self.node_net_cmd("chmod %o %s" % (mode, filename)) + self.node_net_cmd(f"chmod {mode:o} {filename}") diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 3e4f73ef..c8841432 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -344,7 +344,7 @@ class TunTap(CoreInterface): if r == 0: result = True break - msg = "attempt %s failed with nonzero exit status %s" % (i, r) + msg = f"attempt {i} failed with nonzero exit status {r}" if i < attempts + 1: msg += ", retrying..." logging.info(msg) @@ -480,7 +480,7 @@ class GreTap(CoreInterface): self.id = _id sessionid = self.session.short_session_id() # interface name on the local host machine - self.localname = "gt.%s.%s" % (self.id, sessionid) + self.localname = f"gt.{self.id}.{sessionid}" self.transport_type = "raw" if not start: self.up = False diff --git a/daemon/core/nodes/ipaddress.py b/daemon/core/nodes/ipaddress.py index be2ec36d..df2309ab 100644 --- a/daemon/core/nodes/ipaddress.py +++ b/daemon/core/nodes/ipaddress.py @@ -29,7 +29,7 @@ class MacAddress(object): :return: string representation :rtype: str """ - return ":".join("%02x" % x for x in bytearray(self.addr)) + return ":".join(f"{x:02x}" for x in bytearray(self.addr)) def to_link_local(self): """ @@ -217,14 +217,14 @@ class IpPrefix(object): # prefixstr format: address/prefixlen tmp = prefixstr.split("/") if len(tmp) > 2: - raise ValueError("invalid prefix: %s" % prefixstr) + raise ValueError(f"invalid prefix: {prefixstr}") self.af = af if self.af == AF_INET: self.addrlen = 32 elif self.af == AF_INET6: self.addrlen = 128 else: - raise ValueError("invalid address family: %s" % self.af) + raise ValueError(f"invalid address family: {self.af}") if len(tmp) == 2: self.prefixlen = int(tmp[1]) else: @@ -247,7 +247,8 @@ class IpPrefix(object): :return: string representation :rtype: str """ - return "%s/%s" % (socket.inet_ntop(self.af, self.prefix), self.prefixlen) + address = socket.inet_ntop(self.af, self.prefix) + return f"{address}/{self.prefixlen}" def __eq__(self, other): """ @@ -283,7 +284,7 @@ class IpPrefix(object): return NotImplemented a = IpAddress(self.af, self.prefix) + (tmp << (self.addrlen - self.prefixlen)) - prefixstr = "%s/%s" % (a, self.prefixlen) + prefixstr = f"{a}/{self.prefixlen}" if self.__class__ == IpPrefix: return self.__class__(self.af, prefixstr) else: @@ -324,7 +325,7 @@ class IpPrefix(object): self.af == AF_INET and tmp == (1 << (self.addrlen - self.prefixlen)) - 1 ) ): - raise ValueError("invalid hostid for prefix %s: %s" % (self, hostid)) + raise ValueError(f"invalid hostid for prefix {self}: {hostid}") addr = bytes(b"") prefix_endpoint = -1 @@ -374,7 +375,7 @@ class IpPrefix(object): :return: prefix string :rtype: str """ - return "%s" % socket.inet_ntop(self.af, self.prefix) + return socket.inet_ntop(self.af, self.prefix) def netmask_str(self): """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index b11086e7..50588fb2 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -184,13 +184,13 @@ class LxcNode(CoreNode): temp.close() if directory: - self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory)) + self.node_net_cmd(f"mkdir -m {0o755:o} -p {directory}") if self.server is not None: self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) - self.node_net_cmd("chmod %o %s" % (mode, filename)) + self.node_net_cmd(f"chmod {mode:o} {filename}") if self.server is not None: - self.net_cmd("rm -f %s" % temp.name) + self.net_cmd(f"rm -f {temp.name}") os.unlink(temp.name) logging.debug("node(%s) added file: %s; mode: 0%o", self.name, filename, mode) @@ -208,7 +208,7 @@ class LxcNode(CoreNode): "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode ) directory = os.path.dirname(filename) - self.node_net_cmd("mkdir -p %s" % directory) + self.node_net_cmd(f"mkdir -p {directory}") if self.server is None: source = srcfilename @@ -218,7 +218,7 @@ class LxcNode(CoreNode): self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) - self.node_net_cmd("chmod %o %s" % (mode, filename)) + self.node_net_cmd(f"chmod {mode:o} {filename}") def addnetif(self, netif, ifindex): super(LxcNode, self).addnetif(netif, ifindex) diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 9234bef5..94e73e7f 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -41,7 +41,7 @@ class LinuxNetClient(object): :param str name: name for hostname :return: nothing """ - self.run("hostname %s" % name) + self.run(f"hostname {name}") def create_route(self, route, device): """ @@ -51,7 +51,7 @@ class LinuxNetClient(object): :param str device: device to add route to :return: nothing """ - self.run("%s route add %s dev %s" % (IP_BIN, route, device)) + self.run(f"{IP_BIN} route add {route} dev {device}") def device_up(self, device): """ @@ -60,7 +60,7 @@ class LinuxNetClient(object): :param str device: device to bring up :return: nothing """ - self.run("%s link set %s up" % (IP_BIN, device)) + self.run(f"{IP_BIN} link set {device} up") def device_down(self, device): """ @@ -69,7 +69,7 @@ class LinuxNetClient(object): :param str device: device to bring down :return: nothing """ - self.run("%s link set %s down" % (IP_BIN, device)) + self.run(f"{IP_BIN} link set {device} down") def device_name(self, device, name): """ @@ -79,7 +79,7 @@ class LinuxNetClient(object): :param str name: name to set :return: nothing """ - self.run("%s link set %s name %s" % (IP_BIN, device, name)) + self.run(f"{IP_BIN} link set {device} name {name}") def device_show(self, device): """ @@ -89,7 +89,7 @@ class LinuxNetClient(object): :return: device information :rtype: str """ - return self.run("%s link show %s" % (IP_BIN, device)) + return self.run(f"{IP_BIN} link show {device}") def device_ns(self, device, namespace): """ @@ -99,7 +99,7 @@ class LinuxNetClient(object): :param str namespace: namespace to set device to :return: nothing """ - self.run("%s link set %s netns %s" % (IP_BIN, device, namespace)) + self.run(f"{IP_BIN} link set {device} netns {namespace}") def device_flush(self, device): """ @@ -108,7 +108,7 @@ class LinuxNetClient(object): :param str device: device to flush :return: nothing """ - self.run("%s -6 address flush dev %s" % (IP_BIN, device)) + self.run(f"{IP_BIN} -6 address flush dev {device}") def device_mac(self, device, mac): """ @@ -118,7 +118,7 @@ class LinuxNetClient(object): :param str mac: mac to set :return: nothing """ - self.run("%s link set dev %s address %s" % (IP_BIN, device, mac)) + self.run(f"{IP_BIN} link set dev {device} address {mac}") def delete_device(self, device): """ @@ -127,7 +127,7 @@ class LinuxNetClient(object): :param str device: device to delete :return: nothing """ - self.run("%s link delete %s" % (IP_BIN, device)) + self.run(f"{IP_BIN} link delete {device}") def delete_tc(self, device): """ @@ -136,7 +136,7 @@ class LinuxNetClient(object): :param str device: device to remove tc :return: nothing """ - self.run("%s qdisc delete dev %s root" % (TC_BIN, device)) + self.run(f"{TC_BIN} qdisc delete dev {device} root") def checksums_off(self, interface_name): """ @@ -145,7 +145,7 @@ class LinuxNetClient(object): :param str interface_name: interface to update :return: nothing """ - self.run("%s -K %s rx off tx off" % (ETHTOOL_BIN, interface_name)) + self.run(f"{ETHTOOL_BIN} -K {interface_name} rx off tx off") def create_address(self, device, address, broadcast=None): """ @@ -158,11 +158,10 @@ class LinuxNetClient(object): """ if broadcast is not None: self.run( - "%s address add %s broadcast %s dev %s" - % (IP_BIN, address, broadcast, device) + f"{IP_BIN} address add {address} broadcast {broadcast} dev {device}" ) else: - self.run("%s address add %s dev %s" % (IP_BIN, address, device)) + self.run(f"{IP_BIN} address add {address} dev {device}") def delete_address(self, device, address): """ @@ -172,7 +171,7 @@ class LinuxNetClient(object): :param str address: address to remove :return: nothing """ - self.run("%s address delete %s dev %s" % (IP_BIN, address, device)) + self.run(f"{IP_BIN} address delete {address} dev {device}") def create_veth(self, name, peer): """ @@ -182,7 +181,7 @@ class LinuxNetClient(object): :param str peer: peer name :return: nothing """ - self.run("%s link add name %s type veth peer name %s" % (IP_BIN, name, peer)) + self.run(f"{IP_BIN} link add name {name} type veth peer name {peer}") def create_gretap(self, device, address, local, ttl, key): """ @@ -195,13 +194,13 @@ class LinuxNetClient(object): :param int key: key for tap :return: nothing """ - cmd = "%s link add %s type gretap remote %s" % (IP_BIN, device, address) + cmd = f"{IP_BIN} link add {device} type gretap remote {address}" if local is not None: - cmd += " local %s" % local + cmd += f" local {local}" if ttl is not None: - cmd += " ttl %s" % ttl + cmd += f" ttl {ttl}" if key is not None: - cmd += " key %s" % key + cmd += f" key {key}" self.run(cmd) def create_bridge(self, name): @@ -211,13 +210,13 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - self.run("%s addbr %s" % (BRCTL_BIN, name)) - self.run("%s stp %s off" % (BRCTL_BIN, name)) - self.run("%s setfd %s 0" % (BRCTL_BIN, name)) + self.run(f"{BRCTL_BIN} addbr {name}") + self.run(f"{BRCTL_BIN} stp {name} off") + self.run(f"{BRCTL_BIN} setfd {name} 0") self.device_up(name) # turn off multicast snooping so forwarding occurs w/o IGMP joins - snoop = "/sys/devices/virtual/net/%s/bridge/multicast_snooping" % name + snoop = f"/sys/devices/virtual/net/{name}/bridge/multicast_snooping" if os.path.exists(snoop): with open(snoop, "w") as f: f.write("0") @@ -230,7 +229,7 @@ class LinuxNetClient(object): :return: nothing """ self.device_down(name) - self.run("%s delbr %s" % (BRCTL_BIN, name)) + self.run(f"{BRCTL_BIN} delbr {name}") def create_interface(self, bridge_name, interface_name): """ @@ -240,7 +239,7 @@ class LinuxNetClient(object): :param str interface_name: interface name :return: nothing """ - self.run("%s addif %s %s" % (BRCTL_BIN, bridge_name, interface_name)) + self.run(f"{BRCTL_BIN} addif {bridge_name} {interface_name}") self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): @@ -251,7 +250,7 @@ class LinuxNetClient(object): :param str interface_name: interface name :return: nothing """ - self.run("%s delif %s %s" % (BRCTL_BIN, bridge_name, interface_name)) + self.run(f"{BRCTL_BIN} delif {bridge_name} {interface_name}") def existing_bridges(self, _id): """ @@ -259,7 +258,7 @@ class LinuxNetClient(object): :param _id: node id to check bridges for """ - output = self.run("%s show" % BRCTL_BIN) + output = self.run(f"{BRCTL_BIN} show") lines = output.split("\n") for line in lines[1:]: columns = line.split() @@ -278,7 +277,7 @@ class LinuxNetClient(object): :param str name: bridge name :return: nothing """ - self.run("%s setageing %s 0" % (BRCTL_BIN, name)) + self.run(f"{BRCTL_BIN} setageing {name} 0") class OvsNetClient(LinuxNetClient): @@ -293,10 +292,10 @@ class OvsNetClient(LinuxNetClient): :param str name: bridge name :return: nothing """ - self.run("%s add-br %s" % (OVS_BIN, name)) - self.run("%s set bridge %s stp_enable=false" % (OVS_BIN, name)) - self.run("%s set bridge %s other_config:stp-max-age=6" % (OVS_BIN, name)) - self.run("%s set bridge %s other_config:stp-forward-delay=4" % (OVS_BIN, name)) + self.run(f"{OVS_BIN} add-br {name}") + self.run(f"{OVS_BIN} set bridge {name} stp_enable=false") + self.run(f"{OVS_BIN} set bridge {name} other_config:stp-max-age=6") + self.run(f"{OVS_BIN} set bridge {name} other_config:stp-forward-delay=4") self.device_up(name) def delete_bridge(self, name): @@ -307,7 +306,7 @@ class OvsNetClient(LinuxNetClient): :return: nothing """ self.device_down(name) - self.run("%s del-br %s" % (OVS_BIN, name)) + self.run(f"{OVS_BIN} del-br {name}") def create_interface(self, bridge_name, interface_name): """ @@ -317,7 +316,7 @@ class OvsNetClient(LinuxNetClient): :param str interface_name: interface name :return: nothing """ - self.run("%s add-port %s %s" % (OVS_BIN, bridge_name, interface_name)) + self.run(f"{OVS_BIN} add-port {bridge_name} {interface_name}") self.device_up(interface_name) def delete_interface(self, bridge_name, interface_name): @@ -328,7 +327,7 @@ class OvsNetClient(LinuxNetClient): :param str interface_name: interface name :return: nothing """ - self.run("%s del-port %s %s" % (OVS_BIN, bridge_name, interface_name)) + self.run(f"{OVS_BIN} del-port {bridge_name} {interface_name}") def existing_bridges(self, _id): """ @@ -336,7 +335,7 @@ class OvsNetClient(LinuxNetClient): :param _id: node id to check bridges for """ - output = self.run("%s list-br" % OVS_BIN) + output = self.run(f"{OVS_BIN} list-br") if output: for line in output.split("\n"): fields = line.split(".") @@ -351,4 +350,4 @@ class OvsNetClient(LinuxNetClient): :param str name: bridge name :return: nothing """ - self.run("%s set bridge %s other_config:mac-aging-time=0" % (OVS_BIN, name)) + self.run(f"{OVS_BIN} set bridge {name} other_config:mac-aging-time=0") diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 0f9e0217..cae3f298 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -94,7 +94,7 @@ class PhysicalNode(CoreNodeBase): the emulation, no new interface is created; instead, adopt the GreTap netif as the node interface. """ - netif.name = "gt%d" % ifindex + netif.name = f"gt{ifindex}" netif.node = self self.addnetif(netif, ifindex) @@ -161,7 +161,7 @@ class PhysicalNode(CoreNodeBase): ifindex = self.newifindex() if ifname is None: - ifname = "gt%d" % ifindex + ifname = f"gt{ifindex}" if self.up: # this is reached when this node is linked to a network node @@ -177,7 +177,7 @@ class PhysicalNode(CoreNodeBase): def privatedir(self, path): if path[0] != "/": - raise ValueError("path not fully qualified: %s" % path) + raise ValueError(f"path not fully qualified: {path}") hostpath = os.path.join( self.nodedir, os.path.normpath(path).strip("/").replace("/", ".") ) @@ -188,13 +188,13 @@ class PhysicalNode(CoreNodeBase): source = os.path.abspath(source) logging.info("mounting %s at %s", source, target) os.makedirs(target) - self.net_cmd("%s --bind %s %s" % (MOUNT_BIN, source, target), cwd=self.nodedir) + self.net_cmd(f"{MOUNT_BIN} --bind {source} {target}", cwd=self.nodedir) self._mounts.append((source, target)) def umount(self, target): - logging.info("unmounting '%s'" % target) + logging.info("unmounting '%s'", target) try: - self.net_cmd("%s -l %s" % (UMOUNT_BIN, target), cwd=self.nodedir) + self.net_cmd(f"{UMOUNT_BIN} -l {target}", cwd=self.nodedir) except CoreCommandError: logging.exception("unmounting failed for %s", target) @@ -363,7 +363,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): if ifindex == self.ifindex: self.shutdown() else: - raise ValueError("ifindex %s does not exist" % ifindex) + raise ValueError(f"ifindex {ifindex} does not exist") def netif(self, ifindex, net=None): """ @@ -442,7 +442,7 @@ class Rj45Node(CoreNodeBase, CoreInterface): if len(items) < 2: continue - if items[1] == "%s:" % self.localname: + if items[1] == f"{self.localname}:": flags = items[2][1:-1].split(",") if "UP" in flags: self.old_up = True From 79cde8cd59441dc71d2cf2ee6b3796069eaf8571 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 17 Oct 2019 19:25:52 -0700 Subject: [PATCH 093/113] further f string updates --- daemon/core/location/corelocation.py | 2 -- daemon/core/services/xorp.py | 7 ------- daemon/examples/myservices/sample.py | 4 ++-- daemon/examples/python/parser.py | 19 ------------------- daemon/tests/conftest.py | 6 +----- daemon/tests/distributed/test_distributed.py | 6 +++--- daemon/tests/emane/test_emane.py | 2 +- daemon/tests/test_core.py | 2 +- daemon/tests/test_grpc.py | 16 +++++++--------- daemon/tests/test_gui.py | 10 +++++----- daemon/tests/test_nodes.py | 2 +- 11 files changed, 21 insertions(+), 55 deletions(-) diff --git a/daemon/core/location/corelocation.py b/daemon/core/location/corelocation.py index b3e62153..5f9e11e3 100644 --- a/daemon/core/location/corelocation.py +++ b/daemon/core/location/corelocation.py @@ -128,8 +128,6 @@ class CoreLocation(object): z, ) lat, lon = self.refgeo[:2] - # self.info("getgeo(%s,%s,%s) e=%s n=%s zone=%s lat,lon,alt=" \ - # "%.3f,%.3f,%.3f" % (x, y, z, e, n, zone, lat, lon, alt)) return lat, lon, alt def getxyz(self, lat, lon, alt): diff --git a/daemon/core/services/xorp.py b/daemon/core/services/xorp.py index 812fcf9f..1cd62620 100644 --- a/daemon/core/services/xorp.py +++ b/daemon/core/services/xorp.py @@ -312,13 +312,6 @@ class XorpRipng(XorpService): continue cfg += "\tinterface %s {\n" % ifc.name cfg += "\t vif %s {\n" % ifc.name - # for a in ifc.addrlist: - # if a.find(":") < 0: - # continue - # addr = a.split("/")[0] - # cfg += "\t\taddress %s {\n" % addr - # cfg += "\t\t disable: false\n" - # cfg += "\t\t}\n" cfg += "\t\taddress %s {\n" % ifc.hwaddr.tolinklocal() cfg += "\t\t disable: false\n" cfg += "\t\t}\n" diff --git a/daemon/examples/myservices/sample.py b/daemon/examples/myservices/sample.py index d6c111ab..8c6dbe06 100644 --- a/daemon/examples/myservices/sample.py +++ b/daemon/examples/myservices/sample.py @@ -37,7 +37,7 @@ class MyService(CoreService): dependencies = () dirs = () configs = ("myservice1.sh", "myservice2.sh") - startup = ("sh %s" % configs[0], "sh %s" % configs[1]) + startup = tuple(f"sh {x}" for x in configs) validate = () validation_mode = ServiceMode.NON_BLOCKING validation_timer = 5 @@ -81,7 +81,7 @@ class MyService(CoreService): if filename == cls.configs[0]: cfg += "# auto-generated by MyService (sample.py)\n" for ifc in node.netifs(): - cfg += 'echo "Node %s has interface %s"\n' % (node.name, ifc.name) + cfg += f'echo "Node {node.name} has interface {ifc.name}"\n' elif filename == cls.configs[1]: cfg += "echo hello" diff --git a/daemon/examples/python/parser.py b/daemon/examples/python/parser.py index 7bbb0fbc..fdc2591a 100644 --- a/daemon/examples/python/parser.py +++ b/daemon/examples/python/parser.py @@ -24,25 +24,6 @@ def parse_options(name): options = parser.parse_args() - # usagestr = "usage: %prog [-h] [options] [args]" - # parser = optparse.OptionParser(usage=usagestr) - # - # parser.add_option("-n", "--nodes", dest="nodes", type=int, default=DEFAULT_NODES, - # help="number of nodes to create in this example") - # - # parser.add_option("-t", "--time", dest="time", type=int, default=DEFAULT_TIME, - # help="example iperf run time in seconds") - - # def usage(msg=None, err=0): - # print - # if msg: - # print "%s\n" % msg - # parser.print_help() - # sys.exit(err) - - # parse command line options - # options, args = parser.parse_args() - if options.nodes < 2: parser.error("invalid min number of nodes: %s" % options.nodes) if options.time < 1: diff --git a/daemon/tests/conftest.py b/daemon/tests/conftest.py index ead3c2b4..521a2432 100644 --- a/daemon/tests/conftest.py +++ b/daemon/tests/conftest.py @@ -67,11 +67,7 @@ class CoreServerTest(object): self.request_handler.handle_message(message) # add broker server for distributed core - distributed = "%s:%s:%s" % ( - self.distributed_server, - distributed_address, - self.port, - ) + distributed = f"{self.distributed_server}:{distributed_address}:{self.port}" message = CoreConfMessage.create( 0, [ diff --git a/daemon/tests/distributed/test_distributed.py b/daemon/tests/distributed/test_distributed.py index 49271d64..7078d6ed 100644 --- a/daemon/tests/distributed/test_distributed.py +++ b/daemon/tests/distributed/test_distributed.py @@ -206,7 +206,7 @@ class TestDistributed: # test a ping command node_one = cored.session.get_node(1) - message = command_message(node_one, "ping -c 5 %s" % ip4_address) + message = command_message(node_one, f"ping -c 5 {ip4_address}") cored.request_handler.dispatch_replies = validate_response cored.request_handler.handle_message(message) @@ -259,7 +259,7 @@ class TestDistributed: # test a ping command node_one = cored.session.get_node(1) - message = command_message(node_one, "ping -c 5 %s" % ip4_address) + message = command_message(node_one, f"ping -c 5 {ip4_address}") cored.request_handler.dispatch_replies = validate_response cored.request_handler.handle_message(message) @@ -307,7 +307,7 @@ class TestDistributed: # test a ping command node_one = cored.session.get_node(1) - message = command_message(node_one, "ping -c 5 %s" % ip4_address) + message = command_message(node_one, f"ping -c 5 {ip4_address}") cored.request_handler.dispatch_replies = validate_response cored.request_handler.handle_message(message) cored.request_handler.handle_message(message) diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 65065665..a0ae05d1 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -27,7 +27,7 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd("ping -c %s %s" % (count, address)) + from_node.node_net_cmd(f"ping -c {count} {address}") status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 3fc90da8..fa2adc6e 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -20,7 +20,7 @@ _WIRED = [NodeTypes.PEER_TO_PEER, NodeTypes.HUB, NodeTypes.SWITCH] def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd("ping -c 3 %s" % address) + from_node.node_net_cmd(f"ping -c 3 {address}") status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_grpc.py b/daemon/tests/test_grpc.py index 37d8e7ae..b2ea73ca 100644 --- a/daemon/tests/test_grpc.py +++ b/daemon/tests/test_grpc.py @@ -206,8 +206,7 @@ class TestGrpc: # then assert response.node.id == node.id - @pytest.mark.parametrize("node_id, expected", [(1, True), (2, False)]) - def test_edit_node(self, grpc_server, node_id, expected): + def test_edit_node(self, grpc_server): # given client = CoreGrpcClient() session = grpc_server.coreemu.create_session() @@ -217,13 +216,12 @@ class TestGrpc: x, y = 10, 10 with client.context_connect(): position = core_pb2.Position(x=x, y=y) - response = client.edit_node(session.id, node_id, position) + response = client.edit_node(session.id, node.id, position) # then - assert response.result is expected - if expected is True: - assert node.position.x == x - assert node.position.y == y + assert response.result is True + assert node.position.x == x + assert node.position.y == y @pytest.mark.parametrize("node_id, expected", [(1, True), (2, False)]) def test_delete_node(self, grpc_server, node_id, expected): @@ -253,7 +251,7 @@ class TestGrpc: output = "hello world" # then - command = "echo %s" % output + command = f"echo {output}" with client.context_connect(): response = client.node_command(session.id, node.id, command) @@ -863,7 +861,7 @@ class TestGrpc: client.events(session.id, handle_event) time.sleep(0.1) event = EventData( - event_type=EventTypes.RUNTIME_STATE.value, time="%s" % time.time() + event_type=EventTypes.RUNTIME_STATE.value, time=str(time.time()) ) session.broadcast_event(event) diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index c07e2bd3..40c025ee 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -29,7 +29,7 @@ from core.nodes.ipaddress import Ipv4Prefix def dict_to_str(values): - return "|".join("%s=%s" % (x, values[x]) for x in values) + return "|".join(f"{x}={values[x]}" for x in values) class TestGui: @@ -383,7 +383,7 @@ class TestGui: message = coreapi.CoreFileMessage.create( MessageFlags.ADD.value, [ - (FileTlvs.TYPE, "hook:%s" % state), + (FileTlvs.TYPE, f"hook:{state}"), (FileTlvs.NAME, file_name), (FileTlvs.DATA, file_data), ], @@ -406,7 +406,7 @@ class TestGui: MessageFlags.ADD.value, [ (FileTlvs.NODE, node.id), - (FileTlvs.TYPE, "service:%s" % service), + (FileTlvs.TYPE, f"service:{service}"), (FileTlvs.NAME, file_name), (FileTlvs.DATA, file_data), ], @@ -760,7 +760,7 @@ class TestGui: [ (ConfigTlvs.OBJECT, "broker"), (ConfigTlvs.TYPE, ConfigFlags.UPDATE.value), - (ConfigTlvs.VALUES, "%s:%s:%s" % (server, host, port)), + (ConfigTlvs.VALUES, f"{server}:{host}:{port}"), ], ) coreserver.session.distributed.add_server = mock.MagicMock() @@ -844,7 +844,7 @@ class TestGui: (ConfigTlvs.NODE, node.id), (ConfigTlvs.OBJECT, "services"), (ConfigTlvs.TYPE, ConfigFlags.UPDATE.value), - (ConfigTlvs.OPAQUE, "service:%s" % service), + (ConfigTlvs.OPAQUE, f"service:{service}"), (ConfigTlvs.VALUES, dict_to_str(values)), ], ) diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index 1f18c87e..34c426c5 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -67,4 +67,4 @@ class TestNodes: # then assert node assert node.up - assert utils.check_cmd("brctl show %s" % node.brname) + assert utils.check_cmd(f"brctl show {node.brname}") From 7d2a6157167c866a5af3895986f23c1fc15e4826 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 10:33:31 -0700 Subject: [PATCH 094/113] more updates to using f string --- daemon/core/config.py | 24 +++------ daemon/core/emane/commeffect.py | 6 +-- daemon/core/emane/emanemanager.py | 21 ++++---- daemon/core/emane/emanemanifest.py | 2 +- daemon/core/emane/nodes.py | 2 +- daemon/core/emane/tdma.py | 2 +- daemon/core/emulator/distributed.py | 4 +- daemon/core/emulator/emudata.py | 4 +- daemon/core/emulator/session.py | 74 ++++++++++++++-------------- daemon/core/location/mobility.py | 19 ++++--- daemon/core/nodes/base.py | 63 +++++++++++------------ daemon/core/plugins/sdt.py | 34 ++++++------- daemon/core/utils.py | 23 ++++----- daemon/core/xml/corexml.py | 2 +- daemon/core/xml/corexmldeployment.py | 23 ++++----- daemon/core/xml/emanexml.py | 34 +++++++------ daemon/examples/python/emane80211.py | 7 ++- daemon/examples/python/parser.py | 6 +-- daemon/examples/python/switch.py | 16 +++--- daemon/examples/python/wlan.py | 12 +++-- daemon/scripts/core-daemon | 38 ++++++++------ daemon/scripts/core-manage | 29 ++++++----- daemon/scripts/coresendmsg | 54 ++++++++++---------- 23 files changed, 248 insertions(+), 251 deletions(-) diff --git a/daemon/core/config.py b/daemon/core/config.py index f63ad59a..e55d5f17 100644 --- a/daemon/core/config.py +++ b/daemon/core/config.py @@ -40,10 +40,8 @@ class ConfigShim(object): """ group_strings = [] for config_group in config_groups: - group_string = "%s:%s-%s" % ( - config_group.name, - config_group.start, - config_group.stop, + group_string = ( + f"{config_group.name}:{config_group.start}-{config_group.stop}" ) group_strings.append(group_string) return "|".join(group_strings) @@ -74,7 +72,7 @@ class ConfigShim(object): if not captions: captions = configuration.label else: - captions += "|%s" % configuration.label + captions += f"|{configuration.label}" data_types.append(configuration.type.value) @@ -83,11 +81,11 @@ class ConfigShim(object): _id = configuration.id config_value = config.get(_id, configuration.default) - key_value = "%s=%s" % (_id, config_value) + key_value = f"{_id}={config_value}" if not key_values: key_values = key_value else: - key_values += "|%s" % key_value + key_values += f"|{key_value}" groups_str = cls.groups_to_str(configurable_options.config_groups()) return ConfigData( @@ -130,13 +128,7 @@ class Configuration(object): self.label = label def __str__(self): - return "%s(id=%s, type=%s, default=%s, options=%s)" % ( - self.__class__.__name__, - self.id, - self.type, - self.default, - self.options, - ) + return f"{self.__class__.__name__}(id={self.id}, type={self.type}, default={self.default}, options={self.options})" class ConfigurableManager(object): @@ -333,7 +325,7 @@ class ModelManager(ConfigurableManager): # get model class to configure model_class = self.models.get(model_name) if not model_class: - raise ValueError("%s is an invalid model" % model_name) + raise ValueError(f"{model_name} is an invalid model") # retrieve default values model_config = self.get_model_config(node_id, model_name) @@ -361,7 +353,7 @@ class ModelManager(ConfigurableManager): # get model class to configure model_class = self.models.get(model_name) if not model_class: - raise ValueError("%s is an invalid model" % model_name) + raise ValueError(f"{model_name} is an invalid model") config = self.get_configs(node_id=node_id, config_type=model_name) if not config: diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index 13831291..33edc342 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -73,9 +73,7 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): shim_name = emanexml.shim_file_name(self, interface) # create and write nem document - nem_element = etree.Element( - "nem", name="%s NEM" % self.name, type="unstructured" - ) + nem_element = etree.Element("nem", name=f"{self.name} NEM", type="unstructured") transport_type = "virtual" if interface and interface.transport_type == "raw": transport_type = "raw" @@ -90,7 +88,7 @@ class EmaneCommEffectModel(emanemodel.EmaneModel): # create and write shim document shim_element = etree.Element( - "shim", name="%s SHIM" % self.name, library=self.shim_library + "shim", name=f"{self.name} SHIM", library=self.shim_library ) # append all shim options (except filterfile) to shimdoc diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 91553b5a..743f90b2 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -227,7 +227,7 @@ class EmaneManager(ModelManager): with self._emane_node_lock: if emane_net.id in self._emane_nets: raise KeyError( - "non-unique EMANE object id %s for %s" % (emane_net.id, emane_net) + f"non-unique EMANE object id {emane_net.id} for {emane_net}" ) self._emane_nets[emane_net.id] = emane_net @@ -342,7 +342,7 @@ class EmaneManager(ModelManager): try: with open(emane_nems_filename, "w") as f: for nodename, ifname, nemid in nems: - f.write("%s %s %s\n" % (nodename, ifname, nemid)) + f.write(f"{nodename} {ifname} {nemid}\n") except IOError: logging.exception("Error writing EMANE NEMs file: %s") @@ -535,7 +535,7 @@ class EmaneManager(ModelManager): logging.info("setting user-defined EMANE log level: %d", cfgloglevel) loglevel = str(cfgloglevel) - emanecmd = "emane -d -l %s" % loglevel + emanecmd = f"emane -d -l {loglevel}" if realtime: emanecmd += " -r" @@ -580,11 +580,9 @@ class EmaneManager(ModelManager): node.node_net_client.create_route(eventgroup, eventdev) # start emane - args = "%s -f %s %s" % ( - emanecmd, - os.path.join(path, "emane%d.log" % n), - os.path.join(path, "platform%d.xml" % n), - ) + log_file = os.path.join(path, f"emane{n}.log") + platform_xml = os.path.join(path, f"platform{n}.xml") + args = f"{emanecmd} -f {log_file} {platform_xml}" output = node.node_net_cmd(args) logging.info("node(%s) emane daemon running: %s", node.name, args) logging.info("node(%s) emane daemon output: %s", node.name, output) @@ -593,8 +591,9 @@ class EmaneManager(ModelManager): return path = self.session.session_dir - emanecmd += " -f %s" % os.path.join(path, "emane.log") - emanecmd += " %s" % os.path.join(path, "platform.xml") + log_file = os.path.join(path, "emane.log") + platform_xml = os.path.join(path, "platform.xml") + emanecmd += f" -f {log_file} {platform_xml}" utils.check_cmd(emanecmd, cwd=path) self.session.distributed.execute(lambda x: x.remote_cmd(emanecmd, cwd=path)) logging.info("host emane daemon running: %s", emanecmd) @@ -797,7 +796,7 @@ class EmaneManager(ModelManager): node = self.session.get_node(n) except CoreError: logging.exception( - "location event NEM %s has no corresponding node %s" % (nemid, n) + "location event NEM %s has no corresponding node %s", nemid, n ) return False diff --git a/daemon/core/emane/emanemanifest.py b/daemon/core/emane/emanemanifest.py index 13cff8f2..a6583b9e 100644 --- a/daemon/core/emane/emanemanifest.py +++ b/daemon/core/emane/emanemanifest.py @@ -115,7 +115,7 @@ def parse(manifest_path, defaults): # define description and account for gui quirks config_descriptions = config_name if config_name.endswith("uri"): - config_descriptions = "%s file" % config_descriptions + config_descriptions = f"{config_descriptions} file" configuration = Configuration( _id=config_name, diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index 5451506f..e0ceee2f 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -210,7 +210,7 @@ class EmaneNet(CoreNetworkBase): nemid = self.getnemid(netif) ifname = netif.localname if nemid is None: - logging.info("nemid for %s is unknown" % ifname) + logging.info("nemid for %s is unknown", ifname) continue x, y, z = netif.node.getposition() lat, lon, alt = self.session.location.getgeo(x, y, z) diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index 249e81b6..91e662ea 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -62,5 +62,5 @@ class EmaneTdmaModel(emanemodel.EmaneModel): logging.info( "setting up tdma schedule: schedule(%s) device(%s)", schedule, event_device ) - args = "emaneevent-tdmaschedule -i %s %s" % (event_device, schedule) + args = f"emaneevent-tdmaschedule -i {event_device} {schedule}" utils.check_cmd(args) diff --git a/daemon/core/emulator/distributed.py b/daemon/core/emulator/distributed.py index 83576434..03e043eb 100644 --- a/daemon/core/emulator/distributed.py +++ b/daemon/core/emulator/distributed.py @@ -128,7 +128,7 @@ class DistributedController(object): """ server = DistributedServer(name, host) self.servers[name] = server - cmd = "mkdir -p %s" % self.session.session_dir + cmd = f"mkdir -p {self.session.session_dir}" server.remote_cmd(cmd) def execute(self, func): @@ -158,7 +158,7 @@ class DistributedController(object): # remove all remote session directories for name in self.servers: server = self.servers[name] - cmd = "rm -rf %s" % self.session.session_dir + cmd = f"rm -rf {self.session.session_dir}" server.remote_cmd(cmd) # clear tunnels diff --git a/daemon/core/emulator/emudata.py b/daemon/core/emulator/emudata.py index d11d1e0e..5a38c69c 100644 --- a/daemon/core/emulator/emudata.py +++ b/daemon/core/emulator/emudata.py @@ -284,7 +284,7 @@ class InterfaceData(object): :return: ip4 string or None """ if self.has_ip4(): - return "%s/%s" % (self.ip4, self.ip4_mask) + return f"{self.ip4}/{self.ip4_mask}" else: return None @@ -295,7 +295,7 @@ class InterfaceData(object): :return: ip4 string or None """ if self.has_ip6(): - return "%s/%s" % (self.ip6, self.ip6_mask) + return f"{self.ip6}/{self.ip6_mask}" else: return None diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index d962da28..d80e5e25 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -90,7 +90,7 @@ class Session(object): self.master = False # define and create session directory when desired - self.session_dir = os.path.join(tempfile.gettempdir(), "pycore.%s" % self.id) + self.session_dir = os.path.join(tempfile.gettempdir(), f"pycore.{self.id}") if mkdir: os.mkdir(self.session_dir) @@ -164,7 +164,7 @@ class Session(object): """ node_class = NODES.get(_type) if node_class is None: - raise CoreError("invalid node type: %s" % _type) + raise CoreError(f"invalid node type: {_type}") return node_class @classmethod @@ -178,7 +178,7 @@ class Session(object): """ node_type = NODES_TYPE.get(_class) if node_type is None: - raise CoreError("invalid node class: %s" % _class) + raise CoreError(f"invalid node class: {_class}") return node_type def _link_nodes(self, node_one_id, node_two_id): @@ -254,7 +254,7 @@ class Session(object): """ objects = [x for x in objects if x] if len(objects) < 2: - raise CoreError("wireless link failure: %s" % objects) + raise CoreError(f"wireless link failure: {objects}") logging.debug( "handling wireless linking objects(%s) connect(%s)", objects, connect ) @@ -665,13 +665,13 @@ class Session(object): node_options = NodeOptions() name = node_options.name if not name: - name = "%s%s" % (node_class.__name__, _id) + name = f"{node_class.__name__}{_id}" # verify distributed server server = self.distributed.servers.get(node_options.emulation_server) if node_options.emulation_server is not None and server is None: raise CoreError( - "invalid distributed server: %s" % node_options.emulation_server + f"invalid distributed server: {node_options.emulation_server}" ) # create node @@ -854,7 +854,7 @@ class Session(object): :return: nothing """ # hack to conform with old logic until updated - state = ":%s" % state + state = f":{state}" self.set_hook(state, file_name, source_name, data) def add_node_file(self, node_id, source_name, file_name, data): @@ -1066,7 +1066,7 @@ class Session(object): self.run_state_hooks(state_value) if send_event: - event_data = EventData(event_type=state_value, time="%s" % time.time()) + event_data = EventData(event_type=state_value, time=str(time.time())) self.broadcast_event(event_data) def write_state(self, state): @@ -1078,7 +1078,7 @@ class Session(object): """ try: state_file = open(self._state_file, "w") - state_file.write("%d %s\n" % (state, EventTypes(self.state).name)) + state_file.write(f"{state} {EventTypes(self.state).name}\n") state_file.close() except IOError: logging.exception("error writing state file: %s", state) @@ -1195,9 +1195,9 @@ class Session(object): try: hook(state) except Exception: - message = "exception occured when running %s state hook: %s" % ( - EventTypes(self.state).name, - hook, + state_name = EventTypes(self.state).name + message = ( + f"exception occured when running {state_name} state hook: {hook}" ) logging.exception(message) self.exception( @@ -1258,16 +1258,16 @@ class Session(object): :rtype: dict """ env = os.environ.copy() - env["SESSION"] = "%s" % self.id - env["SESSION_SHORT"] = "%s" % self.short_session_id() - env["SESSION_DIR"] = "%s" % self.session_dir - env["SESSION_NAME"] = "%s" % self.name - env["SESSION_FILENAME"] = "%s" % self.file_name - env["SESSION_USER"] = "%s" % self.user - env["SESSION_NODE_COUNT"] = "%s" % self.get_node_count() + env["SESSION"] = str(self.id) + env["SESSION_SHORT"] = self.short_session_id() + env["SESSION_DIR"] = self.session_dir + env["SESSION_NAME"] = str(self.name) + env["SESSION_FILENAME"] = str(self.file_name) + env["SESSION_USER"] = str(self.user) + env["SESSION_NODE_COUNT"] = str(self.get_node_count()) if state: - env["SESSION_STATE"] = "%s" % self.state + env["SESSION_STATE"] = str(self.state) # attempt to read and add environment config file environment_config_file = os.path.join(constants.CORE_CONF_DIR, "environment") @@ -1356,7 +1356,7 @@ class Session(object): with self._nodes_lock: if node.id in self.nodes: node.shutdown() - raise CoreError("duplicate node id %s for %s" % (node.id, node.name)) + raise CoreError(f"duplicate node id {node.id} for {node.name}") self.nodes[node.id] = node return node @@ -1371,7 +1371,7 @@ class Session(object): :raises core.CoreError: when node does not exist """ if _id not in self.nodes: - raise CoreError("unknown node id %s" % _id) + raise CoreError(f"unknown node id {_id}") return self.nodes[_id] def delete_node(self, _id): @@ -1416,9 +1416,7 @@ class Session(object): with open(file_path, "w") as f: for _id in self.nodes.keys(): node = self.nodes[_id] - f.write( - "%s %s %s %s\n" % (_id, node.name, node.apitype, type(node)) - ) + f.write(f"{_id} {node.name} {node.apitype} {type(node)}\n") except IOError: logging.exception("error writing nodes file") @@ -1585,7 +1583,7 @@ class Session(object): interface names, where length may be limited. """ ssid = (self.id >> 8) ^ (self.id & ((1 << 8) - 1)) - return "%x" % ssid + return f"{ssid:x}" def boot_nodes(self): """ @@ -1670,7 +1668,7 @@ class Session(object): def get_control_net(self, net_index): # TODO: all nodes use an integer id and now this wants to use a string - _id = "ctrl%dnet" % net_index + _id = f"ctrl{net_index}net" return self.get_node(_id) def add_remove_control_net(self, net_index, remove=False, conf_required=True): @@ -1718,7 +1716,7 @@ class Session(object): return None # build a new controlnet bridge - _id = "ctrl%dnet" % net_index + _id = f"ctrl{net_index}net" # use the updown script for control net 0 only. updown_script = None @@ -1797,13 +1795,12 @@ class Session(object): control_ip = node.id try: - addrlist = [ - "%s/%s" - % (control_net.prefix.addr(control_ip), control_net.prefix.prefixlen) - ] + address = control_net.prefix.addr(control_ip) + prefix = control_net.prefix.prefixlen + addrlist = [f"{address}/{prefix}"] except ValueError: - msg = "Control interface not added to node %s. " % node.id - msg += "Invalid control network prefix (%s). " % control_net.prefix + msg = f"Control interface not added to node {node.id}. " + msg += f"Invalid control network prefix ({control_net.prefix}). " msg += "A longer prefix length may be required for this many nodes." logging.exception(msg) return @@ -1811,7 +1808,7 @@ class Session(object): interface1 = node.newnetif( net=control_net, ifindex=control_net.CTRLIF_IDX_BASE + net_index, - ifname="ctrl%d" % net_index, + ifname=f"ctrl{net_index}", hwaddr=MacAddress.random(), addrlist=addrlist, ) @@ -1834,7 +1831,7 @@ class Session(object): logging.exception("error retrieving control net node") return - header = "CORE session %s host entries" % self.id + header = f"CORE session {self.id} host entries" if remove: logging.info("Removing /etc/hosts file entries.") utils.file_demunge("/etc/hosts", header) @@ -1844,9 +1841,10 @@ class Session(object): for interface in control_net.netifs(): name = interface.node.name for address in interface.addrlist: - entries.append("%s %s" % (address.split("/")[0], name)) + address = address.split("/")[0] + entries.append(f"{address} {name}") - logging.info("Adding %d /etc/hosts file entries." % len(entries)) + logging.info("Adding %d /etc/hosts file entries.", len(entries)) utils.file_munge("/etc/hosts", header, "\n".join(entries) + "\n") diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index eb2f244f..f2b49818 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -165,15 +165,16 @@ class MobilityManager(ModelManager): elif model.state == model.STATE_PAUSED: event_type = EventTypes.PAUSE.value - data = "start=%d" % int(model.lasttime - model.timezero) - data += " end=%d" % int(model.endtime) + start_time = int(model.lasttime - model.timezero) + end_time = int(model.endtime) + data = f"start={start_time} end={end_time}" event_data = EventData( node=model.id, event_type=event_type, - name="mobility:%s" % model.name, + name=f"mobility:{model.name}", data=data, - time="%s" % time.time(), + time=str(time.time()), ) self.session.broadcast_event(event_data) @@ -991,7 +992,7 @@ class Ns2ScriptedMobility(WayPointMobility): "ns-2 scripted mobility failed to load file: %s", self.file ) return - logging.info("reading ns-2 script file: %s" % filename) + logging.info("reading ns-2 script file: %s", filename) ln = 0 ix = iy = iz = None inodenum = None @@ -1112,7 +1113,7 @@ class Ns2ScriptedMobility(WayPointMobility): :return: nothing """ if self.autostart == "": - logging.info("not auto-starting ns-2 script for %s" % self.wlan.name) + logging.info("not auto-starting ns-2 script for %s", self.wlan.name) return try: t = float(self.autostart) @@ -1124,9 +1125,7 @@ class Ns2ScriptedMobility(WayPointMobility): ) return self.movenodesinitial() - logging.info( - "scheduling ns-2 script for %s autostart at %s" % (self.wlan.name, t) - ) + logging.info("scheduling ns-2 script for %s autostart at %s", self.wlan.name, t) self.state = self.STATE_RUNNING self.session.event_loop.add_event(t, self.run) @@ -1187,7 +1186,7 @@ class Ns2ScriptedMobility(WayPointMobility): if filename is None or filename == "": return filename = self.findfile(filename) - args = "/bin/sh %s %s" % (filename, typestr) + args = f"/bin/sh {filename} {typestr}" utils.check_cmd( args, cwd=self.session.session_dir, env=self.session.get_environment() ) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 4e9de039..0b3bf04b 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -46,7 +46,7 @@ class NodeBase(object): _id = session.get_node_id() self.id = _id if name is None: - name = "o%s" % self.id + name = f"o{self.id}" self.name = name self.server = server @@ -265,7 +265,7 @@ class CoreNodeBase(NodeBase): """ if self.nodedir is None: self.nodedir = os.path.join(self.session.session_dir, self.name + ".conf") - self.net_cmd("mkdir -p %s" % self.nodedir) + self.net_cmd(f"mkdir -p {self.nodedir}") self.tmpnodedir = True else: self.tmpnodedir = False @@ -281,7 +281,7 @@ class CoreNodeBase(NodeBase): return if self.tmpnodedir: - self.net_cmd("rm -rf %s" % self.nodedir) + self.net_cmd(f"rm -rf {self.nodedir}") def addnetif(self, netif, ifindex): """ @@ -292,7 +292,7 @@ class CoreNodeBase(NodeBase): :return: nothing """ if ifindex in self._netif: - raise ValueError("ifindex %s already exists" % ifindex) + raise ValueError(f"ifindex {ifindex} already exists") self._netif[ifindex] = netif # TODO: this should have probably been set ahead, seems bad to me, check for failure and fix netif.netindex = ifindex @@ -305,7 +305,7 @@ class CoreNodeBase(NodeBase): :return: nothing """ if ifindex not in self._netif: - raise ValueError("ifindex %s does not exist" % ifindex) + raise ValueError(f"ifindex {ifindex} does not exist") netif = self._netif.pop(ifindex) netif.shutdown() del netif @@ -334,7 +334,7 @@ class CoreNodeBase(NodeBase): :return: nothing """ if ifindex not in self._netif: - raise ValueError("ifindex %s does not exist" % ifindex) + raise ValueError(f"ifindex {ifindex} does not exist") self._netif[ifindex].attachnet(net) def detachnet(self, ifindex): @@ -345,7 +345,7 @@ class CoreNodeBase(NodeBase): :return: nothing """ if ifindex not in self._netif: - raise ValueError("ifindex %s does not exist" % ifindex) + raise ValueError(f"ifindex {ifindex} does not exist") self._netif[ifindex].detachnet() def setposition(self, x=None, y=None, z=None): @@ -472,7 +472,7 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - self.net_cmd("kill -0 %s" % self.pid) + self.net_cmd(f"kill -0 {self.pid}") except CoreCommandError: return False @@ -496,10 +496,11 @@ class CoreNode(CoreNodeBase): cmd=VNODED_BIN, name=self.ctrlchnlname ) if self.nodedir: - vnoded += " -C %s" % self.nodedir + vnoded += f" -C {self.nodedir}" env = self.session.get_environment(state=False) env["NODE_NUMBER"] = str(self.id) env["NODE_NAME"] = str(self.name) + logging.info("env: %s", env) output = self.net_cmd(vnoded, env=env) self.pid = int(output) @@ -545,13 +546,13 @@ class CoreNode(CoreNodeBase): # kill node process if present try: - self.net_cmd("kill -9 %s" % self.pid) + self.net_cmd(f"kill -9 {self.pid}") except CoreCommandError: logging.exception("error killing process") # remove node directory if present try: - self.net_cmd("rm -rf %s" % self.ctrlchnlname) + self.net_cmd(f"rm -rf {self.ctrlchnlname}") except CoreCommandError: logging.exception("error removing node directory") @@ -604,11 +605,11 @@ class CoreNode(CoreNodeBase): :return: nothing """ if path[0] != "/": - raise ValueError("path not fully qualified: %s" % path) + raise ValueError(f"path not fully qualified: {path}") hostpath = os.path.join( self.nodedir, os.path.normpath(path).strip("/").replace("/", ".") ) - self.net_cmd("mkdir -p %s" % hostpath) + self.net_cmd(f"mkdir -p {hostpath}") self.mount(hostpath, path) def mount(self, source, target): @@ -622,8 +623,8 @@ class CoreNode(CoreNodeBase): """ source = os.path.abspath(source) logging.debug("node(%s) mounting: %s at %s", self.name, source, target) - self.node_net_cmd("mkdir -p %s" % target) - self.node_net_cmd("%s -n --bind %s %s" % (MOUNT_BIN, source, target)) + self.node_net_cmd(f"mkdir -p {target}") + self.node_net_cmd(f"{MOUNT_BIN} -n --bind {source} {target}") self._mounts.append((source, target)) def newifindex(self): @@ -650,22 +651,22 @@ class CoreNode(CoreNodeBase): ifindex = self.newifindex() if ifname is None: - ifname = "eth%d" % ifindex + ifname = f"eth{ifindex}" sessionid = self.session.short_session_id() try: - suffix = "%x.%s.%s" % (self.id, ifindex, sessionid) + suffix = f"{self.id:x}.{ifindex}.{sessionid}" except TypeError: - suffix = "%s.%s.%s" % (self.id, ifindex, sessionid) + suffix = f"{self.id}.{ifindex}.{sessionid}" - localname = "veth" + suffix + localname = f"veth{suffix}" if len(localname) >= 16: - raise ValueError("interface local name (%s) too long" % localname) + raise ValueError(f"interface local name ({localname}) too long") name = localname + "p" if len(name) >= 16: - raise ValueError("interface name (%s) too long" % name) + raise ValueError(f"interface name ({name}) too long") veth = Veth( self.session, self, name, localname, start=self.up, server=self.server @@ -716,10 +717,10 @@ class CoreNode(CoreNodeBase): ifindex = self.newifindex() if ifname is None: - ifname = "eth%d" % ifindex + ifname = f"eth{ifindex}" sessionid = self.session.short_session_id() - localname = "tap%s.%s.%s" % (self.id, ifindex, sessionid) + localname = f"tap{self.id}.{ifindex}.{sessionid}" name = ifname tuntap = TunTap(self.session, self, name, localname, start=self.up) @@ -778,7 +779,7 @@ class CoreNode(CoreNodeBase): try: interface.deladdr(addr) except ValueError: - logging.exception("trying to delete unknown address: %s" % addr) + logging.exception("trying to delete unknown address: %s", addr) if self.up: self.node_net_client.delete_address(interface.name, str(addr)) @@ -850,11 +851,11 @@ class CoreNode(CoreNodeBase): logging.info("adding file from %s to %s", srcname, filename) directory = os.path.dirname(filename) if self.server is None: - self.client.check_cmd("mkdir -p %s" % directory) - self.client.check_cmd("mv %s %s" % (srcname, filename)) + self.client.check_cmd(f"mkdir -p {directory}") + self.client.check_cmd(f"mv {srcname} {filename}") self.client.check_cmd("sync") else: - self.net_cmd("mkdir -p %s" % directory) + self.net_cmd(f"mkdir -p {directory}") self.server.remote_put(srcname, filename) def hostfilename(self, filename): @@ -866,7 +867,7 @@ class CoreNode(CoreNodeBase): """ dirname, basename = os.path.split(filename) if not basename: - raise ValueError("no basename for filename: %s" % filename) + raise ValueError(f"no basename for filename: {filename}") if dirname and dirname[0] == "/": dirname = dirname[1:] dirname = dirname.replace("/", ".") @@ -891,9 +892,9 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - self.net_cmd("mkdir -m %o -p %s" % (0o755, dirname)) + self.net_cmd(f"mkdir -m {0o755:o} -p {dirname}") self.server.remote_put_temp(hostfilename, contents) - self.net_cmd("chmod %o %s" % (mode, hostfilename)) + self.net_cmd(f"chmod {mode:o} {hostfilename}") logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode ) @@ -914,7 +915,7 @@ class CoreNode(CoreNodeBase): else: self.server.remote_put(srcfilename, hostfilename) if mode is not None: - self.net_cmd("chmod %o %s" % (mode, hostfilename)) + self.net_cmd(f"chmod {mode:o} {hostfilename}") logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) diff --git a/daemon/core/plugins/sdt.py b/daemon/core/plugins/sdt.py index 934233e9..280a2bc2 100644 --- a/daemon/core/plugins/sdt.py +++ b/daemon/core/plugins/sdt.py @@ -7,6 +7,7 @@ import socket from urllib.parse import urlparse from core import constants +from core.constants import CORE_DATA_DIR from core.emane.nodes import EmaneNet from core.emulator.enumerations import ( EventTypes, @@ -161,7 +162,7 @@ class Sdt(object): return False self.seturl() - logging.info("connecting to SDT at %s://%s" % (self.protocol, self.address)) + logging.info("connecting to SDT at %s://%s", self.protocol, self.address) if self.sock is None: try: if self.protocol.lower() == "udp": @@ -192,14 +193,14 @@ class Sdt(object): :return: initialize command status :rtype: bool """ - if not self.cmd('path "%s/icons/normal"' % constants.CORE_DATA_DIR): + if not self.cmd(f'path "{CORE_DATA_DIR}/icons/normal"'): return False # send node type to icon mappings for node_type, icon in self.DEFAULT_SPRITES: - if not self.cmd("sprite %s image %s" % (node_type, icon)): + if not self.cmd(f"sprite {node_type} image {icon}"): return False lat, long = self.session.location.refgeo[:2] - return self.cmd("flyto %.6f,%.6f,%d" % (long, lat, self.DEFAULT_ALT)) + return self.cmd(f"flyto {long:.6f},{lat:.6f},{self.DEFAULT_ALT}") def disconnect(self): """ @@ -240,8 +241,8 @@ class Sdt(object): if self.sock is None: return False try: - logging.info("sdt: %s" % cmdstr) - self.sock.sendall("%s\n" % cmdstr) + logging.info("sdt: %s", cmdstr) + self.sock.sendall(f"{cmdstr}\n") return True except IOError: logging.exception("SDT connection error") @@ -266,23 +267,21 @@ class Sdt(object): if not self.connect(): return if flags & MessageFlags.DELETE.value: - self.cmd("delete node,%d" % nodenum) + self.cmd(f"delete node,{nodenum}") return if x is None or y is None: return lat, lon, alt = self.session.location.getgeo(x, y, z) - pos = "pos %.6f,%.6f,%.6f" % (lon, lat, alt) + pos = f"pos {lon:.6f},{lat:.6f},{alt:.6f}" if flags & MessageFlags.ADD.value: if icon is not None: node_type = name icon = icon.replace("$CORE_DATA_DIR", constants.CORE_DATA_DIR) icon = icon.replace("$CORE_CONF_DIR", constants.CORE_CONF_DIR) - self.cmd("sprite %s image %s" % (type, icon)) - self.cmd( - 'node %d type %s label on,"%s" %s' % (nodenum, node_type, name, pos) - ) + self.cmd(f"sprite {node_type} image {icon}") + self.cmd(f'node {nodenum} type {node_type} label on,"{name}" {pos}') else: - self.cmd("node %d %s" % (nodenum, pos)) + self.cmd(f"node {nodenum} {pos}") def updatenodegeo(self, nodenum, lat, long, alt): """ @@ -298,8 +297,8 @@ class Sdt(object): # TODO: received Node Message with lat/long/alt. if not self.connect(): return - pos = "pos %.6f,%.6f,%.6f" % (long, lat, alt) - self.cmd("node %d %s" % (nodenum, pos)) + pos = f"pos {long:.6f},{lat:.6f},{alt:.6f}" + self.cmd(f"node {nodenum} {pos}") def updatelink(self, node1num, node2num, flags, wireless=False): """ @@ -316,14 +315,13 @@ class Sdt(object): if not self.connect(): return if flags & MessageFlags.DELETE.value: - self.cmd("delete link,%s,%s" % (node1num, node2num)) + self.cmd(f"delete link,{node1num},{node2num}") elif flags & MessageFlags.ADD.value: - attr = "" if wireless: attr = " line green,2" else: attr = " line red,2" - self.cmd("link %s,%s%s" % (node1num, node2num, attr)) + self.cmd(f"link {node1num},{node2num}{attr}") def sendobjs(self): """ diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 8f8da19c..7d88e355 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -137,7 +137,7 @@ def which(command, required): break if found_path is None and required: - raise ValueError("failed to find required executable(%s) in path" % command) + raise ValueError(f"failed to find required executable({command}) in path") return found_path @@ -238,12 +238,13 @@ def hex_dump(s, bytes_per_word=2, words_per_line=8): line = s[:total_bytes] s = s[total_bytes:] tmp = map( - lambda x: ("%02x" * bytes_per_word) % x, + lambda x: (f"{bytes_per_word:02x}" * bytes_per_word) % x, zip(*[iter(map(ord, line))] * bytes_per_word), ) if len(line) % 2: - tmp.append("%x" % ord(line[-1])) - dump += "0x%08x: %s\n" % (count, " ".join(tmp)) + tmp.append(f"{ord(line[-1]):x}") + tmp = " ".join(tmp) + dump += f"0x{count:08x}: {tmp}\n" count += len(line) return dump[:-1] @@ -261,9 +262,9 @@ def file_munge(pathname, header, text): file_demunge(pathname, header) with open(pathname, "a") as append_file: - append_file.write("# BEGIN %s\n" % header) + append_file.write(f"# BEGIN {header}\n") append_file.write(text) - append_file.write("# END %s\n" % header) + append_file.write(f"# END {header}\n") def file_demunge(pathname, header): @@ -281,9 +282,9 @@ def file_demunge(pathname, header): end = None for i, line in enumerate(lines): - if line == "# BEGIN %s\n" % header: + if line == f"# BEGIN {header}\n": start = i - elif line == "# END %s\n" % header: + elif line == f"# END {header}\n": end = i + 1 if start is None or end is None: @@ -305,7 +306,7 @@ def expand_corepath(pathname, session=None, node=None): :rtype: str """ if session is not None: - pathname = pathname.replace("~", "/home/%s" % session.user) + pathname = pathname.replace("~", f"/home/{session.user}") pathname = pathname.replace("%SESSION%", str(session.id)) pathname = pathname.replace("%SESSION_DIR%", session.session_dir) pathname = pathname.replace("%SESSION_USER%", session.user) @@ -364,7 +365,7 @@ def load_classes(path, clazz): # validate path exists logging.debug("attempting to load modules from path: %s", path) if not os.path.isdir(path): - logging.warning("invalid custom module directory specified" ": %s" % path) + logging.warning("invalid custom module directory specified" ": %s", path) # check if path is in sys.path parent_path = os.path.dirname(path) if parent_path not in sys.path: @@ -380,7 +381,7 @@ def load_classes(path, clazz): # import and add all service modules in the path classes = [] for module_name in module_names: - import_statement = "%s.%s" % (base_module, module_name) + import_statement = f"{base_module}.{module_name}" logging.debug("importing custom module: %s", import_statement) try: module = importlib.import_module(import_statement) diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 78d1c488..f4a360c6 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -600,7 +600,7 @@ class CoreXmlReader(object): name = hook.get("name") state = hook.get("state") data = hook.text - hook_type = "hook:%s" % state + hook_type = f"hook:{state}" logging.info("reading hook: state(%s) name(%s)", state, name) self.session.set_hook( hook_type, file_name=name, source_name=None, data=data diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index 0a81b75e..239bace2 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -31,20 +31,20 @@ def add_emane_interface(host_element, netif, platform_name="p1", transport_name= host_id = host_element.get("id") # platform data - platform_id = "%s/%s" % (host_id, platform_name) + platform_id = f"{host_id}/{platform_name}" platform_element = etree.SubElement( host_element, "emanePlatform", id=platform_id, name=platform_name ) # transport data - transport_id = "%s/%s" % (host_id, transport_name) + transport_id = f"{host_id}/{transport_name}" etree.SubElement( platform_element, "transport", id=transport_id, name=transport_name ) # nem data - nem_name = "nem%s" % nem_id - nem_element_id = "%s/%s" % (host_id, nem_name) + nem_name = f"nem{nem_id}" + nem_element_id = f"{host_id}/{nem_name}" nem_element = etree.SubElement( platform_element, "nem", id=nem_element_id, name=nem_name ) @@ -68,7 +68,7 @@ def get_address_type(address): def get_ipv4_addresses(hostname): if hostname == "localhost": addresses = [] - args = "%s -o -f inet address show" % IP_BIN + args = f"{IP_BIN} -o -f inet address show" output = utils.check_cmd(args) for line in output.split(os.linesep): split = line.split() @@ -94,13 +94,12 @@ class CoreXmlDeployment(object): self.add_deployment() def find_device(self, name): - device = self.scenario.find("devices/device[@name='%s']" % name) + device = self.scenario.find(f"devices/device[@name='{name}']") return device def find_interface(self, device, name): interface = self.scenario.find( - "devices/device[@name='%s']/interfaces/interface[@name='%s']" - % (device.name, name) + f"devices/device[@name='{device.name}']/interfaces/interface[@name='{name}']" ) return interface @@ -114,7 +113,8 @@ class CoreXmlDeployment(object): def add_physical_host(self, name): # add host - host_id = "%s/%s" % (self.root.get("id"), name) + root_id = self.root.get("id") + host_id = f"{root_id}/{name}" host_element = etree.SubElement(self.root, "testHost", id=host_id, name=name) # add type element @@ -128,10 +128,11 @@ class CoreXmlDeployment(object): def add_virtual_host(self, physical_host, node): if not isinstance(node, CoreNodeBase): - raise TypeError("invalid node type: %s" % node) + raise TypeError(f"invalid node type: {node}") # create virtual host element - host_id = "%s/%s" % (physical_host.get("id"), node.name) + phys_id = physical_host.get("id") + host_id = f"{phys_id}/{node.name}" host_element = etree.SubElement( physical_host, "testHost", id=host_id, name=node.name ) diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 41319ea4..c97c176f 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -57,8 +57,7 @@ def create_file(xml_element, doc_name, file_path, server=None): :return: nothing """ doctype = ( - '' - % {"doc_name": doc_name} + f'' ) if server is not None: temp = NamedTemporaryFile(delete=False) @@ -208,7 +207,7 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x node.setnemid(netif, nem_id) macstr = _hwaddr_prefix + ":00:00:" - macstr += "%02X:%02X" % ((nem_id >> 8) & 0xFF, nem_id & 0xFF) + macstr += f"{(nem_id >> 8) & 0xFF:02X}:{nem_id & 0xFF:02X}" netif.sethwaddr(MacAddress.from_string(macstr)) # increment nem id @@ -222,7 +221,7 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x file_path = os.path.join(emane_manager.session.session_dir, file_name) create_file(platform_element, doc_name, file_path) else: - file_name = "platform%d.xml" % key + file_name = f"platform{key}.xml" file_path = os.path.join(emane_manager.session.session_dir, file_name) linked_node = emane_manager.session.nodes[key] create_file(platform_element, doc_name, file_path, linked_node.server) @@ -290,8 +289,8 @@ def build_transport_xml(emane_manager, node, transport_type): """ transport_element = etree.Element( "transport", - name="%s Transport" % transport_type.capitalize(), - library="trans%s" % transport_type.lower(), + name=f"{transport_type.capitalize()} Transport", + library=f"trans{transport_type.lower()}", ) # add bitrate @@ -330,7 +329,7 @@ def create_phy_xml(emane_model, config, file_path, server): will run on, default is None for localhost :return: nothing """ - phy_element = etree.Element("phy", name="%s PHY" % emane_model.name) + phy_element = etree.Element("phy", name=f"{emane_model.name} PHY") if emane_model.phy_library: phy_element.set("library", emane_model.phy_library) @@ -362,7 +361,7 @@ def create_mac_xml(emane_model, config, file_path, server): raise ValueError("must define emane model library") mac_element = etree.Element( - "mac", name="%s MAC" % emane_model.name, library=emane_model.mac_library + "mac", name=f"{emane_model.name} MAC", library=emane_model.mac_library ) add_configurations( mac_element, emane_model.mac_config, config, emane_model.config_ignore @@ -399,7 +398,7 @@ def create_nem_xml( will run on, default is None for localhost :return: nothing """ - nem_element = etree.Element("nem", name="%s NEM" % emane_model.name) + nem_element = etree.Element("nem", name=f"{emane_model.name} NEM") if is_external(config): nem_element.set("type", "unstructured") else: @@ -450,7 +449,7 @@ def transport_file_name(node_id, transport_type): :param str transport_type: transport type to generate transport file :return: """ - return "n%strans%s.xml" % (node_id, transport_type) + return f"n{node_id}trans{transport_type}.xml" def _basename(emane_model, interface=None): @@ -461,14 +460,14 @@ def _basename(emane_model, interface=None): :return: basename used for file creation :rtype: str """ - name = "n%s" % emane_model.id + name = f"n{emane_model.id}" if interface: node_id = interface.node.id if emane_model.session.emane.getifcconfig(node_id, interface, emane_model.name): name = interface.localname.replace(".", "_") - return "%s%s" % (name, emane_model.name) + return f"{name}{emane_model.name}" def nem_file_name(emane_model, interface=None): @@ -484,7 +483,7 @@ def nem_file_name(emane_model, interface=None): append = "" if interface and interface.transport_type == "raw": append = "_raw" - return "%snem%s.xml" % (basename, append) + return f"{basename}nem{append}.xml" def shim_file_name(emane_model, interface=None): @@ -496,7 +495,8 @@ def shim_file_name(emane_model, interface=None): :return: shim xml filename :rtype: str """ - return "%sshim.xml" % _basename(emane_model, interface) + name = _basename(emane_model, interface) + return f"{name}shim.xml" def mac_file_name(emane_model, interface=None): @@ -508,7 +508,8 @@ def mac_file_name(emane_model, interface=None): :return: mac xml filename :rtype: str """ - return "%smac.xml" % _basename(emane_model, interface) + name = _basename(emane_model, interface) + return f"{name}mac.xml" def phy_file_name(emane_model, interface=None): @@ -520,4 +521,5 @@ def phy_file_name(emane_model, interface=None): :return: phy xml filename :rtype: str """ - return "%sphy.xml" % _basename(emane_model, interface) + name = _basename(emane_model, interface) + return f"{name}phy.xml" diff --git a/daemon/examples/python/emane80211.py b/daemon/examples/python/emane80211.py index 93222f9e..d77252da 100644 --- a/daemon/examples/python/emane80211.py +++ b/daemon/examples/python/emane80211.py @@ -48,12 +48,11 @@ def main(): logging.basicConfig(level=logging.INFO) options = parser.parse_options("emane80211") start = datetime.datetime.now() - print( - "running emane 80211 example: nodes(%s) time(%s)" - % (options.nodes, options.time) + logging.info( + "running emane 80211 example: nodes(%s) time(%s)", options.nodes, options.time ) example(options) - print("elapsed time: %s" % (datetime.datetime.now() - start)) + logging.info("elapsed time: %s", datetime.datetime.now() - start) if __name__ == "__main__" or __name__ == "__builtin__": diff --git a/daemon/examples/python/parser.py b/daemon/examples/python/parser.py index fdc2591a..d9efdab6 100644 --- a/daemon/examples/python/parser.py +++ b/daemon/examples/python/parser.py @@ -6,7 +6,7 @@ DEFAULT_STEP = 1 def parse_options(name): - parser = argparse.ArgumentParser(description="Run %s example" % name) + parser = argparse.ArgumentParser(description=f"Run {name} example") parser.add_argument( "-n", "--nodes", @@ -25,8 +25,8 @@ def parse_options(name): options = parser.parse_args() if options.nodes < 2: - parser.error("invalid min number of nodes: %s" % options.nodes) + parser.error(f"invalid min number of nodes: {options.nodes}") if options.time < 1: - parser.error("invalid test time: %s" % options.time) + parser.error(f"invalid test time: {options.time}") return options diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index 6702802e..80257a4a 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -41,14 +41,12 @@ def example(options): first_node = session.get_node(2) last_node = session.get_node(options.nodes + 1) - print("starting iperf server on node: %s" % first_node.name) + logging.info("starting iperf server on node: %s", first_node.name) first_node.node_net_cmd("iperf -s -D") first_node_address = prefixes.ip4_address(first_node) - print("node %s connecting to %s" % (last_node.name, first_node_address)) - output = last_node.node_net_cmd( - "iperf -t %s -c %s" % (options.time, first_node_address) - ) - print(output) + logging.info("node %s connecting to %s", last_node.name, first_node_address) + output = last_node.node_net_cmd(f"iperf -t {options.time} -c {first_node_address}") + logging.info(output) first_node.node_net_cmd("killall -9 iperf") # shutdown session @@ -59,9 +57,11 @@ def main(): logging.basicConfig(level=logging.INFO) options = parser.parse_options("switch") start = datetime.datetime.now() - print("running switch example: nodes(%s) time(%s)" % (options.nodes, options.time)) + logging.info( + "running switch example: nodes(%s) time(%s)", options.nodes, options.time + ) example(options) - print("elapsed time: %s" % (datetime.datetime.now() - start)) + logging.info("elapsed time: %s", datetime.datetime.now() - start) if __name__ == "__main__": diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index b3b4544e..9506a35c 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -45,11 +45,11 @@ def example(options): first_node = session.get_node(2) last_node = session.get_node(options.nodes + 1) - print("starting iperf server on node: %s" % first_node.name) + logging.info("starting iperf server on node: %s", first_node.name) first_node.node_net_cmd("iperf -s -D") address = prefixes.ip4_address(first_node) - print("node %s connecting to %s" % (last_node.name, address)) - last_node.node_net_cmd("iperf -t %s -c %s" % (options.time, address)) + logging.info("node %s connecting to %s", last_node.name, address) + last_node.node_net_cmd(f"iperf -t {options.time} -c {address}") first_node.node_net_cmd("killall -9 iperf") # shutdown session @@ -61,9 +61,11 @@ def main(): options = parser.parse_options("wlan") start = datetime.datetime.now() - print("running wlan example: nodes(%s) time(%s)" % (options.nodes, options.time)) + logging.info( + "running wlan example: nodes(%s) time(%s)", options.nodes, options.time + ) example(options) - print("elapsed time: %s" % (datetime.datetime.now() - start)) + logging.info("elapsed time: %s", datetime.datetime.now() - start) if __name__ == "__main__": diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 07c6a9a7..49aae2d2 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -17,7 +17,9 @@ from core import constants from core.api.grpc.server import CoreGrpcServer from core.api.tlv.corehandlers import CoreHandler, CoreUdpHandler from core.api.tlv.coreserver import CoreServer, CoreUdpServer +from core.constants import CORE_CONF_DIR, COREDPY_VERSION from core.emulator import enumerations +from core.emulator.enumerations import CORE_API_PORT from core.utils import close_onexec, load_logging_config @@ -67,7 +69,9 @@ def cored(cfg): # initialize grpc api if cfg["grpc"] == "True": grpc_server = CoreGrpcServer(server.coreemu) - grpc_address = "%s:%s" % (cfg["grpcaddress"], cfg["grpcport"]) + address_config = cfg["grpcaddress"] + port_config = cfg["grpcport"] + grpc_address = f"{address_config}:{port_config}" grpc_thread = threading.Thread(target=grpc_server.listen, args=(grpc_address,)) grpc_thread.daemon = True grpc_thread.start() @@ -91,30 +95,34 @@ def get_merged_config(filename): :rtype: dict """ # these are the defaults used in the config file + default_log = os.path.join(constants.CORE_CONF_DIR, "logging.conf") + default_grpc_port = "50051" + default_threads = "1" + default_address = "localhost" defaults = { - "port": "%d" % enumerations.CORE_API_PORT, - "listenaddr": "localhost", - "numthreads": "1", - "grpcport": "50051", - "grpcaddress": "localhost", - "logfile": os.path.join(constants.CORE_CONF_DIR, "logging.conf") + "port": str(CORE_API_PORT), + "listenaddr": default_address, + "numthreads": default_threads, + "grpcport": default_grpc_port, + "grpcaddress": default_address, + "logfile": default_log } parser = argparse.ArgumentParser( - description="CORE daemon v.%s instantiates Linux network namespace nodes." % constants.COREDPY_VERSION) + description=f"CORE daemon v.{COREDPY_VERSION} instantiates Linux network namespace nodes.") parser.add_argument("-f", "--configfile", dest="configfile", - help="read config from specified file; default = %s" % filename) + help=f"read config from specified file; default = {filename}") parser.add_argument("-p", "--port", dest="port", type=int, - help="port number to listen on; default = %s" % defaults["port"]) + help=f"port number to listen on; default = {CORE_API_PORT}") parser.add_argument("-n", "--numthreads", dest="numthreads", type=int, - help="number of server threads; default = %s" % defaults["numthreads"]) + help=f"number of server threads; default = {default_threads}") parser.add_argument("--ovs", action="store_true", help="enable experimental ovs mode, default is false") parser.add_argument("--grpc", action="store_true", help="enable grpc api, default is false") parser.add_argument("--grpc-port", dest="grpcport", - help="grpc port to listen on; default %s" % defaults["grpcport"]) + help=f"grpc port to listen on; default {default_grpc_port}") parser.add_argument("--grpc-address", dest="grpcaddress", - help="grpc address to listen on; default %s" % defaults["grpcaddress"]) - parser.add_argument("-l", "--logfile", help="core logging configuration; default %s" % defaults["logfile"]) + help=f"grpc address to listen on; default {default_address}") + parser.add_argument("-l", "--logfile", help=f"core logging configuration; default {default_log}") # parse command line options args = parser.parse_args() @@ -146,7 +154,7 @@ def main(): :return: nothing """ # get a configuration merged from config file and command-line arguments - cfg = get_merged_config("%s/core.conf" % constants.CORE_CONF_DIR) + cfg = get_merged_config(f"{CORE_CONF_DIR}/core.conf") # load logging configuration load_logging_config(cfg["logfile"]) diff --git a/daemon/scripts/core-manage b/daemon/scripts/core-manage index 93247f6f..d9b9de08 100755 --- a/daemon/scripts/core-manage +++ b/daemon/scripts/core-manage @@ -38,7 +38,7 @@ class FileUpdater(object): txt = "Updating" if self.action == "check": txt = "Checking" - sys.stdout.write("%s file: %s\n" % (txt, self.filename)) + sys.stdout.write(f"{txt} file: {self.filename}\n") if self.target == "service": r = self.update_file(fn=self.update_services) @@ -52,9 +52,9 @@ class FileUpdater(object): if not r: txt = "NOT " if self.action == "check": - sys.stdout.write("String %sfound.\n" % txt) + sys.stdout.write(f"String {txt} found.\n") else: - sys.stdout.write("File %supdated.\n" % txt) + sys.stdout.write(f"File {txt} updated.\n") return r @@ -70,7 +70,7 @@ class FileUpdater(object): r = self.update_keyvals(key, vals) if self.action == "check": return r - valstr = "%s" % r + valstr = str(r) return "= ".join([key, valstr]) + "\n" def update_emane_models(self, line): @@ -125,7 +125,7 @@ class FileUpdater(object): else: raise ValueError("unknown target") if not os.path.exists(filename): - raise ValueError("file %s does not exist" % filename) + raise ValueError(f"file {filename} does not exist") return search, filename def update_file(self, fn=None): @@ -187,18 +187,17 @@ class FileUpdater(object): def main(): + actions = ", ".join(FileUpdater.actions) + targets = ", ".join(FileUpdater.targets) usagestr = "usage: %prog [-h] [options] \n" usagestr += "\nHelper tool to add, remove, or check for " usagestr += "services, models, and node types\nin a CORE installation.\n" usagestr += "\nExamples:\n %prog add service newrouting" usagestr += "\n %prog -v check model RfPipe" usagestr += "\n %prog --userpath=\"$HOME/.core\" add nodetype \"{ftp ftp.gif ftp.gif {DefaultRoute FTP} netns {FTP server} }\" \n" - usagestr += "\nArguments:\n should be one of: %s" % \ - ", ".join(FileUpdater.actions) - usagestr += "\n should be one of: %s" % \ - ", ".join(FileUpdater.targets) - usagestr += "\n is the text to %s" % \ - ", ".join(FileUpdater.actions) + usagestr += f"\nArguments:\n should be one of: {actions}" + usagestr += f"\n should be one of: {targets}" + usagestr += f"\n is the text to {actions}" parser = optparse.OptionParser(usage=usagestr) parser.set_defaults(userpath=None, verbose=False, ) @@ -222,14 +221,14 @@ def main(): action = args[0] if action not in FileUpdater.actions: - usage("invalid action %s" % action, 1) + usage(f"invalid action {action}", 1) target = args[1] if target not in FileUpdater.targets: - usage("invalid target %s" % target, 1) + usage(f"invalid target {target}", 1) if target == "nodetype" and not options.userpath: - usage("user path option required for this target (%s)" % target) + usage(f"user path option required for this target ({target})") data = args[2] @@ -237,7 +236,7 @@ def main(): up = FileUpdater(action, target, data, options) r = up.process() except Exception as e: - sys.stderr.write("Exception: %s\n" % e) + sys.stderr.write(f"Exception: {e}\n") sys.exit(1) if not r: sys.exit(1) diff --git a/daemon/scripts/coresendmsg b/daemon/scripts/coresendmsg index c8813f21..c8dccfd7 100755 --- a/daemon/scripts/coresendmsg +++ b/daemon/scripts/coresendmsg @@ -21,9 +21,9 @@ def print_available_tlvs(t, tlv_class): """ Print a TLV list. """ - print("TLVs available for %s message:" % t) + print(f"TLVs available for {t} message:") for tlv in sorted([tlv for tlv in tlv_class.tlv_type_map], key=lambda x: x.name): - print("%s:%s" % (tlv.value, tlv.name)) + print(f"{tlv.value}:{tlv.name}") def print_examples(name): @@ -54,9 +54,9 @@ def print_examples(name): "srcname=\"./test.log\"", "move a test.log file from host to node 2"), ] - print("Example %s invocations:" % name) + print(f"Example {name} invocations:") for cmd, descr in examples: - print(" %s %s\n\t\t%s" % (name, cmd, descr)) + print(f" {name} {cmd}\n\t\t{descr}") def receive_message(sock): @@ -86,11 +86,11 @@ def receive_message(sock): except KeyError: msg = coreapi.CoreMessage(msgflags, msghdr, msgdata) msg.message_type = msgtype - print("unimplemented CORE message type: %s" % msg.type_str()) + print(f"unimplemented CORE message type: {msg.type_str()}") return msg if len(data) > msglen + coreapi.CoreMessage.header_len: - print("received a message of type %d, dropping %d bytes of extra data" \ - % (msgtype, len(data) - (msglen + coreapi.CoreMessage.header_len))) + data_size = len(data) - (msglen + coreapi.CoreMessage.header_len) + print(f"received a message of type {msgtype}, dropping {data_size} bytes of extra data") return msgcls(msgflags, msghdr, msgdata) @@ -132,7 +132,7 @@ def connect_to_session(sock, requested): print("requested session not found!") return False - print("joining session: %s" % session) + print(f"joining session: {session}") tlvdata = coreapi.CoreSessionTlv.pack(SessionTlvs.NUMBER.value, session) flags = MessageFlags.ADD.value smsg = coreapi.CoreSessionMessage.pack(flags, tlvdata) @@ -147,9 +147,9 @@ def receive_response(sock, opt): print("waiting for response...") msg = receive_message(sock) if msg is None: - print("disconnected from %s:%s" % (opt.address, opt.port)) + print(f"disconnected from {opt.address}:{opt.port}") sys.exit(0) - print("received message: %s" % msg) + print(f"received message: {msg}") def main(): @@ -160,36 +160,36 @@ def main(): flags = [flag.name for flag in MessageFlags] usagestr = "usage: %prog [-h|-H] [options] [message-type] [flags=flags] " usagestr += "[message-TLVs]\n\n" - usagestr += "Supported message types:\n %s\n" % types - usagestr += "Supported message flags (flags=f1,f2,...):\n %s" % flags + usagestr += f"Supported message types:\n {types}\n" + usagestr += f"Supported message flags (flags=f1,f2,...):\n {flags}" parser = optparse.OptionParser(usage=usagestr) + default_address = "localhost" + default_session = None + default_tcp = False parser.set_defaults( port=CORE_API_PORT, - address="localhost", - session=None, + address=default_address, + session=default_session, listen=False, examples=False, tlvs=False, - tcp=False + tcp=default_tcp ) parser.add_option("-H", dest="examples", action="store_true", help="show example usage help message and exit") parser.add_option("-p", "--port", dest="port", type=int, - help="TCP port to connect to, default: %d" % \ - parser.defaults["port"]) + help=f"TCP port to connect to, default: {CORE_API_PORT}") parser.add_option("-a", "--address", dest="address", type=str, - help="Address to connect to, default: %s" % \ - parser.defaults["address"]) + help=f"Address to connect to, default: {default_address}") parser.add_option("-s", "--session", dest="session", type=str, - help="Session to join, default: %s" % \ - parser.defaults["session"]) + help=f"Session to join, default: {default_session}") parser.add_option("-l", "--listen", dest="listen", action="store_true", help="Listen for a response message and print it.") parser.add_option("-t", "--list-tlvs", dest="tlvs", action="store_true", help="List TLVs for the specified message type.") parser.add_option("--tcp", dest="tcp", action="store_true", - help="Use TCP instead of UDP and connect to a session default: %s" % parser.defaults["tcp"]) + help=f"Use TCP instead of UDP and connect to a session default: {default_tcp}") def usage(msg=None, err=0): sys.stdout.write("\n") @@ -209,7 +209,7 @@ def main(): # given a message type t, determine the message and TLV classes t = args.pop(0) if t not in types: - usage("Unknown message type requested: %s" % t) + usage(f"Unknown message type requested: {t}") message_type = MessageTypes[t] msg_cls = coreapi.CLASS_MAP[message_type.value] tlv_cls = msg_cls.tlv_class @@ -225,7 +225,7 @@ def main(): for a in args: typevalue = a.split("=") if len(typevalue) < 2: - usage("Use \"type=value\" syntax instead of \"%s\"." % a) + usage(f"Use \"type=value\" syntax instead of \"{a}\".") tlv_typestr = typevalue[0] tlv_valstr = "=".join(typevalue[1:]) if tlv_typestr == "flags": @@ -237,7 +237,7 @@ def main(): tlv_type = tlv_cls.tlv_type_map[tlv_name] tlvdata += tlv_cls.pack_string(tlv_type.value, tlv_valstr) except KeyError: - usage("Unknown TLV: \"%s\"" % tlv_name) + usage(f"Unknown TLV: \"{tlv_name}\"") flags = 0 for f in flagstr.split(","): @@ -249,7 +249,7 @@ def main(): n = flag_enum.value flags |= n except KeyError: - usage("Invalid flag \"%s\"." % f) + usage(f"Invalid flag \"{f}\".") msg = msg_cls.pack(flags, tlvdata) @@ -264,7 +264,7 @@ def main(): try: sock.connect((opt.address, opt.port)) except Exception as e: - print("Error connecting to %s:%s:\n\t%s" % (opt.address, opt.port, e)) + print(f"Error connecting to {opt.address}:{opt.port}:\n\t{e}") sys.exit(1) if opt.tcp and not connect_to_session(sock, opt.session): From 07b44080763dade1c3f0b0420da3d95a835e0cdf Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 12:44:13 -0700 Subject: [PATCH 095/113] f string updates to all but services complete --- daemon/core/api/tlv/coreapi.py | 34 +++++------ daemon/core/api/tlv/corehandlers.py | 47 +++++++-------- daemon/core/nodes/network.py | 91 +++++++++++++---------------- 3 files changed, 75 insertions(+), 97 deletions(-) diff --git a/daemon/core/api/tlv/coreapi.py b/daemon/core/api/tlv/coreapi.py index ba737fd4..0fd16bf5 100644 --- a/daemon/core/api/tlv/coreapi.py +++ b/daemon/core/api/tlv/coreapi.py @@ -180,7 +180,7 @@ class CoreTlvDataString(CoreTlvData): :rtype: tuple """ if not isinstance(value, str): - raise ValueError("value not a string: %s" % type(value)) + raise ValueError(f"value not a string: {type(value)}") value = value.encode("utf-8") if len(value) < 256: @@ -220,7 +220,7 @@ class CoreTlvDataUint16List(CoreTlvData): :rtype: tuple """ if not isinstance(values, tuple): - raise ValueError("value not a tuple: %s" % values) + raise ValueError(f"value not a tuple: {values}") data = b"" for value in values: @@ -237,7 +237,8 @@ class CoreTlvDataUint16List(CoreTlvData): :param data: data to unpack :return: unpacked data """ - data_format = "!%dH" % (len(data) / 2) + size = int(len(data) / 2) + data_format = f"!{size}H" return struct.unpack(data_format, data) @classmethod @@ -435,7 +436,7 @@ class CoreTlv(object): try: return self.tlv_type_map(self.tlv_type).name except ValueError: - return "unknown tlv type: %s" % str(self.tlv_type) + return f"unknown tlv type: {self.tlv_type}" def __str__(self): """ @@ -444,11 +445,7 @@ class CoreTlv(object): :return: string representation :rtype: str """ - return "%s " % ( - self.__class__.__name__, - self.type_str(), - self.value, - ) + return f"{self.__class__.__name__} " class CoreNodeTlv(CoreTlv): @@ -734,7 +731,7 @@ class CoreMessage(object): :return: nothing """ if key in self.tlv_data: - raise KeyError("key already exists: %s (val=%s)" % (key, value)) + raise KeyError(f"key already exists: {key} (val={value})") self.tlv_data[key] = value @@ -793,7 +790,7 @@ class CoreMessage(object): try: return MessageTypes(self.message_type).name except ValueError: - return "unknown message type: %s" % str(self.message_type) + return f"unknown message type: {self.message_type}" def flag_str(self): """ @@ -810,12 +807,13 @@ class CoreMessage(object): try: message_flags.append(self.flag_map(flag).name) except ValueError: - message_flags.append("0x%x" % flag) + message_flags.append(f"0x{flag:x}") flag <<= 1 if not (self.flags & ~(flag - 1)): break - return "0x%x <%s>" % (self.flags, " | ".join(message_flags)) + message_flags = " | ".join(message_flags) + return f"0x{self.flags:x} <{message_flags}>" def __str__(self): """ @@ -824,20 +822,16 @@ class CoreMessage(object): :return: string representation :rtype: str """ - result = "%s " % ( - self.__class__.__name__, - self.type_str(), - self.flag_str(), - ) + result = f"{self.__class__.__name__} " for key in self.tlv_data: value = self.tlv_data[key] try: tlv_type = self.tlv_class.tlv_type_map(key).name except ValueError: - tlv_type = "tlv type %s" % key + tlv_type = f"tlv type {key}" - result += "\n %s: %s" % (tlv_type, value) + result += f"\n {tlv_type}: {value}" return result diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index f3966ba1..85b7b831 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -75,7 +75,7 @@ class CoreHandler(socketserver.BaseRequestHandler): self.handler_threads = [] num_threads = int(server.config["numthreads"]) if num_threads < 1: - raise ValueError("invalid number of threads: %s" % num_threads) + raise ValueError(f"invalid number of threads: {num_threads}") logging.debug("launching core server handler threads: %s", num_threads) for _ in range(num_threads): @@ -460,7 +460,7 @@ class CoreHandler(socketserver.BaseRequestHandler): try: header = self.request.recv(coreapi.CoreMessage.header_len) except IOError as e: - raise IOError("error receiving header (%s)" % e) + raise IOError(f"error receiving header ({e})") if len(header) != coreapi.CoreMessage.header_len: if len(header) == 0: @@ -478,10 +478,7 @@ class CoreHandler(socketserver.BaseRequestHandler): while len(data) < message_len: data += self.request.recv(message_len - len(data)) if len(data) > message_len: - error_message = ( - "received message length does not match received data (%s != %s)" - % (len(data), message_len) - ) + error_message = f"received message length does not match received data ({len(data)} != {message_len})" logging.error(error_message) raise IOError(error_message) @@ -573,11 +570,7 @@ class CoreHandler(socketserver.BaseRequestHandler): ) except KeyError: # multiple TLVs of same type cause KeyError exception - reply_message = "CoreMessage (type %d flags %d length %d)" % ( - message_type, - message_flags, - message_length, - ) + reply_message = f"CoreMessage (type {message_type} flags {message_flags} length {message_length})" logging.debug("sending reply:\n%s", reply_message) @@ -999,7 +992,7 @@ class CoreHandler(socketserver.BaseRequestHandler): RegisterTlvs.EXECUTE_SERVER.value, execute_server ) tlv_data += coreapi.CoreRegisterTlv.pack( - RegisterTlvs.SESSION.value, "%s" % sid + RegisterTlvs.SESSION.value, str(sid) ) message = coreapi.CoreRegMessage.pack(0, tlv_data) replies.append(message) @@ -1104,7 +1097,7 @@ class CoreHandler(socketserver.BaseRequestHandler): self.session.mobility.config_reset(node_id) self.session.emane.config_reset(node_id) else: - raise Exception("cant handle config all: %s" % message_type) + raise Exception(f"cant handle config all: {message_type}") return replies @@ -1158,7 +1151,7 @@ class CoreHandler(socketserver.BaseRequestHandler): if metadata_configs is None: metadata_configs = {} data_values = "|".join( - ["%s=%s" % (x, metadata_configs[x]) for x in metadata_configs] + [f"{x}={metadata_configs[x]}" for x in metadata_configs] ) data_types = tuple(ConfigDataTypes.STRING.value for _ in metadata_configs) config_response = ConfigData( @@ -1239,7 +1232,7 @@ class CoreHandler(socketserver.BaseRequestHandler): services = sorted(group_map[group], key=lambda x: x.name.lower()) logging.debug("sorted services for group(%s): %s", group, services) end_index = start_index + len(services) - 1 - group_strings.append("%s:%s-%s" % (group, start_index, end_index)) + group_strings.append(f"{group}:{start_index}-{end_index}") start_index += len(services) for service_name in services: captions.append(service_name.name) @@ -1714,24 +1707,24 @@ class CoreHandler(socketserver.BaseRequestHandler): ): status = self.session.services.stop_service(node, service) if status: - fail += "Stop %s," % service.name + fail += f"Stop {service.name}," if ( event_type == EventTypes.START.value or event_type == EventTypes.RESTART.value ): status = self.session.services.startup_service(node, service) if status: - fail += "Start %s(%s)," % service.name + fail += f"Start ({service.name})," if event_type == EventTypes.PAUSE.value: status = self.session.services.validate_service(node, service) if status: - fail += "%s," % service.name + fail += f"{service.name}," if event_type == EventTypes.RECONFIGURE.value: self.session.services.service_reconfigure(node, service) fail_data = "" if len(fail) > 0: - fail_data += "Fail:" + fail + fail_data += f"Fail:{fail}" unknown_data = "" num = len(unknown) if num > 0: @@ -1741,14 +1734,14 @@ class CoreHandler(socketserver.BaseRequestHandler): unknown_data += ", " num -= 1 logging.warning("Event requested for unknown service(s): %s", unknown_data) - unknown_data = "Unknown:" + unknown_data + unknown_data = f"Unknown:{unknown_data}" event_data = EventData( node=node_id, event_type=event_type, name=name, data=fail_data + ";" + unknown_data, - time="%s" % time.time(), + time=str(time.time()), ) self.session.broadcast_event(event_data) @@ -1769,7 +1762,7 @@ class CoreHandler(socketserver.BaseRequestHandler): thumb = message.get_tlv(SessionTlvs.THUMB.value) user = message.get_tlv(SessionTlvs.USER.value) logging.debug( - "SESSION message flags=0x%x sessions=%s" % (message.flags, session_id_str) + "SESSION message flags=0x%x sessions=%s", message.flags, session_id_str ) if message.flags == 0: @@ -1940,7 +1933,7 @@ class CoreHandler(socketserver.BaseRequestHandler): # service customizations service_configs = self.session.services.all_configs() for node_id, service in service_configs: - opaque = "service:%s" % service.name + opaque = f"service:{service.name}" data_types = tuple( repeat(ConfigDataTypes.STRING.value, len(ServiceShim.keys)) ) @@ -1976,7 +1969,7 @@ class CoreHandler(socketserver.BaseRequestHandler): file_data = FileData( message_type=MessageFlags.ADD.value, name=str(file_name), - type="hook:%s" % state, + type=f"hook:{state}", data=str(config_data), ) self.session.broadcast_file(file_data) @@ -1992,7 +1985,7 @@ class CoreHandler(socketserver.BaseRequestHandler): metadata_configs = self.session.metadata.get_configs() if metadata_configs: data_values = "|".join( - ["%s=%s" % (x, metadata_configs[x]) for x in metadata_configs] + [f"{x}={metadata_configs[x]}" for x in metadata_configs] ) data_types = tuple( ConfigDataTypes.STRING.value @@ -2041,7 +2034,7 @@ class CoreUdpHandler(CoreHandler): data = self.request[0] header = data[: coreapi.CoreMessage.header_len] if len(header) < coreapi.CoreMessage.header_len: - raise IOError("error receiving header (received %d bytes)" % len(header)) + raise IOError(f"error receiving header (received {len(header)} bytes)") message_type, message_flags, message_len = coreapi.CoreMessage.unpack_header( header @@ -2136,7 +2129,7 @@ class CoreUdpHandler(CoreHandler): :return: """ raise Exception( - "Unable to queue %s message for later processing using UDP!" % msg + f"Unable to queue {msg} message for later processing using UDP!" ) def sendall(self, data): diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 98bec198..229005c4 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -97,7 +97,7 @@ class EbtablesQueue(object): :return: ebtable atomic command :rtype: list[str] """ - return "%s --atomic-file %s %s" % (EBTABLES_BIN, self.atomic_file, cmd) + return f"{EBTABLES_BIN} --atomic-file {self.atomic_file} {cmd}" def lastupdate(self, wlan): """ @@ -175,7 +175,7 @@ class EbtablesQueue(object): wlan.net_cmd(args) try: - wlan.net_cmd("rm -f %s" % self.atomic_file) + wlan.net_cmd(f"rm -f {self.atomic_file}") except CoreCommandError: logging.exception("error removing atomic file: %s", self.atomic_file) @@ -198,26 +198,22 @@ class EbtablesQueue(object): """ with wlan._linked_lock: # flush the chain - self.cmds.append("-F %s" % wlan.brname) + self.cmds.append(f"-F {wlan.brname}") # rebuild the chain for netif1, v in wlan._linked.items(): for netif2, linked in v.items(): if wlan.policy == "DROP" and linked: self.cmds.extend( [ - "-A %s -i %s -o %s -j ACCEPT" - % (wlan.brname, netif1.localname, netif2.localname), - "-A %s -o %s -i %s -j ACCEPT" - % (wlan.brname, netif1.localname, netif2.localname), + f"-A {wlan.brname} -i {netif1.localname} -o {netif2.localname} -j ACCEPT", + f"-A {wlan.brname} -o {netif1.localname} -i {netif2.localname} -j ACCEPT", ] ) elif wlan.policy == "ACCEPT" and not linked: self.cmds.extend( [ - "-A %s -i %s -o %s -j DROP" - % (wlan.brname, netif1.localname, netif2.localname), - "-A %s -o %s -i %s -j DROP" - % (wlan.brname, netif1.localname, netif2.localname), + f"-A {wlan.brname} -i {netif1.localname} -o {netif2.localname} -j DROP", + f"-A {wlan.brname} -o {netif1.localname} -i {netif2.localname} -j DROP", ] ) @@ -268,7 +264,7 @@ class CoreNetwork(CoreNetworkBase): self.policy = policy self.name = name sessionid = self.session.short_session_id() - self.brname = "b.%s.%s" % (str(self.id), sessionid) + self.brname = f"b.{self.id}.{sessionid}" self.up = False if start: self.startup() @@ -303,9 +299,8 @@ class CoreNetwork(CoreNetworkBase): # create a new ebtables chain for this bridge cmds = [ - "%s -N %s -P %s" % (EBTABLES_BIN, self.brname, self.policy), - "%s -A FORWARD --logical-in %s -j %s" - % (EBTABLES_BIN, self.brname, self.brname), + f"{EBTABLES_BIN} -N {self.brname} -P {self.policy}", + f"{EBTABLES_BIN} -A FORWARD --logical-in {self.brname} -j {self.brname}", ] ebtablescmds(self.net_cmd, cmds) @@ -325,9 +320,8 @@ class CoreNetwork(CoreNetworkBase): try: self.net_client.delete_bridge(self.brname) cmds = [ - "%s -D FORWARD --logical-in %s -j %s" - % (EBTABLES_BIN, self.brname, self.brname), - "%s -X %s" % (EBTABLES_BIN, self.brname), + f"{EBTABLES_BIN} -D FORWARD --logical-in {self.brname} -j {self.brname}", + f"{EBTABLES_BIN} -X {self.brname}", ] ebtablescmds(self.net_cmd, cmds) except CoreCommandError: @@ -379,10 +373,10 @@ class CoreNetwork(CoreNetworkBase): """ # check if the network interfaces are attached to this network if self._netif[netif1.netifi] != netif1: - raise ValueError("inconsistency for netif %s" % netif1.name) + raise ValueError(f"inconsistency for netif {netif1.name}") if self._netif[netif2.netifi] != netif2: - raise ValueError("inconsistency for netif %s" % netif2.name) + raise ValueError(f"inconsistency for netif {netif2.name}") try: linked = self._linked[netif1][netif2] @@ -392,7 +386,7 @@ class CoreNetwork(CoreNetworkBase): elif self.policy == "DROP": linked = False else: - raise Exception("unknown policy: %s" % self.policy) + raise Exception(f"unknown policy: {self.policy}") self._linked[netif1][netif2] = linked return linked @@ -455,7 +449,7 @@ class CoreNetwork(CoreNetworkBase): """ if devname is None: devname = netif.localname - tc = "%s qdisc replace dev %s" % (TC_BIN, devname) + tc = f"{TC_BIN} qdisc replace dev {devname}" parent = "root" changed = False if netif.setparam("bw", bw): @@ -464,16 +458,16 @@ class CoreNetwork(CoreNetworkBase): burst = max(2 * netif.mtu, bw / 1000) # max IP payload limit = 0xFFFF - tbf = "tbf rate %s burst %s limit %s" % (bw, burst, limit) + tbf = f"tbf rate {bw} burst {burst} limit {limit}" if bw > 0: if self.up: - cmd = "%s %s handle 1: %s" % (tc, parent, tbf) + cmd = f"{tc} {parent} handle 1: {tbf}" netif.net_cmd(cmd) netif.setparam("has_tbf", True) changed = True elif netif.getparam("has_tbf") and bw <= 0: if self.up: - cmd = "%s qdisc delete dev %s %s" % (TC_BIN, devname, parent) + cmd = f"{TC_BIN} qdisc delete dev {devname} {parent}" netif.net_cmd(cmd) netif.setparam("has_tbf", False) # removing the parent removes the child @@ -494,17 +488,17 @@ class CoreNetwork(CoreNetworkBase): return # jitter and delay use the same delay statement if delay is not None: - netem += " delay %sus" % delay + netem += f" delay {delay}us" if jitter is not None: if delay is None: - netem += " delay 0us %sus 25%%" % jitter + netem += f" delay 0us {jitter}us 25%" else: - netem += " %sus 25%%" % jitter + netem += f" {jitter}us 25%" if loss is not None and loss > 0: - netem += " loss %s%%" % min(loss, 100) + netem += f" loss {min(loss, 100)}%" if duplicate is not None and duplicate > 0: - netem += " duplicate %s%%" % min(duplicate, 100) + netem += f" duplicate {min(duplicate, 100)}%" delay_check = delay is None or delay <= 0 jitter_check = jitter is None or jitter <= 0 @@ -515,16 +509,13 @@ class CoreNetwork(CoreNetworkBase): if not netif.getparam("has_netem"): return if self.up: - cmd = "%s qdisc delete dev %s %s handle 10:" % (TC_BIN, devname, parent) + cmd = f"{TC_BIN} qdisc delete dev {devname} {parent} handle 10:" netif.net_cmd(cmd) netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: - cmd = "%s qdisc replace dev %s %s handle 10: %s" % ( - TC_BIN, - devname, - parent, - netem, + cmd = ( + f"{TC_BIN} qdisc replace dev {devname} {parent} handle 10: {netem}" ) netif.net_cmd(cmd) netif.setparam("has_netem", True) @@ -540,22 +531,22 @@ class CoreNetwork(CoreNetworkBase): """ sessionid = self.session.short_session_id() try: - _id = "%x" % self.id + _id = f"{self.id:x}" except TypeError: - _id = "%s" % self.id + _id = str(self.id) try: - net_id = "%x" % net.id + net_id = f"{net.id:x}" except TypeError: - net_id = "%s" % net.id + net_id = str(net.id) - localname = "veth%s.%s.%s" % (_id, net_id, sessionid) + localname = f"veth{_id}.{net_id}.{sessionid}" if len(localname) >= 16: - raise ValueError("interface local name %s too long" % localname) + raise ValueError(f"interface local name {localname} too long") - name = "veth%s.%s.%s" % (net_id, _id, sessionid) + name = f"veth{net_id}.{_id}.{sessionid}" if len(name) >= 16: - raise ValueError("interface name %s too long" % name) + raise ValueError(f"interface name {name} too long") netif = Veth(self.session, None, name, localname, start=self.up) self.attach(netif) @@ -689,7 +680,7 @@ class GreTapBridge(CoreNetwork): :return: nothing """ if self.gretap: - raise ValueError("gretap already exists for %s" % self.name) + raise ValueError(f"gretap already exists for {self.name}") remoteip = addrlist[0].split("/")[0] localip = None if len(addrlist) > 1: @@ -773,14 +764,14 @@ class CtrlNet(CoreNetwork): :return: """ use_ovs = self.session.options.get_config("ovs") == "True" - current = "%s/%s" % (address, self.prefix.prefixlen) + current = f"{address}/{self.prefix.prefixlen}" net_client = get_net_client(use_ovs, utils.check_cmd) net_client.create_address(self.brname, current) servers = self.session.distributed.servers for name in servers: server = servers[name] address -= 1 - current = "%s/%s" % (address, self.prefix.prefixlen) + current = f"{address}/{self.prefix.prefixlen}" net_client = get_net_client(use_ovs, server.remote_cmd) net_client.create_address(self.brname, current) @@ -792,7 +783,7 @@ class CtrlNet(CoreNetwork): :raises CoreCommandError: when there is a command exception """ if self.net_client.existing_bridges(self.id): - raise CoreError("old bridges exist for node: %s" % self.id) + raise CoreError(f"old bridges exist for node: {self.id}") CoreNetwork.startup(self) @@ -811,7 +802,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd("%s %s startup" % (self.updown_script, self.brname)) + self.net_cmd(f"{self.updown_script} {self.brname} startup") if self.serverintf: self.net_client.create_interface(self.brname, self.serverintf) @@ -839,7 +830,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd("%s %s shutdown" % (self.updown_script, self.brname)) + self.net_cmd(f"{self.updown_script} {self.brname} shutdown") except CoreCommandError: logging.exception("error issuing shutdown script shutdown") From 5633d4d18ba7e4b2f21dfa8855074532f71c5db9 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 12:55:35 -0700 Subject: [PATCH 096/113] converted format strings to f strings --- daemon/core/api/grpc/server.py | 16 +++--------- daemon/core/nodes/base.py | 8 ++---- daemon/core/nodes/docker.py | 46 ++++++++++------------------------ daemon/core/nodes/lxd.py | 22 +++++++--------- 4 files changed, 28 insertions(+), 64 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index dcea5fb5..11bfcd6b 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -247,9 +247,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): """ session = self.coreemu.sessions.get(session_id) if not session: - context.abort( - grpc.StatusCode.NOT_FOUND, "session {} not found".format(session_id) - ) + context.abort(grpc.StatusCode.NOT_FOUND, f"session {session_id} not found") return session def get_node(self, session, node_id, context): @@ -265,9 +263,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): try: return session.get_node(node_id) except CoreError: - context.abort( - grpc.StatusCode.NOT_FOUND, "node {} not found".format(node_id) - ) + context.abort(grpc.StatusCode.NOT_FOUND, f"node {node_id} not found") def CreateSession(self, request, context): """ @@ -1577,17 +1573,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): nem_one = request.nem_one emane_one, netif = session.emane.nemlookup(nem_one) if not emane_one or not netif: - context.abort( - grpc.StatusCode.NOT_FOUND, "nem one {} not found".format(nem_one) - ) + context.abort(grpc.StatusCode.NOT_FOUND, f"nem one {nem_one} not found") node_one = netif.node nem_two = request.nem_two emane_two, netif = session.emane.nemlookup(nem_two) if not emane_two or not netif: - context.abort( - grpc.StatusCode.NOT_FOUND, "nem two {} not found".format(nem_two) - ) + context.abort(grpc.StatusCode.NOT_FOUND, f"nem two {nem_two} not found") node_two = netif.node if emane_one.id == emane_two.id: diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 0b3bf04b..03147738 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -492,9 +492,7 @@ class CoreNode(CoreNodeBase): raise ValueError("starting a node that is already up") # create a new namespace for this node using vnoded - vnoded = "{cmd} -v -c {name} -l {name}.log -p {name}.pid".format( - cmd=VNODED_BIN, name=self.ctrlchnlname - ) + vnoded = f"{VNODED_BIN} -v -c {self.ctrlchnlname} -l {self.ctrlchnlname}.log -p {self.ctrlchnlname}.pid" if self.nodedir: vnoded += f" -C {self.nodedir}" env = self.session.get_environment(state=False) @@ -593,9 +591,7 @@ class CoreNode(CoreNodeBase): if self.server is None: return terminal else: - return "ssh -X -f {host} xterm -e {terminal}".format( - host=self.server.host, terminal=terminal - ) + return f"ssh -X -f {self.server.host} xterm -e {terminal}" def privatedir(self, path): """ diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index df8422af..369f462b 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -19,23 +19,18 @@ class DockerClient(object): def create_container(self): self.run( - "docker run -td --init --net=none --hostname {name} --name {name} " - "--sysctl net.ipv6.conf.all.disable_ipv6=0 " - "{image} /bin/bash".format( - name=self.name, - image=self.image - )) + f"docker run -td --init --net=none --hostname {self.name} --name {self.name} " + f"--sysctl net.ipv6.conf.all.disable_ipv6=0 {self.image} /bin/bash" + ) self.pid = self.get_pid() return self.pid def get_info(self): - args = "docker inspect {name}".format(name=self.name) + args = f"docker inspect {self.name}" output = self.run(args) data = json.loads(output) if not data: - raise CoreCommandError( - -1, args, "docker({name}) not present".format(name=self.name) - ) + raise CoreCommandError(-1, args, f"docker({self.name}) not present") return data[0] def is_alive(self): @@ -46,43 +41,28 @@ class DockerClient(object): return False def stop_container(self): - self.run("docker rm -f {name}".format( - name=self.name - )) + self.run(f"docker rm -f {self.name}") def check_cmd(self, cmd): logging.info("docker cmd output: %s", cmd) - return utils.check_cmd("docker exec {name} {cmd}".format( - name=self.name, - cmd=cmd - )) + return utils.check_cmd(f"docker exec {self.name} {cmd}") def create_ns_cmd(self, cmd): - return "nsenter -t {pid} -u -i -p -n {cmd}".format( - pid=self.pid, - cmd=cmd - ) + return f"nsenter -t {self.pid} -u -i -p -n {cmd}" def ns_cmd(self, cmd, wait): - args = "nsenter -t {pid} -u -i -p -n {cmd}".format( - pid=self.pid, - cmd=cmd - ) + args = f"nsenter -t {self.pid} -u -i -p -n {cmd}" return utils.check_cmd(args, wait=wait) def get_pid(self): - args = "docker inspect -f '{{{{.State.Pid}}}}' {name}".format(name=self.name) + args = f"docker inspect -f '{{{{.State.Pid}}}}' {self.name}" output = self.run(args) self.pid = output logging.debug("node(%s) pid: %s", self.name, self.pid) return output def copy_file(self, source, destination): - args = "docker cp {source} {name}:{destination}".format( - source=source, - name=self.name, - destination=destination - ) + args = f"docker cp {source} {self.name}:{destination}" return self.run(args) @@ -185,7 +165,7 @@ class DockerNode(CoreNode): :param str sh: shell to execute command in :return: str """ - return "docker exec -it {name} bash".format(name=self.name) + return f"docker exec -it {self.name} bash" def privatedir(self, path): """ @@ -195,7 +175,7 @@ class DockerNode(CoreNode): :return: nothing """ logging.debug("creating node dir: %s", path) - args = "mkdir -p {path}".format(path=path) + args = f"mkdir -p {path}" self.node_net_cmd(args) def mount(self, source, target): diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 50588fb2..323b20a9 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -18,19 +18,17 @@ class LxdClient(object): self.pid = None def create_container(self): - self.run("lxc launch {image} {name}".format(name=self.name, image=self.image)) + self.run(f"lxc launch {self.image} {self.name}") data = self.get_info() self.pid = data["state"]["pid"] return self.pid def get_info(self): - args = "lxc list {name} --format json".format(name=self.name) + args = f"lxc list {self.name} --format json" output = self.run(args) data = json.loads(output) if not data: - raise CoreCommandError( - -1, args, "LXC({name}) not present".format(name=self.name) - ) + raise CoreCommandError(-1, args, f"LXC({self.name}) not present") return data[0] def is_alive(self): @@ -41,13 +39,13 @@ class LxdClient(object): return False def stop_container(self): - self.run("lxc delete --force {name}".format(name=self.name)) + self.run(f"lxc delete --force {self.name}") def create_cmd(self, cmd): - return "lxc exec -nT {name} -- {cmd}".format(name=self.name, cmd=cmd) + return f"lxc exec -nT {self.name} -- {cmd}" def create_ns_cmd(self, cmd): - return "nsenter -t {pid} -m -u -i -p -n {cmd}".format(pid=self.pid, cmd=cmd) + return f"nsenter -t {self.pid} -m -u -i -p -n {cmd}" def check_cmd(self, cmd, wait=True): args = self.create_cmd(cmd) @@ -57,9 +55,7 @@ class LxdClient(object): if destination[0] != "/": destination = os.path.join("/root/", destination) - args = "lxc file push {source} {name}/{destination}".format( - source=source, name=self.name, destination=destination - ) + args = f"lxc file push {source} {self.name}/{destination}" self.run(args) @@ -142,7 +138,7 @@ class LxcNode(CoreNode): :param str sh: shell to execute command in :return: str """ - return "lxc exec {name} -- {sh}".format(name=self.name, sh=sh) + return f"lxc exec {self.name} -- {sh}" def privatedir(self, path): """ @@ -152,7 +148,7 @@ class LxcNode(CoreNode): :return: nothing """ logging.info("creating node dir: %s", path) - args = "mkdir -p {path}".format(path=path) + args = f"mkdir -p {path}" return self.node_net_cmd(args) def mount(self, source, target): From bab5c75cb9f8184e6e82c245eef0fc5dcdbb5dfa Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 13:20:05 -0700 Subject: [PATCH 097/113] removed unwanted logging of container env --- daemon/core/nodes/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 03147738..d8eaae76 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -492,13 +492,15 @@ class CoreNode(CoreNodeBase): raise ValueError("starting a node that is already up") # create a new namespace for this node using vnoded - vnoded = f"{VNODED_BIN} -v -c {self.ctrlchnlname} -l {self.ctrlchnlname}.log -p {self.ctrlchnlname}.pid" + vnoded = ( + f"{VNODED_BIN} -v -c {self.ctrlchnlname} -l {self.ctrlchnlname}.log " + f"-p {self.ctrlchnlname}.pid" + ) if self.nodedir: vnoded += f" -C {self.nodedir}" env = self.session.get_environment(state=False) env["NODE_NUMBER"] = str(self.id) env["NODE_NAME"] = str(self.name) - logging.info("env: %s", env) output = self.net_cmd(vnoded, env=env) self.pid = int(output) From c5ce85b2356b69654a2175bb2d981b3970ac9d52 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 14:22:37 -0700 Subject: [PATCH 098/113] added net client get ifindex and mac functions --- daemon/core/nodes/base.py | 13 +++---------- daemon/core/nodes/netclient.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d8eaae76..fcd75505 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -678,16 +678,9 @@ class CoreNode(CoreNodeBase): veth.name = ifname if self.up: - # TODO: potentially find better way to query interface ID - # retrieve interface information - output = self.node_net_client.device_show(veth.name) - logging.debug("interface command output: %s", output) - output = output.split("\n") - veth.flow_id = int(output[0].strip().split(":")[0]) + 1 - logging.debug("interface flow index: %s - %s", veth.name, veth.flow_id) - # TODO: mimic packed hwaddr - # veth.hwaddr = MacAddress.from_string(output[1].strip().split()[1]) - logging.debug("interface mac: %s - %s", veth.name, veth.hwaddr) + flow_id = self.node_net_client.get_ifindex(veth.name) + veth.flow_id = int(flow_id) + logging.info("interface flow index: %s - %s", veth.name, veth.flow_id) try: # add network interface to the node. If unsuccessful, destroy the diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 94e73e7f..beff4e8e 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -91,6 +91,26 @@ class LinuxNetClient(object): """ return self.run(f"{IP_BIN} link show {device}") + def get_mac(self, device): + """ + Retrieve MAC address for a given device. + + :param str device: device to get mac for + :return: MAC address + :rtype: str + """ + return self.run(f"cat /sys/class/net/{device}/address") + + def get_ifindex(self, device): + """ + Retrieve ifindex for a given device. + + :param str device: device to get ifindex for + :return: ifindex + :rtype: str + """ + return self.run(f"cat /sys/class/net/{device}/ifindex") + def device_ns(self, device, namespace): """ Set netns for a device. From e298a2a5c11282f01a8a354f94c04f03e07fb704 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 14:28:50 -0700 Subject: [PATCH 099/113] grpc will now always be ran, but can be configured through command line or core.conf --- daemon/Pipfile | 2 +- daemon/scripts/core-daemon | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/daemon/Pipfile b/daemon/Pipfile index a1e33a29..a689f4dc 100644 --- a/daemon/Pipfile +++ b/daemon/Pipfile @@ -4,7 +4,7 @@ url = "https://pypi.org/simple" verify_ssl = true [scripts] -core = "python scripts/core-daemon -f data/core.conf -l data/logging.conf --grpc" +core = "python scripts/core-daemon -f data/core.conf -l data/logging.conf" test = "pytest -v tests" test_emane = "pytest -v tests/emane" diff --git a/daemon/scripts/core-daemon b/daemon/scripts/core-daemon index 49aae2d2..6b55c14f 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/scripts/core-daemon @@ -18,7 +18,6 @@ from core.api.grpc.server import CoreGrpcServer from core.api.tlv.corehandlers import CoreHandler, CoreUdpHandler from core.api.tlv.coreserver import CoreServer, CoreUdpServer from core.constants import CORE_CONF_DIR, COREDPY_VERSION -from core.emulator import enumerations from core.emulator.enumerations import CORE_API_PORT from core.utils import close_onexec, load_logging_config @@ -67,14 +66,13 @@ def cored(cfg): sys.exit(1) # initialize grpc api - if cfg["grpc"] == "True": - grpc_server = CoreGrpcServer(server.coreemu) - address_config = cfg["grpcaddress"] - port_config = cfg["grpcport"] - grpc_address = f"{address_config}:{port_config}" - grpc_thread = threading.Thread(target=grpc_server.listen, args=(grpc_address,)) - grpc_thread.daemon = True - grpc_thread.start() + grpc_server = CoreGrpcServer(server.coreemu) + address_config = cfg["grpcaddress"] + port_config = cfg["grpcport"] + grpc_address = f"{address_config}:{port_config}" + grpc_thread = threading.Thread(target=grpc_server.listen, args=(grpc_address,)) + grpc_thread.daemon = True + grpc_thread.start() # start udp server start_udp(server, address) @@ -117,7 +115,6 @@ def get_merged_config(filename): parser.add_argument("-n", "--numthreads", dest="numthreads", type=int, help=f"number of server threads; default = {default_threads}") parser.add_argument("--ovs", action="store_true", help="enable experimental ovs mode, default is false") - parser.add_argument("--grpc", action="store_true", help="enable grpc api, default is false") parser.add_argument("--grpc-port", dest="grpcport", help=f"grpc port to listen on; default {default_grpc_port}") parser.add_argument("--grpc-address", dest="grpcaddress", From 2012105df08830e81e34a5824bbaaea823477a0e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 14:43:36 -0700 Subject: [PATCH 100/113] updated core.conf to contain distributed address and grpc configurations --- daemon/data/core.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon/data/core.conf b/daemon/data/core.conf index 3a111ba0..aa1238d5 100644 --- a/daemon/data/core.conf +++ b/daemon/data/core.conf @@ -1,6 +1,9 @@ [core-daemon] +#distributed_address = 127.0.0.1 listenaddr = localhost port = 4038 +grpcaddress = localhost +grpcport = 50051 numthreads = 1 quagga_bin_search = "/usr/local/bin /usr/bin /usr/lib/quagga" quagga_sbin_search = "/usr/local/sbin /usr/sbin /usr/lib/quagga" From d1e9223d522f9a4f2d4138a02b11efc28956068d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 14:52:08 -0700 Subject: [PATCH 101/113] updates to install docs to remove python2 references --- docs/install.md | 59 ++++++++++--------------------------------------- 1 file changed, 12 insertions(+), 47 deletions(-) diff --git a/docs/install.md b/docs/install.md index 822642fa..12242b4b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -34,7 +34,7 @@ Install Path | Description /usr/bin/core-daemon|Daemon startup command /usr/bin/{core-cleanup, coresendmsg, core-manage}|Misc. helper commands/scripts /usr/lib/core|GUI files -/usr/lib/python{2.7,3}/dist-packages/core|Python modules for daemon/scripts +/usr/lib/python{3.6+}/dist-packages/core|Python modules for daemon/scripts /etc/core/|Daemon and log configuration files ~/.core/|User-specific GUI preferences and scenario files /usr/share/core/|Example scripts and scenarios @@ -48,11 +48,6 @@ You may already have these installed, and can ignore this step if so, but if needed you can run the following to install python and pip. ```shell -# python 2 -sudo apt install python -sudo apt install python-pip - -# python 3 sudo apt install python3 sudo apt install python3-pip ``` @@ -64,23 +59,9 @@ To account for this it would be recommended to install the python dependencies u the latest [CORE Release](https://github.com/coreemu/core/releases). ```shell -# for python 2 -sudo python -m pip install -r requirements.txt -# for python 3 sudo python3 -m pip install -r requirements.txt ``` -## Ubuntu 19.04 - -Ubuntu 19.04 can provide all the packages needed at the system level and can be installed as follows: - -```shell -# python 2 -sudo apt install python-configparser python-enum34 python-future python-grpcio python-lxml -# python 3 -sudo apt install python3-configparser python3-enum34 python3-future python3-grpcio python3-lxml -``` - # Pre-Req Installing OSPF MDR Virtual networks generally require some form of routing in order to work (e.g. to automatically populate routing @@ -134,7 +115,7 @@ this is usually a sign that you have to run ```sudo ldconfig```` to refresh the # Installing from Packages The easiest way to install CORE is using the pre-built packages. The package managers on Ubuntu or Fedora/CentOS -will help in automatically installing most dependencies for you. +will help in automatically installing most dependencies, except for the python ones described previously. You can obtain the CORE packages from [CORE Releases](https://github.com/coreemu/core/releases). @@ -143,10 +124,9 @@ You can obtain the CORE packages from [CORE Releases](https://github.com/coreemu Ubuntu package defaults to using systemd for running as a service. ```shell -# python2 -sudo apt install ./core_python_$VERSION_amd64.deb -# python3 -sudo apt install ./core_python3_$VERSION_amd64.deb +# $PYTHON and $VERSION represent the python and CORE +# versions the package was built for +sudo apt install ./core_$PYTHON_$VERSION_amd64.deb ``` Run the CORE GUI as a normal user: @@ -164,9 +144,6 @@ Messages will print out on the console about connecting to the CORE daemon. on CentOS <= 6, or build from source otherwise** ```shell -# python2 -yum install ./core_python_$VERSION_x86_64.rpm -# python3 yum install ./core_python3_$VERSION_x86_64.rpm ``` @@ -234,10 +211,7 @@ You can obtain the CORE source from the [CORE GitHub](https://github.com/coreemu Python module grpcio-tools is currently needed to generate code from the CORE protobuf file during the build. ```shell -# python2 -pip2 install grpcio-tools -# python3 -pip3 install grpcio-tools +python3 -m pip install grpcio-tools ``` ## Distro Requirements @@ -245,27 +219,26 @@ pip3 install grpcio-tools ### Ubuntu 18.04 Requirements ```shell -sudo apt install automake pkg-config gcc libev-dev bridge-utils ebtables python-dev python-setuptools tk libtk-img ethtool +sudo apt install automake pkg-config gcc libev-dev bridge-utils ebtables python3-dev python3-setuptools tk libtk-img ethtool ``` ### Ubuntu 16.04 Requirements ```shell -sudo apt-get install automake bridge-utils ebtables python-dev libev-dev python-setuptools libtk-img ethtool +sudo apt-get install automake bridge-utils ebtables python3-dev libev-dev python3-setuptools libtk-img ethtool ``` ### CentOS 7 with Gnome Desktop Requirements ```shell -sudo yum -y install automake gcc python-devel libev-devel tk ethtool +sudo yum -y install automake gcc python3-devel python3-devel libev-devel tk ethtool ``` ## Build and Install ```shell ./bootstrap.sh -# $VERSION should be path to python2/3 -PYTHON=$VERSION ./configure +PYTHON=python3 ./configure make sudo make install ``` @@ -275,16 +248,11 @@ sudo make install Building documentation requires python-sphinx not noted above. ```shell -# install python2 sphinx -sudo apt install python-sphinx -sudo yum install python-sphinx -# install python3 sphinx sudo apt install python3-sphinx sudo yum install python3-sphinx ./bootstrap.sh -# $VERSION should be path to python2/3 -PYTHON=$VERSION ./configure +PYTHON=python3 ./configure make doc ``` @@ -297,10 +265,7 @@ Build package commands, DESTDIR is used to make install into and then for packag ```shell ./bootstrap.sh -# for python2 -PYTHON=python2 ./configure -# for python3 -PYTHON=python3 ./configure --enable-python3 +PYTHON=python3 ./configure make mkdir /tmp/core-build make fpm DESTDIR=/tmp/core-build From 83c408359a64b63dc6e820bcf32f9ef83716201a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 14:56:21 -0700 Subject: [PATCH 102/113] set flow id logging to debug --- daemon/core/nodes/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index fcd75505..fd6e0d81 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -680,7 +680,7 @@ class CoreNode(CoreNodeBase): if self.up: flow_id = self.node_net_client.get_ifindex(veth.name) veth.flow_id = int(flow_id) - logging.info("interface flow index: %s - %s", veth.name, veth.flow_id) + logging.debug("interface flow index: %s - %s", veth.name, veth.flow_id) try: # add network interface to the node. If unsuccessful, destroy the From 73b2eff312f4ae3ef6c6b3816d7e1e1ce8f11a28 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 16:25:38 -0700 Subject: [PATCH 103/113] fix for corehandlers.py session_clients access --- daemon/core/api/tlv/corehandlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 85b7b831..c7827e1f 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1016,7 +1016,7 @@ class CoreHandler(socketserver.BaseRequestHandler): # find the session containing this client and set the session to master for _id in self.coreemu.sessions: - clients = self.session_clients[_id] + clients = self.session_clients.get(_id, []) if self in clients: session = self.coreemu.sessions[_id] logging.debug("setting session to master: %s", session.id) From 18e5598203c61e5cfa2ea03d2faade7127c07bf2 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 16:28:13 -0700 Subject: [PATCH 104/113] fixed node data reporting emulation server host instead of name --- daemon/core/nodes/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index fd6e0d81..e963683b 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -196,7 +196,7 @@ class NodeBase(object): model = self.type emulation_server = None if self.server is not None: - emulation_server = self.server.host + emulation_server = self.server.name services = self.services if services is not None: From c0ab9ea4ccfa01742bb4c7e404c3e6d53b6ee4e9 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 21:51:15 -0700 Subject: [PATCH 105/113] small update to grpc docs --- docs/grpc.md | 53 ++-------------------------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/docs/grpc.md b/docs/grpc.md index d9743506..46149c54 100644 --- a/docs/grpc.md +++ b/docs/grpc.md @@ -1,60 +1,11 @@ # Using the gRPC API -By default the gRPC API is currently not turned on by default. There are a couple ways that this can be enabled -to use. +gRPC is the main API for interfacing with CORE. -## Enabling gRPC - -### HTTP Proxy +## HTTP Proxy Since gRPC is HTTP2 based, proxy configurations can cause issue. Clear out your proxy when running if needed. -### Daemon Options - -The gRPC API is enabled through options provided to the **core-daemon**. - -```shell -usage: core-daemon [-h] [-f CONFIGFILE] [-p PORT] [-n NUMTHREADS] [--ovs] - [--grpc] [--grpc-port GRPCPORT] - [--grpc-address GRPCADDRESS] - -CORE daemon v.5.3.0 instantiates Linux network namespace nodes. - -optional arguments: - -h, --help show this help message and exit - -f CONFIGFILE, --configfile CONFIGFILE - read config from specified file; default = - /etc/core/core.conf - -p PORT, --port PORT port number to listen on; default = 4038 - -n NUMTHREADS, --numthreads NUMTHREADS - number of server threads; default = 1 - --ovs enable experimental ovs mode, default is false - --grpc enable grpc api, default is false - --grpc-port GRPCPORT grpc port to listen on; default 50051 - --grpc-address GRPCADDRESS - grpc address to listen on; default localhost -``` - -### Enabling in Service Files - -Modify service files to append the --grpc options as desired. - -For sysv services /etc/init.d/core-daemon -```shell -CMD="PYTHONPATH=/usr/lib/python3.6/site-packages python3 /usr/bin/$NAME --grpc" -``` - -For systemd service /lib/systemd/system/core-daemon.service -```shell -ExecStart=@PYTHON@ @bindir@/core-daemon --grpc -``` - -### Enabling from Command Line - -```shell -sudo core-daemon --grpc -``` - ## Python Client A python client wrapper is provided at **core.api.grpc.client.CoreGrpcClient**. From d4af459653344d78d855bf5824a34b07f79c8bcd Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 22:56:10 -0700 Subject: [PATCH 106/113] update to distributed core doc --- docs/distributed.md | 109 +++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 61 deletions(-) diff --git a/docs/distributed.md b/docs/distributed.md index 076476db..4c5894f8 100644 --- a/docs/distributed.md +++ b/docs/distributed.md @@ -9,39 +9,17 @@ A large emulation scenario can be deployed on multiple emulation servers and controlled by a single GUI. The GUI, representing the entire topology, can be run on one of the emulation servers or on a separate machine. -Each machine that will act as an emulation server would ideally have the - same version of CORE installed. It is not important to have the GUI component - but the CORE Python daemon **core-daemon** needs to be installed. - -**NOTE: The server that the GUI connects with is referred to as -the master server.** +Each machine that will act as an emulation will require the installation of a distributed CORE package and +some configuration to allow SSH as root. -## Configuring Listen Address +## Configuring SSH -First we need to configure the **core-daemon** on all servers to listen on an -interface over the network. The simplest way would be updating the core -configuration file to listen on all interfaces. Alternatively, configure it to -listen to the specific interface you desire by supplying the correct address. +Distributed CORE works using the python fabric library to run commands on remote servers over SSH. -The **listenaddr** configuration should be set to the address of the interface -that should receive CORE API control commands from the other servers; -setting **listenaddr = 0.0.0.0** causes the Python daemon to listen on all -interfaces. CORE uses TCP port **4038** by default to communicate from the -controlling machine (with GUI) to the emulation servers. Make sure that -firewall rules are configured as necessary to allow this traffic. +### Remote GUI Terminals -```shell -# open configuration file -vi /etc/core/core.conf - -# within core.conf -[core-daemon] -listenaddr = 0.0.0.0 -``` - -## Enabling Remote SSH Shells - -### Update GUI Terminal Program +You need to have the same user defined on each server, since the user used +for these remote shells is the same user that is running the CORE GUI. **Edit -> Preferences... -> Terminal program:** @@ -54,31 +32,51 @@ May need to install xterm if, not already installed. sudo apt install xterm ``` -### Setup SSH +### Distributed Server SSH Configuration -In order to easily open shells on the emulation servers, the servers should be -running an SSH server, and public key login should be enabled. This is -accomplished by generating an SSH key for your user on all servers being used -for distributed emulation, if you do not already have one. Then copying your -master server public key to the authorized_keys file on all other servers that -will be used to help drive the distributed emulation. When double-clicking on a -node during runtime, instead of opening a local shell, the GUI will attempt to -SSH to the emulation server to run an interactive shell. +First the distributed servers must be configured to allow passwordless root login over SSH. -You need to have the same user defined on each server, since the user used -for these remote shells is the same user that is running the CORE GUI. - -```shell +On distributed server: +```shelll # install openssh-server sudo apt install openssh-server -# generate ssh if needed -ssh-keygen -o -t rsa -b 4096 +# open sshd config +vi /etc/ssh/sshd_config + +# verify these configurations in file +PermitRootLogin yes +PasswordAuthentication yes + +# restart sshd +sudo systemctl restart sshd +``` + +On master server: +```shell +# install package if needed +sudo apt install openssh-client + +# generate ssh key if needed +ssh-keygen -o -t rsa -b 4096 -f ~/.ssh/core # copy public key to authorized_keys file -ssh-copy-id user@server -# or -scp ~/.ssh/id_rsa.pub username@server:~/.ssh/authorized_keys +ssh-copy-id -i ~/.ssh/core root@server + +# configure fabric to use the core ssh key +sudo vi /etc/fabric.yml +``` + +On distributed server: +```shell +# open sshd config +vi /etc/ssh/sshd_config + +# change configuration for root login to without password +PermitRootLogin without-password + +# restart sshd +sudo systemctl restart sshd ``` ## Add Emulation Servers in GUI @@ -155,27 +153,16 @@ The names before the addresses need to match the servers configured in controlnet = core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 core4:172.16.4.0/24 core5:172.16.5.0/24 ``` -EMANE appears to require location events for nodes to be sync'ed across -all EMANE instances for nodes to find each other. Using an EMANE eel file -for your scenario can help clear this up, which might be desired anyway. - -* https://github.com/adjacentlink/emane/wiki/EEL-Generator - -You can also move nodes within the GUI to help trigger location events from -CORE when the **core.conf** settings below is used. Assuming the nodes -did not find each other by default and you are not using an eel file. - ```shell emane_event_generate = True ``` ## Distributed Checklist -1. Install the same version of the CORE daemon on all servers. -1. Set **listenaddr** configuration in all of the server's core.conf files, -then start (or restart) the daemon. +1. Install CORE on master server +1. Install distributed CORE package on all servers needed 1. Installed and configure public-key SSH access on all servers (if you want to use -double-click shells or Widgets.) +double-click shells or Widgets.) for both the GUI user (for terminals) and root for running CORE commands 1. Choose the servers that participate in distributed emulation. 1. Assign nodes to desired servers, empty for master server. 1. Press the **Start** button to launch the distributed emulation. From 6006710c32f559542bd588e015b2389fda49ce58 Mon Sep 17 00:00:00 2001 From: bharnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 18 Oct 2019 23:28:09 -0700 Subject: [PATCH 107/113] changed net_cmd to host_cmd and node_net_cmd to cmd, for simpler more logical naming --- daemon/core/api/grpc/server.py | 2 +- daemon/core/api/tlv/corehandlers.py | 4 +-- daemon/core/emane/emanemanager.py | 6 ++-- daemon/core/emulator/session.py | 2 +- daemon/core/nodes/base.py | 42 +++++++++++++--------------- daemon/core/nodes/docker.py | 14 +++++----- daemon/core/nodes/interface.py | 8 +++--- daemon/core/nodes/lxd.py | 14 +++++----- daemon/core/nodes/network.py | 26 ++++++++--------- daemon/core/nodes/physical.py | 8 +++--- daemon/core/services/coreservices.py | 6 ++-- daemon/examples/python/switch.py | 6 ++-- daemon/examples/python/wlan.py | 6 ++-- daemon/tests/emane/test_emane.py | 2 +- daemon/tests/test_core.py | 2 +- daemon/tests/test_nodes.py | 2 +- 16 files changed, 74 insertions(+), 76 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 11bfcd6b..08713f71 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -881,7 +881,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) try: - output = node.node_net_cmd(request.command) + output = node.cmd(request.command) except CoreCommandError as e: output = e.stderr return core_pb2.NodeCommandResponse(output=output) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index c7827e1f..1cb3a45d 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -885,7 +885,7 @@ class CoreHandler(socketserver.BaseRequestHandler): status = e.returncode else: try: - res = node.node_net_cmd(command) + res = node.cmd(command) status = 0 except CoreCommandError as e: res = e.stderr @@ -911,7 +911,7 @@ class CoreHandler(socketserver.BaseRequestHandler): if message.flags & MessageFlags.LOCAL.value: utils.mute_detach(command) else: - node.node_net_cmd(command, wait=False) + node.cmd(command, wait=False) except CoreError: logging.exception("error getting object: %s", node_num) # XXX wait and queue this message to try again later diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 743f90b2..a03e8e2a 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -583,7 +583,7 @@ class EmaneManager(ModelManager): log_file = os.path.join(path, f"emane{n}.log") platform_xml = os.path.join(path, f"platform{n}.xml") args = f"{emanecmd} -f {log_file} {platform_xml}" - output = node.node_net_cmd(args) + output = node.cmd(args) logging.info("node(%s) emane daemon running: %s", node.name, args) logging.info("node(%s) emane daemon output: %s", node.name, output) @@ -613,7 +613,7 @@ class EmaneManager(ModelManager): continue if node.up: - node.node_net_cmd(kill_emaned, wait=False) + node.cmd(kill_emaned, wait=False) # TODO: RJ45 node if stop_emane_on_host: @@ -813,7 +813,7 @@ class EmaneManager(ModelManager): """ args = "pkill -0 -x emane" try: - node.node_net_cmd(args) + node.cmd(args) result = True except CoreCommandError: result = False diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index d80e5e25..076b79b3 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -1915,4 +1915,4 @@ class Session(object): utils.mute_detach(data) else: node = self.get_node(node_id) - node.node_net_cmd(data, wait=False) + node.cmd(data, wait=False) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index e963683b..9a91c5ce 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -61,7 +61,7 @@ class NodeBase(object): self.position = Position() use_ovs = session.options.get_config("ovs") == "True" - self.net_client = get_net_client(use_ovs, self.net_cmd) + self.net_client = get_net_client(use_ovs, self.host_cmd) def startup(self): """ @@ -79,10 +79,9 @@ class NodeBase(object): """ raise NotImplementedError - def net_cmd(self, args, env=None, cwd=None, wait=True): + def host_cmd(self, args, env=None, cwd=None, wait=True): """ - Runs a command that is used to configure and setup the network on the host - system. + Runs a command on the host system or distributed server. :param str args: command to run :param dict env: environment to run command with @@ -265,7 +264,7 @@ class CoreNodeBase(NodeBase): """ if self.nodedir is None: self.nodedir = os.path.join(self.session.session_dir, self.name + ".conf") - self.net_cmd(f"mkdir -p {self.nodedir}") + self.host_cmd(f"mkdir -p {self.nodedir}") self.tmpnodedir = True else: self.tmpnodedir = False @@ -281,7 +280,7 @@ class CoreNodeBase(NodeBase): return if self.tmpnodedir: - self.net_cmd(f"rm -rf {self.nodedir}") + self.host_cmd(f"rm -rf {self.nodedir}") def addnetif(self, netif, ifindex): """ @@ -383,10 +382,9 @@ class CoreNodeBase(NodeBase): return common - def node_net_cmd(self, args, wait=True): + def cmd(self, args, wait=True): """ - Runs a command that is used to configure and setup the network within a - node. + Runs a command within a node container. :param str args: command to run :param bool wait: True to wait for status, False otherwise @@ -462,7 +460,7 @@ class CoreNode(CoreNodeBase): :param bool use_ovs: True for OVS bridges, False for Linux bridges :return:node network client """ - return get_net_client(use_ovs, self.node_net_cmd) + return get_net_client(use_ovs, self.cmd) def alive(self): """ @@ -472,7 +470,7 @@ class CoreNode(CoreNodeBase): :rtype: bool """ try: - self.net_cmd(f"kill -0 {self.pid}") + self.host_cmd(f"kill -0 {self.pid}") except CoreCommandError: return False @@ -502,7 +500,7 @@ class CoreNode(CoreNodeBase): env["NODE_NUMBER"] = str(self.id) env["NODE_NAME"] = str(self.name) - output = self.net_cmd(vnoded, env=env) + output = self.host_cmd(vnoded, env=env) self.pid = int(output) logging.debug("node(%s) pid: %s", self.name, self.pid) @@ -546,13 +544,13 @@ class CoreNode(CoreNodeBase): # kill node process if present try: - self.net_cmd(f"kill -9 {self.pid}") + self.host_cmd(f"kill -9 {self.pid}") except CoreCommandError: logging.exception("error killing process") # remove node directory if present try: - self.net_cmd(f"rm -rf {self.ctrlchnlname}") + self.host_cmd(f"rm -rf {self.ctrlchnlname}") except CoreCommandError: logging.exception("error removing node directory") @@ -565,7 +563,7 @@ class CoreNode(CoreNodeBase): finally: self.rmnodedir() - def node_net_cmd(self, args, wait=True): + def cmd(self, args, wait=True): """ Runs a command that is used to configure and setup the network within a node. @@ -607,7 +605,7 @@ class CoreNode(CoreNodeBase): hostpath = os.path.join( self.nodedir, os.path.normpath(path).strip("/").replace("/", ".") ) - self.net_cmd(f"mkdir -p {hostpath}") + self.host_cmd(f"mkdir -p {hostpath}") self.mount(hostpath, path) def mount(self, source, target): @@ -621,8 +619,8 @@ class CoreNode(CoreNodeBase): """ source = os.path.abspath(source) logging.debug("node(%s) mounting: %s at %s", self.name, source, target) - self.node_net_cmd(f"mkdir -p {target}") - self.node_net_cmd(f"{MOUNT_BIN} -n --bind {source} {target}") + self.cmd(f"mkdir -p {target}") + self.cmd(f"{MOUNT_BIN} -n --bind {source} {target}") self._mounts.append((source, target)) def newifindex(self): @@ -846,7 +844,7 @@ class CoreNode(CoreNodeBase): self.client.check_cmd(f"mv {srcname} {filename}") self.client.check_cmd("sync") else: - self.net_cmd(f"mkdir -p {directory}") + self.host_cmd(f"mkdir -p {directory}") self.server.remote_put(srcname, filename) def hostfilename(self, filename): @@ -883,9 +881,9 @@ class CoreNode(CoreNodeBase): open_file.write(contents) os.chmod(open_file.name, mode) else: - self.net_cmd(f"mkdir -m {0o755:o} -p {dirname}") + self.host_cmd(f"mkdir -m {0o755:o} -p {dirname}") self.server.remote_put_temp(hostfilename, contents) - self.net_cmd(f"chmod {mode:o} {hostfilename}") + self.host_cmd(f"chmod {mode:o} {hostfilename}") logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode ) @@ -906,7 +904,7 @@ class CoreNode(CoreNodeBase): else: self.server.remote_put(srcfilename, hostfilename) if mode is not None: - self.net_cmd(f"chmod {mode:o} {hostfilename}") + self.host_cmd(f"chmod {mode:o} {hostfilename}") logging.info( "node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode ) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 369f462b..d4850d8a 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -131,7 +131,7 @@ class DockerNode(CoreNode): if self.up: raise ValueError("starting a node that is already up") self.makenodedir() - self.client = DockerClient(self.name, self.image, self.net_cmd) + self.client = DockerClient(self.name, self.image, self.host_cmd) self.pid = self.client.create_container() self.up = True @@ -176,7 +176,7 @@ class DockerNode(CoreNode): """ logging.debug("creating node dir: %s", path) args = f"mkdir -p {path}" - self.node_net_cmd(args) + self.cmd(args) def mount(self, source, target): """ @@ -206,13 +206,13 @@ class DockerNode(CoreNode): temp.close() if directory: - self.node_net_cmd(f"mkdir -m {0o755:o} -p {directory}") + self.cmd(f"mkdir -m {0o755:o} -p {directory}") if self.server is not None: self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) - self.node_net_cmd(f"chmod {mode:o} {filename}") + self.cmd(f"chmod {mode:o} {filename}") if self.server is not None: - self.net_cmd(f"rm -f {temp.name}") + self.host_cmd(f"rm -f {temp.name}") os.unlink(temp.name) logging.debug( "node(%s) added file: %s; mode: 0%o", self.name, filename, mode @@ -232,7 +232,7 @@ class DockerNode(CoreNode): "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode ) directory = os.path.dirname(filename) - self.node_net_cmd(f"mkdir -p {directory}") + self.cmd(f"mkdir -p {directory}") if self.server is None: source = srcfilename @@ -242,4 +242,4 @@ class DockerNode(CoreNode): self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) - self.node_net_cmd(f"chmod {mode:o} {filename}") + self.cmd(f"chmod {mode:o} {filename}") diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index c8841432..cf215e5b 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -17,7 +17,7 @@ class CoreInterface(object): def __init__(self, session, node, name, mtu, server=None): """ - Creates a PyCoreNetIf instance. + Creates a CoreInterface instance. :param core.emulator.session.Session session: core session instance :param core.nodes.base.CoreNode node: node for interface @@ -46,11 +46,11 @@ class CoreInterface(object): self.flow_id = None self.server = server use_ovs = session.options.get_config("ovs") == "True" - self.net_client = get_net_client(use_ovs, self.net_cmd) + self.net_client = get_net_client(use_ovs, self.host_cmd) - def net_cmd(self, args, env=None, cwd=None, wait=True): + def host_cmd(self, args, env=None, cwd=None, wait=True): """ - Runs a command on the host system or distributed servers. + Runs a command on the host system or distributed server. :param str args: command to run :param dict env: environment to run command with diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 323b20a9..77078047 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -112,7 +112,7 @@ class LxcNode(CoreNode): if self.up: raise ValueError("starting a node that is already up") self.makenodedir() - self.client = LxdClient(self.name, self.image, self.net_cmd) + self.client = LxdClient(self.name, self.image, self.host_cmd) self.pid = self.client.create_container() self.up = True @@ -149,7 +149,7 @@ class LxcNode(CoreNode): """ logging.info("creating node dir: %s", path) args = f"mkdir -p {path}" - return self.node_net_cmd(args) + return self.cmd(args) def mount(self, source, target): """ @@ -180,13 +180,13 @@ class LxcNode(CoreNode): temp.close() if directory: - self.node_net_cmd(f"mkdir -m {0o755:o} -p {directory}") + self.cmd(f"mkdir -m {0o755:o} -p {directory}") if self.server is not None: self.server.remote_put(temp.name, temp.name) self.client.copy_file(temp.name, filename) - self.node_net_cmd(f"chmod {mode:o} {filename}") + self.cmd(f"chmod {mode:o} {filename}") if self.server is not None: - self.net_cmd(f"rm -f {temp.name}") + self.host_cmd(f"rm -f {temp.name}") os.unlink(temp.name) logging.debug("node(%s) added file: %s; mode: 0%o", self.name, filename, mode) @@ -204,7 +204,7 @@ class LxcNode(CoreNode): "node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode ) directory = os.path.dirname(filename) - self.node_net_cmd(f"mkdir -p {directory}") + self.cmd(f"mkdir -p {directory}") if self.server is None: source = srcfilename @@ -214,7 +214,7 @@ class LxcNode(CoreNode): self.server.remote_put(source, temp.name) self.client.copy_file(source, filename) - self.node_net_cmd(f"chmod {mode:o} {filename}") + self.cmd(f"chmod {mode:o} {filename}") def addnetif(self, netif, ifindex): super(LxcNode, self).addnetif(netif, ifindex) diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 229005c4..9b122650 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -162,20 +162,20 @@ class EbtablesQueue(object): """ # save kernel ebtables snapshot to a file args = self.ebatomiccmd("--atomic-save") - wlan.net_cmd(args) + wlan.host_cmd(args) # modify the table file using queued ebtables commands for c in self.cmds: args = self.ebatomiccmd(c) - wlan.net_cmd(args) + wlan.host_cmd(args) self.cmds = [] # commit the table file to the kernel args = self.ebatomiccmd("--atomic-commit") - wlan.net_cmd(args) + wlan.host_cmd(args) try: - wlan.net_cmd(f"rm -f {self.atomic_file}") + wlan.host_cmd(f"rm -f {self.atomic_file}") except CoreCommandError: logging.exception("error removing atomic file: %s", self.atomic_file) @@ -270,7 +270,7 @@ class CoreNetwork(CoreNetworkBase): self.startup() ebq.startupdateloop(self) - def net_cmd(self, args, env=None, cwd=None, wait=True): + def host_cmd(self, args, env=None, cwd=None, wait=True): """ Runs a command that is used to configure and setup the network on the host system and all configured distributed servers. @@ -302,7 +302,7 @@ class CoreNetwork(CoreNetworkBase): f"{EBTABLES_BIN} -N {self.brname} -P {self.policy}", f"{EBTABLES_BIN} -A FORWARD --logical-in {self.brname} -j {self.brname}", ] - ebtablescmds(self.net_cmd, cmds) + ebtablescmds(self.host_cmd, cmds) self.up = True @@ -323,7 +323,7 @@ class CoreNetwork(CoreNetworkBase): f"{EBTABLES_BIN} -D FORWARD --logical-in {self.brname} -j {self.brname}", f"{EBTABLES_BIN} -X {self.brname}", ] - ebtablescmds(self.net_cmd, cmds) + ebtablescmds(self.host_cmd, cmds) except CoreCommandError: logging.exception("error during shutdown") @@ -462,13 +462,13 @@ class CoreNetwork(CoreNetworkBase): if bw > 0: if self.up: cmd = f"{tc} {parent} handle 1: {tbf}" - netif.net_cmd(cmd) + netif.host_cmd(cmd) netif.setparam("has_tbf", True) changed = True elif netif.getparam("has_tbf") and bw <= 0: if self.up: cmd = f"{TC_BIN} qdisc delete dev {devname} {parent}" - netif.net_cmd(cmd) + netif.host_cmd(cmd) netif.setparam("has_tbf", False) # removing the parent removes the child netif.setparam("has_netem", False) @@ -510,14 +510,14 @@ class CoreNetwork(CoreNetworkBase): return if self.up: cmd = f"{TC_BIN} qdisc delete dev {devname} {parent} handle 10:" - netif.net_cmd(cmd) + netif.host_cmd(cmd) netif.setparam("has_netem", False) elif len(netem) > 1: if self.up: cmd = ( f"{TC_BIN} qdisc replace dev {devname} {parent} handle 10: {netem}" ) - netif.net_cmd(cmd) + netif.host_cmd(cmd) netif.setparam("has_netem", True) def linknet(self, net): @@ -802,7 +802,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd(f"{self.updown_script} {self.brname} startup") + self.host_cmd(f"{self.updown_script} {self.brname} startup") if self.serverintf: self.net_client.create_interface(self.brname, self.serverintf) @@ -830,7 +830,7 @@ class CtrlNet(CoreNetwork): self.brname, self.updown_script, ) - self.net_cmd(f"{self.updown_script} {self.brname} shutdown") + self.host_cmd(f"{self.updown_script} {self.brname} shutdown") except CoreCommandError: logging.exception("error issuing shutdown script shutdown") diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index cae3f298..c1d6328b 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -188,13 +188,13 @@ class PhysicalNode(CoreNodeBase): source = os.path.abspath(source) logging.info("mounting %s at %s", source, target) os.makedirs(target) - self.net_cmd(f"{MOUNT_BIN} --bind {source} {target}", cwd=self.nodedir) + self.host_cmd(f"{MOUNT_BIN} --bind {source} {target}", cwd=self.nodedir) self._mounts.append((source, target)) def umount(self, target): logging.info("unmounting '%s'", target) try: - self.net_cmd(f"{UMOUNT_BIN} -l {target}", cwd=self.nodedir) + self.host_cmd(f"{UMOUNT_BIN} -l {target}", cwd=self.nodedir) except CoreCommandError: logging.exception("unmounting failed for %s", target) @@ -220,8 +220,8 @@ class PhysicalNode(CoreNodeBase): os.chmod(node_file.name, mode) logging.info("created nodefile: '%s'; mode: 0%o", node_file.name, mode) - def node_net_cmd(self, args, wait=True): - return self.net_cmd(args, wait=wait) + def cmd(self, args, wait=True): + return self.host_cmd(args, wait=wait) class Rj45Node(CoreNodeBase, CoreInterface): diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index 6db2d8ed..20553eb1 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -598,7 +598,7 @@ class CoreServices(object): for cmd in cmds: logging.debug("validating service(%s) using: %s", service.name, cmd) try: - node.node_net_cmd(cmd) + node.cmd(cmd) except CoreCommandError as e: logging.debug( "node(%s) service(%s) validate failed", node.name, service.name @@ -631,7 +631,7 @@ class CoreServices(object): status = 0 for args in service.shutdown: try: - node.node_net_cmd(args) + node.cmd(args) except CoreCommandError: logging.exception("error running stop command %s", args) status = -1 @@ -729,7 +729,7 @@ class CoreServices(object): status = 0 for cmd in cmds: try: - node.node_net_cmd(cmd, wait) + node.cmd(cmd, wait) except CoreCommandError: logging.exception("error starting command") status = -1 diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index 80257a4a..252253f8 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -42,12 +42,12 @@ def example(options): last_node = session.get_node(options.nodes + 1) logging.info("starting iperf server on node: %s", first_node.name) - first_node.node_net_cmd("iperf -s -D") + first_node.cmd("iperf -s -D") first_node_address = prefixes.ip4_address(first_node) logging.info("node %s connecting to %s", last_node.name, first_node_address) - output = last_node.node_net_cmd(f"iperf -t {options.time} -c {first_node_address}") + output = last_node.cmd(f"iperf -t {options.time} -c {first_node_address}") logging.info(output) - first_node.node_net_cmd("killall -9 iperf") + first_node.cmd("killall -9 iperf") # shutdown session coreemu.shutdown() diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index 9506a35c..4b4bc724 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -46,11 +46,11 @@ def example(options): last_node = session.get_node(options.nodes + 1) logging.info("starting iperf server on node: %s", first_node.name) - first_node.node_net_cmd("iperf -s -D") + first_node.cmd("iperf -s -D") address = prefixes.ip4_address(first_node) logging.info("node %s connecting to %s", last_node.name, address) - last_node.node_net_cmd(f"iperf -t {options.time} -c {address}") - first_node.node_net_cmd("killall -9 iperf") + last_node.cmd(f"iperf -t {options.time} -c {address}") + first_node.cmd("killall -9 iperf") # shutdown session coreemu.shutdown() diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index a0ae05d1..3eb87596 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -27,7 +27,7 @@ _DIR = os.path.dirname(os.path.abspath(__file__)) def ping(from_node, to_node, ip_prefixes, count=3): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd(f"ping -c {count} {address}") + from_node.cmd(f"ping -c {count} {address}") status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index fa2adc6e..47368740 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -20,7 +20,7 @@ _WIRED = [NodeTypes.PEER_TO_PEER, NodeTypes.HUB, NodeTypes.SWITCH] def ping(from_node, to_node, ip_prefixes): address = ip_prefixes.ip4_address(to_node) try: - from_node.node_net_cmd(f"ping -c 3 {address}") + from_node.cmd(f"ping -c 3 {address}") status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index 34c426c5..e495dea9 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -30,7 +30,7 @@ class TestNodes: assert os.path.exists(node.nodedir) assert node.alive() assert node.up - assert node.node_net_cmd("ip address show lo") + assert node.cmd("ip address show lo") def test_node_update(self, session): # given From d056578e9d6d888766d472e5ede15b7c3a17b9c3 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 21 Oct 2019 09:36:07 -0700 Subject: [PATCH 108/113] modified ctrlnets to use an id starting at 9001, to avoid string based ids --- daemon/core/emulator/session.py | 7 +++---- daemon/core/nodes/network.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index d80e5e25..a2f72647 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -71,6 +71,7 @@ NODES = { NodeTypes.LXC: LxcNode, } NODES_TYPE = {NODES[x]: x for x in NODES} +CTRL_NET_ID = 9001 class Session(object): @@ -1667,9 +1668,7 @@ class Session(object): return -1 def get_control_net(self, net_index): - # TODO: all nodes use an integer id and now this wants to use a string - _id = f"ctrl{net_index}net" - return self.get_node(_id) + return self.get_node(CTRL_NET_ID + net_index) def add_remove_control_net(self, net_index, remove=False, conf_required=True): """ @@ -1716,7 +1715,7 @@ class Session(object): return None # build a new controlnet bridge - _id = f"ctrl{net_index}net" + _id = CTRL_NET_ID + net_index # use the updown script for control net 0 only. updown_script = None diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 229005c4..46730212 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -723,7 +723,7 @@ class CtrlNet(CoreNetwork): def __init__( self, session, - _id="ctrlnet", + _id=None, name=None, prefix=None, hostid=None, From 3fc0ca5cec5c388250bdd6cefb771ed5a57708ac Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 21 Oct 2019 09:51:52 -0700 Subject: [PATCH 109/113] fix to get_node over grpc to avoid issues with nodes that dont have services --- daemon/core/api/grpc/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 08713f71..0ab4806a 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -807,7 +807,10 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): if isinstance(node, EmaneNet): emane_model = node.model.name - services = [x.name for x in getattr(node, "services", [])] + services = [] + if node.services: + services = [x.name for x in node.services] + position = core_pb2.Position( x=node.position.x, y=node.position.y, z=node.position.z ) From 16b7e70c33acd97160f1a7d4a601f03eacd6e470 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 21 Oct 2019 10:08:41 -0700 Subject: [PATCH 110/113] update to add config example for fabric.yml --- docs/distributed.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/distributed.md b/docs/distributed.md index 4c5894f8..b6ef9f77 100644 --- a/docs/distributed.md +++ b/docs/distributed.md @@ -48,6 +48,10 @@ vi /etc/ssh/sshd_config PermitRootLogin yes PasswordAuthentication yes +# if desired add/modify the following line to allow SSH to +# accept all env variables +AcceptEnv * + # restart sshd sudo systemctl restart sshd ``` @@ -65,6 +69,9 @@ ssh-copy-id -i ~/.ssh/core root@server # configure fabric to use the core ssh key sudo vi /etc/fabric.yml + +# set configuration +connect_kwargs: {"key_filename": "/home/user/.ssh/core"} ``` On distributed server: From 78f981463d5dd03bc0f784c1236c9d365e127fa0 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 21 Oct 2019 10:32:42 -0700 Subject: [PATCH 111/113] renamed utils.check_cmd to utils.cmd, updated host_cmd to allow for shell commands for output redirection --- daemon/core/api/tlv/corehandlers.py | 2 +- daemon/core/emane/emanemanager.py | 8 ++++---- daemon/core/emane/tdma.py | 2 +- daemon/core/location/mobility.py | 2 +- daemon/core/nodes/base.py | 5 +++-- daemon/core/nodes/client.py | 2 +- daemon/core/nodes/docker.py | 6 +++--- daemon/core/nodes/interface.py | 5 +++-- daemon/core/nodes/lxd.py | 2 +- daemon/core/nodes/netclient.py | 10 ++++------ daemon/core/nodes/network.py | 7 ++++--- daemon/core/services/utility.py | 2 +- daemon/core/utils.py | 8 +++++--- daemon/core/xml/corexmldeployment.py | 2 +- daemon/tests/test_nodes.py | 2 +- 15 files changed, 34 insertions(+), 31 deletions(-) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 1cb3a45d..41bac314 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -878,7 +878,7 @@ class CoreHandler(socketserver.BaseRequestHandler): ): if message.flags & MessageFlags.LOCAL.value: try: - res = utils.check_cmd(command) + res = utils.cmd(command) status = 0 except CoreCommandError as e: res = e.stderr diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index a03e8e2a..b40ed118 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -140,7 +140,7 @@ class EmaneManager(ModelManager): try: # check for emane args = "emane --version" - emane_version = utils.check_cmd(args) + emane_version = utils.cmd(args) logging.info("using EMANE: %s", emane_version) self.session.distributed.execute(lambda x: x.remote_cmd(args)) @@ -594,7 +594,7 @@ class EmaneManager(ModelManager): log_file = os.path.join(path, "emane.log") platform_xml = os.path.join(path, "platform.xml") emanecmd += f" -f {log_file} {platform_xml}" - utils.check_cmd(emanecmd, cwd=path) + utils.cmd(emanecmd, cwd=path) self.session.distributed.execute(lambda x: x.remote_cmd(emanecmd, cwd=path)) logging.info("host emane daemon running: %s", emanecmd) @@ -618,8 +618,8 @@ class EmaneManager(ModelManager): if stop_emane_on_host: try: - utils.check_cmd(kill_emaned) - utils.check_cmd(kill_transortd) + utils.cmd(kill_emaned) + utils.cmd(kill_transortd) self.session.distributed.execute(lambda x: x.remote_cmd(kill_emaned)) self.session.distributed.execute(lambda x: x.remote_cmd(kill_transortd)) except CoreCommandError: diff --git a/daemon/core/emane/tdma.py b/daemon/core/emane/tdma.py index 91e662ea..afad9d10 100644 --- a/daemon/core/emane/tdma.py +++ b/daemon/core/emane/tdma.py @@ -63,4 +63,4 @@ class EmaneTdmaModel(emanemodel.EmaneModel): "setting up tdma schedule: schedule(%s) device(%s)", schedule, event_device ) args = f"emaneevent-tdmaschedule -i {event_device} {schedule}" - utils.check_cmd(args) + utils.cmd(args) diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index f2b49818..92bd1b4b 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -1187,6 +1187,6 @@ class Ns2ScriptedMobility(WayPointMobility): return filename = self.findfile(filename) args = f"/bin/sh {filename} {typestr}" - utils.check_cmd( + utils.cmd( args, cwd=self.session.session_dir, env=self.session.get_environment() ) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 9a91c5ce..8b35410d 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -79,7 +79,7 @@ class NodeBase(object): """ raise NotImplementedError - def host_cmd(self, args, env=None, cwd=None, wait=True): + def host_cmd(self, args, env=None, cwd=None, wait=True, shell=False): """ Runs a command on the host system or distributed server. @@ -87,12 +87,13 @@ class NodeBase(object): :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise + :param bool shell: True to use shell, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - return utils.check_cmd(args, env, cwd, wait) + return utils.cmd(args, env, cwd, wait, shell) else: return self.server.remote_cmd(args, env, cwd, wait) diff --git a/daemon/core/nodes/client.py b/daemon/core/nodes/client.py index 299b8135..3596bfa7 100644 --- a/daemon/core/nodes/client.py +++ b/daemon/core/nodes/client.py @@ -65,4 +65,4 @@ class VnodeClient(object): """ self._verify_connection() args = self.create_cmd(args) - return utils.check_cmd(args, wait=wait) + return utils.cmd(args, wait=wait) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index d4850d8a..a70cfb39 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -45,14 +45,14 @@ class DockerClient(object): def check_cmd(self, cmd): logging.info("docker cmd output: %s", cmd) - return utils.check_cmd(f"docker exec {self.name} {cmd}") + return utils.cmd(f"docker exec {self.name} {cmd}") def create_ns_cmd(self, cmd): return f"nsenter -t {self.pid} -u -i -p -n {cmd}" def ns_cmd(self, cmd, wait): args = f"nsenter -t {self.pid} -u -i -p -n {cmd}" - return utils.check_cmd(args, wait=wait) + return utils.cmd(args, wait=wait) def get_pid(self): args = f"docker inspect -f '{{{{.State.Pid}}}}' {self.name}" @@ -153,7 +153,7 @@ class DockerNode(CoreNode): def nsenter_cmd(self, args, wait=True): if self.server is None: args = self.client.create_ns_cmd(args) - return utils.check_cmd(args, wait=wait) + return utils.cmd(args, wait=wait) else: args = self.client.create_ns_cmd(args) return self.server.remote_cmd(args, wait=wait) diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index cf215e5b..a32103b8 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -48,7 +48,7 @@ class CoreInterface(object): use_ovs = session.options.get_config("ovs") == "True" self.net_client = get_net_client(use_ovs, self.host_cmd) - def host_cmd(self, args, env=None, cwd=None, wait=True): + def host_cmd(self, args, env=None, cwd=None, wait=True, shell=False): """ Runs a command on the host system or distributed server. @@ -56,12 +56,13 @@ class CoreInterface(object): :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise + :param bool shell: True to use shell, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ if self.server is None: - return utils.check_cmd(args, env, cwd, wait) + return utils.cmd(args, env, cwd, wait, shell) else: return self.server.remote_cmd(args, env, cwd, wait) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 77078047..9d5dedc4 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -49,7 +49,7 @@ class LxdClient(object): def check_cmd(self, cmd, wait=True): args = self.create_cmd(cmd) - return utils.check_cmd(args, wait=wait) + return utils.cmd(args, wait=wait) def copy_file(self, source, destination): if destination[0] != "/": diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index beff4e8e..8bd58be7 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -2,8 +2,6 @@ Clients for dealing with bridge/interface commands. """ -import os - from core.constants import BRCTL_BIN, ETHTOOL_BIN, IP_BIN, OVS_BIN, TC_BIN @@ -236,10 +234,10 @@ class LinuxNetClient(object): self.device_up(name) # turn off multicast snooping so forwarding occurs w/o IGMP joins - snoop = f"/sys/devices/virtual/net/{name}/bridge/multicast_snooping" - if os.path.exists(snoop): - with open(snoop, "w") as f: - f.write("0") + snoop_file = "multicast_snooping" + snoop = f"/sys/devices/virtual/net/{name}/bridge/{snoop_file}" + self.run(f"echo 0 > /tmp/{snoop_file}", shell=True) + self.run(f"cp /tmp/{snoop_file} {snoop}") def delete_bridge(self, name): """ diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 892460fd..f85235f1 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -270,7 +270,7 @@ class CoreNetwork(CoreNetworkBase): self.startup() ebq.startupdateloop(self) - def host_cmd(self, args, env=None, cwd=None, wait=True): + def host_cmd(self, args, env=None, cwd=None, wait=True, shell=False): """ Runs a command that is used to configure and setup the network on the host system and all configured distributed servers. @@ -279,12 +279,13 @@ class CoreNetwork(CoreNetworkBase): :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise + :param bool shell: True to use shell, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when a non-zero exit status occurs """ logging.info("network node(%s) cmd", self.name) - output = utils.check_cmd(args, env, cwd, wait) + output = utils.cmd(args, env, cwd, wait, shell) self.session.distributed.execute(lambda x: x.remote_cmd(args, env, cwd, wait)) return output @@ -765,7 +766,7 @@ class CtrlNet(CoreNetwork): """ use_ovs = self.session.options.get_config("ovs") == "True" current = f"{address}/{self.prefix.prefixlen}" - net_client = get_net_client(use_ovs, utils.check_cmd) + net_client = get_net_client(use_ovs, utils.cmd) net_client.create_address(self.brname, current) servers = self.session.distributed.servers for name in servers: diff --git a/daemon/core/services/utility.py b/daemon/core/services/utility.py index 14bd5a90..e408b182 100644 --- a/daemon/core/services/utility.py +++ b/daemon/core/services/utility.py @@ -415,7 +415,7 @@ class HttpService(UtilService): Detect the apache2 version using the 'a2query' command. """ try: - result = utils.check_cmd("a2query -v") + result = utils.cmd("a2query -v") status = 0 except CoreCommandError as e: status = e.returncode diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 7d88e355..2e2296c0 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -191,7 +191,7 @@ def mute_detach(args, **kwargs): return Popen(args, **kwargs).pid -def check_cmd(args, env=None, cwd=None, wait=True): +def cmd(args, env=None, cwd=None, wait=True, shell=False): """ Execute a command on the host and return a tuple containing the exit status and result string. stderr output is folded into the stdout result string. @@ -200,15 +200,17 @@ def check_cmd(args, env=None, cwd=None, wait=True): :param dict env: environment to run command with :param str cwd: directory to run command in :param bool wait: True to wait for status, False otherwise + :param bool shell: True to use shell, False otherwise :return: combined stdout and stderr :rtype: str :raises CoreCommandError: when there is a non-zero exit status or the file to execute is not found """ logging.info("command cwd(%s) wait(%s): %s", cwd, wait, args) - args = shlex.split(args) + if shell is False: + args = shlex.split(args) try: - p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd) + p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd, shell=shell) if wait: stdout, stderr = p.communicate() status = p.wait() diff --git a/daemon/core/xml/corexmldeployment.py b/daemon/core/xml/corexmldeployment.py index 239bace2..83bf3333 100644 --- a/daemon/core/xml/corexmldeployment.py +++ b/daemon/core/xml/corexmldeployment.py @@ -69,7 +69,7 @@ def get_ipv4_addresses(hostname): if hostname == "localhost": addresses = [] args = f"{IP_BIN} -o -f inet address show" - output = utils.check_cmd(args) + output = utils.cmd(args) for line in output.split(os.linesep): split = line.split() if not split: diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index e495dea9..70525f70 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -67,4 +67,4 @@ class TestNodes: # then assert node assert node.up - assert utils.check_cmd(f"brctl show {node.brname}") + assert utils.cmd(f"brctl show {node.brname}") From 630b44627c8e5282828aff4777d37887e4472541 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 21 Oct 2019 11:36:59 -0700 Subject: [PATCH 112/113] updated distributed python examples a bit to clean things up --- daemon/examples/python/distributed_emane.py | 22 +++----- daemon/examples/python/distributed_lxd.py | 17 +++--- daemon/examples/python/distributed_parser.py | 15 +++++ daemon/examples/python/distributed_ptp.py | 17 +++--- .../{distributed.py => distributed_switch.py} | 19 +++---- .../examples/python/distributed_switches.py | 43 -------------- daemon/examples/python/distributed_wlan.py | 56 ------------------- daemon/examples/python/emane80211.py | 4 -- daemon/examples/python/switch.py | 7 --- daemon/examples/python/switch_inject.py | 6 -- daemon/examples/python/wlan.py | 7 --- 11 files changed, 46 insertions(+), 167 deletions(-) create mode 100644 daemon/examples/python/distributed_parser.py rename daemon/examples/python/{distributed.py => distributed_switch.py} (81%) delete mode 100644 daemon/examples/python/distributed_switches.py delete mode 100644 daemon/examples/python/distributed_wlan.py diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py index 4ef50ccb..74c7c93b 100644 --- a/daemon/examples/python/distributed_emane.py +++ b/daemon/examples/python/distributed_emane.py @@ -1,17 +1,13 @@ import logging -import pdb -import sys +import distributed_parser from core.emane.ieee80211abg import EmaneIeee80211abgModel from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes -def main(): - address = sys.argv[1] - remote = sys.argv[2] - +def main(args): # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") @@ -20,14 +16,14 @@ def main(): { "controlnet": "core1:172.16.1.0/24 core2:172.16.2.0/24 core3:172.16.3.0/24 " "core4:172.16.4.0/24 core5:172.16.5.0/24", - "distributed_address": address, + "distributed_address": args.address, } ) session = coreemu.create_session() # initialize distributed server_name = "core2" - session.distributed.add_server(server_name, remote) + session.distributed.add_server(server_name, args.server) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -48,13 +44,10 @@ def main(): session.add_link(node_two.id, emane_net.id, interface_one=interface_two) # instantiate session - try: - session.instantiate() - except Exception: - logging.exception("error during instantiate") + session.instantiate() # pause script for verification - pdb.set_trace() + input("press enter for shutdown") # shutdown session coreemu.shutdown() @@ -62,4 +55,5 @@ def main(): if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - main() + args = distributed_parser.parse(__file__) + main(args) diff --git a/daemon/examples/python/distributed_lxd.py b/daemon/examples/python/distributed_lxd.py index 130942ea..80366a14 100644 --- a/daemon/examples/python/distributed_lxd.py +++ b/daemon/examples/python/distributed_lxd.py @@ -1,26 +1,22 @@ import logging -import pdb -import sys +import distributed_parser from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes -def main(): - address = sys.argv[1] - remote = sys.argv[2] - +def main(args): # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu({"distributed_address": address}) + coreemu = CoreEmu({"distributed_address": args.address}) session = coreemu.create_session() # initialize distributed server_name = "core2" - session.distributed.add_server(server_name, remote) + session.distributed.add_server(server_name, args.server) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -40,7 +36,7 @@ def main(): session.instantiate() # pause script for verification - pdb.set_trace() + input("press enter for shutdown") # shutdown session coreemu.shutdown() @@ -48,4 +44,5 @@ def main(): if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - main() + args = distributed_parser.parse(__file__) + main(args) diff --git a/daemon/examples/python/distributed_parser.py b/daemon/examples/python/distributed_parser.py new file mode 100644 index 00000000..5557efd8 --- /dev/null +++ b/daemon/examples/python/distributed_parser.py @@ -0,0 +1,15 @@ +import argparse + + +def parse(name): + parser = argparse.ArgumentParser(description=f"Run {name} example") + parser.add_argument( + "-a", + "--address", + help="local address that distributed servers will use for gre tunneling", + ) + parser.add_argument( + "-s", "--server", help="distributed server to use for creating nodes" + ) + options = parser.parse_args() + return options diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index 62e7df64..887fdae4 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -1,26 +1,22 @@ import logging -import pdb -import sys +import distributed_parser from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes -def main(): - address = sys.argv[1] - remote = sys.argv[2] - +def main(args): # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu({"distributed_address": address}) + coreemu = CoreEmu({"distributed_address": args.address}) session = coreemu.create_session() # initialize distributed server_name = "core2" - session.distributed.add_server(server_name, remote) + session.distributed.add_server(server_name, args.server) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -40,7 +36,7 @@ def main(): session.instantiate() # pause script for verification - pdb.set_trace() + input("press enter for shutdown") # shutdown session coreemu.shutdown() @@ -48,4 +44,5 @@ def main(): if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - main() + args = distributed_parser.parse(__file__) + main(args) diff --git a/daemon/examples/python/distributed.py b/daemon/examples/python/distributed_switch.py similarity index 81% rename from daemon/examples/python/distributed.py rename to daemon/examples/python/distributed_switch.py index 8eb23b2c..e87cd2c9 100644 --- a/daemon/examples/python/distributed.py +++ b/daemon/examples/python/distributed_switch.py @@ -1,26 +1,24 @@ import logging -import pdb -import sys +import distributed_parser from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes -def main(): - address = sys.argv[1] - remote = sys.argv[2] - +def main(args): # ip generator for example prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu({"controlnet": "172.16.0.0/24", "distributed_address": address}) + coreemu = CoreEmu( + {"controlnet": "172.16.0.0/24", "distributed_address": args.address} + ) session = coreemu.create_session() # initialize distributed server_name = "core2" - session.distributed.add_server(server_name, remote) + session.distributed.add_server(server_name, args.server) # must be in configuration state for nodes to start, when using "node_add" below session.set_state(EventTypes.CONFIGURATION_STATE) @@ -42,7 +40,7 @@ def main(): session.instantiate() # pause script for verification - pdb.set_trace() + input("press enter for shutdown") # shutdown session coreemu.shutdown() @@ -50,4 +48,5 @@ def main(): if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - main() + args = distributed_parser.parse(__file__) + main(args) diff --git a/daemon/examples/python/distributed_switches.py b/daemon/examples/python/distributed_switches.py deleted file mode 100644 index f9b69757..00000000 --- a/daemon/examples/python/distributed_switches.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging -import pdb -import sys - -from core.emulator.coreemu import CoreEmu -from core.emulator.enumerations import EventTypes, NodeTypes - - -def main(): - address = sys.argv[1] - remote = sys.argv[2] - - # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu({"distributed_address": address}) - session = coreemu.create_session() - - # initialize distributed - server_name = "core2" - session.distributed.add_server(server_name, remote) - - # must be in configuration state for nodes to start, when using "node_add" below - session.set_state(EventTypes.CONFIGURATION_STATE) - - # create local node, switch, and remote nodes - switch_one = session.add_node(_type=NodeTypes.SWITCH) - switch_two = session.add_node(_type=NodeTypes.SWITCH) - - # create node interfaces and link - session.add_link(switch_one.id, switch_two.id) - - # instantiate session - session.instantiate() - - # pause script for verification - pdb.set_trace() - - # shutdown session - coreemu.shutdown() - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/daemon/examples/python/distributed_wlan.py b/daemon/examples/python/distributed_wlan.py deleted file mode 100644 index 10f25aa8..00000000 --- a/daemon/examples/python/distributed_wlan.py +++ /dev/null @@ -1,56 +0,0 @@ -import logging -import pdb -import sys - -from core.emulator.coreemu import CoreEmu -from core.emulator.emudata import IpPrefixes, NodeOptions -from core.emulator.enumerations import EventTypes, NodeTypes -from core.location.mobility import BasicRangeModel - - -def main(): - address = sys.argv[1] - remote = sys.argv[2] - - # ip generator for example - prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - - # create emulator instance for creating sessions and utility methods - coreemu = CoreEmu({"distributed_address": address}) - session = coreemu.create_session() - - # initialize distributed - server_name = "core2" - session.distributed.add_server(server_name, remote) - - # must be in configuration state for nodes to start, when using "node_add" below - session.set_state(EventTypes.CONFIGURATION_STATE) - - # create local node, switch, and remote nodes - options = NodeOptions() - options.set_position(0, 0) - options.emulation_server = server_name - node_one = session.add_node(node_options=options) - wlan = session.add_node(_type=NodeTypes.WIRELESS_LAN) - session.mobility.set_model(wlan, BasicRangeModel) - node_two = session.add_node(node_options=options) - - # create node interfaces and link - interface_one = prefixes.create_interface(node_one) - interface_two = prefixes.create_interface(node_two) - session.add_link(node_one.id, wlan.id, interface_one=interface_one) - session.add_link(node_two.id, wlan.id, interface_one=interface_two) - - # instantiate session - session.instantiate() - - # pause script for verification - pdb.set_trace() - - # shutdown session - coreemu.shutdown() - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/daemon/examples/python/emane80211.py b/daemon/examples/python/emane80211.py index d77252da..75098398 100644 --- a/daemon/examples/python/emane80211.py +++ b/daemon/examples/python/emane80211.py @@ -1,7 +1,3 @@ -#!/usr/bin/python -i -# -# Example CORE Python script that attaches N nodes to an EMANE 802.11abg network. - import datetime import logging import parser diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index 252253f8..0d952fda 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -1,10 +1,3 @@ -#!/usr/bin/python -# -# run iperf to measure the effective throughput between two nodes when -# n nodes are connected to a virtual wlan; run test for testsec -# and repeat for minnodes <= n <= maxnodes with a step size of -# nodestep - import datetime import logging import parser diff --git a/daemon/examples/python/switch_inject.py b/daemon/examples/python/switch_inject.py index 1b7b634c..0a87afd2 100644 --- a/daemon/examples/python/switch_inject.py +++ b/daemon/examples/python/switch_inject.py @@ -1,9 +1,3 @@ -#!/usr/bin/python -# -# run iperf to measure the effective throughput between two nodes when -# n nodes are connected to a virtual wlan; run test for testsec -# and repeat for minnodes <= n <= maxnodes with a step size of -# nodestep import logging from core.emulator.emudata import IpPrefixes diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index 4b4bc724..1ef1a5d1 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -1,10 +1,3 @@ -#!/usr/bin/python -# -# run iperf to measure the effective throughput between two nodes when -# n nodes are connected to a virtual wlan; run test for testsec -# and repeat for minnodes <= n <= maxnodes with a step size of -# nodestep - import datetime import logging import parser From 233ca92fd2cdebbabe0896ae4dac81a0d1e8cb24 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 21 Oct 2019 12:51:38 -0700 Subject: [PATCH 113/113] update grpc to allow for configuring and created distributed nodes --- daemon/Pipfile.lock | 70 +++++++++--------- daemon/core/api/grpc/client.py | 16 ++++ daemon/core/api/grpc/server.py | 16 ++++ daemon/examples/grpc/distributed_switch.py | 86 ++++++++++++++++++++++ daemon/proto/core/api/grpc/core.proto | 13 ++++ 5 files changed, 167 insertions(+), 34 deletions(-) create mode 100644 daemon/examples/grpc/distributed_switch.py diff --git a/daemon/Pipfile.lock b/daemon/Pipfile.lock index 4bdaea3f..5a19aae7 100644 --- a/daemon/Pipfile.lock +++ b/daemon/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6195c89ec6e2e449fcbd7f3fa41cbab79c02d952984a913e0f80114e1904bf11" + "sha256": "d702e6eed5a1362bf261543572bbffd2e8a87140b8d8cb07b99fb0d25220a2b5" }, "pipfile-spec": 6, "requires": {}, @@ -20,6 +20,7 @@ "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", + "sha256:6305557019906466fc42dbc53b46da004e72fd7a551c044a827e572c82191752", "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", @@ -30,6 +31,7 @@ "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", + "sha256:ce4e4f0deb51d38b1611a27f330426154f2980e66582dc5f438aad38b5f24fc1", "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" ], @@ -37,40 +39,38 @@ }, "cffi": { "hashes": [ - "sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8", - "sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d", - "sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36", - "sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45", - "sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b", - "sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2", - "sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b", - "sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5", - "sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a", - "sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4", - "sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2", - "sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5", - "sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f", - "sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa", - "sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a", - "sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c", - "sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1", - "sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24", - "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226", - "sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895", - "sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529", - "sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1", - "sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27", - "sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165", - "sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5", - "sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa", - "sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2", - "sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89", - "sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59", - "sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df", - "sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f", - "sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1" + "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa", + "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a", + "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400", + "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365", + "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98", + "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526", + "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14", + "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5", + "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e", + "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1", + "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434", + "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b", + "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730", + "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43", + "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4", + "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331", + "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36", + "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599", + "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", + "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", + "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", + "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", + "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", + "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", + "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", + "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", + "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", + "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", + "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05", + "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2" ], - "version": "==1.13.0" + "version": "==1.13.1" }, "core": { "editable": true, @@ -226,6 +226,7 @@ "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", + "sha256:53126cd91356342dcae7e209f840212a58dcf1177ad52c1d938d428eebc9fee5", "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", @@ -234,6 +235,7 @@ "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", + "sha256:bf459128feb543cfca16a95f8da31e2e65e4c5257d2f3dfa8c0c1031139c9c92", "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" ], diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index 54f77fc6..f14bd064 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -276,6 +276,22 @@ class CoreGrpcClient(object): request = core_pb2.SetSessionStateRequest(session_id=session_id, state=state) return self.stub.SetSessionState(request) + def add_session_server(self, session_id, name, host): + """ + Add distributed session server. + + :param int session_id: id of session + :param str name: name of server to add + :param str host: host address to connect to + :return: response with result of success or failure + :rtype: core_pb2.AddSessionServerResponse + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.AddSessionServerRequest( + session_id=session_id, name=name, host=host + ) + return self.stub.AddSessionServer(request) + def events(self, session_id, handler): """ Listen for session events. diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 0ab4806a..a92a72e5 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -473,6 +473,20 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): session_proto = core_pb2.Session(state=session.state, nodes=nodes, links=links) return core_pb2.GetSessionResponse(session=session_proto) + def AddSessionServer(self, request, context): + """ + Add distributed server to a session. + + :param core.api.grpc.core_pb2.AddSessionServerRequest request: get-session + request + :param grpc.ServicerContext context: context object + :return: add session server response + :rtype: core.api.grpc.core_bp2.AddSessionServerResponse + """ + session = self.get_session(request.session_id, context) + session.distributed.add_server(request.name, request.host) + return core_pb2.AddSessionServerResponse(result=True) + def Events(self, request, context): session = self.get_session(request.session_id, context) queue = Queue() @@ -761,6 +775,8 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): node_options.opaque = node_proto.opaque node_options.image = node_proto.image node_options.services = node_proto.services + if node_proto.server: + node_options.emulation_server = node_proto.server position = node_proto.position node_options.set_position(position.x, position.y) diff --git a/daemon/examples/grpc/distributed_switch.py b/daemon/examples/grpc/distributed_switch.py new file mode 100644 index 00000000..9cc35f72 --- /dev/null +++ b/daemon/examples/grpc/distributed_switch.py @@ -0,0 +1,86 @@ +import argparse +import logging + +from core.api.grpc import client, core_pb2 + + +def log_event(event): + logging.info("event: %s", event) + + +def main(args): + core = client.CoreGrpcClient() + + with core.context_connect(): + # create session + response = core.create_session() + session_id = response.session_id + logging.info("created session: %s", response) + + # add distributed server + server_name = "core2" + response = core.add_session_server(session_id, server_name, args.server) + logging.info("added session server: %s", response) + + # handle events session may broadcast + core.events(session_id, log_event) + + # change session state + response = core.set_session_state( + session_id, core_pb2.SessionState.CONFIGURATION + ) + logging.info("set session state: %s", response) + + # create switch node + switch = core_pb2.Node(type=core_pb2.NodeType.SWITCH) + response = core.add_node(session_id, switch) + logging.info("created switch: %s", response) + switch_id = response.node_id + + # helper to create interfaces + interface_helper = client.InterfaceHelper(ip4_prefix="10.83.0.0/16") + + # create node one + position = core_pb2.Position(x=100, y=50) + node = core_pb2.Node(position=position) + response = core.add_node(session_id, node) + logging.info("created node one: %s", response) + node_one_id = response.node_id + + # create link + interface_one = interface_helper.create_interface(node_one_id, 0) + response = core.add_link(session_id, node_one_id, switch_id, interface_one) + logging.info("created link from node one to switch: %s", response) + + # create node two + position = core_pb2.Position(x=200, y=50) + node = core_pb2.Node(position=position, server=server_name) + response = core.add_node(session_id, node) + logging.info("created node two: %s", response) + node_two_id = response.node_id + + # create link + interface_one = interface_helper.create_interface(node_two_id, 0) + response = core.add_link(session_id, node_two_id, switch_id, interface_one) + logging.info("created link from node two to switch: %s", response) + + # change session state + response = core.set_session_state( + session_id, core_pb2.SessionState.INSTANTIATION + ) + logging.info("set session state: %s", response) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + parser = argparse.ArgumentParser(description="Run distributed_switch example") + parser.add_argument( + "-a", + "--address", + help="local address that distributed servers will use for gre tunneling", + ) + parser.add_argument( + "-s", "--server", help="distributed server to use for creating nodes" + ) + args = parser.parse_args() + main(args) diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 1e17b327..d5528b52 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -25,6 +25,8 @@ service CoreApi { } rpc SetSessionState (SetSessionStateRequest) returns (SetSessionStateResponse) { } + rpc AddSessionServer (AddSessionServerRequest) returns (AddSessionServerResponse) { + } // streams rpc Events (EventsRequest) returns (stream Event) { @@ -201,6 +203,16 @@ message SetSessionStateResponse { bool result = 1; } +message AddSessionServerRequest { + int32 session_id = 1; + string name = 2; + string host = 3; +} + +message AddSessionServerResponse { + bool result = 1; +} + message EventsRequest { int32 session_id = 1; } @@ -802,6 +814,7 @@ message Node { string icon = 8; string opaque = 9; string image = 10; + string server = 11; } message Link {