refactor fabric distributed to use a class and update sessions to create and provide these to nodes

This commit is contained in:
Blake Harnden 2019-10-14 15:43:57 -07:00
parent 5f282bb695
commit 6570f22ccf
10 changed files with 153 additions and 133 deletions

View file

@ -18,7 +18,6 @@ from core.emane.ieee80211abg import EmaneIeee80211abgModel
from core.emane.nodes import EmaneNet
from core.emane.rfpipe import EmaneRfPipeModel
from core.emane.tdma import EmaneTdmaModel
from core.emulator import distributed
from core.emulator.enumerations import (
ConfigDataTypes,
ConfigFlags,
@ -155,9 +154,9 @@ class EmaneManager(ModelManager):
args = "emane --version"
emane_version = utils.check_cmd(args)
logging.info("using EMANE: %s", emane_version)
for server in self.session.servers:
conn = self.session.servers[server]
distributed.remote_cmd(conn, args)
for host in self.session.servers:
server = self.session.servers[host]
server.remote_cmd(args)
# load default emane models
self.load_models(EMANE_MODELS)
@ -757,9 +756,9 @@ class EmaneManager(ModelManager):
emanecmd += " -f %s" % os.path.join(path, "emane.log")
emanecmd += " %s" % os.path.join(path, "platform.xml")
utils.check_cmd(emanecmd, cwd=path)
for server in self.session.servers:
conn = self.session.servers[server]
distributed.remote_cmd(conn, emanecmd, cwd=path)
for host in self.session.servers:
server = self.session.servers[host]
server.remote_cmd(emanecmd, cwd=path)
logging.info("host emane daemon running: %s", emanecmd)
def stopdaemons(self):
@ -784,10 +783,10 @@ class EmaneManager(ModelManager):
try:
utils.check_cmd(kill_emaned)
utils.check_cmd(kill_transortd)
for server in self.session.servers:
conn = self.session[server]
distributed.remote_cmd(conn, kill_emaned)
distributed.remote_cmd(conn, kill_transortd)
for host in self.session.servers:
server = self.session[host]
server.remote_cmd(kill_emaned)
server.remote_cmd(kill_transortd)
except CoreCommandError:
logging.exception("error shutting down emane daemons")

View file

@ -1,8 +1,13 @@
"""
Defines distributed server functionality.
"""
import logging
import os
import threading
from tempfile import NamedTemporaryFile
from fabric import Connection
from invoke import UnexpectedExit
from core.errors import CoreCommandError
@ -10,52 +15,80 @@ from core.errors import CoreCommandError
LOCK = threading.Lock()
def remote_cmd(server, cmd, env=None, cwd=None, wait=True):
class DistributedServer(object):
"""
Run command remotely using server connection.
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param str cmd: command to run
:param dict env: environment for remote command, default is None
:param str cwd: directory to run command in, defaults to None, which is the user's
home directory
:param bool wait: True to wait for status, False to background process
:return: stdout when success
:rtype: str
:raises CoreCommandError: when a non-zero exit status occurs
Provides distributed server interactions.
"""
replace_env = env is not None
if not wait:
cmd += " &"
logging.info(
"remote cmd server(%s) cwd(%s) wait(%s): %s", server.host, cwd, wait, cmd
)
try:
with LOCK:
if cwd is None:
result = server.run(cmd, hide=False, env=env, replace_env=replace_env)
else:
with server.cd(cwd):
result = server.run(
def __init__(self, host):
"""
Create a DistributedServer instance.
:param str host: host to connect to
"""
self.host = host
self.conn = Connection(host, user="root")
self.lock = threading.Lock()
def remote_cmd(self, cmd, env=None, cwd=None, wait=True):
"""
Run command remotely using server connection.
:param str cmd: command to run
:param dict env: environment for remote command, default is None
:param str cwd: directory to run command in, defaults to None, which is the user's
home directory
:param bool wait: True to wait for status, False to background process
:return: stdout when success
:rtype: str
:raises CoreCommandError: when a non-zero exit status occurs
"""
replace_env = env is not None
if not wait:
cmd += " &"
logging.info(
"remote cmd server(%s) cwd(%s) wait(%s): %s", self.host, cwd, wait, cmd
)
try:
with self.lock:
if cwd is None:
result = self.conn.run(
cmd, hide=False, env=env, replace_env=replace_env
)
return result.stdout.strip()
except UnexpectedExit as e:
stdout, stderr = e.streams_for_display()
raise CoreCommandError(e.result.exited, cmd, stdout, stderr)
else:
with self.conn.cd(cwd):
result = self.conn.run(
cmd, hide=False, env=env, replace_env=replace_env
)
return result.stdout.strip()
except UnexpectedExit as e:
stdout, stderr = e.streams_for_display()
raise CoreCommandError(e.result.exited, cmd, stdout, stderr)
def remote_put(self, source, destination):
"""
Push file to remote server.
def remote_put(server, source, destination):
with LOCK:
server.put(source, destination)
:param str source: source file to push
:param str destination: destination file location
:return: nothing
"""
with self.lock:
self.conn.put(source, destination)
def remote_put_temp(self, destination, data):
"""
Remote push file contents to a remote server, using a temp file as an
intermediate step.
def remote_put_temp(server, destination, data):
with LOCK:
temp = NamedTemporaryFile(delete=False)
temp.write(data.encode("utf-8"))
temp.close()
server.put(temp.name, destination)
os.unlink(temp.name)
:param str destination: file destination for data
:param str data: data to store in remote file
:return: nothing
"""
with self.lock:
temp = NamedTemporaryFile(delete=False)
temp.write(data.encode("utf-8"))
temp.close()
self.conn.put(temp.name, destination)
os.unlink(temp.name)

View file

@ -14,14 +14,13 @@ import threading
import time
from multiprocessing.pool import ThreadPool
from fabric import Connection
from core import constants, utils
from core.api.tlv import coreapi
from core.api.tlv.broker import CoreBroker
from core.emane.emanemanager import EmaneManager
from core.emane.nodes import EmaneNet
from core.emulator.data import EventData, ExceptionData, NodeData
from core.emulator.distributed import DistributedServer
from core.emulator.emudata import (
IdGen,
LinkOptions,
@ -162,11 +161,11 @@ class Session(object):
"host": ("DefaultRoute", "SSH"),
}
def add_distributed(self, server):
conn = Connection(server, user="root")
self.servers[server] = conn
def add_distributed(self, host):
server = DistributedServer(host)
self.servers[host] = server
cmd = "mkdir -p %s" % self.session_dir
conn.run(cmd, hide=False)
server.remote_cmd(cmd)
def shutdown_distributed(self):
# shutdown all tunnels
@ -176,10 +175,10 @@ class Session(object):
tunnel.shutdown()
# remove all remote session directories
for server in self.servers:
conn = self.servers[server]
for host in self.servers:
server = self.servers[host]
cmd = "rm -rf %s" % self.session_dir
conn.run(cmd, hide=False)
server.remote_cmd(cmd)
# clear tunnels
self.tunnels.clear()
@ -194,18 +193,15 @@ class Session(object):
if isinstance(node, CtrlNet) and node.serverintf is not None:
continue
for server in self.servers:
conn = self.servers[server]
key = self.tunnelkey(node_id, IpAddress.to_int(server))
for host in self.servers:
server = self.servers[host]
key = self.tunnelkey(node_id, IpAddress.to_int(host))
# local to server
logging.info(
"local tunnel node(%s) to remote(%s) key(%s)",
node.name,
server,
key,
"local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key
)
local_tap = GreTap(session=self, remoteip=server, key=key)
local_tap = GreTap(session=self, remoteip=host, key=key)
local_tap.net_client.create_interface(node.brname, local_tap.localname)
# server to local
@ -216,7 +212,7 @@ class Session(object):
key,
)
remote_tap = GreTap(
session=self, remoteip=self.address, key=key, server=conn
session=self, remoteip=self.address, key=key, server=server
)
remote_tap.net_client.create_interface(
node.brname, remote_tap.localname

View file

@ -14,7 +14,6 @@ from socket import AF_INET, AF_INET6
from core import utils
from core.constants import MOUNT_BIN, VNODED_BIN
from core.emulator import distributed
from core.emulator.data import LinkData, NodeData
from core.emulator.enumerations import LinkTypes, NodeTypes
from core.errors import CoreCommandError
@ -41,8 +40,8 @@ class NodeBase(object):
:param int _id: id
:param str name: object name
:param bool start: start value
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
"""
self.session = session
@ -101,7 +100,7 @@ class NodeBase(object):
if self.server is None:
return utils.check_cmd(args, env, cwd, wait)
else:
return distributed.remote_cmd(self.server, args, env, cwd, wait)
return self.server.remote_cmd(args, env, cwd, wait)
def setposition(self, x=None, y=None, z=None):
"""
@ -200,7 +199,7 @@ class NodeBase(object):
x, y, _ = self.getposition()
model = self.type
emulation_server = self.server
emulation_server = self.server.host
services = self.services
if services is not None:
@ -253,8 +252,8 @@ class CoreNodeBase(NodeBase):
:param int _id: object id
:param str name: object name
:param bool start: boolean for starting
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
"""
super(CoreNodeBase, self).__init__(session, _id, name, start, server)
self.services = []
@ -437,8 +436,8 @@ class CoreNode(CoreNodeBase):
:param str nodedir: node directory
:param str bootsh: boot shell to use
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
"""
super(CoreNode, self).__init__(session, _id, name, start, server)
self.nodedir = nodedir
@ -585,7 +584,7 @@ class CoreNode(CoreNodeBase):
return self.client.check_cmd(args, wait=wait)
else:
args = self.client.create_cmd(args)
return distributed.remote_cmd(self.server, args, wait=wait)
return self.server.remote_cmd(args, wait=wait)
def termcmdstring(self, sh="/bin/sh"):
"""
@ -888,7 +887,7 @@ class CoreNode(CoreNodeBase):
self.client.check_cmd("sync")
else:
self.net_cmd("mkdir -p %s" % directory)
distributed.remote_put(self.server, srcname, filename)
self.server.remote_put(srcname, filename)
def hostfilename(self, filename):
"""
@ -925,7 +924,7 @@ class CoreNode(CoreNodeBase):
os.chmod(open_file.name, mode)
else:
self.net_cmd("mkdir -m %o -p %s" % (0o755, dirname))
distributed.remote_put_temp(self.server, hostfilename, contents)
self.server.remote_put_temp(hostfilename, contents)
self.net_cmd("chmod %o %s" % (mode, hostfilename))
logging.debug(
"node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode
@ -944,12 +943,10 @@ class CoreNode(CoreNodeBase):
hostfilename = self.hostfilename(filename)
if self.server is None:
shutil.copy2(srcfilename, hostfilename)
if mode is not None:
os.chmod(hostfilename, mode)
else:
distributed.remote_put(self.server, srcfilename, hostfilename)
if mode is not None:
self.net_cmd("chmod %o %s" % (mode, hostfilename))
self.server.remote_put(srcfilename, hostfilename)
if mode is not None:
self.net_cmd("chmod %o %s" % (mode, hostfilename))
logging.info(
"node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode
)
@ -971,8 +968,8 @@ class CoreNetworkBase(NodeBase):
:param int _id: object id
:param str name: object name
:param bool start: should object start
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
"""
super(CoreNetworkBase, self).__init__(session, _id, name, start, server)
self._linked = {}

View file

@ -4,7 +4,6 @@ import os
from tempfile import NamedTemporaryFile
from core import utils
from core.emulator import distributed
from core.emulator.enumerations import NodeTypes
from core.errors import CoreCommandError
from core.nodes.base import CoreNode
@ -159,7 +158,7 @@ class DockerNode(CoreNode):
return utils.check_cmd(args, wait=wait)
else:
args = self.client.create_ns_cmd(args)
return distributed.remote_cmd(self.server, args, wait=wait)
return self.server.remote_cmd(args, wait=wait)
def termcmdstring(self, sh="/bin/sh"):
"""
@ -211,7 +210,7 @@ class DockerNode(CoreNode):
if directory:
self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory))
if self.server is not None:
distributed.remote_put(self.server, temp.name, temp.name)
self.server.remote_put(temp.name, temp.name)
self.client.copy_file(temp.name, filename)
self.node_net_cmd("chmod %o %s" % (mode, filename))
if self.server is not None:
@ -242,7 +241,7 @@ class DockerNode(CoreNode):
else:
temp = NamedTemporaryFile(delete=False)
source = temp.name
distributed.remote_put(self.server, source, temp.name)
self.server.remote_put(source, temp.name)
self.client.copy_file(source, filename)
self.node_net_cmd("chmod %o %s" % (mode, filename))

View file

@ -7,7 +7,6 @@ import time
from builtins import int, range
from core import utils
from core.emulator import distributed
from core.errors import CoreCommandError
from core.nodes.netclient import LinuxNetClient
@ -24,8 +23,8 @@ class CoreInterface(object):
:param core.nodes.base.CoreNode node: node for interface
:param str name: interface name
:param mtu: mtu value
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
"""
self.node = node
@ -63,7 +62,7 @@ class CoreInterface(object):
if self.server is None:
return utils.check_cmd(args, env, cwd, wait)
else:
return distributed.remote_cmd(self.server, args, env, cwd, wait)
return self.server.remote_cmd(args, env, cwd, wait)
def startup(self):
"""
@ -220,8 +219,8 @@ class Veth(CoreInterface):
:param str name: interface name
:param str localname: interface local name
:param mtu: interface mtu
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:param bool start: start flag
:raises CoreCommandError: when there is a command exception
"""
@ -280,8 +279,8 @@ class TunTap(CoreInterface):
:param str name: interface name
:param str localname: local interface name
:param mtu: interface mtu
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:param bool start: start flag
"""
CoreInterface.__init__(self, node, name, mtu, server)
@ -463,8 +462,8 @@ class GreTap(CoreInterface):
:param int ttl: ttl value
:param int key: gre tap key
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:raises CoreCommandError: when there is a command exception
"""
CoreInterface.__init__(self, node, name, mtu, server)

View file

@ -5,7 +5,6 @@ import time
from tempfile import NamedTemporaryFile
from core import utils
from core.emulator import distributed
from core.emulator.enumerations import NodeTypes
from core.errors import CoreCommandError
from core.nodes.base import CoreNode
@ -182,7 +181,7 @@ class LxcNode(CoreNode):
if directory:
self.node_net_cmd("mkdir -m %o -p %s" % (0o755, directory))
if self.server is not None:
distributed.remote_put(self.server, temp.name, temp.name)
self.server.remote_put(temp.name, temp.name)
self.client.copy_file(temp.name, filename)
self.node_net_cmd("chmod %o %s" % (mode, filename))
if self.server is not None:
@ -211,7 +210,7 @@ class LxcNode(CoreNode):
else:
temp = NamedTemporaryFile(delete=False)
source = temp.name
distributed.remote_put(self.server, source, temp.name)
self.server.remote_put(source, temp.name)
self.client.copy_file(source, filename)
self.node_net_cmd("chmod %o %s" % (mode, filename))

View file

@ -10,7 +10,6 @@ from socket import AF_INET, AF_INET6
from core import utils
from core.constants import EBTABLES_BIN, TC_BIN
from core.emulator import distributed
from core.emulator.data import LinkData
from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs
from core.errors import CoreCommandError, CoreError
@ -257,8 +256,8 @@ class CoreNetwork(CoreNetworkBase):
:param int _id: object id
:param str name: object name
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:param policy: network policy
"""
CoreNetworkBase.__init__(self, session, _id, name, start, server)
@ -289,9 +288,9 @@ class CoreNetwork(CoreNetworkBase):
"""
logging.info("network node(%s) cmd", self.name)
output = utils.check_cmd(args, env, cwd, wait)
for server in self.session.servers:
conn = self.session.servers[server]
distributed.remote_cmd(conn, args, env, cwd, wait)
for host in self.session.servers:
server = self.session.servers[host]
server.remote_cmd(args, env, cwd, wait)
return output
def startup(self):
@ -632,8 +631,8 @@ class GreTapBridge(CoreNetwork):
:param ttl: ttl value
:param key: gre tap key
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
"""
CoreNetwork.__init__(self, session, _id, name, False, server, policy)
self.grekey = key
@ -753,8 +752,8 @@ class CtrlNet(CoreNetwork):
:param prefix: control network ipv4 prefix
:param hostid: host id
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:param str assign_address: assigned address
:param str updown_script: updown script
:param serverintf: server interface
@ -1008,8 +1007,8 @@ class HubNode(CoreNetwork):
:param int _id: node id
:param str name: node namee
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:raises CoreCommandError: when there is a command exception
"""
CoreNetwork.__init__(self, session, _id, name, start, server)
@ -1039,8 +1038,8 @@ class WlanNode(CoreNetwork):
:param int _id: node id
:param str name: node name
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:param policy: wlan policy
"""
CoreNetwork.__init__(self, session, _id, name, start, server, policy)

View file

@ -242,8 +242,8 @@ class Rj45Node(CoreNodeBase, CoreInterface):
:param str name: node name
:param mtu: rj45 mtu
:param bool start: start flag
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
"""
CoreNodeBase.__init__(self, session, _id, name, start, server)
CoreInterface.__init__(self, node=self, name=name, mtu=mtu)

View file

@ -5,7 +5,6 @@ from tempfile import NamedTemporaryFile
from lxml import etree
from core import utils
from core.emulator import distributed
from core.nodes.ipaddress import MacAddress
from core.xml import corexml
@ -53,8 +52,8 @@ def create_file(xml_element, doc_name, file_path, server=None):
:param lxml.etree.Element xml_element: root element to write to file
:param str doc_name: name to use in the emane doctype
:param str file_path: file path to write xml file to
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:return: nothing
"""
doctype = (
@ -65,7 +64,7 @@ def create_file(xml_element, doc_name, file_path, server=None):
temp = NamedTemporaryFile(delete=False)
create_file(xml_element, doc_name, temp.name)
temp.close()
distributed.remote_put(server, temp.name, file_path)
server.remote_put(temp.name, file_path)
os.unlink(temp.name)
else:
corexml.write_xml_file(xml_element, file_path, doctype=doctype)
@ -327,8 +326,8 @@ def create_phy_xml(emane_model, config, file_path, server):
:param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml
:param dict config: all current configuration values
:param str file_path: path to write file to
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:return: nothing
"""
phy_element = etree.Element("phy", name="%s PHY" % emane_model.name)
@ -355,8 +354,8 @@ def create_mac_xml(emane_model, config, file_path, server):
:param core.emane.emanemodel.EmaneModel emane_model: emane model to create xml
:param dict config: all current configuration values
:param str file_path: path to write file to
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:return: nothing
"""
if not emane_model.mac_library:
@ -396,8 +395,8 @@ def create_nem_xml(
:param str transport_definition: transport file definition path
:param str mac_definition: mac file definition path
:param str phy_definition: phy file definition path
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:return: nothing
"""
nem_element = etree.Element("nem", name="%s NEM" % emane_model.name)
@ -424,8 +423,8 @@ def create_event_service_xml(group, port, device, file_directory, server=None):
:param str port: event port
:param str device: event device
:param str file_directory: directory to create file in
:param fabric.connection.Connection server: remote server node will run on,
default is None for localhost
:param core.emulator.distributed.DistributedServer server: remote server node
will run on, default is None for localhost
:return: nothing
"""
event_element = etree.Element("emaneeventmsgsvc")