catching up with commits: daemon: Add an instantiation-complete CORE API event type

This commit is contained in:
Blake J. Harnden 2017-04-25 11:38:53 -07:00
parent 00f4ebf5a9
commit 2fc6345138
8 changed files with 377 additions and 332 deletions

View file

@ -38,6 +38,31 @@ from core.phys.pnodes import PhysicalNode
logger = log.get_logger(__name__)
class CoreServer(object):
def __init__(self, name, host, port):
self.name = name
self.host = host
self.port = port
self.sock = None
self.instantiation_complete = False
def connect(self):
assert self.sock is None
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.host, self.port))
except:
sock.close()
raise
self.sock = sock
def close(self):
if self.sock is not None:
self.sock.close()
self.sock = None
class CoreBroker(ConfigurableManager):
"""
Member of pycore session class for handling global emulation server data.
@ -69,12 +94,12 @@ class CoreBroker(ConfigurableManager):
# reference counts of nodes on servers
self.nodecounts = {}
self.bootcount = 0
# list of node numbers that are link-layer nodes (networks)
self.network_nodes = []
# list of node numbers that are PhysicalNode nodes
self.physical_nodes = []
# set of node numbers that are link-layer nodes (networks)
self.network_nodes = set()
# set of node numbers that are PhysicalNode nodes
self.physical_nodes = set()
# allows for other message handlers to process API messages (e.g. EMANE)
self.handlers = ()
self.handlers = set()
# dict with tunnel key to tunnel device mapping
self.tunnels = {}
self.dorecvloop = False
@ -96,12 +121,11 @@ class CoreBroker(ConfigurableManager):
"""
with self.servers_lock:
while len(self.servers) > 0:
server, v = self.servers.popitem()
host, port, sock = v
if sock is None:
continue
logger.info("closing connection with %s @ %s:%s", server, host, port)
sock.close()
name, server = self.servers.popitem()
if server.sock is not None:
logger.info("closing connection with %s @ %s:%s" %
(name, server.host, server.port))
server.close()
self.reset()
self.dorecvloop = False
if self.recvthread is not None:
@ -113,21 +137,21 @@ class CoreBroker(ConfigurableManager):
"""
self.nodemap_lock.acquire()
self.nodemap.clear()
for server in self.nodecounts:
if self.nodecounts[server] < 1:
for server, count in self.nodecounts.iteritems():
if count < 1:
self.delserver(server)
self.nodecounts.clear()
self.bootcount = 0
self.nodemap_lock.release()
del self.network_nodes[:]
del self.physical_nodes[:]
self.network_nodes.clear()
self.physical_nodes.clear()
while len(self.tunnels) > 0:
key, gt = self.tunnels.popitem()
gt.shutdown()
def startrecvloop(self):
"""
Spawn the recvloop() thread if it hasn't been already started.
Spawn the recvloop() thread if it hasn"t been already started.
"""
if self.recvthread is not None:
if self.recvthread.isAlive():
@ -151,53 +175,55 @@ class CoreBroker(ConfigurableManager):
rlist = []
with self.servers_lock:
# build a socket list for select call
for name in self.servers:
(h, p, sock) = self.servers[name]
if sock is not None:
rlist.append(sock.fileno())
for server in self.servers.itervalues():
if server.sock is not None:
rlist.append(server.sock)
r, w, x = select.select(rlist, [], [], 1.0)
for sockfd in r:
try:
(h, p, sock, name) = self.getserverbysock(sockfd)
except KeyError:
for sock in r:
server = self.getserverbysock(sock)
if server is None:
# servers may have changed; loop again
logger.exception("get server by sock error")
break
rcvlen = self.recv(sock, h)
continue
rcvlen = self.recv(server)
if rcvlen == 0:
logger.info("connection with %s @ %s:%s has closed", name, h, p)
self.servers[name] = (h, p, None)
logger.info("connection with %s @ %s:%s has closed" % (
server.name, server.host, server.port))
def recv(self, sock, host):
def recv(self, server):
"""
Receive data on an emulation server socket and broadcast it to
all connected session handlers. Returns the length of data recevied
and forwarded. Return value of zero indicates the socket has closed
and should be removed from the self.servers dict.
:param CoreServer server: server to receive from
:return: message length
:rtype: int
"""
msghdr = sock.recv(coreapi.CoreMessage.header_len)
msghdr = server.sock.recv(coreapi.CoreMessage.header_len)
if len(msghdr) == 0:
# server disconnected
sock.close()
server.close()
return 0
if len(msghdr) != coreapi.CoreMessage.header_len:
logger.info("warning: broker received not enough data len=%s" % len(msghdr))
return len(msghdr)
msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(msghdr)
msgdata = sock.recv(msglen)
msgdata = server.sock.recv(msglen)
data = msghdr + msgdata
count = None
# snoop exec response for remote interactive TTYs
if msgtype == MessageTypes.EXECUTE.value and msgflags & MessageFlags.TTY.value:
data = self.fixupremotetty(msghdr, msgdata, host)
data = self.fixupremotetty(msghdr, msgdata, server.host)
elif msgtype == MessageTypes.NODE.value:
# snoop node delete response to decrement node counts
if msgflags & MessageFlags.DELETE.value:
msg = coreapi.CoreNodeMessage(msgflags, msghdr, msgdata)
nodenum = msg.get_tlv(NodeTlvs.NUMBER.value)
if nodenum is not None:
count = self.delnodemap(sock, nodenum)
count = self.delnodemap(server, nodenum)
# snoop node add response to increment booted node count
# (only CoreNodes send these response messages)
elif msgflags & (MessageFlags.ADD.value | MessageFlags.LOCAL.value):
@ -207,6 +233,13 @@ class CoreBroker(ConfigurableManager):
# this allows green link lines for remote WLANs
msg = coreapi.CoreLinkMessage(msgflags, msghdr, msgdata)
self.session.sdt.handle_distributed(msg)
elif msgtype == MessageTypes.EVENT.value:
msg = coreapi.CoreEventMessage(msgflags, msghdr, msgdata)
eventtype = msg.get_tlv(EventTlvs.TYPE.value)
if eventtype == EventTypes.INSTANTIATION_COMPLETE.value:
server.instantiation_complete = True
if self.instantiation_complete():
self.session.check_runtime()
else:
logger.error("unknown message type received: %s", msgtype)
@ -222,85 +255,88 @@ class CoreBroker(ConfigurableManager):
def addserver(self, name, host, port):
"""
Add a new server, and try to connect to it. If we're already
Add a new server, and try to connect to it. If we"re already
connected to this (host, port), then leave it alone. When host,port
is None, do not try to connect.
"""
self.servers_lock.acquire()
if name in self.servers:
oldhost, oldport, sock = self.servers[name]
if host == oldhost or port == oldport:
# leave this socket connected
if sock is not None:
self.servers_lock.release()
with self.servers_lock:
server = self.servers.get(name)
if server is not None:
if host == server.host and port == server.port and \
server.sock is not None:
# leave this socket connected
return
if host is not None and sock is not None:
logger.info("closing connection with %s @ %s:%s", name, host, port)
if sock is not None:
sock.close()
self.servers_lock.release()
if host is not None:
logger.info("adding server %s @ %s:%s", name, host, port)
if host is None:
sock = None
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# sock.setblocking(0)
# error = sock.connect_ex((host, port))
try:
sock.connect((host, port))
self.startrecvloop()
except IOError:
logger.exception("error connecting to server %s:%s", host, port)
sock.close()
sock = None
self.servers_lock.acquire()
self.servers[name] = (host, port, sock)
self.servers_lock.release()
def delserver(self, name):
logger.info("closing connection with %s @ %s:%s" % (name, server.host, server.port))
server.close()
del self.servers[name]
logger.info("adding server %s @ %s:%s" % (name, host, port))
server = CoreServer(name, host, port)
if host is not None and port is not None:
try:
server.connect()
except IOError:
logger.exception("error connecting to server %s:%s" % (host, port))
if server.sock is not None:
self.startrecvloop()
self.servers[name] = server
def delserver(self, server):
"""
Remove a server and hang up any connection.
"""
self.servers_lock.acquire()
if name not in self.servers:
self.servers_lock.release()
return
(host, port, sock) = self.servers.pop(name)
if sock is not None:
logger.info("closing connection with %s @ %s:%s", name, host, port)
sock.close()
self.servers_lock.release()
def getserver(self, name):
"""
Return the (host, port, sock) tuple, or raise a KeyError exception.
"""
if name not in self.servers:
raise KeyError("emulation server %s not found" % name)
return self.servers[name]
def getserverbysock(self, sockfd):
"""
Return a (host, port, sock, name) tuple based on socket file
descriptor, or raise a KeyError exception.
:param CoreServer server: server to delete
:return:
"""
with self.servers_lock:
for name in self.servers:
(host, port, sock) = self.servers[name]
if sock is None:
continue
if sock.fileno() == sockfd:
return host, port, sock, name
raise KeyError("socket fd %s not found" % sockfd)
try:
s = self.servers.pop(server.name)
assert s == server
except KeyError:
pass
if server.sock is not None:
logger.info("closing connection with %s @ %s:%s" % (server.name, server.host, server.port))
server.close()
def getserverlist(self):
def getserverbyname(self, name):
"""
Return the list of server names (keys from self.servers).
Return the server object having the given name, or None.
:param str name: name of server to retrieve
:return: server for given name
:rtype: CoreServer
"""
with self.servers_lock:
serverlist = sorted(self.servers.keys())
return serverlist
return self.servers.get(name)
def getserverbysock(self, sock):
"""
Return the server object corresponding to the given socket, or None.
:param sock: socket associated with a server
:return: core server associated wit the socket
:rtype: CoreServer
"""
with self.servers_lock:
for server in self.servers.itervalues():
if server.sock == sock:
return server
return None
def getservers(self):
"""
Return a list of servers sorted by name.
"""
with self.servers_lock:
return sorted(self.servers.values(), key=lambda x: x.name)
def getservernames(self):
"""
Return a sorted list of server names (keys from self.servers).
"""
with self.servers_lock:
return sorted(self.servers.keys())
def tunnelkey(self, n1num, n2num):
"""
@ -353,12 +389,12 @@ class CoreBroker(ConfigurableManager):
try:
net = self.session.get_object(n)
except KeyError:
raise KeyError, "network node %s not found" % n
raise KeyError("network node %s not found" % n)
# add other nets here that do not require tunnels
if nodeutils.is_node(net, NodeTypes.EMANE_NET):
return None
if nodeutils.is_node(net, NodeTypes.CONTROL_NET):
if hasattr(net, 'serverintf'):
if hasattr(net, "serverintf"):
if net.serverintf is not None:
return None
@ -367,17 +403,12 @@ class CoreBroker(ConfigurableManager):
return None
hosts = []
for server in servers:
(host, port, sock) = self.getserver(server)
if host is None:
if server.host is None:
continue
hosts.append(host)
if len(hosts) == 0:
hosts.append(server.host)
if len(hosts) == 0 and self.session_handler.client_address != "":
# get IP address from API message sender (master)
self.session._handlerslock.acquire()
for h in self.session._handlers:
if h.client_address != "":
hosts.append(h.client_address[0])
self.session._handlerslock.release()
hosts.append(self.session_handler.client_address[0])
r = []
for host in hosts:
@ -426,45 +457,33 @@ class CoreBroker(ConfigurableManager):
"""
Record a node number to emulation server mapping.
"""
self.nodemap_lock.acquire()
if nodenum in self.nodemap:
if server in self.nodemap[nodenum]:
self.nodemap_lock.release()
return
self.nodemap[nodenum].append(server)
else:
self.nodemap[nodenum] = [server, ]
if server in self.nodecounts:
self.nodecounts[server] += 1
else:
self.nodecounts[server] = 1
self.nodemap_lock.release()
with self.nodemap_lock:
if nodenum in self.nodemap:
if server in self.nodemap[nodenum]:
return
self.nodemap[nodenum].add(server)
else:
self.nodemap[nodenum] = {server}
if server in self.nodecounts:
self.nodecounts[server] += 1
else:
self.nodecounts[server] = 1
def delnodemap(self, sock, nodenum):
def delnodemap(self, server, nodenum):
"""
Remove a node number to emulation server mapping.
Return the number of nodes left on this server.
"""
self.nodemap_lock.acquire()
count = None
if nodenum not in self.nodemap:
self.nodemap_lock.release()
return count
found = False
for server in self.nodemap[nodenum]:
(host, port, srvsock) = self.getserver(server)
if srvsock == sock:
found = True
break
if server in self.nodecounts:
count = self.nodecounts[server]
if found:
with self.nodemap_lock:
if nodenum not in self.nodemap:
return count
self.nodemap[nodenum].remove(server)
if server in self.nodecounts:
count = self.nodecounts[server]
count -= 1
self.nodecounts[server] = count
self.nodemap_lock.release()
return count
return count
def incrbootcount(self):
"""
@ -481,29 +500,24 @@ class CoreBroker(ConfigurableManager):
def getserversbynode(self, nodenum):
"""
Retrieve a list of emulation servers given a node number.
Retrieve a set of emulation servers given a node number.
"""
self.nodemap_lock.acquire()
if nodenum not in self.nodemap:
self.nodemap_lock.release()
return []
r = self.nodemap[nodenum]
self.nodemap_lock.release()
return r
with self.nodemap_lock:
if nodenum not in self.nodemap:
return set()
return self.nodemap[nodenum]
def addnet(self, nodenum):
"""
Add a node number to the list of link-layer nodes.
"""
if nodenum not in self.network_nodes:
self.network_nodes.append(nodenum)
self.network_nodes.add(nodenum)
def addphys(self, nodenum):
"""
Add a node number to the list of physical nodes.
"""
if nodenum not in self.physical_nodes:
self.physical_nodes.append(nodenum)
self.physical_nodes.add(nodenum)
def configure_reset(self, config_data):
"""
@ -518,7 +532,7 @@ class CoreBroker(ConfigurableManager):
def configure_values(self, config_data):
"""
Receive configuration message with a list of server:host:port
combinations that we'll need to connect with.
combinations that we"ll need to connect with.
:param core.conf.ConfigData config_data: configuration data for carrying out a configuration
:return: None
@ -529,27 +543,27 @@ class CoreBroker(ConfigurableManager):
if values is None:
logger.info("emulation server data missing")
return None
values = values.split('|')
values = values.split("|")
# string of "server:ip:port,server:ip:port,..."
server_strings = values[0]
server_list = server_strings.split(',')
server_list = server_strings.split(",")
for server in server_list:
server_items = server.split(':')
server_items = server.split(":")
(name, host, port) = server_items[:3]
if host == '':
if host == "":
host = None
if port == '':
if port == "":
port = None
else:
port = int(port)
if session_id is not None:
# receive session ID and my IP from master
self.session_id_master = int(session_id.split('|')[0])
self.session_id_master = int(session_id.split("|")[0])
self.myip = host
host = None
port = None
@ -568,67 +582,70 @@ class CoreBroker(ConfigurableManager):
Returns True when message does not need to be handled locally,
and performs forwarding if required.
Returning False indicates this message should be handled locally.
:param core.api.coreapi.CoreMessage message: message to handle
:return: true or false for handling locally
:rtype: bool
"""
serverlist = []
handle_locally = False
servers = set()
# Do not forward messages when in definition state
# (for e.g. configuring services)
if self.session.state == EventTypes.DEFINITION_STATE.value:
handle_locally = True
return not handle_locally
return False
# Decide whether message should be handled locally or forwarded, or both
if message.message_type == MessageTypes.NODE.value:
(handle_locally, serverlist) = self.handlenodemsg(message)
servers = self.handlenodemsg(message)
elif message.message_type == MessageTypes.EVENT.value:
# broadcast events everywhere
serverlist = self.getserverlist()
servers = self.getservers()
elif message.message_type == MessageTypes.CONFIG.value:
# broadcast location and services configuration everywhere
confobj = message.get_tlv(ConfigTlvs.OBJECT.value)
if confobj == "location" or confobj == "services" or \
confobj == "session" or confobj == "all":
serverlist = self.getserverlist()
servers = self.getservers()
elif message.message_type == MessageTypes.FILE.value:
# broadcast hook scripts and custom service files everywhere
filetype = message.get_tlv(FileTlvs.TYPE.value)
if filetype is not None and (filetype[:5] == "hook:" or filetype[:8] == "service:"):
serverlist = self.getserverlist()
servers = self.getservers()
if message.message_type == MessageTypes.LINK.value:
# prepare a serverlist from two node numbers in link message
(handle_locally, serverlist, message) = self.handlelinkmsg(message)
elif len(serverlist) == 0:
# prepare a server list from two node numbers in link message
servers, message = self.handlelinkmsg(message)
elif len(servers) == 0:
# check for servers based on node numbers in all messages but link
nn = message.node_numbers()
if len(nn) == 0:
return False
serverlist = self.getserversbynode(nn[0])
servers = self.getserversbynode(nn[0])
if len(serverlist) == 0:
handle_locally = True
# allow other handlers to process this message
# (this is used by e.g. EMANE to use the link add message to keep counts
# of interfaces on other servers)
# allow other handlers to process this message (this is used
# by e.g. EMANE to use the link add message to keep counts of
# interfaces on other servers)
for handler in self.handlers:
handler(message)
# Perform any message forwarding
handle_locally = self.forwardmsg(message, serverlist, handle_locally)
handle_locally = self.forwardmsg(message, servers)
return not handle_locally
def setupserver(self, server):
def setupserver(self, servername):
"""
Send the appropriate API messages for configuring the specified
emulation server.
"""
host, port, sock = self.getserver(server)
if host is None or sock is None:
server = self.getserverbyname(servername)
if server is None:
logger.warn("ignoring unknown server: %s" % servername)
return
if server.sock is None or server.host is None or server.port is None:
logger.info("ignoring disconnected server: %s" % servername)
return
# communicate this session's current state to the server
# communicate this session"s current state to the server
tlvdata = coreapi.CoreEventTlv.pack(EventTlvs.TYPE.value, self.session.state)
msg = coreapi.CoreEventMessage.pack(0, tlvdata)
sock.send(msg)
server.sock.send(msg)
# send a Configuration message for the broker object and inform the
# server of its local name
@ -636,10 +653,11 @@ class CoreBroker(ConfigurableManager):
tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.OBJECT.value, "broker")
tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.TYPE.value, ConfigFlags.UPDATE.value)
tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.DATA_TYPES.value, (ConfigDataTypes.STRING.value,))
tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.VALUES.value, "%s:%s:%s" % (server, host, port))
tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.VALUES.value,
"%s:%s:%s" % (server.name, server.host, server.port))
tlvdata += coreapi.CoreConfigTlv.pack(ConfigTlvs.SESSION.value, "%s" % self.session.session_id)
msg = coreapi.CoreConfMessage.pack(0, tlvdata)
sock.send(msg)
server.sock.send(msg)
@staticmethod
def fixupremotetty(msghdr, msgdata, host):
@ -666,46 +684,47 @@ class CoreBroker(ConfigurableManager):
return coreapi.CoreExecMessage.pack(msgflags, tlvdata)
def handlenodemsg(self, msg):
def handlenodemsg(self, message):
"""
Determine and return the servers to which this node message should
be forwarded. Also keep track of link-layer nodes and the mapping of
nodes to servers.
:param core.api.coreapi.CoreMessage message: message to handle
:return:
"""
serverlist = []
handle_locally = False
servers = set()
serverfiletxt = None
# snoop Node Message for emulation server TLV and record mapping
n = msg.tlv_data[NodeTlvs.NUMBER.value]
n = message.tlv_data[NodeTlvs.NUMBER.value]
# replicate link-layer nodes on all servers
nodetype = msg.get_tlv(NodeTlvs.TYPE.value)
nodetype = message.get_tlv(NodeTlvs.TYPE.value)
if nodetype is not None:
try:
nodecls = nodeutils.get_node_class(NodeTypes(nodetype))
except KeyError:
logger.exception("broker invalid node type %s", nodetype)
return False, serverlist
logger.warn("broker invalid node type %s" % nodetype)
return servers
if nodecls is None:
logger.warn("broker unimplemented node type %s" % nodetype)
return False, serverlist
return servers
if issubclass(nodecls, PyCoreNet) and nodetype != NodeTypes.WIRELESS_LAN.value:
# network node replicated on all servers; could be optimized
# don't replicate WLANs, because ebtables rules won't work
serverlist = self.getserverlist()
handle_locally = True
# don"t replicate WLANs, because ebtables rules won"t work
servers = self.getservers()
self.addnet(n)
for server in serverlist:
for server in servers:
self.addnodemap(server, n)
# do not record server name for networks since network
# nodes are replicated across all server
return handle_locally, serverlist
return servers
if issubclass(nodecls, PyCoreNet) and nodetype == NodeTypes.WIRELESS_LAN.value:
# special case where remote WLANs not in session._objs, and no
# node response message received, so they are counted here
if msg.get_tlv(NodeTlvs.EMULATION_SERVER.value) is not None:
if message.get_tlv(NodeTlvs.EMULATION_SERVER.value) is not None:
self.incrbootcount()
elif issubclass(nodecls, PyCoreNode):
name = msg.get_tlv(NodeTlvs.NAME.value)
name = message.get_tlv(NodeTlvs.NAME.value)
if name:
serverfiletxt = "%s %s %s" % (n, name, nodecls)
if issubclass(nodecls, PhysicalNode):
@ -713,101 +732,109 @@ class CoreBroker(ConfigurableManager):
self.addphys(n)
# emulation server TLV specifies server
server = msg.get_tlv(NodeTlvs.EMULATION_SERVER.value)
servername = message.get_tlv(NodeTlvs.EMULATION_SERVER.value)
server = self.getserverbyname(servername)
if server is not None:
self.addnodemap(server, n)
if server not in serverlist:
serverlist.append(server)
if server not in servers:
servers.add(server)
if serverfiletxt and self.session.master:
self.writenodeserver(serverfiletxt, server)
# hook to update coordinates of physical nodes
if n in self.physical_nodes:
self.session.mobility.physnodeupdateposition(msg)
return handle_locally, serverlist
self.session.mobility.physnodeupdateposition(message)
return servers
def handlelinkmsg(self, msg):
def handlelinkmsg(self, message):
"""
Determine and return the servers to which this link message should
be forwarded. Also build tunnels between different servers or add
opaque data to the link message before forwarding.
:param core.api.coreapi.CoreMessage message: message to handle
:return:
"""
serverlist = []
servers = set()
handle_locally = False
# determine link message destination using non-network nodes
nn = msg.node_numbers()
nn = message.node_numbers()
if nn[0] in self.network_nodes:
if nn[1] in self.network_nodes:
# two network nodes linked together - prevent loops caused by
# the automatic tunnelling
handle_locally = True
else:
serverlist = self.getserversbynode(nn[1])
servers = self.getserversbynode(nn[1])
elif nn[1] in self.network_nodes:
serverlist = self.getserversbynode(nn[0])
servers = self.getserversbynode(nn[0])
else:
serverset1 = set(self.getserversbynode(nn[0]))
serverset2 = set(self.getserversbynode(nn[1]))
servers1 = self.getserversbynode(nn[0])
servers2 = self.getserversbynode(nn[1])
# nodes are on two different servers, build tunnels as needed
if serverset1 != serverset2:
if servers1 != servers2:
localn = None
if len(serverset1) == 0 or len(serverset2) == 0:
if len(servers1) == 0 or len(servers2) == 0:
handle_locally = True
serverlist = list(serverset1 | serverset2)
servers = servers1.union(servers2)
host = None
# get the IP of remote server and decide which node number
# is for a local node
for server in serverlist:
(host, port, sock) = self.getserver(server)
for server in servers:
host = server.host
if host is None:
# named server is local
# server is local
handle_locally = True
if server in serverset1:
if server in servers1:
localn = nn[0]
else:
localn = nn[1]
if handle_locally and localn is None:
# having no local node at this point indicates local node is
# the one with the empty serverset
if len(serverset1) == 0:
# the one with the empty server set
if len(servers1) == 0:
localn = nn[0]
elif len(serverset2) == 0:
elif len(servers2) == 0:
localn = nn[1]
if host is None:
host = self.getlinkendpoint(msg, localn == nn[0])
host = self.getlinkendpoint(message, localn == nn[0])
if localn is None:
msg = self.addlinkendpoints(msg, serverset1, serverset2)
elif msg.flags & MessageFlags.ADD.value:
message = self.addlinkendpoints(message, servers1, servers2)
elif message.flags & MessageFlags.ADD.value:
self.addtunnel(host, nn[0], nn[1], localn)
elif msg.flags & MessageFlags.DELETE.value:
elif message.flags & MessageFlags.DELETE.value:
self.deltunnel(nn[0], nn[1])
handle_locally = False
else:
serverlist = list(serverset1 | serverset2)
servers = servers1.union(servers2)
return handle_locally, serverlist, msg
return servers, message
def addlinkendpoints(self, msg, serverset1, serverset2):
def addlinkendpoints(self, message, servers1, servers2):
"""
For a link message that is not handled locally, inform the remote
servers of the IP addresses used as tunnel endpoints by adding
opaque data to the link message.
:param core.api.coreapi.CoreMessage message: message to link end points
:param servers1:
:param servers2:
:return:
"""
ip1 = ""
for server in serverset1:
(host, port, sock) = self.getserver(server)
if host is not None:
ip1 = host
for server in servers1:
if server.host is not None:
ip1 = server.host
break
ip2 = ""
for server in serverset2:
(host, port, sock) = self.getserver(server)
if host is not None:
ip2 = host
tlvdata = msg.rawmsg[coreapi.CoreMessage.header_len:]
for server in servers2:
if server.host is not None:
ip2 = server.host
break
tlvdata = message.raw_message[coreapi.CoreMessage.header_len:]
tlvdata += coreapi.CoreLinkTlv.pack(LinkTlvs.OPAQUE.value, "%s:%s" % (ip1, ip2))
newraw = coreapi.CoreLinkMessage.pack(msg.flags, tlvdata)
newraw = coreapi.CoreLinkMessage.pack(message.flags, tlvdata)
msghdr = newraw[:coreapi.CoreMessage.header_len]
return coreapi.CoreLinkMessage(msg.flags, msghdr, tlvdata)
return coreapi.CoreLinkMessage(message.flags, msghdr, tlvdata)
def getlinkendpoint(self, msg, first_is_local):
"""
@ -820,18 +847,14 @@ class CoreBroker(ConfigurableManager):
opaque = msg.get_tlv(LinkTlvs.OPAQUE.value)
if opaque is not None:
if first_is_local:
host = opaque.split(':')[1]
host = opaque.split(":")[1]
else:
host = opaque.split(':')[0]
host = opaque.split(":")[0]
if host == "":
host = None
if host is None:
if host is None and self.session_handler.client_address != "":
# get IP address from API message sender (master)
self.session._handlerslock.acquire()
for h in self.session._handlers:
if h.client_address != "":
host = h.client_address[0]
self.session._handlerslock.release()
host = self.session_handler.client_address[0]
return host
def handlerawmsg(self, msg):
@ -843,35 +866,27 @@ class CoreBroker(ConfigurableManager):
msgcls = coreapi.CLASS_MAP[msgtype]
return self.handle_message(msgcls(flags, hdr, msg[coreapi.CoreMessage.header_len:]))
def forwardmsg(self, message, serverlist, handle_locally):
def forwardmsg(self, message, servers):
"""
Forward API message to all servers in serverlist; if an empty
host/port is encountered, set the handle_locally flag. Returns the
value of the handle_locally flag, which may be unchanged.
Forward API message to all given servers.
:param coreapi.CoreMessage message: core message to forward
:param list serverlist: server list to forward to
:param bool handle_locally: used to determine if this should be handled locally
:return: should message be handled locally
:rtype: bool
Return True if an empty host/port is encountered, indicating
the message should be handled locally.
:param core.api.coreapi.CoreMessage message: message to forward
:param list servers: server to forward message to
:return:
"""
for server in serverlist:
try:
(host, port, sock) = self.getserver(server)
except KeyError:
# server not found, don't handle this message locally
logger.exception("broker could not find server %s, message with type %s dropped",
server, message.message_type)
continue
if host is None and port is None:
handle_locally = len(servers) == 0
for server in servers:
if server.host is None and server.port is None:
# local emulation server, handle this locally
handle_locally = True
elif server.sock is None:
logger.info("server %s @ %s:%s is disconnected" % (
server.name, server.host, server.port))
else:
if sock is None:
logger.info("server %s @ %s:%s is disconnected", server, host, port)
else:
sock.send(message.raw_message)
server.sock.send(message.raw_message)
return handle_locally
def writeservers(self):
@ -879,28 +894,24 @@ class CoreBroker(ConfigurableManager):
Write the server list to a text file in the session directory upon
startup: /tmp/pycore.nnnnn/servers
"""
servers = self.getservers()
filename = os.path.join(self.session.session_dir, "servers")
master = self.session_id_master
if master is None:
master = self.session.session_id
try:
f = open(filename, "w")
master = self.session_id_master
if master is None:
master = self.session.session_id
f.write("master=%s\n" % master)
self.servers_lock.acquire()
for name in sorted(self.servers.keys()):
if name == "localhost":
continue
(host, port, sock) = self.servers[name]
try:
(lhost, lport) = sock.getsockname()
except:
lhost, lport = None, None
f.write("%s %s %s %s %s\n" % (name, host, port, lhost, lport))
f.close()
with open(filename, "w") as f:
f.write("master=%s\n" % master)
for server in servers:
if server.name == "localhost":
continue
try:
lhost, lport = server.sock.getsockname()
except IOError:
lhost, lport = None, None
f.write("%s %s %s %s %s\n" % (server.name, server.host, server.port, lhost, lport))
except IOError:
logger.exception("Error writing server list to the file: %s", filename)
finally:
self.servers_lock.release()
logger.exception("error writing server list to the file: %s" % filename)
def writenodeserver(self, nodestr, server):
"""
@ -909,24 +920,45 @@ class CoreBroker(ConfigurableManager):
other machines, much like local nodes may be accessed via the
VnodeClient class.
"""
(host, port, sock) = self.getserver(server)
serverstr = "%s %s %s" % (server, host, port)
serverstr = "%s %s %s" % (server.name, server.host, server.port)
name = nodestr.split()[1]
dirname = os.path.join(self.session.session_dir, name + ".conf")
filename = os.path.join(dirname, "server")
try:
os.makedirs(dirname)
except OSError:
# directory may already exist from previous distributed run
logger.exception("error creating directory: %s", dirname)
try:
f = open(filename, "w")
f.write("%s\n%s\n" % (serverstr, nodestr))
f.close()
return True
with open(filename, "w") as f:
f.write("%s\n%s\n" % (serverstr, nodestr))
except IOError:
msg = "Error writing server file '%s'" % filename
msg += "for node %s" % name
logger.exception(msg)
return False
logger.exception("error writing server file %s for node %s" % (filename, name))
def local_instantiation_complete(self):
"""
Set the local server"s instantiation-complete status to True.
"""
# TODO: do we really want to allow a localhost to not exist?
with self.servers_lock:
server = self.servers.get("localhost")
if server is not None:
server.instantiation_complete = True
if self.session.is_connected():
tlvdata = ""
tlvdata += coreapi.CoreEventTlv.pack(EventTlvs.TYPE.value, EventTypes.INSTANTIATION_COMPLETE.value)
msg = coreapi.CoreEventMessage.pack(0, tlvdata)
self.session_handler.sendall(msg)
def instantiation_complete(self):
"""
Return True if all servers have completed instantiation, False
otherwise.
"""
with self.servers_lock:
for server in self.servers.itervalues():
if not server.instantiation_complete:
return False
return True

View file

@ -88,7 +88,7 @@ class EmaneManager(ConfigurableManager):
self.logversion()
# model for global EMANE configuration options
self.emane_config = EmaneGlobalModel(session, None)
session.broker.handlers += (self.handledistributed,)
session.broker.handlers.add(self.handledistributed)
self.service = None
self._modelclsmap = {
self.emane_config.name: self.emane_config
@ -458,22 +458,22 @@ class EmaneManager(ConfigurableManager):
self._objslock.release()
for server in servers:
if server == "localhost":
if server.name == "localhost":
continue
(host, port, sock) = self.session.broker.getserver(server)
if sock is None:
if server.sock is None:
continue
platformid += 1
typeflags = ConfigFlags.UPDATE.value
values[names.index("platform_id_start")] = str(platformid)
values[names.index("nem_id_start")] = str(nemid)
msg = EmaneGlobalModel.config_data(flags=0, node_id=None, type_flags=typeflags, values=values)
sock.send(msg)
server.sock.send(msg)
# increment nemid for next server by number of interfaces
self._ifccountslock.acquire()
if server in self._ifccounts:
nemid += self._ifccounts[server]
self._ifccountslock.release()
with self._ifccountslock:
if server in self._ifccounts:
nemid += self._ifccounts[server]
return False
@ -511,16 +511,19 @@ class EmaneManager(ConfigurableManager):
using the default list of prefixes.
"""
session = self.session
# slave server
if not session.master:
return # slave server
servers = session.broker.getserverlist()
return
servers = session.broker.getservernames()
# not distributed
if len(servers) < 2:
return # not distributed
return
prefix = session.config.get('controlnet')
prefix = getattr(session.options, 'controlnet', prefix)
prefixes = prefix.split()
# normal Config messaging will distribute controlnets
if len(prefixes) >= len(servers):
return # normal Config messaging will distribute controlnets
return
# this generates a config message having controlnet prefix assignments
logger.info("Setting up default controlnet prefixes for distributed (%d configured)" % len(prefixes))
prefixes = ctrlnet.DEFAULT_PREFIX_LIST[0]

View file

@ -277,6 +277,7 @@ class EventTypes(Enum):
FILE_SAVE = 12
SCHEDULED = 13
RECONFIGURE = 14
INSTANTIATION_COMPLETE = 15
# Session Message TLV Types

View file

@ -53,7 +53,7 @@ class MobilityManager(ConfigurableManager):
# dummy node objects for tracking position of nodes on other servers
self.phys = {}
self.physnets = {}
self.session.broker.handlers += (self.physnodehandlelink,)
self.session.broker.handlers.add(self.physnodehandlelink)
def startup(self, node_ids=None):
"""
@ -254,12 +254,13 @@ class MobilityManager(ConfigurableManager):
for nodenum in nodenums:
node = self.phys[nodenum]
servers = self.session.broker.getserversbynode(nodenum)
(host, port, sock) = self.session.broker.getserver(servers[0])
netif = self.session.broker.gettunnel(net.objid, IpAddress.to_int(host))
# TODO: fix this bad logic, relating to depending on a break to get a valid server
for server in self.session.broker.getserversbynode(nodenum):
break
netif = self.session.broker.gettunnel(net.objid, IpAddress.to_int(server.host))
node.addnetif(netif, 0)
netif.node = node
(x, y, z) = netif.node.position.get()
x, y, z = netif.node.position.get()
netif.poshook(netif, x, y, z)

View file

@ -66,7 +66,7 @@ class Sdt(object):
# node information for remote nodes not in session._objs
# local nodes also appear here since their obj may not exist yet
self.remotes = {}
session.broker.handlers += (self.handledistributed,)
session.broker.handlers.add(self.handledistributed)
def is_enabled(self):
"""

View file

@ -225,6 +225,7 @@ class Session(object):
self.xen = XenConfigManager(session=self)
self.add_config_object(XenConfigManager.name, XenConfigManager.config_type, self.xen.configure)
# setup sdt
self.sdt = Sdt(session=self)
# future parameters set by the GUI may go here
@ -782,8 +783,9 @@ class Session(object):
# controlnet may be needed by some EMANE models
self.add_remove_control_interface(node=None, remove=False)
# instantiate will be invoked again upon Emane configure
if self.emane.startup() == self.emane.NOT_READY:
return # instantiate() will be invoked again upon Emane.configure()
return
# startup broker
self.broker.startup()
@ -800,6 +802,9 @@ class Session(object):
# validate nodes
self.validate_nodes()
# set broker local instantiation to complete
self.broker.local_instantiation_complete()
# assume either all nodes have booted already, or there are some
# nodes on slave servers that will be booted and those servers will
# send a node status response message
@ -851,7 +856,10 @@ class Session(object):
# return # do not have information on all nodes yet
# information on all nodes has been received and they have been started enter the runtime state
# TODO: more sophisticated checks to verify that all nodes and networks are running
# check to verify that all nodes and networks are running
if not self.broker.instantiation_complete():
return
# start event loop and set to runtime
self.event_loop.run()
@ -1029,7 +1037,7 @@ class Session(object):
prefix = prefixes[0]
else:
# slave servers have their name and localhost in the serverlist
servers = self.broker.getserverlist()
servers = self.broker.getservernames()
servers.remove("localhost")
prefix = None
@ -1066,7 +1074,7 @@ class Session(object):
# tunnels between controlnets will be built with Broker.addnettunnels()
self.broker.addnet(object_id)
for server in self.broker.getserverlist():
for server in self.broker.getservers():
self.broker.addnodemap(server, object_id)
return control_net
@ -1299,7 +1307,7 @@ class SessionConfig(ConfigurableManager, Configurable):
"""
ConfigurableManager.__init__(self)
self.session = session
self.session.broker.handlers += (self.handle_distributed,)
self.session.broker.handlers.add(self.handle_distributed)
self.reset()
def reset(self):
@ -1376,7 +1384,7 @@ class SessionConfig(ConfigurableManager, Configurable):
logger.warn("multiple controlnet prefixes do not exist")
return
servers = self.session.broker.getserverlist()
servers = self.session.broker.getservernames()
if len(servers) < 2:
logger.warn("not distributed")
return

View file

@ -94,7 +94,7 @@ class CoreDeploymentWriter(object):
for n in nodelist:
self.add_virtual_host(testhost, n)
# TODO: handle other servers
# servers = self.session.broker.getserverlist()
# servers = self.session.broker.getservernames()
# servers.remove('localhost')
def add_child_element(self, parent, tag_name):

View file

@ -54,15 +54,15 @@ def cmd(node, exec_cmd):
exec_num += 1
# Now wait for the response
h, p, sock = node.session.broker.servers["localhost"]
sock.settimeout(50.0)
server = node.session.broker.servers["localhost"]
server.sock.settimeout(50.0)
# receive messages until we get our execute response
result = None
while True:
msghdr = sock.recv(coreapi.CoreMessage.header_len)
msghdr = server.sock.recv(coreapi.CoreMessage.header_len)
msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(msghdr)
msgdata = sock.recv(msglen)
msgdata = server.sock.recv(msglen)
# If we get the right response return the results
print "received response message: %s" % MessageTypes(msgtype)