remerged with git pending fix to distributed configuration problem

This commit is contained in:
Rod A Santiago 2016-10-19 16:21:06 -07:00
parent 5d0aa4ac1a
commit d17ed889d2
9 changed files with 348 additions and 345 deletions

View file

@ -286,6 +286,7 @@ event_types = dict(enumerate([
"CORE_EVENT_FILE_SAVE", "CORE_EVENT_FILE_SAVE",
"CORE_EVENT_SCHEDULED", "CORE_EVENT_SCHEDULED",
"CORE_EVENT_RECONFIGURE", "CORE_EVENT_RECONFIGURE",
"CORE_EVENT_INSTANTIATION_COMPLETE",
])) ]))
enumdict(event_types) enumdict(event_types)

View file

@ -24,7 +24,30 @@ from core.conf import ConfigurableManager
if os.uname()[0] == "Linux": if os.uname()[0] == "Linux":
from core.netns.vif import GreTap from core.netns.vif import GreTap
from core.netns.vnet import GreTapBridge from core.netns.vnet import GreTapBridge
class CoreServer(object):
def __init__(self, name, host, port):
self.name = name
self.host = host
self.port = port
self.sock = None
self.instantiation_complete = False
def connect(self):
assert self.sock is None
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#sock.setblocking(0)
try:
sock.connect((self.host, self.port))
except:
sock.close()
raise
self.sock = sock
def close(self):
if self.sock is not None:
self.sock.close()
self.sock = None
class CoreBroker(ConfigurableManager): class CoreBroker(ConfigurableManager):
''' Member of pycore session class for handling global emulation server ''' Member of pycore session class for handling global emulation server
@ -47,14 +70,13 @@ class CoreBroker(ConfigurableManager):
# this lock also protects self.nodecounts # this lock also protects self.nodecounts
self.nodemap_lock = threading.Lock() self.nodemap_lock = threading.Lock()
# reference counts of nodes on servers # reference counts of nodes on servers
self.nodecounts = { } self.nodecounts = {}
self.bootcount = 0 # set of node numbers that are link-layer nodes (networks)
# list of node numbers that are link-layer nodes (networks) self.nets = set()
self.nets = [] # set of node numbers that are PhysicalNode nodes
# list of node numbers that are PhysicalNode nodes self.phys = set()
self.phys = []
# allows for other message handlers to process API messages (e.g. EMANE) # allows for other message handlers to process API messages (e.g. EMANE)
self.handlers = () self.handlers = set()
# dict with tunnel key to tunnel device mapping # dict with tunnel key to tunnel device mapping
self.tunnels = {} self.tunnels = {}
self.dorecvloop = False self.dorecvloop = False
@ -67,21 +89,19 @@ class CoreBroker(ConfigurableManager):
''' '''
self.addnettunnels() self.addnettunnels()
self.writeservers() self.writeservers()
def shutdown(self): def shutdown(self):
''' Close all active sockets; called when the session enters the ''' Close all active sockets; called when the session enters the
data collect state data collect state
''' '''
with self.servers_lock: with self.servers_lock:
while len(self.servers) > 0: while len(self.servers) > 0:
(server, v) = self.servers.popitem() name, server = self.servers.popitem()
(host, port, sock) = v if server.sock is not None:
if sock is None: if self.verbose:
continue self.session.info("closing connection with %s @ %s:%s" % \
if self.verbose: (name, server.host, server.port))
self.session.info("closing connection with %s @ %s:%s" % \ server.close()
(server, host, port))
sock.close()
self.reset() self.reset()
self.dorecvloop = False self.dorecvloop = False
if self.recvthread is not None: if self.recvthread is not None:
@ -92,14 +112,13 @@ class CoreBroker(ConfigurableManager):
''' '''
self.nodemap_lock.acquire() self.nodemap_lock.acquire()
self.nodemap.clear() self.nodemap.clear()
for server in self.nodecounts: for server, count in self.nodecounts.iteritems():
if self.nodecounts[server] < 1: if count < 1:
self.delserver(server) self.delserver(server)
self.nodecounts.clear() self.nodecounts.clear()
self.bootcount = 0
self.nodemap_lock.release() self.nodemap_lock.release()
del self.nets[:] self.nets.clear()
del self.phys[:] self.phys.clear()
while len(self.tunnels) > 0: while len(self.tunnels) > 0:
(key, gt) = self.tunnels.popitem() (key, gt) = self.tunnels.popitem()
gt.shutdown() gt.shutdown()
@ -128,35 +147,32 @@ class CoreBroker(ConfigurableManager):
rlist = [] rlist = []
with self.servers_lock: with self.servers_lock:
# build a socket list for select call # build a socket list for select call
for name in self.servers: for server in self.servers.itervalues():
(h, p, sock) = self.servers[name] if server.sock is not None:
if sock is not None: rlist.append(server.sock)
rlist.append(sock.fileno())
r, w, x = select.select(rlist, [], [], 1.0) r, w, x = select.select(rlist, [], [], 1.0)
for sockfd in r: for sock in r:
try: server = self.getserverbysock(sock)
(h, p, sock, name) = self.getserverbysock(sockfd) if server is None:
except KeyError:
# servers may have changed; loop again # servers may have changed; loop again
break continue
rcvlen = self.recv(sock, h) rcvlen = self.recv(server)
if rcvlen == 0: if rcvlen == 0:
if self.verbose: if self.verbose:
self.session.info("connection with %s @ %s:%s" \ msg = 'connection with %s @ %s:%s has closed' % \
" has closed" % (name, h, p)) (server.name, server.host, server.port)
self.servers[name] = (h, p, None) self.session.info(msg)
def recv(self, server):
def recv(self, sock, host):
''' Receive data on an emulation server socket and broadcast it to ''' Receive data on an emulation server socket and broadcast it to
all connected session handlers. Returns the length of data recevied all connected session handlers. Returns the length of data recevied
and forwarded. Return value of zero indicates the socket has closed and forwarded. Return value of zero indicates the socket has closed
and should be removed from the self.servers dict. and should be removed from the self.servers dict.
''' '''
msghdr = sock.recv(coreapi.CoreMessage.hdrsiz) msghdr = server.sock.recv(coreapi.CoreMessage.hdrsiz)
if len(msghdr) == 0: if len(msghdr) == 0:
# server disconnected # server disconnected
sock.close() server.close()
return 0 return 0
if len(msghdr) != coreapi.CoreMessage.hdrsiz: if len(msghdr) != coreapi.CoreMessage.hdrsiz:
if self.verbose: if self.verbose:
@ -165,30 +181,31 @@ class CoreBroker(ConfigurableManager):
return len(msghdr) return len(msghdr)
msgtype, msgflags, msglen = coreapi.CoreMessage.unpackhdr(msghdr) msgtype, msgflags, msglen = coreapi.CoreMessage.unpackhdr(msghdr)
msgdata = sock.recv(msglen) msgdata = server.sock.recv(msglen)
data = msghdr + msgdata data = msghdr + msgdata
count = None count = None
# snoop exec response for remote interactive TTYs # snoop exec response for remote interactive TTYs
if msgtype == coreapi.CORE_API_EXEC_MSG and \ if msgtype == coreapi.CORE_API_EXEC_MSG and \
msgflags & coreapi.CORE_API_TTY_FLAG: msgflags & coreapi.CORE_API_TTY_FLAG:
data = self.fixupremotetty(msghdr, msgdata, host) data = self.fixupremotetty(msghdr, msgdata, server.host)
elif msgtype == coreapi.CORE_API_NODE_MSG: elif msgtype == coreapi.CORE_API_NODE_MSG:
# snoop node delete response to decrement node counts # snoop node delete response to decrement node counts
if msgflags & coreapi.CORE_API_DEL_FLAG: if msgflags & coreapi.CORE_API_DEL_FLAG:
msg = coreapi.CoreNodeMessage(msgflags, msghdr, msgdata) msg = coreapi.CoreNodeMessage(msgflags, msghdr, msgdata)
nodenum = msg.gettlv(coreapi.CORE_TLV_NODE_NUMBER) nodenum = msg.gettlv(coreapi.CORE_TLV_NODE_NUMBER)
if nodenum is not None: if nodenum is not None:
count = self.delnodemap(sock, nodenum) count = self.delnodemap(server, nodenum)
# snoop node add response to increment booted node count
# (only CoreNodes send these response messages)
elif msgflags & \
(coreapi.CORE_API_ADD_FLAG | coreapi.CORE_API_LOC_FLAG):
self.incrbootcount()
self.session.checkruntime()
elif msgtype == coreapi.CORE_API_LINK_MSG: elif msgtype == coreapi.CORE_API_LINK_MSG:
# this allows green link lines for remote WLANs # this allows green link lines for remote WLANs
msg = coreapi.CoreLinkMessage(msgflags, msghdr, msgdata) msg = coreapi.CoreLinkMessage(msgflags, msghdr, msgdata)
self.session.sdt.handledistributed(msg) self.session.sdt.handledistributed(msg)
elif msgtype == coreapi.CORE_API_EVENT_MSG:
msg = coreapi.CoreEventMessage(msgflags, msghdr, msgdata)
eventtype = msg.gettlv(coreapi.CORE_TLV_EVENT_TYPE)
if eventtype == coreapi.CORE_EVENT_INSTANTIATION_COMPLETE:
server.instantiation_complete = True
if self.instantiation_complete():
self.session.checkruntime()
self.session.broadcastraw(None, data) self.session.broadcastraw(None, data)
if count is not None and count < 1: if count is not None and count < 1:
@ -196,87 +213,99 @@ class CoreBroker(ConfigurableManager):
else: else:
return len(data) return len(data)
def local_instantiation_complete(self):
'''\
Set the local server's instantiation-complete status to True.
'''
with self.servers_lock:
server = self.servers.get('localhost')
if server is not None:
server.instantiation_complete = True
def instantiation_complete(self):
'''\
Return True if all servers have completed instantiation, False
otherwise.
'''
with self.servers_lock:
for server in self.servers.itervalues():
if not server.instantiation_complete:
return False
return True
def addserver(self, name, host, port): def addserver(self, name, host, port):
''' Add a new server, and try to connect to it. If we're already ''' Add a new server, and try to connect to it. If we're already
connected to this (host, port), then leave it alone. When host,port connected to this (host, port), then leave it alone. When host,port
is None, do not try to connect. is None, do not try to connect.
''' '''
self.servers_lock.acquire() with self.servers_lock:
if name in self.servers: server = self.servers.get(name)
(oldhost, oldport, sock) = self.servers[name] if server is not None:
if host == oldhost or port == oldport: if host == server.host and port == server.port and \
# leave this socket connected server.sock is not None:
if sock is not None: # leave this socket connected
self.servers_lock.release()
return return
if self.verbose and host is not None and sock is not None: if self.verbose:
self.session.info("closing connection with %s @ %s:%s" % \ self.session.info('closing connection with %s @ %s:%s' % \
(name, server.host, server.port))
server.close()
del self.servers[name]
if self.verbose:
self.session.info('adding server %s @ %s:%s' % \
(name, host, port)) (name, host, port))
if sock is not None: server = CoreServer(name, host, port)
sock.close() if host is not None and port is not None:
self.servers_lock.release() try:
if self.verbose and host is not None: server.connect()
self.session.info("adding server %s @ %s:%s" % (name, host, port)) except Exception as e:
if host is None: self.session.warn('error connecting to server %s:%s:\n\t%s' % \
sock = None (host, port, e))
else: if server.sock is not None:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.startrecvloop()
#sock.setblocking(0) self.servers[name] = server
#error = sock.connect_ex((host, port))
try: def delserver(self, server):
sock.connect((host, port))
self.startrecvloop()
except Exception, e:
self.session.warn("error connecting to server %s:%s:\n\t%s" % \
(host, port, e))
sock.close()
sock = None
self.servers_lock.acquire()
self.servers[name] = (host, port, sock)
self.servers_lock.release()
def delserver(self, name):
''' Remove a server and hang up any connection. ''' Remove a server and hang up any connection.
''' '''
self.servers_lock.acquire() with self.servers_lock:
if name not in self.servers: try:
self.servers_lock.release() s = self.servers.pop(server.name)
return assert s == server
(host, port, sock) = self.servers.pop(name) except KeyError:
if sock is not None: pass
if server.sock is not None:
if self.verbose: if self.verbose:
self.session.info("closing connection with %s @ %s:%s" % \ self.session.info("closing connection with %s @ %s:%s" % \
(name, host, port)) (server.name, server.host, server.port))
sock.close() server.close()
self.servers_lock.release()
def getserver(self, name): def getserverbyname(self, name):
''' Return the (host, port, sock) tuple, or raise a KeyError exception. ''' Return the server object having the given name, or None.
'''
if name not in self.servers:
raise KeyError, "emulation server %s not found" % name
return self.servers[name]
def getserverbysock(self, sockfd):
''' Return a (host, port, sock, name) tuple based on socket file
descriptor, or raise a KeyError exception.
''' '''
with self.servers_lock: with self.servers_lock:
for name in self.servers: return self.servers.get(name)
(host, port, sock) = self.servers[name]
if sock is None: def getserverbysock(self, sock):
continue ''' Return the server object corresponding to the given socket,
if sock.fileno() == sockfd: or None.
return (host, port, sock, name)
raise KeyError, "socket fd %s not found" % sockfd
def getserverlist(self):
''' Return the list of server names (keys from self.servers).
''' '''
with self.servers_lock: with self.servers_lock:
serverlist = sorted(self.servers.keys()) for server in self.servers.itervalues():
return serverlist if server.sock == sock:
return server
return None
def getservers(self):
'''Return a list of servers sorted by name.'''
with self.servers_lock:
return sorted(self.servers.values(), key = lambda x: x.name)
def getservernames(self):
''' Return a sorted list of server names (keys from self.servers).
'''
with self.servers_lock:
return sorted(self.servers.keys())
def tunnelkey(self, n1num, n2num): def tunnelkey(self, n1num, n2num):
''' Compute a 32-bit key used to uniquely identify a GRE tunnel. ''' Compute a 32-bit key used to uniquely identify a GRE tunnel.
The hash(n1num), hash(n2num) values are used, so node numbers may be The hash(n1num), hash(n2num) values are used, so node numbers may be
@ -322,7 +351,7 @@ class CoreBroker(ConfigurableManager):
''' '''
for n in self.nets: for n in self.nets:
self.addnettunnel(n) self.addnettunnel(n)
def addnettunnel(self, n): def addnettunnel(self, n):
try: try:
net = self.session.obj(n) net = self.session.obj(n)
@ -335,16 +364,15 @@ class CoreBroker(ConfigurableManager):
if hasattr(net, 'serverintf'): if hasattr(net, 'serverintf'):
if net.serverintf is not None: if net.serverintf is not None:
return None return None
servers = self.getserversbynode(n) servers = self.getserversbynode(n)
if len(servers) < 2: if len(servers) < 2:
return None return None
hosts = [] hosts = []
for server in servers: for server in servers:
(host, port, sock) = self.getserver(server) if server.host is None:
if host is None:
continue continue
hosts.append(host) hosts.append(server.host)
if len(hosts) == 0: if len(hosts) == 0:
# get IP address from API message sender (master) # get IP address from API message sender (master)
self.session._handlerslock.acquire() self.session._handlerslock.acquire()
@ -352,7 +380,7 @@ class CoreBroker(ConfigurableManager):
if h.client_address != "": if h.client_address != "":
hosts.append(h.client_address[0]) hosts.append(h.client_address[0])
self.session._handlerslock.release() self.session._handlerslock.release()
r = [] r = []
for host in hosts: for host in hosts:
if self.myip: if self.myip:
@ -370,11 +398,11 @@ class CoreBroker(ConfigurableManager):
remoteip=host, key=key) remoteip=host, key=key)
self.tunnels[key] = gt self.tunnels[key] = gt
r.append(gt) r.append(gt)
# attaching to net will later allow gt to be destroyed # attaching to net will later allow gt to be destroyed
# during net.shutdown() # during net.shutdown()
net.attach(gt) net.attach(gt)
return r return r
def deltunnel(self, n1num, n2num): def deltunnel(self, n1num, n2num):
''' Cleanup of the GreTapBridge. ''' Cleanup of the GreTapBridge.
''' '''
@ -399,78 +427,50 @@ class CoreBroker(ConfigurableManager):
def addnodemap(self, server, nodenum): def addnodemap(self, server, nodenum):
''' Record a node number to emulation server mapping. ''' Record a node number to emulation server mapping.
''' '''
self.nodemap_lock.acquire() with self.nodemap_lock:
if nodenum in self.nodemap: if nodenum in self.nodemap:
if server in self.nodemap[nodenum]: if server in self.nodemap[nodenum]:
self.nodemap_lock.release() return
return self.nodemap[nodenum].add(server)
self.nodemap[nodenum].append(server) else:
else: self.nodemap[nodenum] = {server}
self.nodemap[nodenum] = [server,] if server in self.nodecounts:
if server in self.nodecounts: self.nodecounts[server] += 1
self.nodecounts[server] += 1 else:
else: self.nodecounts[server] = 1
self.nodecounts[server] = 1
self.nodemap_lock.release() def delnodemap(self, server, nodenum):
def delnodemap(self, sock, nodenum):
''' Remove a node number to emulation server mapping. ''' Remove a node number to emulation server mapping.
Return the number of nodes left on this server. Return the number of nodes left on this server.
''' '''
self.nodemap_lock.acquire()
count = None count = None
if nodenum not in self.nodemap: with self.nodemap_lock:
self.nodemap_lock.release() if nodenum not in self.nodemap:
return count return count
found = False
for server in self.nodemap[nodenum]:
(host, port, srvsock) = self.getserver(server)
if srvsock == sock:
found = True
break
if server in self.nodecounts:
count = self.nodecounts[server]
if found:
self.nodemap[nodenum].remove(server) self.nodemap[nodenum].remove(server)
if server in self.nodecounts: if server in self.nodecounts:
count = self.nodecounts[server]
count -= 1 count -= 1
self.nodecounts[server] = count self.nodecounts[server] = count
self.nodemap_lock.release() return count
return count
def incrbootcount(self):
''' Count a node that has booted.
'''
self.bootcount += 1
return self.bootcount
def getbootcount(self):
''' Return the number of booted nodes.
'''
return self.bootcount
def getserversbynode(self, nodenum): def getserversbynode(self, nodenum):
''' Retrieve a list of emulation servers given a node number. ''' Retrieve a set of emulation servers given a node number.
''' '''
self.nodemap_lock.acquire() with self.nodemap_lock:
if nodenum not in self.nodemap: if nodenum not in self.nodemap:
self.nodemap_lock.release() return set()
return [] return self.nodemap[nodenum]
r = self.nodemap[nodenum]
self.nodemap_lock.release()
return r
def addnet(self, nodenum): def addnet(self, nodenum):
''' Add a node number to the list of link-layer nodes. ''' Add a node number to the list of link-layer nodes.
''' '''
if nodenum not in self.nets: self.nets.add(nodenum)
self.nets.append(nodenum)
def addphys(self, nodenum): def addphys(self, nodenum):
''' Add a node number to the list of physical nodes. ''' Add a node number to the list of physical nodes.
''' '''
if nodenum not in self.phys: self.phys.add(nodenum)
self.phys.append(nodenum)
def configure_reset(self, msg): def configure_reset(self, msg):
''' Ignore reset messages, because node delete responses may still ''' Ignore reset messages, because node delete responses may still
@ -516,72 +516,74 @@ class CoreBroker(ConfigurableManager):
def handlemsg(self, msg): def handlemsg(self, msg):
''' Handle an API message. Determine whether this needs to be handled ''' Handle an API message. Determine whether this needs to be handled
by the local server or forwarded on to another one. by the local server or forwarded on to another one.
Returns True when message does not need to be handled locally, Returns True when message does not need to be handled locally,
and performs forwarding if required. and performs forwarding if required.
Returning False indicates this message should be handled locally. Returning False indicates this message should be handled locally.
''' '''
serverlist = [] servers = set()
handle_locally = False # Do not forward messages when in definition state
# Do not forward messages when in definition state
# (for e.g. configuring services) # (for e.g. configuring services)
if self.session.getstate() == coreapi.CORE_EVENT_DEFINITION_STATE: if self.session.getstate() == coreapi.CORE_EVENT_DEFINITION_STATE:
handle_locally = True return False
return not handle_locally
# Decide whether message should be handled locally or forwarded, or both # Decide whether message should be handled locally or forwarded, or both
if msg.msgtype == coreapi.CORE_API_NODE_MSG: if msg.msgtype == coreapi.CORE_API_NODE_MSG:
(handle_locally, serverlist) = self.handlenodemsg(msg) servers = self.handlenodemsg(msg)
elif msg.msgtype == coreapi.CORE_API_EVENT_MSG: elif msg.msgtype == coreapi.CORE_API_EVENT_MSG:
# broadcast events everywhere # broadcast events everywhere
serverlist = self.getserverlist() servers = self.getservers()
elif msg.msgtype == coreapi.CORE_API_CONF_MSG: elif msg.msgtype == coreapi.CORE_API_CONF_MSG:
# broadcast location and services configuration everywhere # broadcast location and services configuration everywhere
confobj = msg.gettlv(coreapi.CORE_TLV_CONF_OBJ) confobj = msg.gettlv(coreapi.CORE_TLV_CONF_OBJ)
if confobj == "location" or confobj == "services" or \ if confobj == "location" or confobj == "services" or \
confobj == "session" or confobj == "all": confobj == "session" or confobj == "all":
serverlist = self.getserverlist() servers = self.getservers()
elif msg.msgtype == coreapi.CORE_API_FILE_MSG: elif msg.msgtype == coreapi.CORE_API_FILE_MSG:
# broadcast hook scripts and custom service files everywhere # broadcast hook scripts and custom service files everywhere
filetype = msg.gettlv(coreapi.CORE_TLV_FILE_TYPE) filetype = msg.gettlv(coreapi.CORE_TLV_FILE_TYPE)
if filetype is not None and \ if filetype is not None and \
(filetype[:5] == "hook:" or filetype[:8] == "service:"): (filetype[:5] == "hook:" or filetype[:8] == "service:"):
serverlist = self.getserverlist() servers = self.getservers()
if msg.msgtype == coreapi.CORE_API_LINK_MSG: if msg.msgtype == coreapi.CORE_API_LINK_MSG:
# prepare a serverlist from two node numbers in link message # prepare a server list from two node numbers in link message
(handle_locally, serverlist, msg) = self.handlelinkmsg(msg) servers, msg = self.handlelinkmsg(msg)
elif len(serverlist) == 0: elif len(servers) == 0:
# check for servers based on node numbers in all messages but link # check for servers based on node numbers in all messages but link
nn = msg.nodenumbers() nn = msg.nodenumbers()
if len(nn) == 0: if len(nn) == 0:
return False return False
serverlist = self.getserversbynode(nn[0]) servers = self.getserversbynode(nn[0])
if len(serverlist) == 0: # allow other handlers to process this message (this is used
handle_locally = True # by e.g. EMANE to use the link add message to keep counts of
# interfaces on other servers)
# allow other handlers to process this message
# (this is used by e.g. EMANE to use the link add message to keep counts
# of interfaces on other servers)
for handler in self.handlers: for handler in self.handlers:
handler(msg) handler(msg)
# Perform any message forwarding # Perform any message forwarding
handle_locally = self.forwardmsg(msg, serverlist, handle_locally) handle_locally = self.forwardmsg(msg, servers)
return not handle_locally return not handle_locally
def setupserver(self, server): def setupserver(self, servername):
''' Send the appropriate API messages for configuring the specified ''' Send the appropriate API messages for configuring the specified
emulation server. emulation server.
''' '''
(host, port, sock) = self.getserver(server) server = self.getserverbyname(servername)
if host is None or sock is None: if server is None:
msg = 'ignoring unknown server: \'%s\'' % servername
self.session.warn(msg)
return
if server.sock is None or server.host is None or server.port is None:
if self.verbose:
msg = 'ignoring disconnected server: \'%s\'' % servername
self.session.info(msg)
return return
# communicate this session's current state to the server # communicate this session's current state to the server
tlvdata = coreapi.CoreEventTlv.pack(coreapi.CORE_TLV_EVENT_TYPE, tlvdata = coreapi.CoreEventTlv.pack(coreapi.CORE_TLV_EVENT_TYPE,
self.session.getstate()) self.session.getstate())
msg = coreapi.CoreEventMessage.pack(0, tlvdata) msg = coreapi.CoreEventMessage.pack(0, tlvdata)
sock.send(msg) server.sock.send(msg)
# send a Configuration message for the broker object and inform the # send a Configuration message for the broker object and inform the
# server of its local name # server of its local name
tlvdata = "" tlvdata = ""
@ -591,11 +593,11 @@ class CoreBroker(ConfigurableManager):
tlvdata += coreapi.CoreConfTlv.pack(coreapi.CORE_TLV_CONF_DATA_TYPES, tlvdata += coreapi.CoreConfTlv.pack(coreapi.CORE_TLV_CONF_DATA_TYPES,
(coreapi.CONF_DATA_TYPE_STRING,)) (coreapi.CONF_DATA_TYPE_STRING,))
tlvdata += coreapi.CoreConfTlv.pack(coreapi.CORE_TLV_CONF_VALUES, tlvdata += coreapi.CoreConfTlv.pack(coreapi.CORE_TLV_CONF_VALUES,
"%s:%s:%s" % (server, host, port)) "%s:%s:%s" % (server.name, server.host, server.port))
tlvdata += coreapi.CoreConfTlv.pack(coreapi.CORE_TLV_CONF_SESSION, tlvdata += coreapi.CoreConfTlv.pack(coreapi.CORE_TLV_CONF_SESSION,
"%s" % self.session.sessionid) "%s" % self.session.sessionid)
msg = coreapi.CoreConfMessage.pack(0, tlvdata) msg = coreapi.CoreConfMessage.pack(0, tlvdata)
sock.send(msg) server.sock.send(msg)
@staticmethod @staticmethod
def fixupremotetty(msghdr, msgdata, host): def fixupremotetty(msghdr, msgdata, host):
@ -626,8 +628,7 @@ class CoreBroker(ConfigurableManager):
be forwarded. Also keep track of link-layer nodes and the mapping of be forwarded. Also keep track of link-layer nodes and the mapping of
nodes to servers. nodes to servers.
''' '''
serverlist = [] servers = set()
handle_locally = False
serverfiletxt = None serverfiletxt = None
# snoop Node Message for emulation server TLV and record mapping # snoop Node Message for emulation server TLV and record mapping
n = msg.tlvdata[coreapi.CORE_TLV_NODE_NUMBER] n = msg.tlvdata[coreapi.CORE_TLV_NODE_NUMBER]
@ -638,28 +639,21 @@ class CoreBroker(ConfigurableManager):
nodecls = coreapi.node_class(nodetype) nodecls = coreapi.node_class(nodetype)
except KeyError: except KeyError:
self.session.warn("broker invalid node type %s" % nodetype) self.session.warn("broker invalid node type %s" % nodetype)
return (False, serverlist) return servers
if nodecls is None: if nodecls is None:
self.session.warn("broker unimplemented node type %s" % nodetype) self.session.warn("broker unimplemented node type %s" % nodetype)
return (False, serverlist) return servers
if issubclass(nodecls, PyCoreNet) and \ if issubclass(nodecls, PyCoreNet) and \
nodetype != coreapi.CORE_NODE_WLAN: nodetype != coreapi.CORE_NODE_WLAN:
# network node replicated on all servers; could be optimized # network node replicated on all servers; could be optimized
# don't replicate WLANs, because ebtables rules won't work # don't replicate WLANs, because ebtables rules won't work
serverlist = self.getserverlist() servers = self.getservers()
handle_locally = True
self.addnet(n) self.addnet(n)
for server in serverlist: for server in servers:
self.addnodemap(server, n) self.addnodemap(server, n)
# do not record server name for networks since network # do not record server name for networks since network
# nodes are replicated across all server # nodes are replicated across all server
return (handle_locally, serverlist) return servers
if issubclass(nodecls, PyCoreNet) and \
nodetype == coreapi.CORE_NODE_WLAN:
# special case where remote WLANs not in session._objs, and no
# node response message received, so they are counted here
if msg.gettlv(coreapi.CORE_TLV_NODE_EMUSRV) is not None:
self.incrbootcount()
elif issubclass(nodecls, PyCoreNode): elif issubclass(nodecls, PyCoreNode):
name = msg.gettlv(coreapi.CORE_TLV_NODE_NAME) name = msg.gettlv(coreapi.CORE_TLV_NODE_NAME)
if name: if name:
@ -667,26 +661,27 @@ class CoreBroker(ConfigurableManager):
if issubclass(nodecls, PhysicalNode): if issubclass(nodecls, PhysicalNode):
# remember physical nodes # remember physical nodes
self.addphys(n) self.addphys(n)
# emulation server TLV specifies server # emulation server TLV specifies server
server = msg.gettlv(coreapi.CORE_TLV_NODE_EMUSRV) servername = msg.gettlv(coreapi.CORE_TLV_NODE_EMUSRV)
server = self.getserverbyname(servername)
if server is not None: if server is not None:
self.addnodemap(server, n) self.addnodemap(server, n)
if server not in serverlist: if server not in servers:
serverlist.append(server) servers.add(server)
if serverfiletxt and self.session.master: if serverfiletxt and self.session.master:
self.writenodeserver(serverfiletxt, server) self.writenodeserver(serverfiletxt, server)
# hook to update coordinates of physical nodes # hook to update coordinates of physical nodes
if n in self.phys: if n in self.phys:
self.session.mobility.physnodeupdateposition(msg) self.session.mobility.physnodeupdateposition(msg)
return (handle_locally, serverlist) return servers
def handlelinkmsg(self, msg): def handlelinkmsg(self, msg):
''' Determine and return the servers to which this link message should ''' Determine and return the servers to which this link message should
be forwarded. Also build tunnels between different servers or add be forwarded. Also build tunnels between different servers or add
opaque data to the link message before forwarding. opaque data to the link message before forwarding.
''' '''
serverlist = [] servers = set()
handle_locally = False handle_locally = False
# determine link message destination using non-network nodes # determine link message destination using non-network nodes
@ -697,71 +692,70 @@ class CoreBroker(ConfigurableManager):
# the automatic tunnelling # the automatic tunnelling
handle_locally = True handle_locally = True
else: else:
serverlist = self.getserversbynode(nn[1]) servers = self.getserversbynode(nn[1])
elif nn[1] in self.nets: elif nn[1] in self.nets:
serverlist = self.getserversbynode(nn[0]) servers = self.getserversbynode(nn[0])
else: else:
serverset1 = set(self.getserversbynode(nn[0])) servers1 = self.getserversbynode(nn[0])
serverset2 = set(self.getserversbynode(nn[1])) servers2 = self.getserversbynode(nn[1])
# nodes are on two different servers, build tunnels as needed # nodes are on two different servers, build tunnels as needed
if serverset1 != serverset2: if servers1 != servers2:
localn = None localn = None
if len(serverset1) == 0 or len(serverset2) == 0: if len(servers1) == 0 or len(servers2) == 0:
handle_locally = True handle_locally = True
serverlist = list(serverset1 | serverset2) servers = servers1.union(servers2)
host = None host = None
# get the IP of remote server and decide which node number # get the IP of remote server and decide which node number
# is for a local node # is for a local node
for server in serverlist: for server in servers:
(host, port, sock) = self.getserver(server) host = server.host
if host is None: if host is None:
# named server is local # server is local
handle_locally = True handle_locally = True
if server in serverset1: if server in servers1:
localn = nn[0] localn = nn[0]
else: else:
localn = nn[1] localn = nn[1]
if handle_locally and localn is None: if handle_locally and localn is None:
# having no local node at this point indicates local node is # having no local node at this point indicates local node is
# the one with the empty serverset # the one with the empty server set
if len(serverset1) == 0: if len(servers1) == 0:
localn = nn[0] localn = nn[0]
elif len(serverset2) == 0: elif len(servers2) == 0:
localn = nn[1] localn = nn[1]
if host is None: if host is None:
host = self.getlinkendpoint(msg, localn == nn[0]) host = self.getlinkendpoint(msg, localn == nn[0])
if localn is None: if localn is None:
msg = self.addlinkendpoints(msg, serverset1, serverset2) msg = self.addlinkendpoints(msg, servers1, servers2)
elif msg.flags & coreapi.CORE_API_ADD_FLAG: elif msg.flags & coreapi.CORE_API_ADD_FLAG:
self.addtunnel(host, nn[0], nn[1], localn) self.addtunnel(host, nn[0], nn[1], localn)
elif msg.flags & coreapi.CORE_API_DEL_FLAG: elif msg.flags & coreapi.CORE_API_DEL_FLAG:
self.deltunnel(nn[0], nn[1]) self.deltunnel(nn[0], nn[1])
handle_locally = False
else: else:
serverlist = list(serverset1 | serverset2) servers = servers1.union(servers2)
return (handle_locally, serverlist, msg) return servers, msg
def addlinkendpoints(self, msg, serverset1, serverset2): def addlinkendpoints(self, msg, servers1, servers2):
''' For a link message that is not handled locally, inform the remote ''' For a link message that is not handled locally, inform the remote
servers of the IP addresses used as tunnel endpoints by adding servers of the IP addresses used as tunnel endpoints by adding
opaque data to the link message. opaque data to the link message.
''' '''
ip1 = "" ip1 = ''
for server in serverset1: for server in servers1:
(host, port, sock) = self.getserver(server) if server.host is not None:
if host is not None: ip1 = server.host
ip1 = host break
ip2 = "" ip2 = ''
for server in serverset2: for server in servers2:
(host, port, sock) = self.getserver(server) if server.host is not None:
if host is not None: ip2 = server.host
ip2 = host break
tlvdata = msg.rawmsg[coreapi.CoreMessage.hdrsiz:] tlvdata = msg.rawmsg[coreapi.CoreMessage.hdrsiz:]
tlvdata += coreapi.CoreLinkTlv.pack(coreapi.CORE_TLV_LINK_OPAQUE, tlvdata += coreapi.CoreLinkTlv.pack(coreapi.CORE_TLV_LINK_OPAQUE,
"%s:%s" % (ip1, ip2)) "%s:%s" % (ip1, ip2))
newraw = coreapi.CoreLinkMessage.pack(msg.flags, tlvdata) newraw = coreapi.CoreLinkMessage.pack(msg.flags, tlvdata)
msghdr = newraw[:coreapi.CoreMessage.hdrsiz] msghdr = newraw[:coreapi.CoreMessage.hdrsiz]
return coreapi.CoreLinkMessage(msg.flags, msghdr, tlvdata) return coreapi.CoreLinkMessage(msg.flags, msghdr, tlvdata)
def getlinkendpoint(self, msg, first_is_local): def getlinkendpoint(self, msg, first_is_local):
@ -795,59 +789,50 @@ class CoreBroker(ConfigurableManager):
msgtype, flags, msglen = coreapi.CoreMessage.unpackhdr(hdr) msgtype, flags, msglen = coreapi.CoreMessage.unpackhdr(hdr)
msgcls = coreapi.msg_class(msgtype) msgcls = coreapi.msg_class(msgtype)
return self.handlemsg(msgcls(flags, hdr, msg[coreapi.CoreMessage.hdrsiz:])) return self.handlemsg(msgcls(flags, hdr, msg[coreapi.CoreMessage.hdrsiz:]))
def forwardmsg(self, msg, serverlist, handle_locally): def forwardmsg(self, msg, servers):
''' Forward API message to all servers in serverlist; if an empty ''' Forward API message to all given servers.
host/port is encountered, set the handle_locally flag. Returns the
value of the handle_locally flag, which may be unchanged. Return True if an empty host/port is encountered, indicating
the message should be handled locally.
''' '''
for server in serverlist: handle_locally = len(servers) == 0
try: for server in servers:
(host, port, sock) = self.getserver(server) if server.host is None and server.port is None:
except KeyError:
# server not found, don't handle this message locally
self.session.info("broker could not find server %s, message " \
"with type %s dropped" % \
(server, msg.msgtype))
continue
if host is None and port is None:
# local emulation server, handle this locally # local emulation server, handle this locally
handle_locally = True handle_locally = True
elif server.sock is None:
self.session.info("server %s @ %s:%s is disconnected" % \
(server.name, server.host, server.port))
else: else:
if sock is None: server.sock.send(msg.rawmsg)
self.session.info("server %s @ %s:%s is disconnected" % \
(server, host, port))
else:
sock.send(msg.rawmsg)
return handle_locally return handle_locally
def writeservers(self): def writeservers(self):
''' Write the server list to a text file in the session directory upon ''' Write the server list to a text file in the session directory upon
startup: /tmp/pycore.nnnnn/servers startup: /tmp/pycore.nnnnn/servers
''' '''
servers = self.getservers()
filename = os.path.join(self.session.sessiondir, "servers") filename = os.path.join(self.session.sessiondir, "servers")
master = self.session_id_master
if master is None:
master = self.session.sessionid
try: try:
f = open(filename, "w") with open(filename, 'w') as f:
master = self.session_id_master f.write("master=%s\n" % master)
if master is None: for server in servers:
master = self.session.sessionid if server.name == "localhost":
f.write("master=%s\n" % master) continue
self.servers_lock.acquire() try:
for name in sorted(self.servers.keys()): (lhost, lport) = server.sock.getsockname()
if name == "localhost": except:
continue lhost, lport = None, None
(host, port, sock) = self.servers[name] f.write('%s %s %s %s %s\n' % (server.name, server.host,
try: server.port, lhost, lport))
(lhost, lport) = sock.getsockname() except Exception as e:
except: msg = 'Error writing server list to the file: \'%s\'\n%s' % \
lhost, lport = None, None (filename, e)
f.write("%s %s %s %s %s\n" % (name, host, port, lhost, lport)) self.session.warn(msg)
f.close()
except Exception, e:
self.session.warn("Error writing server list to the file: %s\n%s" \
% (filename, e))
finally:
self.servers_lock.release()
def writenodeserver(self, nodestr, server): def writenodeserver(self, nodestr, server):
''' Creates a /tmp/pycore.nnnnn/nX.conf/server file having the node ''' Creates a /tmp/pycore.nnnnn/nX.conf/server file having the node
@ -855,8 +840,7 @@ class CoreBroker(ConfigurableManager):
other machines, much like local nodes may be accessed via the other machines, much like local nodes may be accessed via the
VnodeClient class. VnodeClient class.
''' '''
(host, port, sock) = self.getserver(server) serverstr = "%s %s %s" % (server.name, server.host, server.port)
serverstr = "%s %s %s" % (server, host, port)
name = nodestr.split()[1] name = nodestr.split()[1]
dirname = os.path.join(self.session.sessiondir, name + ".conf") dirname = os.path.join(self.session.sessiondir, name + ".conf")
filename = os.path.join(dirname, "server") filename = os.path.join(dirname, "server")
@ -866,14 +850,9 @@ class CoreBroker(ConfigurableManager):
# directory may already exist from previous distributed run # directory may already exist from previous distributed run
pass pass
try: try:
f = open(filename, "w") with open(filename, 'w') as f:
f.write("%s\n%s\n" % (serverstr, nodestr)) f.write('%s\n%s\n' % (serverstr, nodestr))
f.close() except Exception as e:
return True msg = 'Error writing server file \'%s\' for node %s:\n%s' % \
except Exception, e: (filename, name, e)
msg = "Error writing server file '%s'" % filename
msg += "for node %s:\n%s" % (name, e)
self.session.warn(msg) self.session.warn(msg)
return False

View file

@ -67,7 +67,7 @@ class Emane(ConfigurableManager):
self.logversion() self.logversion()
# model for global EMANE configuration options # model for global EMANE configuration options
self.emane_config = EmaneGlobalModel(session, None, self.verbose) self.emane_config = EmaneGlobalModel(session, None, self.verbose)
session.broker.handlers += (self.handledistributed, ) session.broker.handlers.add(self.handledistributed)
self.loadmodels() self.loadmodels()
self.service = None self.service = None
@ -455,11 +455,11 @@ class Emane(ConfigurableManager):
servers.append(s) servers.append(s)
self._objslock.release() self._objslock.release()
servers.sort(key = lambda x: x.name)
for server in servers: for server in servers:
if server == "localhost": if server.name == "localhost":
continue continue
(host, port, sock) = self.session.broker.getserver(server) if server.sock is None:
if sock is None:
continue continue
platformid += 1 platformid += 1
typeflags = coreapi.CONF_TYPE_FLAGS_UPDATE typeflags = coreapi.CONF_TYPE_FLAGS_UPDATE
@ -467,12 +467,11 @@ class Emane(ConfigurableManager):
values[names.index("nem_id_start")] = str(nemid) values[names.index("nem_id_start")] = str(nemid)
msg = EmaneGlobalModel.toconfmsg(flags=0, nodenum=None, msg = EmaneGlobalModel.toconfmsg(flags=0, nodenum=None,
typeflags=typeflags, values=values) typeflags=typeflags, values=values)
sock.send(msg) server.sock.send(msg)
# increment nemid for next server by number of interfaces # increment nemid for next server by number of interfaces
self._ifccountslock.acquire() with self._ifccountslock:
if server in self._ifccounts: if server in self._ifccounts:
nemid += self._ifccounts[server] nemid += self._ifccounts[server]
self._ifccountslock.release()
return False return False
@ -511,7 +510,7 @@ class Emane(ConfigurableManager):
session = self.session session = self.session
if not session.master: if not session.master:
return # slave server return # slave server
servers = session.broker.getserverlist() servers = session.broker.getservernames()
if len(servers) < 2: if len(servers) < 2:
return # not distributed return # not distributed
prefix = session.cfg.get('controlnet') prefix = session.cfg.get('controlnet')
@ -1155,6 +1154,22 @@ class Emane(ConfigurableManager):
self.session.sdt.updatenodegeo(node.objid, lat, long, alt) self.session.sdt.updatenodegeo(node.objid, lat, long, alt)
return True return True
def emanerunning(self, node):
'''\
Return True if an EMANE process associated with the given node
is running, False otherwise.
'''
status = -1
cmd = ['pkill', '-0', '-x', 'emane']
try:
if self.version < self.EMANE092:
status = subprocess.call(cmd)
else:
status = node.cmd(cmd, wait=True)
except:
pass
return status == 0
def emane_version(): def emane_version():
'Return the locally installed EMANE version identifier and string.' 'Return the locally installed EMANE version identifier and string.'
cmd = ('emane', '--version') cmd = ('emane', '--version')

View file

@ -87,7 +87,7 @@ class CoreDeploymentWriter(object):
for n in nodelist: for n in nodelist:
self.add_virtual_host(testhost, n) self.add_virtual_host(testhost, n)
# TODO: handle other servers # TODO: handle other servers
# servers = self.session.broker.getserverlist() # servers = self.session.broker.getservernames()
# servers.remove('localhost') # servers.remove('localhost')
def add_child_element(self, parent, tagName): def add_child_element(self, parent, tagName):

View file

@ -33,7 +33,7 @@ class MobilityManager(ConfigurableManager):
# dummy node objects for tracking position of nodes on other servers # dummy node objects for tracking position of nodes on other servers
self.phys = {} self.phys = {}
self.physnets = {} self.physnets = {}
self.session.broker.handlers += (self.physnodehandlelink, ) self.session.broker.handlers.add(self.physnodehandlelink)
self.register() self.register()
def startup(self, nodenums=None): def startup(self, nodenums=None):
@ -237,9 +237,10 @@ class MobilityManager(ConfigurableManager):
return return
for nodenum in nodenums: for nodenum in nodenums:
node = self.phys[nodenum] node = self.phys[nodenum]
servers = self.session.broker.getserversbynode(nodenum) for server in self.session.broker.getserversbynode(nodenum):
(host, port, sock) = self.session.broker.getserver(servers[0]) break
netif = self.session.broker.gettunnel(net.objid, IPAddr.toint(host)) netif = self.session.broker.gettunnel(net.objid,
IPAddr.toint(server.host))
node.addnetif(netif, 0) node.addnetif(netif, 0)
netif.node = node netif.node = node
(x,y,z) = netif.node.position.get() (x,y,z) = netif.node.position.get()

View file

@ -116,7 +116,19 @@ class TunTap(PyCoreNetIf):
def nodedevexists(): def nodedevexists():
cmd = (IP_BIN, 'link', 'show', self.name) cmd = (IP_BIN, 'link', 'show', self.name)
return self.node.cmd(cmd) return self.node.cmd(cmd)
self.waitfor(nodedevexists) count = 0
while True:
try:
self.waitfor(nodedevexists)
break
except RuntimeError:
# check if this is an EMANE interface; if so, continue
# waiting if EMANE is still running
if count < 5 and isinstance(self.net, EmaneNode) and \
self.node.session.emane.emanerunning(self.node):
count += 1
else:
raise
def install(self): def install(self):
''' Install this TAP into its namespace. This is not done from the ''' Install this TAP into its namespace. This is not done from the

View file

@ -50,7 +50,7 @@ class Sdt(object):
# node information for remote nodes not in session._objs # node information for remote nodes not in session._objs
# local nodes also appear here since their obj may not exist yet # local nodes also appear here since their obj may not exist yet
self.remotes = {} self.remotes = {}
session.broker.handlers += (self.handledistributed, ) session.broker.handlers.add(self.handledistributed)
def is_enabled(self): def is_enabled(self):
''' Check for 'enablesdt' session option. Return False by default if ''' Check for 'enablesdt' session option. Return False by default if

View file

@ -638,9 +638,16 @@ class Session(object):
# allow time for processes to start # allow time for processes to start
time.sleep(0.125) time.sleep(0.125)
self.validatenodes() self.validatenodes()
self.broker.local_instantiation_complete()
if self.isconnected():
tlvdata = ''
tlvdata += coreapi.CoreEventTlv.pack(coreapi.CORE_TLV_EVENT_TYPE,
coreapi.CORE_EVENT_INSTANTIATION_COMPLETE)
msg = coreapi.CoreEventMessage.pack(0, tlvdata)
self.broadcastraw(None, msg)
# assume either all nodes have booted already, or there are some # assume either all nodes have booted already, or there are some
# nodes on slave servers that will be booted and those servers will # nodes on slave servers that will be booted and those servers will
# send a node status response message # send a status response message
self.checkruntime() self.checkruntime()
def getnodecount(self): def getnodecount(self):
@ -673,22 +680,9 @@ class Session(object):
return return
if self.getstate() == coreapi.CORE_EVENT_RUNTIME_STATE: if self.getstate() == coreapi.CORE_EVENT_RUNTIME_STATE:
return return
session_node_count = int(self.node_count) # check if all servers have completed instantiation
nc = self.getnodecount() if not self.broker.instantiation_complete():
# count booted nodes not emulated on this server return
# TODO: let slave server determine RUNTIME and wait for Event Message
# broker.getbootocunt() counts all CoreNodes from status reponse
# messages, plus any remote WLANs; remote EMANE, hub, switch, etc.
# are already counted in self._objs
nc += self.broker.getbootcount()
self.info("Checking for runtime with %d of %d session nodes" % \
(nc, session_node_count))
if nc < session_node_count:
return # do not have information on all nodes yet
# information on all nodes has been received and they have been started
# enter the runtime state
# TODO: more sophisticated checks to verify that all nodes and networks
# are running
state = coreapi.CORE_EVENT_RUNTIME_STATE state = coreapi.CORE_EVENT_RUNTIME_STATE
self.evq.run() self.evq.run()
self.setstate(state, info=True, sendevent=True) self.setstate(state, info=True, sendevent=True)
@ -890,7 +884,7 @@ class Session(object):
prefix = prefixes[0] prefix = prefixes[0]
else: else:
# slave servers have their name and localhost in the serverlist # slave servers have their name and localhost in the serverlist
servers = self.broker.getserverlist() servers = self.broker.getservernames()
servers.remove('localhost') servers.remove('localhost')
prefix = None prefix = None
for server_prefix in prefixes: for server_prefix in prefixes:
@ -927,7 +921,7 @@ class Session(object):
# tunnels between controlnets will be built with Broker.addnettunnels() # tunnels between controlnets will be built with Broker.addnettunnels()
self.broker.addnet(oid) self.broker.addnet(oid)
for server in self.broker.getserverlist(): for server in self.broker.getservers():
self.broker.addnodemap(server, oid) self.broker.addnodemap(server, oid)
return ctrlnet return ctrlnet
@ -1132,7 +1126,7 @@ class SessionConfig(ConfigurableManager, Configurable):
def __init__(self, session): def __init__(self, session):
ConfigurableManager.__init__(self, session) ConfigurableManager.__init__(self, session)
session.broker.handlers += (self.handledistributed, ) session.broker.handlers.add(self.handledistributed)
self.reset() self.reset()
def reset(self): def reset(self):
@ -1190,7 +1184,7 @@ class SessionConfig(ConfigurableManager, Configurable):
controlnets = value.split() controlnets = value.split()
if len(controlnets) < 2: if len(controlnets) < 2:
return # multiple controlnet prefixes do not exist return # multiple controlnet prefixes do not exist
servers = self.session.broker.getserverlist() servers = self.session.broker.getservernames()
if len(servers) < 2: if len(servers) < 2:
return # not distributed return # not distributed
servers.remove("localhost") servers.remove("localhost")

View file

@ -62,3 +62,4 @@ emane_models = RfPipe, Ieee80211abg, CommEffect, Bypass
#emane_log_level = 2 #emane_log_level = 2
emane_realtime = True emane_realtime = True
aux_request_handler = core.addons.api2handler.CoreApi2RequestHandler:12222