afonso/core-extra
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #210 from coreemu/cleanup/codacy-cleanup

Browse source 
Cleanup/codacy cleanup
This commit is contained in:
bharnden 2018-10-18 16:36:13 -07:00 committed by GitHub
commit 3c8fbc73ad
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 528 additions and 4062 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -20,6 +20,9 @@ stamp-h1
# python build directory
dist
# vscode
.vscode
# intellij
*.iml
.idea

9
README.md
View file

@ -1,4 +1,4 @@
# CORE
# CORE [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d94eb0244ade4510a106b4af76077a92)](https://www.codacy.com/app/blakeharnden/core?utm_source=github.com&utm_medium=referral&utm_content=coreemu/core&utm_campaign=Badge_Grade)
CORE: Common Open Research Emulator
@ -17,7 +17,7 @@ scripting network emulation.
## Documentation and Examples
* Documentation hosted on GitHub
* http://coreemu.github.io/core/
* <http://coreemu.github.io/core/>
* Basic Script Examples
* [Examples](daemon/examples/api)
* Custom Service Example
@ -30,7 +30,7 @@ scripting network emulation.
We are leveraging Discord for persistent chat rooms, voice chat, and
GitHub integration. This allows for more dynamic conversations and the
capability to respond faster. Feel free to join us at the link below.
https://discord.gg/AKd7kmP
<https://discord.gg/AKd7kmP>
You can also get help with questions, comments, or trouble, by using
the CORE mailing lists:
@ -42,7 +42,6 @@ the CORE mailing lists:
See [CORE Installation](http://coreemu.github.io/core/install.html) for detailed build instructions.
Running CORE
------------
### Running CORE
See [Using the CORE GUI](http://coreemu.github.io/core/usage.html) for more details on running CORE.

12
daemon/core/api/coreapi.py
View file

@ -91,7 +91,7 @@ class CoreTlvDataObj(CoreTlvData):
"""
@classmethod
def pack(cls, obj):
def pack(cls, value):
"""
Convenience method for packing custom object data.
@ -99,7 +99,7 @@ class CoreTlvDataObj(CoreTlvData):
:return: length of data and the packed data itself
:rtype: tuple
"""
value = cls.get_value(obj)
value = cls.get_value(value)
return super(CoreTlvDataObj, cls).pack(value)
@classmethod
@ -244,7 +244,7 @@ class CoreTlvDataUint16List(CoreTlvData):
:return: unint 16 list
:rtype: list
"""
return tuple(map(lambda (x): int(x), value.split()))
return tuple(int(x) for x in value.split())
class CoreTlvDataIpv4Addr(CoreTlvDataObj):
@ -266,7 +266,7 @@ class CoreTlvDataIpv4Addr(CoreTlvDataObj):
return obj.addr
@staticmethod
def new_obj(value):
def new_obj(obj):
"""
Retrieve Ipv4 address from a string representation.
@ -274,7 +274,7 @@ class CoreTlvDataIpv4Addr(CoreTlvDataObj):
:return: Ipv4 address
:rtype: core.misc.ipaddress.IpAddress
"""
return IpAddress(af=socket.AF_INET, address=value)
return IpAddress(af=socket.AF_INET, address=obj)
class CoreTlvDataIPv6Addr(CoreTlvDataObj):
@ -380,7 +380,7 @@ class CoreTlv(object):
tlv_type, tlv_len = struct.unpack(cls.header_format, data[:cls.header_len])
header_len = cls.header_len
if tlv_len == 0:
tlv_type, zero, tlv_len = struct.unpack(cls.long_header_format, data[:cls.long_header_len])
tlv_type, _zero, tlv_len = struct.unpack(cls.long_header_format, data[:cls.long_header_len])
header_len = cls.long_header_len
tlv_size = header_len + tlv_len
# for 32-bit alignment

20
daemon/core/broker.py
View file

@ -58,7 +58,9 @@ class CoreDistributedServer(object):
:return: nothing
"""
assert self.sock is None
if self.sock:
raise ValueError("socket already connected")
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
@ -168,7 +170,7 @@ class CoreBroker(object):
self.network_nodes.clear()
self.physical_nodes.clear()
while len(self.tunnels) > 0:
key, gt = self.tunnels.popitem()
_key, gt = self.tunnels.popitem()
gt.shutdown()
def startrecvloop(self):
@ -202,7 +204,7 @@ class CoreBroker(object):
for server in self.servers.itervalues():
if server.sock is not None:
rlist.append(server.sock)
r, w, x = select.select(rlist, [], [], 1.0)
r, _w, _x = select.select(rlist, [], [], 1.0)
for sock in r:
server = self.getserverbysock(sock)
logger.info("attempting to receive from server: peer:%s remote:%s",
@ -319,7 +321,8 @@ class CoreBroker(object):
with self.servers_lock:
try:
s = self.servers.pop(server.name)
assert s == server
if s != server:
raise ValueError("server removed was not the server provided")
except KeyError:
logger.exception("error deleting server")
@ -709,7 +712,7 @@ class CoreBroker(object):
:param str host: host address
:return: packed core execute tlv data
"""
msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(msghdr)
msgtype, msgflags, _msglen = coreapi.CoreMessage.unpack_header(msghdr)
msgcls = coreapi.CLASS_MAP[msgtype]
msg = msgcls(msgflags, msghdr, msgdata)
@ -722,7 +725,6 @@ class CoreBroker(object):
tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.NODE.value, nodenum)
tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.NUMBER.value, execnum)
tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.COMMAND.value, cmd)
title = "\\\"CORE: n%s @ %s\\\"" % (nodenum, host)
res = "ssh -X -f " + host + " xterm -e " + res
tlvdata += coreapi.CoreExecuteTlv.pack(ExecuteTlvs.RESULT.value, res)
@ -931,7 +933,7 @@ class CoreBroker(object):
:rtype: bool
"""
hdr = msg[:coreapi.CoreMessage.header_len]
msgtype, flags, msglen = coreapi.CoreMessage.unpack_header(hdr)
msgtype, flags, _msglen = coreapi.CoreMessage.unpack_header(hdr)
msgcls = coreapi.CLASS_MAP[msgtype]
return self.handle_message(msgcls(flags, hdr, msg[coreapi.CoreMessage.header_len:]))
@ -1067,7 +1069,7 @@ class CoreBroker(object):
value_strings = values_str.split("|")
for value_string in value_strings:
key, value = value_string.split("=", 1)
key, _value = value_string.split("=", 1)
if key == "controlnet":
self.handle_distributed_control_net(message, value_strings, value_strings.index(value_string))
@ -1083,7 +1085,7 @@ class CoreBroker(object):
:return: nothing
"""
key_value = values[index]
key, value = key_value.split("=", 1)
_key, value = key_value.split("=", 1)
control_nets = value.split()
if len(control_nets) < 2:

11
daemon/core/corehandlers.py
View file

@ -1005,17 +1005,17 @@ class CoreHandler(SocketServer.BaseRequestHandler):
if not config_data.data_values:
logger.warn("location data missing")
else:
values = config_data.data_values.split("|")
values = [float(x) for x in config_data.data_values.split("|")]
# Cartesian coordinate reference point
refx, refy = map(lambda x: float(x), values[0:2])
refx, refy = values[0], values[1]
refz = 0.0
lat, lon, alt = map(lambda x: float(x), values[2:5])
lat, lon, alt = values[2], values[3], values[4]
# xyz point
self.session.location.refxyz = (refx, refy, refz)
# geographic reference point
self.session.location.setrefgeo(lat, lon, alt)
self.session.location.refscale = float(values[5])
self.session.location.refscale = values[5]
logger.info("location configured: %s = %s scale=%s", self.session.location.refxyz,
self.session.location.refgeo, self.session.location.refscale)
logger.info("location configured: UTM%s", self.session.location.refutm)
@ -1475,8 +1475,7 @@ class CoreHandler(SocketServer.BaseRequestHandler):
return ()
elif event_type == EventTypes.FILE_SAVE:
filename = event_data.name
xml_version = self.session.options.get_config("xmlfilever")
self.session.save_xml(filename, xml_version)
self.session.save_xml(filename)
elif event_type == EventTypes.SCHEDULED:
etime = event_data.time
node = event_data.node

60
daemon/core/coreobj.py
View file

@ -85,6 +85,9 @@ class PyCoreObj(object):
if name is None:
name = "o%s" % self.objid
self.name = name
self.type = None
self.server = None
self.services = None
# ifindex is key, PyCoreNetIf instance is value
self._netif = {}
self.ifindex = 0
@ -205,21 +208,12 @@ class PyCoreObj(object):
return None
x, y, _ = self.getposition()
model = self.type
emulation_server = self.server
model = None
if hasattr(self, "type"):
model = self.type
emulation_server = None
if hasattr(self, "server"):
emulation_server = self.server
services = None
if hasattr(self, "services") and len(self.services) != 0:
nodeservices = []
for s in self.services:
nodeservices.append(s.name)
services = "|".join(nodeservices)
services = self.services
if services is not None:
services = "|".join([service.name for service in services])
node_data = NodeData(
message_type=message_type,
@ -269,10 +263,8 @@ class PyCoreNode(PyCoreObj):
:param str name: object name
:param bool start: boolean for starting
"""
PyCoreObj.__init__(self, session, objid, name, start=start)
super(PyCoreNode, self).__init__(session, objid, name, start=start)
self.services = []
if not hasattr(self, "type"):
self.type = None
self.nodedir = None
self.tmpnodedir = False
@ -460,6 +452,19 @@ class PyCoreNet(PyCoreObj):
"""
linktype = LinkTypes.WIRED.value
def __init__(self, session, objid, name, start=True):
"""
Create a PyCoreNet instance.
:param core.session.Session session: CORE session object
:param int objid: object id
:param str name: object name
:param bool start: should object start
"""
super(PyCoreNet, self).__init__(session, objid, name, start=start)
self._linked = {}
self._linked_lock = threading.Lock()
def startup(self):
"""
Each object implements its own startup method.
@ -476,19 +481,6 @@ class PyCoreNet(PyCoreObj):
"""
raise NotImplementedError
def __init__(self, session, objid, name, start=True):
"""
Create a PyCoreNet instance.
:param core.session.Session session: CORE session object
:param int objid: object id
:param str name: object name
:param bool start: should object start
"""
PyCoreObj.__init__(self, session, objid, name, start=start)
self._linked = {}
self._linked_lock = threading.Lock()
def attach(self, netif):
"""
Attach network interface.
@ -550,7 +542,7 @@ class PyCoreNet(PyCoreObj):
interface2_ip6 = None
interface2_ip6_mask = None
for address in netif.addrlist:
ip, sep, mask = address.partition('/')
ip, _sep, mask = address.partition("/")
mask = int(mask)
if ipaddress.is_ipv4_address(ip):
family = AF_INET
@ -627,7 +619,8 @@ class PyCoreNetIf(object):
self._params = {}
self.addrlist = []
self.hwaddr = None
self.poshook = None
# placeholder position hook
self.poshook = lambda a, b, c, d: None
# used with EMANE
self.transport_type = None
# interface index on the network
@ -760,5 +753,4 @@ class PyCoreNetIf(object):
:param z: z position
:return: nothing
"""
if self.poshook is not None:
self.poshook(self, x, y, z)
self.poshook(self, x, y, z)

2
daemon/core/emane/commeffect.py
View file

@ -25,7 +25,7 @@ def convert_none(x):
"""
Helper to use 0 for None values.
"""
if type(x) is str:
if isinstance(x, basestring):
x = float(x)
if x is None:
return 0

8
daemon/core/emane/emanemanager.py
View file

@ -627,11 +627,11 @@ class EmaneManager(ModelManager):
if realtime:
emanecmd += "-r",
otagroup, otaport = self.get_config("otamanagergroup").split(":")
otagroup, _otaport = self.get_config("otamanagergroup").split(":")
otadev = self.get_config("otamanagerdevice")
otanetidx = self.session.get_control_net_index(otadev)
eventgroup, eventport = self.get_config("eventservicegroup").split(":")
eventgroup, _eventport = self.get_config("eventservicegroup").split(":")
eventdev = self.get_config("eventservicedevice")
eventservicenetidx = self.session.get_control_net_index(eventdev)
@ -781,7 +781,7 @@ class EmaneManager(ModelManager):
return
logger.info("subscribing to EMANE location events. (%s)", threading.currentThread().getName())
while self.doeventloop is True:
uuid, seq, events = self.service.nextEvent()
_uuid, _seq, events = self.service.nextEvent()
# this occurs with 0.9.1 event service
if not self.doeventloop:
@ -820,7 +820,7 @@ class EmaneManager(ModelManager):
Returns True if successfully parsed and a Node Message was sent.
"""
# convert nemid to node number
emanenode, netif = self.nemlookup(nemid)
_emanenode, netif = self.nemlookup(nemid)
if netif is None:
logger.info("location event for unknown NEM %s", nemid)
return False

2
daemon/core/emane/nodes.py
View file

@ -37,7 +37,7 @@ class EmaneNode(EmaneNet):
"""
def __init__(self, session, objid=None, name=None, start=True):
PyCoreNet.__init__(self, session, objid, name, start)
super(EmaneNode, self).__init__(session, objid, name, start)
self.conf = ""
self.up = False
self.nemidmap = {}

9
daemon/core/emulator/coreemu.py
View file

@ -262,7 +262,7 @@ class EmuSession(Session):
# network to network
if net_one and net_two:
logger.info("adding link from network to network: %s", net_one.name, net_two.name)
logger.info("adding link from network to network: %s - %s", net_one.name, net_two.name)
if nodeutils.is_node(net_two, NodeTypes.RJ45):
interface = net_two.linknet(net_one)
else:
@ -326,7 +326,7 @@ class EmuSession(Session):
:return: nothing
"""
# get node objects identified by link data
node_one, node_two, net_one, net_two, tunnel = self._link_nodes(node_one_id, node_two_id)
node_one, node_two, net_one, net_two, _tunnel = self._link_nodes(node_one_id, node_two_id)
if node_one:
node_one.lock.acquire()
@ -388,7 +388,7 @@ class EmuSession(Session):
:return: nothing
"""
# get node objects identified by link data
node_one, node_two, net_one, net_two, tunnel = self._link_nodes(node_one_id, node_two_id)
node_one, node_two, net_one, net_two, _tunnel = self._link_nodes(node_one_id, node_two_id)
if node_one:
node_one.lock.acquire()
@ -670,12 +670,11 @@ class EmuSession(Session):
self.file_name = file_name
self.instantiate()
def save_xml(self, file_name, version):
def save_xml(self, file_name):
"""
Export a session to the EmulationScript XML format.
:param str file_name: file name to write session xml to
:param str version: xml version type
:return: nothing
"""
CoreXmlWriter(self).write(file_name)

16
daemon/core/location.py
View file

@ -139,7 +139,7 @@ class CoreLocation(object):
"""
# convert lat/lon to UTM coordinates in meters
e, n, zonen, zonel = utm.from_latlon(lat, lon)
rlat, rlon, ralt = self.refgeo
_rlat, _rlon, ralt = self.refgeo
xshift = self.geteastingshift(zonen, zonel)
if xshift is None:
xm = e - self.refutm[1]
@ -180,11 +180,11 @@ class CoreLocation(object):
if z in self.zoneshifts and self.zoneshifts[z][0] is not None:
return self.zoneshifts[z][0]
rlat, rlon, ralt = self.refgeo
rlat, rlon, _ralt = self.refgeo
# ea. zone is 6deg band
lon2 = rlon + 6 * (zonen - rzonen)
# ignore northing
e2, n2, zonen2, zonel2 = utm.from_latlon(rlat, lon2)
e2, _n2, _zonen2, _zonel2 = utm.from_latlon(rlat, lon2)
# NOTE: great circle distance used here, not reference ellipsoid!
xshift = utm.haversine(rlon, rlat, lon2, rlat) - e2
# cache the return value
@ -216,12 +216,12 @@ class CoreLocation(object):
if z in self.zoneshifts and self.zoneshifts[z][1] is not None:
return self.zoneshifts[z][1]
rlat, rlon, ralt = self.refgeo
rlat, rlon, _ralt = self.refgeo
# zonemap is used to calculate degrees difference between zone letters
latshift = self.zonemap[zonel] - self.zonemap[rzonel]
# ea. latitude band is 8deg high
lat2 = rlat + latshift
e2, n2, zonen2, zonel2 = utm.from_latlon(lat2, rlon)
_e2, n2, _zonen2, _zonel2 = utm.from_latlon(lat2, rlon)
# NOTE: great circle distance used here, not reference ellipsoid
yshift = -(utm.haversine(rlon, rlat, rlon, lat2) + n2)
# cache the return value
@ -244,12 +244,12 @@ class CoreLocation(object):
:rtype: tuple
"""
zone = self.refutm[0]
rlat, rlon, ralt = self.refgeo
rlat, rlon, _ralt = self.refgeo
if e > 834000 or e < 166000:
num_zones = (int(e) - 166000) / (utm.R / 10)
# estimate number of zones to shift, E (positive) or W (negative)
rlon2 = self.refgeo[1] + (num_zones * 6)
e2, n2, zonen2, zonel2 = utm.from_latlon(rlat, rlon2)
_e2, _n2, zonen2, zonel2 = utm.from_latlon(rlat, rlon2)
xshift = utm.haversine(rlon, rlat, rlon2, rlat)
# after >3 zones away from refpt, the above estimate won't work
# (the above estimate could be improved)
@ -257,7 +257,7 @@ class CoreLocation(object):
# move one more zone away
num_zones = (abs(num_zones) + 1) * (abs(num_zones) / num_zones)
rlon2 = self.refgeo[1] + (num_zones * 6)
e2, n2, zonen2, zonel2 = utm.from_latlon(rlat, rlon2)
_e2, _n2, zonen2, zonel2 = utm.from_latlon(rlat, rlon2)
xshift = utm.haversine(rlon, rlat, rlon2, rlat)
e = e - xshift
zone = (zonen2, zonel2)

9
daemon/core/misc/event.py
View file

@ -155,7 +155,8 @@ class EventLoop(object):
schedule = True
break
event = heapq.heappop(self.queue)
assert event.time <= now
if event.time > now:
raise ValueError("invalid event time: %s > %s", event.time, now)
event.run()
with self.lock:
@ -170,11 +171,13 @@ class EventLoop(object):
:return: nothing
"""
with self.lock:
assert self.running
if not self.running:
raise ValueError("scheduling event while not running")
if not self.queue:
return
delay = self.queue[0].time - time.time()
assert self.timer is None
if self.timer:
raise ValueError("timer was already set")
self.timer = Timer(delay, self.__run_events)
self.timer.daemon = True
self.timer.start()

12
daemon/core/misc/ipaddress.py
View file

@ -31,7 +31,7 @@ class MacAddress(object):
:return: string representation
:rtype: str
"""
return ":".join(map(lambda x: "%02x" % ord(x), self.addr))
return ":".join("%02x" % ord(x) for x in self.addr)
def to_link_local(self):
"""
@ -61,7 +61,7 @@ class MacAddress(object):
:return: mac address class
:rtype: MacAddress
"""
addr = "".join(map(lambda x: chr(int(x, 16)), s.split(":")))
addr = "".join(chr(int(x, 16)) for x in s.split(":"))
return cls(addr)
@classmethod
@ -154,14 +154,14 @@ class IpAddress(object):
logger.exception("error during addition")
return NotImplemented
tmp = map(lambda x: ord(x), self.addr)
tmp = [ord(x) for x in self.addr]
for i in xrange(len(tmp) - 1, -1, -1):
x = tmp[i] + carry
tmp[i] = x & 0xff
carry = x >> 8
if carry == 0:
break
addr = "".join(map(lambda x: chr(x), tmp))
addr = "".join(chr(x) for x in tmp)
return self.__class__(self.af, addr)
def __sub__(self, other):
@ -200,8 +200,8 @@ class IpAddress(object):
:return: integer value
:rtype: int
"""
bin = socket.inet_pton(AF_INET, s)
return struct.unpack("!I", bin)[0]
value = socket.inet_pton(AF_INET, s)
return struct.unpack("!I", value)[0]
class IpPrefix(object):

6
daemon/core/misc/utils.py
View file

@ -293,10 +293,10 @@ def file_demunge(pathname, header):
start = None
end = None
for i in range(len(lines)):
if lines[i] == "# BEGIN %s\n" % header:
for i, line in enumerate(lines):
if line == "# BEGIN %s\n" % header:
start = i
elif lines[i] == "# END %s\n" % header:
elif line == "# END %s\n" % header:
end = i + 1
if start is None or end is None:

8
daemon/core/netns/nodes.py
View file

@ -211,9 +211,7 @@ class PtpNet(LxBrNet):
if len(self._netif) != 2:
return all_links
if1, if2 = self._netif.items()
if1 = if1[1]
if2 = if2[1]
if1, if2 = self._netif.values()
unidirectional = 0
if if1.getparams() != if2.getparams():
@ -224,7 +222,7 @@ class PtpNet(LxBrNet):
interface1_ip6 = None
interface1_ip6_mask = None
for address in if1.addrlist:
ip, sep, mask = address.partition("/")
ip, _sep, mask = address.partition("/")
mask = int(mask)
if ipaddress.is_ipv4_address(ip):
family = AF_INET
@ -242,7 +240,7 @@ class PtpNet(LxBrNet):
interface2_ip6 = None
interface2_ip6_mask = None
for address in if2.addrlist:
ip, sep, mask = address.partition("/")
ip, _sep, mask = address.partition("/")
mask = int(mask)
if ipaddress.is_ipv4_address(ip):
family = AF_INET

8
daemon/core/netns/openvswitch.py
View file

@ -449,9 +449,7 @@ class OvsPtpNet(OvsNet):
if len(self._netif) != 2:
return all_links
if1, if2 = self._netif.items()
if1 = if1[1]
if2 = if2[1]
if1, if2 = self._netif.values()
unidirectional = 0
if if1.getparams() != if2.getparams():
@ -462,7 +460,7 @@ class OvsPtpNet(OvsNet):
interface1_ip6 = None
interface1_ip6_mask = None
for address in if1.addrlist:
ip, sep, mask = address.partition("/")
ip, _sep, mask = address.partition("/")
mask = int(mask)
if ipaddress.is_ipv4_address(ip):
family = AF_INET
@ -480,7 +478,7 @@ class OvsPtpNet(OvsNet):
interface2_ip6 = None
interface2_ip6_mask = None
for address in if2.addrlist:
ip, sep, mask = address.partition("/")
ip, _sep, mask = address.partition("/")
mask = int(mask)
if ipaddress.is_ipv4_address(ip):
family = AF_INET

8
daemon/core/netns/vnet.py
View file

@ -140,11 +140,9 @@ class EbtablesQueue(object):
while self.doupdateloop:
with self.updatelock:
for wlan in self.updates:
"""
Check if wlan is from a previously closed session. Because of the
rate limiting scheme employed here, this may happen if a new session
is started soon after closing a previous session.
"""
# Check if wlan is from a previously closed session. Because of the
# rate limiting scheme employed here, this may happen if a new session
# is started soon after closing a previous session.
# TODO: if these are WlanNodes, this will never throw an exception
try:
wlan.session

7
daemon/core/netns/vnode.py
View file

@ -364,7 +364,7 @@ class SimpleLxcNode(PyCoreNode):
if self.up:
self.check_cmd([constants.IP_BIN, "addr", "del", str(addr), "dev", self.ifname(ifindex)])
def delalladdr(self, ifindex, address_types=valid_address_types):
def delalladdr(self, ifindex, address_types=None):
"""
Delete all addresses from an interface.
@ -373,6 +373,9 @@ class SimpleLxcNode(PyCoreNode):
:return: nothing
:raises CoreCommandError: when a non-zero exit status occurs
"""
if not address_types:
address_types = self.valid_address_types
interface_name = self.ifname(ifindex)
addresses = self.client.getaddr(interface_name, rescan=True)
@ -570,7 +573,7 @@ class LxcNode(SimpleLxcNode):
:rtype: file
"""
hostfilename = self.hostfilename(filename)
dirname, basename = os.path.split(hostfilename)
dirname, _basename = os.path.split(hostfilename)
if not os.path.isdir(dirname):
os.makedirs(dirname, mode=0755)
return open(hostfilename, mode)

2
daemon/core/phys/pnodes.py
View file

@ -35,7 +35,7 @@ class PhysicalNode(PyCoreNode):
with self.lock:
while self._mounts:
source, target = self._mounts.pop(-1)
_source, target = self._mounts.pop(-1)
self.umount(target)
for netif in self.netifs():

32
daemon/core/sdt.py
View file

@ -183,10 +183,10 @@ class Sdt(object):
if not self.cmd("path \"%s/icons/normal\"" % constants.CORE_DATA_DIR):
return False
# send node type to icon mappings
for type, icon in self.DEFAULT_SPRITES:
if not self.cmd("sprite %s image %s" % (type, icon)):
for node_type, icon in self.DEFAULT_SPRITES:
if not self.cmd("sprite %s image %s" % (node_type, icon)):
return False
(lat, long) = self.session.location.refgeo[:2]
lat, long = self.session.location.refgeo[:2]
return self.cmd("flyto %.6f,%.6f,%d" % (long, lat, self.DEFAULT_ALT))
def disconnect(self):
@ -238,7 +238,7 @@ class Sdt(object):
self.connected = False
return False
def updatenode(self, nodenum, flags, x, y, z, name=None, type=None, icon=None):
def updatenode(self, nodenum, flags, x, y, z, name=None, node_type=None, icon=None):
"""
Node is updated from a Node Message or mobility script.
@ -248,7 +248,7 @@ class Sdt(object):
:param y: y position
:param z: z position
:param str name: node name
:param type: node type
:param node_type: node type
:param icon: node icon
:return: nothing
"""
@ -263,11 +263,11 @@ class Sdt(object):
pos = "pos %.6f,%.6f,%.6f" % (lon, lat, alt)
if flags & MessageFlags.ADD.value:
if icon is not None:
type = name
node_type = name
icon = icon.replace("$CORE_DATA_DIR", constants.CORE_DATA_DIR)
icon = icon.replace("$CORE_CONF_DIR", constants.CORE_CONF_DIR)
self.cmd("sprite %s image %s" % (type, icon))
self.cmd("node %d type %s label on,\"%s\" %s" % (nodenum, type, name, pos))
self.cmd("node %d type %s label on,\"%s\" %s" % (nodenum, node_type, name, pos))
else:
self.cmd("node %d %s" % (nodenum, pos))
@ -403,12 +403,12 @@ class Sdt(object):
nodetype == NodeTypes.PHYSICAL.value:
if model is None:
model = "router"
type = model
nodetype = model
elif nodetype is not None:
type = nodeutils.get_node_class(NodeTypes(nodetype)).type
nodetype = nodeutils.get_node_class(NodeTypes(nodetype)).type
net = True
else:
type = None
nodetype = None
try:
node = self.session.get_object(nodenum)
@ -421,15 +421,15 @@ class Sdt(object):
remote = self.remotes[nodenum]
if name is None:
name = remote.name
if type is None:
type = remote.type
if nodetype is None:
nodetype = remote.type
if icon is None:
icon = remote.icon
else:
remote = Bunch(objid=nodenum, type=type, icon=icon, name=name, net=net, links=set())
remote = Bunch(objid=nodenum, type=nodetype, icon=icon, name=name, net=net, links=set())
self.remotes[nodenum] = remote
remote.pos = (x, y, z)
self.updatenode(nodenum, msg.flags, x, y, z, name, type, icon)
self.updatenode(nodenum, msg.flags, x, y, z, name, nodetype, icon)
def handlelinkmsg(self, msg):
"""
@ -467,8 +467,8 @@ class Sdt(object):
:rtype: bool
"""
if nodenum in self.remotes:
type = self.remotes[nodenum].type
if type in ("wlan", "emane"):
node_type = self.remotes[nodenum].type
if node_type in ("wlan", "emane"):
return True
else:
try:

4
daemon/core/service.py
View file

@ -138,7 +138,7 @@ class ServiceShim(object):
if not service.custom:
valmap[1] = service.get_configs(node)
valmap[3] = service.get_startup(node)
vals = map(lambda a, b: "%s=%s" % (a, str(b)), cls.keys, valmap)
vals = ["%s=%s" % (x, y) for x, y in zip(cls.keys, valmap)]
return "|".join(vals)
@classmethod
@ -469,7 +469,7 @@ class CoreServices(object):
try:
node.privatedir(directory)
except (CoreCommandError, ValueError) as e:
logger.warn("error mounting private dir '%s' for service '%s': %s",
logger.warn("error mounting private dir '%s' for service '%s': %s",
directory, service.name, e)
# create service files

3
daemon/core/services/nrl.py
View file

@ -602,8 +602,7 @@ class MgenActor(NrlService):
comments = ""
cmd = "mgenBasicActor.py -n %s -a 0.0.0.0" % node.name
servicenames = map(lambda x: x.name, node.services)
netifs = filter(lambda x: not getattr(x, 'control', False), node.netifs())
netifs = [x for x in node.netifs() if not getattr(x, "control", False)]
if len(netifs) == 0:
return ""

1
daemon/core/services/quagga.py
View file

@ -551,7 +551,6 @@ class Babel(QuaggaService):
@classmethod
def generatequaggaifcconfig(cls, node, ifc):
type = "wired"
if ifc.net and ifc.net.linktype == LinkTypes.WIRELESS.value:
return " babel wireless\n no babel split-horizon\n"
else:

6
daemon/core/services/sdn.py
View file

@ -35,12 +35,6 @@ class OvsService(SdnService):
if s.name == "zebra":
has_zebra = 1
# Check whether the node is running an SDN controller
has_sdn_ctrlr = 0
for s in node.services:
if s.name == "ryuService":
has_sdn_ctrlr = 1
cfg = "#!/bin/sh\n"
cfg += "# auto-generated by OvsService (OvsService.py)\n"
cfg += "/etc/init.d/openvswitch-switch start < /dev/null\n"

14
daemon/core/services/ucarp.py
View file

@ -107,11 +107,6 @@ ${UCARP_EXEC} -B ${UCARP_OPTS}
"""
Generate a shell script used to boot the Ucarp daemons.
"""
try:
ucarp_bin = node.session.cfg['ucarp_bin']
except KeyError:
ucarp_bin = "/usr/sbin/ucarp"
return """\
#!/bin/sh
# Location of the UCARP config directory
@ -129,11 +124,6 @@ ${UCARP_CFGDIR}/default.sh
"""
Generate a shell script used to start the virtual ip
"""
try:
ucarp_bin = node.session.cfg['ucarp_bin']
except KeyError:
ucarp_bin = "/usr/sbin/ucarp"
return """\
#!/bin/bash
@ -156,10 +146,6 @@ fi
"""
Generate a shell script used to stop the virtual ip
"""
try:
ucarp_bin = node.session.cfg['ucarp_bin']
except KeyError:
ucarp_bin = "/usr/sbin/ucarp"
return """\
#!/bin/bash

4
daemon/core/services/utility.py
View file

@ -83,10 +83,8 @@ class DefaultRouteService(UtilService):
def addrstr(x):
if x.find(":") >= 0:
net = Ipv6Prefix(x)
fam = "inet6 ::"
else:
net = Ipv4Prefix(x)
fam = "inet 0.0.0.0"
if net.max_addr() == net.min_addr():
return ""
else:
@ -145,11 +143,9 @@ class StaticRouteService(UtilService):
def routestr(x):
if x.find(":") >= 0:
net = Ipv6Prefix(x)
fam = "inet6"
dst = "3ffe:4::/64"
else:
net = Ipv4Prefix(x)
fam = "inet"
dst = "10.9.8.0/24"
if net.max_addr() == net.min_addr():
return ""

17
daemon/core/session.py
View file

@ -252,8 +252,9 @@ class Session(object):
hooks = self._hooks.get(state, [])
# execute all state hooks
for hook in hooks:
self.run_hook(hook)
if hooks:
for hook in hooks:
self.run_hook(hook)
else:
logger.info("no state hooks for %s", state)
@ -269,7 +270,7 @@ class Session(object):
"""
logger.info("setting state hook: %s - %s from %s", hook_type, file_name, source_name)
hook_id, state = hook_type.split(':')[:2]
_hook_id, state = hook_type.split(':')[:2]
if not state.isdigit():
logger.error("error setting hook having state '%s'", state)
return
@ -352,7 +353,8 @@ class Session(object):
:return: nothing
"""
hooks = self._state_hooks.setdefault(state, [])
assert hook not in hooks
if hook in hooks:
raise ValueError("attempting to add duplicate state hook")
hooks.append(hook)
if self.state == state:
@ -629,13 +631,10 @@ class Session(object):
"""
with self._objects_lock:
count = len(filter(lambda x: not nodeutils.is_node(x, (NodeTypes.PEER_TO_PEER, NodeTypes.CONTROL_NET)),
self.objects))
count = len([x for x in self.objects if not nodeutils.is_node(x, (NodeTypes.PEER_TO_PEER, NodeTypes.CONTROL_NET))])
# on Linux, GreTapBridges are auto-created, not part of GUI's node count
count -= len(filter(
lambda (x): nodeutils.is_node(x, NodeTypes.TAP_BRIDGE) and not nodeutils.is_node(x, NodeTypes.TUNNEL),
self.objects))
count -= len([x for x in self.objects if nodeutils.is_node(x, NodeTypes.TAP_BRIDGE) and not nodeutils.is_node(x, NodeTypes.TUNNEL)])
return count

160
daemon/core/xml/corexml.py
View file

@ -6,7 +6,6 @@ from core.emulator.emudata import InterfaceData
from core.emulator.emudata import LinkOptions
from core.emulator.emudata import NodeOptions
from core.enumerations import NodeTypes
from core.misc import ipaddress
from core.misc import nodeutils
from core.misc.ipaddress import MacAddress
from core.netns import nodes
@ -98,79 +97,6 @@ def add_configuration(parent, name, value):
add_attribute(config_element, "value", value)
def get_endpoints(node):
endpoints = []
for interface in node.netifs(sort=True):
endpoint = get_endpoint(node, interface)
endpoints.append(endpoint)
return endpoints
def get_endpoint(node, interface):
l2devport = None
othernet = getattr(interface, "othernet", None)
# reference interface of node that is part of this network
if interface.net.objid == node.objid and interface.node:
params = interface.getparams()
if nodeutils.is_node(interface.net, (NodeTypes.HUB, NodeTypes.SWITCH)):
l2devport = "%s/e%s" % (interface.net.name, interface.netindex)
endpoint_id = "%s/%s" % (interface.node.name, interface.name)
endpoint = Endpoint(
node,
interface,
"interface",
endpoint_id,
l2devport,
params
)
# references another node connected to this network
elif othernet and othernet.objid == node.objid:
interface.swapparams("_params_up")
params = interface.getparams()
interface.swapparams("_params_up")
l2devport = "%s/e%s" % (othernet.name, interface.netindex)
endpoint_id = "%s/%s/%s" % (node.name, interface.node.name, interface.netindex)
endpoint = Endpoint(
interface.net,
interface,
"interface",
endpoint_id,
l2devport,
params
)
else:
endpoint = Endpoint(
node,
interface,
)
return endpoint
def get_downstream_l2_devices(node):
all_endpoints = []
l2_devices = [node]
current_endpoint = get_endpoints(node)
all_endpoints.extend(current_endpoint)
for endpoint in current_endpoint:
if endpoint.type and endpoint.network.objid != node.objid:
new_l2_devices, new_endpoints = get_downstream_l2_devices(endpoint.network)
l2_devices.extend(new_l2_devices)
all_endpoints.extend(new_endpoints)
return l2_devices, all_endpoints
class Endpoint(object):
def __init__(self, network, interface, _type=None, _id=None, l2devport=None, params=None):
self.network = network
self.interface = interface
self.type = _type
self.id = _id
self.l2devport = l2devport
self.params = params
class NodeElement(object):
def __init__(self, session, node, element_name):
self.session = session
@ -198,50 +124,6 @@ class NodeElement(object):
add_attribute(position, "alt", alt)
class InterfaceElement(object):
def __init__(self, session, node, interface):
self.session = session
self.node = node
self.interface = interface
self.element = etree.Element("interface")
add_attribute(self.element, "id", interface.netindex)
add_attribute(self.element, "name", interface.name)
mac = etree.SubElement(self.element, "mac")
mac.text = str(interface.hwaddr)
self.add_mtu()
self.addresses = etree.SubElement(self.element, "addresses")
self.add_addresses()
self.add_model()
def add_mtu(self):
# check to add mtu
if self.interface.mtu and self.interface.mtu != 1500:
add_attribute(self.element, "mtu", self.interface.mtu)
def add_model(self):
# check for emane specific interface configuration
net_model = None
if self.interface.net and hasattr(self.interface.net, "model"):
net_model = self.interface.net.model
if net_model and net_model.name.startswith("emane_"):
config = self.session.emane.getifcconfig(self.node.objid, self.interface, net_model.name)
if config:
emane_element = create_emane_model_config(net_model, config)
self.element.append(emane_element)
def add_addresses(self):
for address in self.interface.addrlist:
ip, mask = address.split("/")
if ipaddress.is_ipv4_address(ip):
address_type = "IPv4"
else:
address_type = "IPv6"
address_element = etree.SubElement(self.addresses, "address")
add_attribute(address_element, "type", address_type)
address_element.text = str(address)
class ServiceElement(object):
def __init__(self, service):
self.service = service
@ -309,7 +191,6 @@ class DeviceElement(NodeElement):
def __init__(self, session, node):
super(DeviceElement, self).__init__(session, node, "device")
add_attribute(self.element, "type", node.type)
# self.add_interfaces()
self.add_services()
def add_services(self):
@ -320,15 +201,6 @@ class DeviceElement(NodeElement):
if service_elements.getchildren():
self.element.append(service_elements)
def add_interfaces(self):
interfaces = etree.Element("interfaces")
for interface in self.node.netifs(sort=True):
interface_element = InterfaceElement(self.session, self.node, interface)
interfaces.append(interface_element.element)
if interfaces.getchildren():
self.element.append(interfaces)
class NetworkElement(NodeElement):
def __init__(self, session, node):
@ -343,9 +215,6 @@ class NetworkElement(NodeElement):
if grekey and grekey is not None:
add_attribute(self.element, "grekey", grekey)
self.add_type()
# self.endpoints = get_endpoints(self.node)
# self.l2_devices = self.get_l2_devices()
# self.add_configs()
def add_type(self):
if self.node.apitype:
@ -354,35 +223,6 @@ class NetworkElement(NodeElement):
node_type = self.node.__class__.__name__
add_attribute(self.element, "type", node_type)
def get_l2_devices(self):
l2_devices = []
found_l2_devices = []
found_endpoints = []
if nodeutils.is_node(self.node, (NodeTypes.SWITCH, NodeTypes.HUB)):
for endpoint in self.endpoints:
if endpoint.type and endpoint.network.objid != self.node.objid:
downstream_l2_devices, downstream_endpoints = get_downstream_l2_devices(endpoint.network)
found_l2_devices.extend(downstream_l2_devices)
found_endpoints.extend(downstream_endpoints)
for l2_device in found_l2_devices:
pass
self.endpoints.extend(found_endpoints)
return l2_devices
def add_peer_to_peer_config(self):
pass
def add_switch_hub_tunnel_config(self):
pass
def add_configs(self):
if nodeutils.is_node(self.node, NodeTypes.PEER_TO_PEER):
self.add_peer_to_peer_config()
elif nodeutils.is_node(self.node, (NodeTypes.SWITCH, NodeTypes.HUB, NodeTypes.TUNNEL)):
self.add_switch_hub_tunnel_config()
class CoreXmlWriter(object):
def __init__(self, session):

17
daemon/core/xml/corexmldeployment.py
View file

@ -49,7 +49,7 @@ def add_emane_interface(host_element, netif, platform_name="p1", transport_name=
def get_address_type(address):
addr, slash, prefixlen = address.partition("/")
addr, _slash, _prefixlen = address.partition("/")
if ipaddress.is_ipv4_address(addr):
address_type = "IPv4"
elif ipaddress.is_ipv6_address(addr):
@ -120,20 +120,13 @@ class CoreXmlDeployment(object):
return host_element
def add_virtual_host(self, physical_host, node):
assert isinstance(node, PyCoreNode)
if not isinstance(node, PyCoreNode):
raise TypeError("invalid node type: %s" % node)
# create virtual host element
host_id = "%s/%s" % (physical_host.get("id"), node.name)
host_element = etree.SubElement(physical_host, "testHost", id=host_id, name=node.name)
# TODO: need to inject mapping into device element?
self.find_device(node.name)
# device = self.find_device(self.root.base_element, obj.name)
# if device is None:
# logger.warn("corresponding XML device not found for %s", obj.name)
# return
# add_mapping(device, "testHost", host_id)
# add host type
add_type(host_element, "virtual")
@ -149,7 +142,3 @@ class CoreXmlDeployment(object):
for address in netif.addrlist:
address_type = get_address_type(address)
add_address(parent_element, address_type, address, netif.name)
# TODO: need to inject mapping in interface?
# interface = self.find_interface(device, netif.name)
# add_mapping(interface, "nem", nem.getAttribute("id"))

191
daemon/core/xml/xmldeployment.py
View file

@ -1,191 +0,0 @@
import os
import socket
from core import constants
from core import logger
from core.enumerations import NodeTypes
from core.misc import ipaddress
from core.misc import nodeutils
from core.misc import utils
from core.netns import nodes
from core.xml import xmlutils
class CoreDeploymentWriter(object):
def __init__(self, dom, root, session):
self.dom = dom
self.root = root
self.session = session
self.hostname = socket.gethostname()
@staticmethod
def get_ipv4_addresses(hostname):
if hostname == 'localhost':
addr_list = []
args = [constants.IP_BIN, '-o', '-f', 'inet', 'addr', 'show']
output = utils.check_cmd(args)
for line in output.split(os.linesep):
split = line.split()
if not split:
continue
addr = split[3]
if not addr.startswith('127.'):
addr_list.append(addr)
return addr_list
else:
# TODO: handle other hosts
raise NotImplementedError
@staticmethod
def get_interface_names(hostname):
"""
Uses same methodology of get_ipv4_addresses() to get
parallel list of interface names to go with ...
"""
if hostname == 'localhost':
iface_list = []
args = [constants.IP_BIN, '-o', '-f', 'inet', 'addr', 'show']
output = utils.check_cmd(args)
for line in output.split(os.linesep):
split = line.split()
if not split:
continue
interface_name = split[1]
addr = split[3]
if not addr.startswith('127.'):
iface_list.append(interface_name)
return iface_list
else:
# TODO: handle other hosts
raise NotImplementedError
@staticmethod
def find_device(scenario, name):
tag_name = ('device', 'host', 'router')
for d in xmlutils.iter_descendants_with_attribute(scenario, tag_name, 'name', name):
return d
return None
@staticmethod
def find_interface(device, name):
for i in xmlutils.iter_descendants_with_attribute(device, 'interface', 'name', name):
return i
return None
def add_deployment(self):
testbed = self.dom.createElement('container')
testbed.setAttribute('name', 'TestBed')
testbed.setAttribute('id', 'TestBed')
self.root.base_element.appendChild(testbed)
nodelist = []
for obj in self.session.objects.itervalues():
if isinstance(obj, nodes.PyCoreNode):
nodelist.append(obj)
name = self.hostname
ipv4_addresses = self.get_ipv4_addresses('localhost')
iface_names = self.get_interface_names('localhost')
testhost = self.add_physical_host(testbed, name, ipv4_addresses, iface_names)
for n in nodelist:
self.add_virtual_host(testhost, n)
# TODO: handle other servers
# servers = self.session.broker.getservernames()
# servers.remove('localhost')
def add_child_element(self, parent, tag_name):
el = self.dom.createElement(tag_name)
parent.appendChild(el)
return el
def add_child_element_with_nameattr(self, parent, tag_name, name, setid=True):
el = self.add_child_element(parent, tag_name)
el.setAttribute('name', name)
if setid:
el.setAttribute('id', '%s/%s' % (parent.getAttribute('id'), name))
return el
def add_address(self, parent, address_type, address_str, address_iface=None):
el = self.add_child_element(parent, 'address')
el.setAttribute('type', address_type)
if address_iface is not None:
el.setAttribute('iface', address_iface)
el.appendChild(self.dom.createTextNode(address_str))
return el
def add_type(self, parent, type_str):
el = self.add_child_element(parent, 'type')
el.appendChild(self.dom.createTextNode(type_str))
return el
def add_platform(self, parent, name):
el = self.add_child_element_with_nameattr(parent, 'emanePlatform', name)
return el
def add_transport(self, parent, name):
el = self.add_child_element_with_nameattr(parent, 'transport', name)
return el
def add_nem(self, parent, name):
el = self.add_child_element_with_nameattr(parent, 'nem', name)
return el
def add_parameter(self, parent, name, val):
el = self.add_child_element_with_nameattr(parent, 'parameter', name, False)
el.appendChild(self.dom.createTextNode(val))
return el
def add_mapping(self, parent, maptype, mapref):
el = self.add_child_element(parent, 'mapping')
el.setAttribute('type', maptype)
el.setAttribute('ref', mapref)
return el
def add_host(self, parent, name):
el = self.add_child_element_with_nameattr(parent, 'testHost', name)
return el
def add_physical_host(self, parent, name, ipv4_addresses, iface_names):
el = self.add_host(parent, name)
self.add_type(el, 'physical')
for i in range(0, len(ipv4_addresses)):
addr = ipv4_addresses[i]
if iface_names:
interface_name = iface_names[i]
else:
interface_name = None
self.add_address(el, 'IPv4', addr, interface_name)
return el
def add_virtual_host(self, parent, obj):
assert isinstance(obj, nodes.PyCoreNode)
el = self.add_host(parent, obj.name)
device = self.find_device(self.root.base_element, obj.name)
if device is None:
logger.warn('corresponding XML device not found for %s' % obj.name)
return
self.add_mapping(device, 'testHost', el.getAttribute('id'))
self.add_type(el, 'virtual')
for netif in obj.netifs():
for address in netif.addrlist:
addr, slash, prefixlen = address.partition('/')
if ipaddress.is_ipv4_address(addr):
addr_type = 'IPv4'
elif ipaddress.is_ipv6_address(addr):
addr_type = 'IPv6'
else:
raise NotImplementedError
self.add_address(el, addr_type, address, netif.name)
if nodeutils.is_node(netif.net, NodeTypes.EMANE):
nem = self.add_emane_interface(parent, el, netif)
interface = self.find_interface(device, netif.name)
self.add_mapping(interface, 'nem', nem.getAttribute('id'))
return el
def add_emane_interface(self, physical_host, virtual_host, netif, platform_name='p1', transport_name='t1'):
nemid = netif.net.nemidmap[netif]
platform = self.add_platform(virtual_host, name=platform_name)
transport = self.add_transport(virtual_host, name=transport_name)
nem_name = 'nem%s' % nemid
nem = self.add_nem(platform, nem_name)
self.add_parameter(nem, 'nemid', str(nemid))
self.add_mapping(transport, 'nem', nem.getAttribute('id'))
return nem

54
daemon/core/xml/xmlparser.py
View file

@ -1,54 +0,0 @@
from xml.dom.minidom import parse
from core.xml.xmlparser0 import CoreDocumentParser0
from core.xml.xmlparser1 import CoreDocumentParser1
from core.xml.xmlutils import get_first_child_by_tag_name
class CoreVersionParser(object):
"""
Helper class to check the version of Network Plan document. This
simply looks for a "Scenario" element; when present, this
indicates a 0.0 version document. The dom member is set in order
to prevent parsing a file twice (it can be passed to the
appropriate CoreDocumentParser class.)
"""
DEFAULT_SCENARIO_VERSION = '1.0'
def __init__(self, filename, options):
if 'dom' in options:
self.dom = options['dom']
else:
self.dom = parse(filename)
scenario = get_first_child_by_tag_name(self.dom, 'scenario')
if scenario:
version = scenario.getAttribute('version')
if not version:
version = self.DEFAULT_SCENARIO_VERSION
self.version = version
elif get_first_child_by_tag_name(self.dom, 'Scenario'):
self.version = '0.0'
else:
self.version = 'unknown'
def core_document_parser(session, filename, options):
"""
Retrieves the xml document parser.
:param core.session.Session session: core
:param str filename: name of file to save to or load from
:param dict options: parsing options
:return: xml document parser
"""
vp = CoreVersionParser(filename, options)
if 'dom' not in options:
options['dom'] = vp.dom
if vp.version == '0.0':
doc = CoreDocumentParser0(session, filename, options)
elif vp.version == '1.0':
doc = CoreDocumentParser1(session, filename, options)
else:
raise ValueError('unsupported document version: %s' % vp.version)
return doc

420
daemon/core/xml/xmlparser0.py
View file

@ -1,420 +0,0 @@
from xml.dom.minidom import parse
from core import logger
from core.conf import ConfigShim
from core.enumerations import NodeTypes
from core.misc import nodeutils
from core.service import ServiceManager, ServiceShim
from core.xml import xmlutils
class CoreDocumentParser0(object):
def __init__(self, session, filename, options):
self.session = session
self.filename = filename
if 'dom' in options:
# this prevents parsing twice when detecting file versions
self.dom = options['dom']
else:
self.dom = parse(filename)
self.start = options['start']
self.nodecls = options['nodecls']
self.np = xmlutils.get_one_element(self.dom, "NetworkPlan")
if self.np is None:
raise ValueError, "missing NetworkPlan!"
self.mp = xmlutils.get_one_element(self.dom, "MotionPlan")
self.sp = xmlutils.get_one_element(self.dom, "ServicePlan")
self.meta = xmlutils.get_one_element(self.dom, "CoreMetaData")
self.coords = self.getmotiondict(self.mp)
# link parameters parsed in parsenets(), applied in parsenodes()
self.linkparams = {}
self.parsedefaultservices()
self.parseorigin()
self.parsenets()
self.parsenodes()
self.parseservices()
self.parsemeta()
def getmotiondict(self, mp):
"""
Parse a MotionPlan into a dict with node names for keys and coordinates
for values.
"""
if mp is None:
return {}
coords = {}
for node in mp.getElementsByTagName("Node"):
nodename = str(node.getAttribute("name"))
if nodename == '':
continue
for m in node.getElementsByTagName("motion"):
if m.getAttribute("type") != "stationary":
continue
point = m.getElementsByTagName("point")
if len(point) == 0:
continue
txt = point[0].firstChild
if txt is None:
continue
xyz = map(int, txt.nodeValue.split(','))
z = None
x, y = xyz[0:2]
if len(xyz) == 3:
z = xyz[2]
coords[nodename] = (x, y, z)
return coords
@staticmethod
def getcommonattributes(obj):
"""
Helper to return tuple of attributes common to nodes and nets.
"""
node_id = obj.getAttribute("id")
try:
node_id = int(node_id)
except:
logger.debug("parsing node without integer id: %s", node_id)
name = str(obj.getAttribute("name"))
node_type = str(obj.getAttribute("type"))
return node_id, name, node_type
def parsenets(self):
linkednets = []
for net in self.np.getElementsByTagName("NetworkDefinition"):
node_id, name, node_type = self.getcommonattributes(net)
nodecls = xmlutils.xml_type_to_node_class(node_type)
if not nodecls:
logger.warn("skipping unknown network node '%s' type '%s'", name, node_type)
continue
n = self.session.add_object(cls=nodecls, objid=node_id, name=name, start=self.start)
if name in self.coords:
x, y, z = self.coords[name]
n.setposition(x, y, z)
xmlutils.get_params_set_attrs(net, ("icon", "canvas", "opaque"), n)
if hasattr(n, "canvas") and n.canvas is not None:
n.canvas = int(n.canvas)
# links between two nets (e.g. switch-switch)
for ifc in net.getElementsByTagName("interface"):
netid = str(ifc.getAttribute("net"))
ifcname = str(ifc.getAttribute("name"))
linkednets.append((n, netid, ifcname))
self.parsemodels(net, n)
# link networks together now that they all have been parsed
for n, netid, ifcname in linkednets:
try:
n2 = n.session.get_object_by_name(netid)
except KeyError:
logger.warn("skipping net %s interface: unknown net %s", n.name, netid)
continue
upstream = False
netif = n.getlinknetif(n2)
if netif is None:
netif = n2.linknet(n)
else:
netif.swapparams('_params_up')
upstream = True
key = (n2.name, ifcname)
if key in self.linkparams:
for k, v in self.linkparams[key]:
netif.setparam(k, v)
if upstream:
netif.swapparams('_params_up')
def parsenodes(self):
for node in self.np.getElementsByTagName("Node"):
id, name, type = self.getcommonattributes(node)
if type == "rj45":
nodecls = nodeutils.get_node_class(NodeTypes.RJ45)
else:
nodecls = self.nodecls
n = self.session.add_object(cls=nodecls, objid=id, name=name, start=self.start)
if name in self.coords:
x, y, z = self.coords[name]
n.setposition(x, y, z)
n.type = type
xmlutils.get_params_set_attrs(node, ("icon", "canvas", "opaque"), n)
if hasattr(n, "canvas") and n.canvas is not None:
n.canvas = int(n.canvas)
for ifc in node.getElementsByTagName("interface"):
self.parseinterface(n, ifc)
def parseinterface(self, n, ifc):
"""
Parse a interface block such as:
<interface name="eth0" net="37278">
<address type="mac">00:00:00:aa:00:01</address>
<address>10.0.0.2/24</address>
<address>2001::2/64</address>
</interface>
"""
name = str(ifc.getAttribute("name"))
netid = str(ifc.getAttribute("net"))
hwaddr = None
addrlist = []
try:
net = n.session.get_object_by_name(netid)
except KeyError:
logger.warn("skipping node %s interface %s: unknown net %s", n.name, name, netid)
return
for addr in ifc.getElementsByTagName("address"):
addrstr = xmlutils.get_text_child(addr)
if addrstr is None:
continue
if addr.getAttribute("type") == "mac":
hwaddr = addrstr
else:
addrlist.append(addrstr)
i = n.newnetif(net, addrlist=addrlist, hwaddr=hwaddr, ifindex=None, ifname=name)
for model in ifc.getElementsByTagName("model"):
self.parsemodel(model, n, n.objid)
key = (n.name, name)
if key in self.linkparams:
netif = n.netif(i)
for k, v in self.linkparams[key]:
netif.setparam(k, v)
def parsemodels(self, dom, obj):
"""
Mobility/wireless model config is stored in a ConfigurableManager's
config dict.
"""
nodenum = int(dom.getAttribute("id"))
for model in dom.getElementsByTagName("model"):
self.parsemodel(model, obj, nodenum)
def parsemodel(self, model, obj, nodenum):
"""
Mobility/wireless model config is stored in a ConfigurableManager's
config dict.
"""
name = model.getAttribute("name")
if name == '':
return
type = model.getAttribute("type")
# convert child text nodes into key=value pairs
kvs = xmlutils.get_text_elements_to_list(model)
mgr = self.session.mobility
# TODO: the session.confobj() mechanism could be more generic;
# it only allows registering Conf Message callbacks, but here
# we want access to the ConfigurableManager, not the callback
if name[:5] == "emane":
mgr = self.session.emane
elif name[:5] == "netem":
mgr = None
self.parsenetem(model, obj, kvs)
# TODO: assign other config managers here
if mgr:
for k, v in kvs:
mgr.set_config(k, v, node_id=nodenum, config_type=name)
def parsenetem(self, model, obj, kvs):
"""
Determine interface and invoke setparam() using the parsed
(key, value) pairs.
"""
ifname = model.getAttribute("netif")
peer = model.getAttribute("peer")
key = (peer, ifname)
# nodes and interfaces do not exist yet, at this point of the parsing,
# save (key, value) pairs for later
try:
kvs = map(self.numericvalue, kvs)
except ValueError:
logger.warn("error parsing link parameters for '%s' on '%s'", ifname, peer)
self.linkparams[key] = kvs
@staticmethod
def numericvalue(keyvalue):
(key, value) = keyvalue
if '.' in str(value):
value = float(value)
else:
value = int(value)
return key, value
def parseorigin(self):
"""
Parse any origin tag from the Mobility Plan and set the CoreLocation
reference point appropriately.
"""
origin = xmlutils.get_one_element(self.mp, "origin")
if not origin:
return
location = self.session.location
geo = []
attrs = ("lat", "lon", "alt")
for i in xrange(3):
a = origin.getAttribute(attrs[i])
if a is not None:
a = float(a)
geo.append(a)
location.setrefgeo(geo[0], geo[1], geo[2])
scale = origin.getAttribute("scale100")
if scale is not None and scale:
location.refscale = float(scale)
point = xmlutils.get_one_element(origin, "point")
if point is not None and point.firstChild is not None:
xyz = point.firstChild.nodeValue.split(',')
if len(xyz) == 2:
xyz.append('0.0')
if len(xyz) == 3:
xyz = map(lambda (x): float(x), xyz)
location.refxyz = (xyz[0], xyz[1], xyz[2])
def parsedefaultservices(self):
"""
Prior to parsing nodes, use session.services manager to store
default services for node types
"""
for node in self.sp.getElementsByTagName("Node"):
type = node.getAttribute("type")
if type == '':
continue # node-specific service config
services = []
for service in node.getElementsByTagName("Service"):
services.append(str(service.getAttribute("name")))
self.session.services.default_services[type] = services
logger.info("default services for type %s set to %s" % (type, services))
def parseservices(self):
"""
After node objects exist, parse service customizations and add them
to the nodes.
"""
svclists = {}
# parse services and store configs into session.services.configs
for node in self.sp.getElementsByTagName("Node"):
name = node.getAttribute("name")
if name == '':
continue # node type without name
n = self.session.get_object_by_name(name)
if n is None:
logger.warn("skipping service config for unknown node '%s'" % name)
continue
for service in node.getElementsByTagName("Service"):
svcname = service.getAttribute("name")
if self.parseservice(service, n):
if n.objid in svclists:
svclists[n.objid] += "|" + svcname
else:
svclists[n.objid] = svcname
# nodes in NetworkPlan but not in ServicePlan use the
# default services for their type
for node in self.np.getElementsByTagName("Node"):
id, name, type = self.getcommonattributes(node)
if id in svclists:
continue # custom config exists
else:
svclists[int(id)] = None # use defaults
# associate nodes with services
for objid in sorted(svclists.keys()):
n = self.session.get_object(objid)
services = svclists[objid]
if services:
services = services.split("|")
self.session.services.add_services(node=n, node_type=n.type, services=services)
def parseservice(self, service, n):
"""
Use session.services manager to store service customizations before
they are added to a node.
"""
name = service.getAttribute("name")
svc = ServiceManager.get(name)
if svc is None:
return False
values = []
startup_idx = service.getAttribute("startup_idx")
if startup_idx is not None:
values.append("startidx=%s" % startup_idx)
startup_time = service.getAttribute("start_time")
if startup_time is not None:
values.append("starttime=%s" % startup_time)
dirs = []
for dir in service.getElementsByTagName("Directory"):
dirname = dir.getAttribute("name")
dirs.append(dirname)
if len(dirs):
values.append("dirs=%s" % dirs)
startup = []
shutdown = []
validate = []
for cmd in service.getElementsByTagName("Command"):
type = cmd.getAttribute("type")
cmdstr = xmlutils.get_text_child(cmd)
if cmdstr is None:
continue
if type == "start":
startup.append(cmdstr)
elif type == "stop":
shutdown.append(cmdstr)
elif type == "validate":
validate.append(cmdstr)
if len(startup):
values.append("cmdup=%s" % startup)
if len(shutdown):
values.append("cmddown=%s" % shutdown)
if len(validate):
values.append("cmdval=%s" % validate)
files = []
for file in service.getElementsByTagName("File"):
filename = file.getAttribute("name")
files.append(filename)
data = xmlutils.get_text_child(file)
self.session.services.set_service_file(node_id=n.objid, service_name=name, file_name=filename, data=data)
if len(files):
values.append("files=%s" % files)
if not bool(service.getAttribute("custom")):
return True
self.session.services.set_service(n.objid, svc)
# set custom values for custom service
svc = self.session.services.get_service(n.objid, None)
if not svc:
raise ValueError("custom service(%s) for node(%s) does not exist", svc.name, n.objid)
values = ConfigShim.str_to_dict("|".join(values))
for name, value in values.iteritems():
ServiceShim.setvalue(svc, name, value)
return True
def parsehooks(self, hooks):
''' Parse hook scripts from XML into session._hooks.
'''
for hook in hooks.getElementsByTagName("Hook"):
filename = hook.getAttribute("name")
state = hook.getAttribute("state")
data = xmlutils.get_text_child(hook)
if data is None:
data = "" # allow for empty file
type = "hook:%s" % state
self.session.set_hook(type, file_name=filename, source_name=None, data=data)
def parsemeta(self):
opt = xmlutils.get_one_element(self.meta, "SessionOptions")
if opt:
for param in opt.getElementsByTagName("param"):
k = str(param.getAttribute("name"))
v = str(param.getAttribute("value"))
if v == '':
v = xmlutils.get_text_child(param) # allow attribute/text for newlines
self.session.options.set_config(k, v)
hooks = xmlutils.get_one_element(self.meta, "Hooks")
if hooks:
self.parsehooks(hooks)
meta = xmlutils.get_one_element(self.meta, "MetaData")
if meta:
for param in meta.getElementsByTagName("param"):
k = str(param.getAttribute("name"))
v = str(param.getAttribute("value"))
if v == '':
v = xmlutils.get_text_child(param)
self.session.metadata.set_config(k, v)

889
daemon/core/xml/xmlparser1.py
View file

@ -1,889 +0,0 @@
import random
from xml.dom.minidom import Node
from xml.dom.minidom import parse
from core import constants
from core import logger
from core.conf import ConfigShim
from core.enumerations import NodeTypes
from core.misc import nodeutils
from core.misc.ipaddress import MacAddress
from core.service import ServiceManager, ServiceShim
from core.xml import xmlutils
class CoreDocumentParser1(object):
layer2_device_types = 'hub', 'switch'
layer3_device_types = 'host', 'router'
device_types = layer2_device_types + layer3_device_types
# TODO: support CORE interface classes:
# RJ45Node
# TunnelNode
def __init__(self, session, filename, options):
"""
Creates an CoreDocumentParser1 object.
:param core.session.Session session:
:param str filename: file name to open and parse
:param dict options: parsing options
:return:
"""
logger.info("creating xml parser: file (%s) options(%s)", filename, options)
self.session = session
self.filename = filename
if 'dom' in options:
# this prevents parsing twice when detecting file versions
self.dom = options['dom']
else:
self.dom = parse(filename)
self.start = options['start']
self.nodecls = options['nodecls']
self.scenario = self.get_scenario(self.dom)
self.location_refgeo_set = False
self.location_refxyz_set = False
# saved link parameters saved when parsing networks and applied later
self.link_params = {}
# map from id-string to objid, for files having node names but
# not node numbers
self.objidmap = {}
self.objids = set()
self.default_services = {}
if self.scenario:
self.parse_scenario()
@staticmethod
def get_scenario(dom):
scenario = xmlutils.get_first_child_by_tag_name(dom, 'scenario')
if not scenario:
raise ValueError('no scenario element found')
version = scenario.getAttribute('version')
if version and version != '1.0':
raise ValueError('unsupported scenario version found: \'%s\'' % version)
return scenario
def parse_scenario(self):
self.parse_default_services()
self.parse_session_config()
self.parse_network_plan()
def assign_id(self, idstr, idval):
if idstr in self.objidmap:
assert self.objidmap[idstr] == idval and idval in self.objids
return
self.objidmap[idstr] = idval
self.objids.add(idval)
def rand_id(self):
while True:
x = random.randint(0, 0xffff)
if x not in self.objids:
return x
def get_id(self, idstr):
"""
Get a, possibly new, object id (node number) corresponding to
the given XML string id.
"""
if not idstr:
idn = self.rand_id()
self.objids.add(idn)
return idn
elif idstr in self.objidmap:
return self.objidmap[idstr]
else:
try:
idn = int(idstr)
except ValueError:
idn = self.rand_id()
self.assign_id(idstr, idn)
return idn
def get_common_attributes(self, node):
"""
Return id, name attributes for the given XML element. These
attributes are common to nodes and networks.
"""
idstr = node.getAttribute('id')
# use an explicit set COREID if it exists
coreid = self.find_core_id(node)
if coreid:
idn = int(coreid)
if idstr:
self.assign_id(idstr, idn)
else:
idn = self.get_id(idstr)
# TODO: consider supporting unicode; for now convert to an
# ascii string
namestr = str(node.getAttribute('name'))
return idn, namestr
def iter_network_member_devices(self, element):
# element can be a network or a channel
for interface in xmlutils.iter_children_with_attribute(element, 'member', 'type', 'interface'):
if_id = xmlutils.get_child_text_trim(interface)
assert if_id # XXX for testing
if not if_id:
continue
device, if_name = self.find_device_with_interface(if_id)
assert device, 'no device for if_id: %s' % if_id # XXX for testing
if device:
yield device, if_name
def network_class(self, network, network_type):
"""
Return the corresponding CORE network class for the given
network/network_type.
"""
if network_type in ['ethernet', 'satcom']:
return nodeutils.get_node_class(NodeTypes.PEER_TO_PEER)
elif network_type == 'wireless':
channel = xmlutils.get_first_child_by_tag_name(network, 'channel')
if channel:
# use an explicit CORE type if it exists
coretype = xmlutils.get_first_child_text_trim_with_attribute(channel, 'type', 'domain', 'CORE')
if coretype:
if coretype == 'basic_range':
return nodeutils.get_node_class(NodeTypes.WIRELESS_LAN)
elif coretype.startswith('emane'):
return nodeutils.get_node_class(NodeTypes.EMANE)
else:
logger.warn('unknown network type: \'%s\'', coretype)
return xmlutils.xml_type_to_node_class(coretype)
return nodeutils.get_node_class(NodeTypes.WIRELESS_LAN)
logger.warn('unknown network type: \'%s\'', network_type)
return None
def create_core_object(self, objcls, objid, objname, element, node_type):
obj = self.session.add_object(cls=objcls, objid=objid, name=objname, start=self.start)
logger.info('added object objid=%s name=%s cls=%s' % (objid, objname, objcls))
self.set_object_position(obj, element)
self.set_object_presentation(obj, element, node_type)
return obj
def get_core_object(self, idstr):
if idstr and idstr in self.objidmap:
objid = self.objidmap[idstr]
return self.session.get_object(objid)
return None
def parse_network_plan(self):
# parse the scenario in the following order:
# 1. layer-2 devices
# 2. other networks (ptp/wlan)
# 3. layer-3 devices
self.parse_layer2_devices()
self.parse_networks()
self.parse_layer3_devices()
def set_ethernet_link_parameters(self, channel, link_params, mobility_model_name, mobility_params):
# save link parameters for later use, indexed by the tuple
# (device_id, interface_name)
for dev, if_name in self.iter_network_member_devices(channel):
if self.device_type(dev) in self.device_types:
dev_id = dev.getAttribute('id')
key = (dev_id, if_name)
self.link_params[key] = link_params
if mobility_model_name or mobility_params:
raise NotImplementedError
def set_wireless_link_parameters(self, channel, link_params, mobility_model_name, mobility_params):
network = self.find_channel_network(channel)
network_id = network.getAttribute('id')
if network_id in self.objidmap:
nodenum = self.objidmap[network_id]
else:
logger.warn('unknown network: %s', network.toxml('utf-8'))
assert False # XXX for testing
model_name = xmlutils.get_first_child_text_trim_with_attribute(channel, 'type', 'domain', 'CORE')
if not model_name:
model_name = 'basic_range'
if model_name == 'basic_range':
mgr = self.session.mobility
elif model_name.startswith('emane'):
mgr = self.session.emane
else:
# TODO: any other config managers?
raise NotImplementedError
logger.info("setting wireless link params: node(%s) model(%s) mobility_model(%s)",
nodenum, model_name, mobility_model_name)
mgr.set_model_config(node_id=nodenum, model_name=model_name, config=link_params)
if mobility_model_name and mobility_params:
self.session.mobility.set_model_config(node_id=nodenum, model_name=mobility_model_name,
config=mobility_params)
def link_layer2_devices(self, device1, ifname1, device2, ifname2):
"""
Link two layer-2 devices together.
"""
devid1 = device1.getAttribute('id')
dev1 = self.get_core_object(devid1)
devid2 = device2.getAttribute('id')
dev2 = self.get_core_object(devid2)
assert dev1 and dev2 # XXX for testing
if dev1 and dev2:
# TODO: review this
if nodeutils.is_node(dev2, NodeTypes.RJ45):
# RJ45 nodes have different linknet()
netif = dev2.linknet(dev1)
else:
netif = dev1.linknet(dev2)
self.set_wired_link_parameters(dev1, netif, devid1, ifname1)
@classmethod
def parse_xml_value(cls, valtext):
if not valtext:
return None
try:
if not valtext.translate(None, '0123456789'):
val = int(valtext)
else:
val = float(valtext)
except ValueError:
val = str(valtext)
return val
@classmethod
def parse_parameter_children(cls, parent):
params = {}
for parameter in xmlutils.iter_children_with_name(parent, 'parameter'):
param_name = parameter.getAttribute('name')
assert param_name # XXX for testing
if not param_name:
continue
# TODO: consider supporting unicode; for now convert
# to an ascii string
param_name = str(param_name)
param_val = cls.parse_xml_value(xmlutils.get_child_text_trim(parameter))
# TODO: check if the name already exists?
if param_name and param_val:
params[param_name] = param_val
return params
def parse_network_channel(self, channel):
element = self.search_for_element(channel, 'type', lambda x: not x.hasAttributes())
channel_type = xmlutils.get_child_text_trim(element)
link_params = self.parse_parameter_children(channel)
mobility = xmlutils.get_first_child_by_tag_name(channel, 'CORE:mobility')
if mobility:
mobility_model_name = xmlutils.get_first_child_text_trim_by_tag_name(mobility, 'type')
mobility_params = self.parse_parameter_children(mobility)
else:
mobility_model_name = None
mobility_params = None
if channel_type == 'wireless':
self.set_wireless_link_parameters(channel, link_params, mobility_model_name, mobility_params)
elif channel_type == 'ethernet':
# TODO: maybe this can be done in the loop below to avoid
# iterating through channel members multiple times
self.set_ethernet_link_parameters(channel, link_params, mobility_model_name, mobility_params)
else:
raise NotImplementedError
layer2_device = []
for dev, if_name in self.iter_network_member_devices(channel):
if self.device_type(dev) in self.layer2_device_types:
layer2_device.append((dev, if_name))
assert len(layer2_device) <= 2
if len(layer2_device) == 2:
self.link_layer2_devices(layer2_device[0][0], layer2_device[0][1],
layer2_device[1][0], layer2_device[1][1])
def parse_network(self, network):
"""
Each network element should have an 'id' and 'name' attribute
and include the following child elements:
type (one)
member (zero or more with type="interface" or type="channel")
channel (zero or more)
"""
layer2_members = set()
layer3_members = 0
for dev, if_name in self.iter_network_member_devices(network):
if not dev:
continue
devtype = self.device_type(dev)
if devtype in self.layer2_device_types:
layer2_members.add(dev)
elif devtype in self.layer3_device_types:
layer3_members += 1
else:
raise NotImplementedError
if len(layer2_members) == 0:
net_type = xmlutils.get_first_child_text_trim_by_tag_name(network, 'type')
if not net_type:
logger.warn('no network type found for network: \'%s\'', network.toxml('utf-8'))
assert False # XXX for testing
net_cls = self.network_class(network, net_type)
objid, net_name = self.get_common_attributes(network)
logger.info('parsing network: name=%s id=%s' % (net_name, objid))
if objid in self.session.objects:
return
n = self.create_core_object(net_cls, objid, net_name, network, None)
# handle channel parameters
for channel in xmlutils.iter_children_with_name(network, 'channel'):
self.parse_network_channel(channel)
def parse_networks(self):
"""
Parse all 'network' elements.
"""
for network in xmlutils.iter_descendants_with_name(self.scenario, 'network'):
self.parse_network(network)
def parse_addresses(self, interface):
mac = []
ipv4 = []
ipv6 = []
hostname = []
for address in xmlutils.iter_children_with_name(interface, 'address'):
addr_type = address.getAttribute('type')
if not addr_type:
msg = 'no type attribute found for address ' \
'in interface: \'%s\'' % interface.toxml('utf-8')
logger.warn(msg)
assert False # XXX for testing
addr_text = xmlutils.get_child_text_trim(address)
if not addr_text:
msg = 'no text found for address ' \
'in interface: \'%s\'' % interface.toxml('utf-8')
logger.warn(msg)
assert False # XXX for testing
if addr_type == 'mac':
mac.append(addr_text)
elif addr_type == 'IPv4':
ipv4.append(addr_text)
elif addr_type == 'IPv6':
ipv6.append(addr_text)
elif addr_type == 'hostname':
hostname.append(addr_text)
else:
msg = 'skipping unknown address type \'%s\' in ' \
'interface: \'%s\'' % (addr_type, interface.toxml('utf-8'))
logger.warn(msg)
assert False # XXX for testing
return mac, ipv4, ipv6, hostname
def parse_interface(self, node, device_id, interface):
"""
Each interface can have multiple 'address' elements.
"""
if_name = interface.getAttribute('name')
network = self.find_interface_network_object(interface)
if not network:
msg = 'skipping node \'%s\' interface \'%s\': ' \
'unknown network' % (node.name, if_name)
logger.warn(msg)
assert False # XXX for testing
mac, ipv4, ipv6, hostname = self.parse_addresses(interface)
if mac:
hwaddr = MacAddress.from_string(mac[0])
else:
hwaddr = None
ifindex = node.newnetif(network, addrlist=ipv4 + ipv6, hwaddr=hwaddr, ifindex=None, ifname=if_name)
# TODO: 'hostname' addresses are unused
msg = 'node \'%s\' interface \'%s\' connected ' \
'to network \'%s\'' % (node.name, if_name, network.name)
logger.info(msg)
# set link parameters for wired links
if nodeutils.is_node(network, (NodeTypes.HUB, NodeTypes.PEER_TO_PEER, NodeTypes.SWITCH)):
netif = node.netif(ifindex)
self.set_wired_link_parameters(network, netif, device_id)
def set_wired_link_parameters(self, network, netif, device_id, netif_name=None):
if netif_name is None:
netif_name = netif.name
key = (device_id, netif_name)
if key in self.link_params:
link_params = self.link_params[key]
if self.start:
bw = link_params.get('bw')
delay = link_params.get('delay')
loss = link_params.get('loss')
duplicate = link_params.get('duplicate')
jitter = link_params.get('jitter')
network.linkconfig(netif, bw=bw, delay=delay, loss=loss, duplicate=duplicate, jitter=jitter)
else:
for k, v in link_params.iteritems():
netif.setparam(k, v)
@staticmethod
def search_for_element(node, tag_name, match=None):
"""
Search the given node and all ancestors for an element named
tagName that satisfies the given matching function.
"""
while True:
for child in xmlutils.iter_children(node, Node.ELEMENT_NODE):
if child.tagName == tag_name and (match is None or match(child)):
return child
node = node.parentNode
if not node:
break
return None
@classmethod
def find_core_id(cls, node):
def match(x):
domain = x.getAttribute('domain')
return domain == 'COREID'
alias = cls.search_for_element(node, 'alias', match)
if alias:
return xmlutils.get_child_text_trim(alias)
return None
@classmethod
def find_point(cls, node):
return cls.search_for_element(node, 'point')
@staticmethod
def find_channel_network(channel):
p = channel.parentNode
if p and p.tagName == 'network':
return p
return None
def find_interface_network_object(self, interface):
network_id = xmlutils.get_first_child_text_trim_with_attribute(interface, 'member', 'type', 'network')
if not network_id:
# support legacy notation: <interface net="netid" ...
network_id = interface.getAttribute('net')
obj = self.get_core_object(network_id)
if obj:
# the network_id should exist for ptp or wlan/emane networks
return obj
# the network should correspond to a layer-2 device if the
# network_id does not exist
channel_id = xmlutils.get_first_child_text_trim_with_attribute(interface, 'member', 'type', 'channel')
if not network_id or not channel_id:
return None
network = xmlutils.get_first_child_with_attribute(self.scenario, 'network', 'id', network_id)
if not network:
return None
channel = xmlutils.get_first_child_with_attribute(network, 'channel', 'id', channel_id)
if not channel:
return None
device = None
for dev, if_name in self.iter_network_member_devices(channel):
if self.device_type(dev) in self.layer2_device_types:
assert not device # XXX
device = dev
if device:
obj = self.get_core_object(device.getAttribute('id'))
if obj:
return obj
return None
def set_object_position_pixel(self, obj, point):
x = float(point.getAttribute('x'))
y = float(point.getAttribute('y'))
z = point.getAttribute('z')
if z:
z = float(z)
else:
z = 0.0
# TODO: zMode is unused
# z_mode = point.getAttribute('zMode'))
if x < 0.0:
logger.warn('limiting negative x position of \'%s\' to zero: %s' % (obj.name, x))
x = 0.0
if y < 0.0:
logger.warn('limiting negative y position of \'%s\' to zero: %s' % (obj.name, y))
y = 0.0
obj.setposition(x, y, z)
def set_object_position_gps(self, obj, point):
lat = float(point.getAttribute('lat'))
lon = float(point.getAttribute('lon'))
zalt = point.getAttribute('z')
if zalt:
zalt = float(zalt)
else:
zalt = 0.0
# TODO: zMode is unused
# z_mode = point.getAttribute('zMode'))
if not self.location_refgeo_set:
# for x,y,z conversion, we need a reasonable refpt; this
# picks the first coordinates as the origin
self.session.location.setrefgeo(lat, lon, zalt)
self.location_refgeo_set = True
x, y, z = self.session.location.getxyz(lat, lon, zalt)
if x < 0.0:
logger.warn('limiting negative x position of \'%s\' to zero: %s' % (obj.name, x))
x = 0.0
if y < 0.0:
logger.warn('limiting negative y position of \'%s\' to zero: %s' % (obj.name, y))
y = 0.0
obj.setposition(x, y, z)
def set_object_position_cartesian(self, obj, point):
# TODO: review this
xm = float(point.getAttribute('x'))
ym = float(point.getAttribute('y'))
zm = point.getAttribute('z')
if zm:
zm = float(zm)
else:
zm = 0.0
# TODO: zMode is unused
# z_mode = point.getAttribute('zMode'))
if not self.location_refxyz_set:
self.session.location.refxyz = xm, ym, zm
self.location_refxyz_set = True
# need to convert meters to pixels
x = self.session.location.m2px(xm) + self.session.location.refxyz[0]
y = self.session.location.m2px(ym) + self.session.location.refxyz[1]
z = self.session.location.m2px(zm) + self.session.location.refxyz[2]
if x < 0.0:
logger.warn('limiting negative x position of \'%s\' to zero: %s' % (obj.name, x))
x = 0.0
if y < 0.0:
logger.warn('limiting negative y position of \'%s\' to zero: %s' % (obj.name, y))
y = 0.0
obj.setposition(x, y, z)
def set_object_position(self, obj, element):
"""
Set the x,y,x position of obj from the point associated with
the given element.
"""
point = self.find_point(element)
if not point:
return False
point_type = point.getAttribute('type')
if not point_type:
msg = 'no type attribute found for point: \'%s\'' % \
point.toxml('utf-8')
logger.warn(msg)
assert False # XXX for testing
elif point_type == 'pixel':
self.set_object_position_pixel(obj, point)
elif point_type == 'gps':
self.set_object_position_gps(obj, point)
elif point_type == 'cart':
self.set_object_position_cartesian(obj, point)
else:
logger.warn("skipping unknown point type: '%s'" % point_type)
assert False # XXX for testing
logger.info('set position of %s from point element: \'%s\'', obj.name, point.toxml('utf-8'))
return True
def parse_device_service(self, service, node):
name = service.getAttribute('name')
session_service = ServiceManager.get(name)
if not session_service:
assert False # XXX for testing
values = []
startup_idx = service.getAttribute('startup_idx')
if startup_idx:
values.append('startidx=%s' % startup_idx)
startup_time = service.getAttribute('start_time')
if startup_time:
values.append('starttime=%s' % startup_time)
dirs = []
for directory in xmlutils.iter_children_with_name(service, 'directory'):
dirname = directory.getAttribute('name')
dirs.append(str(dirname))
if dirs:
values.append("dirs=%s" % dirs)
startup = []
shutdown = []
validate = []
for command in xmlutils.iter_children_with_name(service, 'command'):
command_type = command.getAttribute('type')
command_text = xmlutils.get_child_text_trim(command)
if not command_text:
continue
if command_type == 'start':
startup.append(str(command_text))
elif command_type == 'stop':
shutdown.append(str(command_text))
elif command_type == 'validate':
validate.append(str(command_text))
if startup:
values.append('cmdup=%s' % startup)
if shutdown:
values.append('cmddown=%s' % shutdown)
if validate:
values.append('cmdval=%s' % validate)
filenames = []
files = []
for f in xmlutils.iter_children_with_name(service, 'file'):
filename = f.getAttribute('name')
if not filename:
continue
filenames.append(filename)
data = xmlutils.get_child_text_trim(f)
if data:
data = str(data)
else:
data = None
typestr = 'service:%s:%s' % (name, filename)
files.append((typestr, filename, data))
if filenames:
values.append('files=%s' % filenames)
custom = service.getAttribute('custom')
if custom and custom.lower() == 'true':
self.session.services.set_service(node.objid, session_service.name)
values = ConfigShim.str_to_dict("|".join(values))
for key, value in values.iteritems():
ServiceShim.setvalue(session_service, key, value)
# NOTE: if a custom service is used, setservicefile() must be
# called after the custom service exists
for typestr, filename, data in files:
svcname = typestr.split(":")[1]
self.session.services.set_service_file(
node_id=node.objid,
service_name=svcname,
file_name=filename,
data=data
)
return str(name)
def parse_device_services(self, services, node):
"""
Use session.services manager to store service customizations
before they are added to a node.
"""
service_names = []
for service in xmlutils.iter_children_with_name(services, 'service'):
name = self.parse_device_service(service, node)
if name:
service_names.append(name)
return '|'.join(service_names)
def add_device_services(self, node, device, node_type):
"""
Add services to the given node.
"""
services = xmlutils.get_first_child_by_tag_name(device, 'CORE:services')
if services:
services_str = self.parse_device_services(services, node)
logger.info('services for node \'%s\': %s' % (node.name, services_str))
elif node_type in self.default_services:
services_str = None # default services will be added
else:
return
if services_str:
services_str = services_str.split("|")
self.session.services.add_services(
node=node,
node_type=node_type,
services=services_str
)
def set_object_presentation(self, obj, element, node_type):
# defaults from the CORE GUI
default_icons = {
'router': 'router.gif',
'host': 'host.gif',
'PC': 'pc.gif',
'mdr': 'mdr.gif',
}
icon_set = False
for child in xmlutils.iter_children_with_name(element, 'CORE:presentation'):
canvas = child.getAttribute('canvas')
if canvas:
obj.canvas = int(canvas)
icon = child.getAttribute('icon')
if icon:
icon = str(icon).replace("$CORE_DATA_DIR",
constants.CORE_DATA_DIR)
obj.icon = icon
icon_set = True
if not icon_set and node_type in default_icons:
obj.icon = default_icons[node_type]
def device_type(self, device):
if device.tagName in self.device_types:
return device.tagName
return None
def core_node_type(self, device):
# use an explicit CORE type if it exists
coretype = xmlutils.get_first_child_text_trim_with_attribute(device, 'type', 'domain', 'CORE')
if coretype:
return coretype
return self.device_type(device)
def find_device_with_interface(self, interface_id):
# TODO: suport generic 'device' elements
for device in xmlutils.iter_descendants_with_name(self.scenario, self.device_types):
interface = xmlutils.get_first_child_with_attribute(device, 'interface', 'id', interface_id)
if interface:
if_name = interface.getAttribute('name')
return device, if_name
return None, None
def parse_layer2_device(self, device):
objid, device_name = self.get_common_attributes(device)
logger.info('parsing layer-2 device: name=%s id=%s' % (device_name, objid))
try:
return self.session.get_object(objid)
except KeyError:
logger.exception("error geting object: %s", objid)
device_type = self.device_type(device)
if device_type == 'hub':
device_class = nodeutils.get_node_class(NodeTypes.HUB)
elif device_type == 'switch':
device_class = nodeutils.get_node_class(NodeTypes.SWITCH)
else:
logger.warn('unknown layer-2 device type: \'%s\'' % device_type)
assert False # XXX for testing
n = self.create_core_object(device_class, objid, device_name, device, None)
return n
def parse_layer3_device(self, device):
objid, device_name = self.get_common_attributes(device)
logger.info('parsing layer-3 device: name=%s id=%s', device_name, objid)
try:
return self.session.get_object(objid)
except KeyError:
logger.exception("error getting session object: %s", objid)
device_cls = self.nodecls
core_node_type = self.core_node_type(device)
n = self.create_core_object(device_cls, objid, device_name, device, core_node_type)
n.type = core_node_type
self.add_device_services(n, device, core_node_type)
for interface in xmlutils.iter_children_with_name(device, 'interface'):
self.parse_interface(n, device.getAttribute('id'), interface)
return n
def parse_layer2_devices(self):
"""
Parse all layer-2 device elements. A device can be: 'switch',
'hub'.
"""
# TODO: suport generic 'device' elements
for device in xmlutils.iter_descendants_with_name(self.scenario, self.layer2_device_types):
self.parse_layer2_device(device)
def parse_layer3_devices(self):
"""
Parse all layer-3 device elements. A device can be: 'host',
'router'.
"""
# TODO: suport generic 'device' elements
for device in xmlutils.iter_descendants_with_name(self.scenario, self.layer3_device_types):
self.parse_layer3_device(device)
def parse_session_origin(self, session_config):
"""
Parse the first origin tag and set the CoreLocation reference
point appropriately.
"""
# defaults from the CORE GUI
self.session.location.setrefgeo(47.5791667, -122.132322, 2.0)
self.session.location.refscale = 150.0
origin = xmlutils.get_first_child_by_tag_name(session_config, 'origin')
if not origin:
return
lat = origin.getAttribute('lat')
lon = origin.getAttribute('lon')
alt = origin.getAttribute('alt')
if lat and lon and alt:
self.session.location.setrefgeo(float(lat), float(lon), float(alt))
self.location_refgeo_set = True
scale100 = origin.getAttribute("scale100")
if scale100:
self.session.location.refscale = float(scale100)
point = xmlutils.get_first_child_text_trim_by_tag_name(origin, 'point')
if point:
xyz = point.split(',')
if len(xyz) == 2:
xyz.append('0.0')
if len(xyz) == 3:
self.session.location.refxyz = (float(xyz[0]), float(xyz[1]), float(xyz[2]))
self.location_refxyz_set = True
def parse_session_options(self, session_config):
options = xmlutils.get_first_child_by_tag_name(session_config, 'options')
if not options:
return
params = self.parse_parameter_children(options)
for name, value in params.iteritems():
if name and value:
self.session.options.set_config(str(name), str(value))
def parse_session_hooks(self, session_config):
"""
Parse hook scripts.
"""
hooks = xmlutils.get_first_child_by_tag_name(session_config, 'hooks')
if not hooks:
return
for hook in xmlutils.iter_children_with_name(hooks, 'hook'):
filename = hook.getAttribute('name')
state = hook.getAttribute('state')
data = xmlutils.get_child_text_trim(hook)
if data is None:
data = '' # allow for empty file
hook_type = "hook:%s" % state
self.session.set_hook(hook_type, file_name=str(filename), source_name=None, data=str(data))
def parse_session_metadata(self, session_config):
metadata = xmlutils.get_first_child_by_tag_name(session_config, 'metadata')
if not metadata:
return
params = self.parse_parameter_children(metadata)
for name, value in params.iteritems():
if name and value:
self.session.metadata.set_config(str(name), str(value))
def parse_session_config(self):
session_config = xmlutils.get_first_child_by_tag_name(self.scenario, 'CORE:sessionconfig')
if not session_config:
return
self.parse_session_origin(session_config)
self.parse_session_options(session_config)
self.parse_session_hooks(session_config)
self.parse_session_metadata(session_config)
def parse_default_services(self):
# defaults from the CORE GUI
self.default_services = {
'router': ['zebra', 'OSPFv2', 'OSPFv3', 'IPForward'],
'host': ['DefaultRoute', 'SSH'],
'PC': ['DefaultRoute', ],
'mdr': ['zebra', 'OSPFv3MDR', 'IPForward'],
}
default_services = xmlutils.get_first_child_by_tag_name(self.scenario, 'CORE:defaultservices')
if not default_services:
return
for device in xmlutils.iter_children_with_name(default_services, 'device'):
device_type = device.getAttribute('type')
if not device_type:
logger.warn('parse_default_services: no type attribute found for device')
continue
services = []
for service in xmlutils.iter_children_with_name(device, 'service'):
name = service.getAttribute('name')
if name:
services.append(str(name))
self.default_services[device_type] = services
# store default services for the session
for t, s in self.default_services.iteritems():
self.session.services.default_services[t] = s
logger.info('default services for node type \'%s\' set to: %s' % (t, s))

36
daemon/core/xml/xmlsession.py
View file

@ -1,36 +0,0 @@
"""
Helpers for loading and saving XML files. savesessionxml(session, filename) is
the main public interface here.
"""
import os.path
from core.enumerations import NodeTypes
from core.misc import nodeutils
from core.xml.xmlparser import core_document_parser
from core.xml.xmlwriter import core_document_writer
def open_session_xml(session, filename, start=False, nodecls=None):
"""
Import a session from the EmulationScript XML format.
"""
# set default node class when one is not provided
if not nodecls:
nodecls = nodeutils.get_node_class(NodeTypes.DEFAULT)
options = {'start': start, 'nodecls': nodecls}
doc = core_document_parser(session, filename, options)
if start:
session.name = os.path.basename(filename)
session.filename = filename
session.instantiate()
def save_session_xml(session, filename, version):
"""
Export a session to the EmulationScript XML format.
"""
doc = core_document_writer(session, version)
doc.writexml(filename)

350
daemon/core/xml/xmlutils.py
View file

@ -1,350 +0,0 @@
from xml.dom.minidom import Node
from core import logger
from core.netns import nodes
_NODE_MAP = {
nodes.CoreNode.__name__: nodes.CoreNode,
nodes.SwitchNode.__name__: nodes.SwitchNode,
nodes.HubNode.__name__: nodes.HubNode,
nodes.WlanNode.__name__: nodes.WlanNode,
nodes.RJ45Node.__name__: nodes.RJ45Node,
nodes.TunnelNode.__name__: nodes.TunnelNode,
nodes.GreTapBridge.__name__: nodes.GreTapBridge,
nodes.PtpNet.__name__: nodes.PtpNet,
nodes.CtrlNet.__name__: nodes.CtrlNet
}
def add_elements_from_list(dom, parent, iterable, name, attr_name):
"""
XML helper to iterate through a list and add items to parent using tags
of the given name and the item value as an attribute named attr_name.
Example: addelementsfromlist(dom, parent, ('a','b','c'), "letter", "value")
<parent>
<letter value="a"/>
<letter value="b"/>
<letter value="c"/>
</parent>
"""
for item in iterable:
element = dom.createElement(name)
element.setAttribute(attr_name, item)
parent.appendChild(element)
def add_text_elements_from_list(dom, parent, iterable, name, attrs):
"""
XML helper to iterate through a list and add items to parent using tags
of the given name, attributes specified in the attrs tuple, and having the
text of the item within the tags.
Example: addtextelementsfromlist(dom, parent, ('a','b','c'), "letter",
(('show','True'),))
<parent>
<letter show="True">a</letter>
<letter show="True">b</letter>
<letter show="True">c</letter>
</parent>
"""
for item in iterable:
element = dom.createElement(name)
for k, v in attrs:
element.setAttribute(k, v)
parent.appendChild(element)
txt = dom.createTextNode(item)
element.appendChild(txt)
def add_text_elements_from_tuples(dom, parent, iterable, attrs=()):
"""
XML helper to iterate through a list of tuples and add items to
parent using tags named for the first tuple element,
attributes specified in the attrs tuple, and having the
text of second tuple element.
Example: addtextelementsfromtuples(dom, parent,
(('first','a'),('second','b'),('third','c')),
(('show','True'),))
<parent>
<first show="True">a</first>
<second show="True">b</second>
<third show="True">c</third>
</parent>
"""
for name, value in iterable:
element = dom.createElement(name)
for k, v in attrs:
element.setAttribute(k, v)
parent.appendChild(element)
txt = dom.createTextNode(value)
element.appendChild(txt)
def get_text_elements_to_list(parent):
"""
XML helper to parse child text nodes from the given parent and return
a list of (key, value) tuples.
"""
r = []
for n in parent.childNodes:
if n.nodeType != Node.ELEMENT_NODE:
continue
k = str(n.nodeName)
# sometimes want None here?
v = ''
for c in n.childNodes:
if c.nodeType != Node.TEXT_NODE:
continue
v = str(c.nodeValue)
break
r.append((k, v))
return r
def add_param_to_parent(dom, parent, name, value):
"""
XML helper to add a <param name="name" value="value"/> tag to the parent
element, when value is not None.
"""
if value is None:
return None
p = dom.createElement("param")
parent.appendChild(p)
p.setAttribute("name", name)
p.setAttribute("value", "%s" % value)
return p
def add_text_param_to_parent(dom, parent, name, value):
"""
XML helper to add a <param name="name">value</param> tag to the parent
element, when value is not None.
"""
if value is None:
return None
p = dom.createElement("param")
parent.appendChild(p)
p.setAttribute("name", name)
txt = dom.createTextNode(value)
p.appendChild(txt)
return p
def add_param_list_to_parent(dom, parent, name, values):
"""
XML helper to return a parameter list and optionally add it to the
parent element:
<paramlist name="name">
<item value="123">
<item value="456">
</paramlist>
"""
if values is None:
return None
p = dom.createElement("paramlist")
if parent:
parent.appendChild(p)
p.setAttribute("name", name)
for v in values:
item = dom.createElement("item")
item.setAttribute("value", str(v))
p.appendChild(item)
return p
def get_one_element(dom, name):
e = dom.getElementsByTagName(name)
if len(e) == 0:
return None
return e[0]
def iter_descendants(dom, max_depth=0):
"""
Iterate over all descendant element nodes in breadth first order.
Only consider nodes up to max_depth deep when max_depth is greater
than zero.
"""
nodes = [dom]
depth = 0
current_depth_nodes = 1
next_depth_nodes = 0
while nodes:
n = nodes.pop(0)
for child in n.childNodes:
if child.nodeType == Node.ELEMENT_NODE:
yield child
nodes.append(child)
next_depth_nodes += 1
current_depth_nodes -= 1
if current_depth_nodes == 0:
depth += 1
if max_depth > 0 and depth == max_depth:
return
current_depth_nodes = next_depth_nodes
next_depth_nodes = 0
def iter_matching_descendants(dom, match_function, max_depth=0):
"""
Iterate over descendant elements where matchFunction(descendant)
returns true. Only consider nodes up to max_depth deep when
max_depth is greater than zero.
"""
for d in iter_descendants(dom, max_depth):
if match_function(d):
yield d
def iter_descendants_with_name(dom, tag_name, max_depth=0):
"""
Iterate over descendant elements whose name is contained in
tagName (or is named tagName if tagName is a string). Only
consider nodes up to max_depth deep when max_depth is greater than
zero.
"""
if isinstance(tag_name, basestring):
tag_name = (tag_name,)
def match(d):
return d.tagName in tag_name
return iter_matching_descendants(dom, match, max_depth)
def iter_descendants_with_attribute(dom, tag_name, attr_name, attr_value, max_depth=0):
"""
Iterate over descendant elements whose name is contained in
tagName (or is named tagName if tagName is a string) and have an
attribute named attrName with value attrValue. Only consider
nodes up to max_depth deep when max_depth is greater than zero.
"""
if isinstance(tag_name, basestring):
tag_name = (tag_name,)
def match(d):
return d.tagName in tag_name and \
d.getAttribute(attr_name) == attr_value
return iter_matching_descendants(dom, match, max_depth)
def iter_children(dom, node_type):
"""
Iterate over all child elements of the given type.
"""
for child in dom.childNodes:
if child.nodeType == node_type:
yield child
def get_text_child(dom):
"""
Return the text node of the given element.
"""
for child in iter_children(dom, Node.TEXT_NODE):
return str(child.nodeValue)
return None
def get_child_text_trim(dom):
text = get_text_child(dom)
if text:
text = text.strip()
return text
def get_params_set_attrs(dom, param_names, target):
"""
XML helper to get <param name="name" value="value"/> tags and set
the attribute in the target object. String type is used. Target object
attribute is unchanged if the XML attribute is not present.
"""
params = dom.getElementsByTagName("param")
for param in params:
param_name = param.getAttribute("name")
value = param.getAttribute("value")
if value is None:
# never reached?
continue
if param_name in param_names:
setattr(target, param_name, str(value))
def xml_type_to_node_class(node_type):
"""
Helper to convert from a type string to a class name in nodes.*.
"""
logger.error("xml type to node type: %s", node_type)
if hasattr(nodes, node_type):
return _NODE_MAP[node_type]
else:
return None
def iter_children_with_name(dom, tag_name):
return iter_descendants_with_name(dom, tag_name, 1)
def iter_children_with_attribute(dom, tag_name, attr_name, attr_value):
return iter_descendants_with_attribute(dom, tag_name, attr_name, attr_value, 1)
def get_first_child_by_tag_name(dom, tag_name):
"""
Return the first child element whose name is contained in tagName
(or is named tagName if tagName is a string).
"""
for child in iter_children_with_name(dom, tag_name):
return child
return None
def get_first_child_text_by_tag_name(dom, tag_name):
"""
Return the corresponding text of the first child element whose
name is contained in tagName (or is named tagName if tagName is a
string).
"""
child = get_first_child_by_tag_name(dom, tag_name)
if child:
return get_text_child(child)
return None
def get_first_child_text_trim_by_tag_name(dom, tag_name):
text = get_first_child_text_by_tag_name(dom, tag_name)
if text:
text = text.strip()
return text
def get_first_child_with_attribute(dom, tag_name, attr_name, attr_value):
"""
Return the first child element whose name is contained in tagName
(or is named tagName if tagName is a string) that has an attribute
named attrName with value attrValue.
"""
for child in \
iter_children_with_attribute(dom, tag_name, attr_name, attr_value):
return child
return None
def get_first_child_text_with_attribute(dom, tag_name, attr_name, attr_value):
"""
Return the corresponding text of the first child element whose
name is contained in tagName (or is named tagName if tagName is a
string) that has an attribute named attrName with value attrValue.
"""
child = get_first_child_with_attribute(dom, tag_name, attr_name, attr_value)
if child:
return get_text_child(child)
return None
def get_first_child_text_trim_with_attribute(dom, tag_name, attr_name, attr_value):
text = get_first_child_text_with_attribute(dom, tag_name, attr_name, attr_value)
if text:
text = text.strip()
return text

12
daemon/core/xml/xmlwriter.py
View file

@ -1,12 +0,0 @@
from core.xml.xmlwriter0 import CoreDocumentWriter0
from core.xml.xmlwriter1 import CoreDocumentWriter1
def core_document_writer(session, version):
if version == '0.0':
doc = CoreDocumentWriter0(session)
elif version == '1.0':
doc = CoreDocumentWriter1(session)
else:
raise ValueError('unsupported document version: %s' % version)
return doc

386
daemon/core/xml/xmlwriter0.py
View file

@ -1,386 +0,0 @@
import os
from xml.dom.minidom import Document
import pwd
from core import logger
from core.coreobj import PyCoreNet
from core.coreobj import PyCoreNode
from core.enumerations import RegisterTlvs, EventTypes
from core.xml import xmlutils
class CoreDocumentWriter0(Document):
"""
Utility class for writing a CoreSession to XML. The init method builds
an xml.dom.minidom.Document, and the writexml() method saves the XML file.
"""
def __init__(self, session):
"""
Create an empty Scenario XML Document, then populate it with
objects from the given session.
"""
Document.__init__(self)
self.session = session
self.scenario = self.createElement("Scenario")
self.np = self.createElement("NetworkPlan")
self.mp = self.createElement("MotionPlan")
self.sp = self.createElement("ServicePlan")
self.meta = self.createElement("CoreMetaData")
self.appendChild(self.scenario)
self.scenario.appendChild(self.np)
self.scenario.appendChild(self.mp)
self.scenario.appendChild(self.sp)
self.scenario.appendChild(self.meta)
self.populatefromsession()
def populatefromsession(self):
if self.session.state != EventTypes.RUNTIME_STATE.value:
self.session.emane.setup() # not during runtime?
self.addorigin()
self.adddefaultservices()
self.addnets()
self.addnodes()
self.addmetadata()
def writexml(self, filename):
logger.info("saving session XML file %s", filename)
f = open(filename, "w")
Document.writexml(self, writer=f, indent="", addindent=" ", newl="\n", encoding="UTF-8")
f.close()
if self.session.user is not None:
uid = pwd.getpwnam(self.session.user).pw_uid
gid = os.stat(self.session.sessiondir).st_gid
os.chown(filename, uid, gid)
def addnets(self):
"""
Add PyCoreNet objects as NetworkDefinition XML elements.
"""
with self.session._objects_lock:
for net in self.session.objects.itervalues():
if not isinstance(net, PyCoreNet):
continue
self.addnet(net)
def addnet(self, net):
"""
Add one PyCoreNet object as a NetworkDefinition XML element.
"""
n = self.createElement("NetworkDefinition")
self.np.appendChild(n)
n.setAttribute("name", net.name)
# could use net.brname
n.setAttribute("id", "%s" % net.objid)
n.setAttribute("type", "%s" % net.__class__.__name__)
self.addnetinterfaces(n, net)
# key used with tunnel node
if hasattr(net, 'grekey') and net.grekey is not None:
n.setAttribute("key", "%s" % net.grekey)
# link parameters
for netif in net.netifs(sort=True):
self.addnetem(n, netif)
# wireless/mobility models
modelconfigs = net.session.mobility.get_models(net)
modelconfigs += net.session.emane.get_models(net)
self.addmodels(n, modelconfigs)
self.addposition(net)
def addnetem(self, n, netif):
"""
Similar to addmodels(); used for writing netem link effects
parameters. TODO: Interface parameters should be moved to the model
construct, then this separate method shouldn't be required.
"""
params = netif.getparams()
if len(params) == 0:
return
model = self.createElement("model")
model.setAttribute("name", "netem")
model.setAttribute("netif", netif.name)
if hasattr(netif, "node") and netif.node is not None:
model.setAttribute("peer", netif.node.name)
# link between switches uses one veth interface
elif hasattr(netif, "othernet") and netif.othernet is not None:
if netif.othernet.name == n.getAttribute("name"):
model.setAttribute("peer", netif.net.name)
else:
model.setAttribute("peer", netif.othernet.name)
model.setAttribute("netif", netif.localname)
# hack used for upstream parameters for link between switches
# (see LxBrNet.linknet())
if netif.othernet.objid == int(n.getAttribute("id")):
netif.swapparams('_params_up')
params = netif.getparams()
netif.swapparams('_params_up')
has_params = False
for k, v in params:
# default netem parameters are 0 or None
if v is None or v == 0:
continue
if k == "has_netem" or k == "has_tbf":
continue
key = self.createElement(k)
key.appendChild(self.createTextNode("%s" % v))
model.appendChild(key)
has_params = True
if has_params:
n.appendChild(model)
def addmodels(self, n, configs):
"""
Add models from a list of model-class, config values tuples.
"""
for m, conf in configs:
model = self.createElement("model")
n.appendChild(model)
model.setAttribute("name", m.name)
type = "wireless"
if m.config_type == RegisterTlvs.MOBILITY.value:
type = "mobility"
model.setAttribute("type", type)
for k, value in conf.iteritems():
key = self.createElement(k)
if value is None:
value = ""
key.appendChild(self.createTextNode("%s" % value))
model.appendChild(key)
def addnodes(self):
"""
Add PyCoreNode objects as node XML elements.
"""
with self.session._objects_lock:
for node in self.session.objects.itervalues():
if not isinstance(node, PyCoreNode):
continue
self.addnode(node)
def addnode(self, node):
"""
Add a PyCoreNode object as node XML elements.
"""
n = self.createElement("Node")
self.np.appendChild(n)
n.setAttribute("name", node.name)
n.setAttribute("id", "%s" % node.objid)
if node.type:
n.setAttribute("type", node.type)
self.addinterfaces(n, node)
self.addposition(node)
xmlutils.add_param_to_parent(self, n, "icon", node.icon)
xmlutils.add_param_to_parent(self, n, "canvas", node.canvas)
self.addservices(node)
def addinterfaces(self, n, node):
"""
Add PyCoreNetIfs to node XML elements.
"""
for ifc in node.netifs(sort=True):
i = self.createElement("interface")
n.appendChild(i)
i.setAttribute("name", ifc.name)
netmodel = None
if ifc.net:
i.setAttribute("net", ifc.net.name)
if hasattr(ifc.net, "model"):
netmodel = ifc.net.model
if ifc.mtu and ifc.mtu != 1500:
i.setAttribute("mtu", "%s" % ifc.mtu)
# could use ifc.params, transport_type
self.addaddresses(i, ifc)
# per-interface models
if netmodel and netmodel.name[:6] == "emane_":
cfg = self.session.emane.getifcconfig(node.objid, ifc, netmodel.name)
if cfg:
self.addmodels(i, ((netmodel, cfg),))
def addnetinterfaces(self, n, net):
"""
Similar to addinterfaces(), but only adds interface elements to the
supplied XML node that would not otherwise appear in the Node elements.
These are any interfaces that link two switches/hubs together.
"""
for ifc in net.netifs(sort=True):
if not hasattr(ifc, "othernet") or not ifc.othernet:
continue
i = self.createElement("interface")
n.appendChild(i)
if net.objid == ifc.net.objid:
i.setAttribute("name", ifc.localname)
i.setAttribute("net", ifc.othernet.name)
else:
i.setAttribute("name", ifc.name)
i.setAttribute("net", ifc.net.name)
def addposition(self, node):
"""
Add object coordinates as location XML element.
"""
(x, y, z) = node.position.get()
if x is None or y is None:
return
# <Node name="n1">
mpn = self.createElement("Node")
mpn.setAttribute("name", node.name)
self.mp.appendChild(mpn)
# <motion type="stationary">
motion = self.createElement("motion")
motion.setAttribute("type", "stationary")
mpn.appendChild(motion)
# <point>$X$,$Y$,$Z$</point>
pt = self.createElement("point")
motion.appendChild(pt)
coordstxt = "%s,%s" % (x, y)
if z:
coordstxt += ",%s" % z
coords = self.createTextNode(coordstxt)
pt.appendChild(coords)
def addorigin(self):
"""
Add origin to Motion Plan using canvas reference point.
The CoreLocation class maintains this reference point.
"""
refgeo = self.session.location.refgeo
origin = self.createElement("origin")
attrs = ("lat", "lon", "alt")
have_origin = False
for i in xrange(3):
if refgeo[i] is not None:
origin.setAttribute(attrs[i], str(refgeo[i]))
have_origin = True
if not have_origin:
return
if self.session.location.refscale != 1.0: # 100 pixels = refscale m
origin.setAttribute("scale100", str(self.session.location.refscale))
if self.session.location.refxyz != (0.0, 0.0, 0.0):
pt = self.createElement("point")
origin.appendChild(pt)
x, y, z = self.session.location.refxyz
coordstxt = "%s,%s" % (x, y)
if z:
coordstxt += ",%s" % z
coords = self.createTextNode(coordstxt)
pt.appendChild(coords)
self.mp.appendChild(origin)
def adddefaultservices(self):
"""
Add default services and node types to the ServicePlan.
"""
for type in self.session.services.default_services:
defaults = self.session.services.get_default_services(type)
spn = self.createElement("Node")
spn.setAttribute("type", type)
self.sp.appendChild(spn)
for svc in defaults:
s = self.createElement("Service")
spn.appendChild(s)
s.setAttribute("name", str(svc.name))
def addservices(self, node):
"""
Add services and their customizations to the ServicePlan.
"""
if len(node.services) == 0:
return
defaults = self.session.services.get_default_services(node.type)
if node.services == defaults:
return
spn = self.createElement("Node")
spn.setAttribute("name", node.name)
self.sp.appendChild(spn)
for svc in node.services:
s = self.createElement("Service")
spn.appendChild(s)
s.setAttribute("name", str(svc.name))
# only record service names if not a customized service
if not svc.custom:
continue
s.setAttribute("custom", str(svc.custom))
xmlutils.add_elements_from_list(self, s, svc.dirs, "Directory", "name")
for fn in svc.configs:
if len(fn) == 0:
continue
f = self.createElement("File")
f.setAttribute("name", fn)
# all file names are added to determine when a file has been deleted
s.appendChild(f)
data = svc.config_data.get(fn)
if data is None:
# this includes only customized file contents and skips
# the auto-generated files
continue
txt = self.createTextNode(data)
f.appendChild(txt)
xmlutils.add_text_elements_from_list(self, s, svc.startup, "Command", (("type", "start"),))
xmlutils.add_text_elements_from_list(self, s, svc.shutdown, "Command", (("type", "stop"),))
xmlutils.add_text_elements_from_list(self, s, svc.validate, "Command", (("type", "validate"),))
def addaddresses(self, i, netif):
"""
Add MAC and IP addresses to interface XML elements.
"""
if netif.hwaddr:
h = self.createElement("address")
i.appendChild(h)
h.setAttribute("type", "mac")
htxt = self.createTextNode("%s" % netif.hwaddr)
h.appendChild(htxt)
for addr in netif.addrlist:
a = self.createElement("address")
i.appendChild(a)
# a.setAttribute("type", )
atxt = self.createTextNode("%s" % addr)
a.appendChild(atxt)
def addhooks(self):
"""
Add hook script XML elements to the metadata tag.
"""
hooks = self.createElement("Hooks")
for state in sorted(self.session._hooks.keys()):
for filename, data in self.session._hooks[state]:
hook = self.createElement("Hook")
hook.setAttribute("name", filename)
hook.setAttribute("state", str(state))
txt = self.createTextNode(data)
hook.appendChild(txt)
hooks.appendChild(hook)
if hooks.hasChildNodes():
self.meta.appendChild(hooks)
def addmetadata(self):
"""
Add CORE-specific session meta-data XML elements.
"""
# options
options = self.createElement("SessionOptions")
defaults = self.session.options.default_values()
for name, current_value in self.session.options.get_configs().iteritems():
default_value = defaults[name]
if current_value != default_value:
xmlutils.add_text_param_to_parent(self, options, name, current_value)
if options.hasChildNodes():
self.meta.appendChild(options)
# hook scripts
self.addhooks()
# meta
meta = self.createElement("MetaData")
self.meta.appendChild(meta)
for name, current_value in self.session.metadata.get_configs().iteritems():
xmlutils.add_text_param_to_parent(self, meta, name, current_value)

998
daemon/core/xml/xmlwriter1.py
View file

@ -1,998 +0,0 @@
import collections
import os
from xml.dom.minidom import Document
import pwd
from core import coreobj
from core import logger
from core.enumerations import EventTypes
from core.enumerations import NodeTypes
from core.enumerations import RegisterTlvs
from core.misc import ipaddress
from core.misc import nodeutils
from core.netns import nodes
from core.xml import xmlutils
from core.xml.xmldeployment import CoreDeploymentWriter
class Alias:
ID = "COREID"
class MembType:
INTERFACE = "interface"
CHANNEL = "channel"
SWITCH = "switch"
HUB = "hub"
TUNNEL = "tunnel"
NETWORK = "network"
class NodeType:
ROUTER = "router"
HOST = "host"
MDR = "mdr"
PC = "PC"
RJ45 = "rj45"
SWITCH = "lanswitch"
HUB = "hub"
class DevType:
HOST = "host"
ROUTER = "router"
SWITCH = "switch"
HUB = "hub"
class NetType:
WIRELESS = "wireless"
ETHERNET = "ethernet"
PTP_WIRED = "point-to-point-wired"
PTP_WIRELESS = "point-to-point-wireless"
"""
A link endpoint in CORE
net: the network that the endpoint belongs to
netif: the network interface at this end
id: the identifier for the endpoint
l2devport: if the other end is a layer 2 device, this is the assigned port in that device
params: link/interface parameters
"""
Endpoint = collections.namedtuple('Endpoint',
['net', 'netif', 'type', 'id', 'l2devport', 'params'])
class CoreDocumentWriter1(Document):
"""
Utility class for writing a CoreSession to XML in the NMF scenPlan schema. The init
method builds an xml.dom.minidom.Document, and the writexml() method saves the XML file.
"""
def __init__(self, session):
"""
Create an empty Scenario XML Document, then populate it with
objects from the given session.
"""
Document.__init__(self)
logger.debug('Exporting to NMF XML version 1.0')
with session._objects_lock:
self.scenarioPlan = ScenarioPlan(self, session)
if session.state == EventTypes.RUNTIME_STATE.value:
deployment = CoreDeploymentWriter(self, self.scenarioPlan, session)
deployment.add_deployment()
self.scenarioPlan.setAttribute('deployed', 'true')
def writexml(self, filename):
"""
Commit to file
"""
logger.info("saving session XML file %s", filename)
f = open(filename, "w")
Document.writexml(self, writer=f, indent="", addindent=" ", newl="\n", encoding="UTF-8")
f.close()
if self.scenarioPlan.coreSession.user is not None:
uid = pwd.getpwnam(self.scenarioPlan.coreSession.user).pw_uid
gid = os.stat(self.scenarioPlan.coreSession.session_dir).st_gid
os.chown(filename, uid, gid)
class XmlElement(object):
"""
The base class for all XML elements in the scenario plan. Includes
convenience functions.
"""
def __init__(self, document, parent, element_type):
self.document = document
self.parent = parent
self.base_element = document.createElement("%s" % element_type)
if self.parent is not None:
self.parent.appendChild(self.base_element)
def createElement(self, element_tag):
return self.document.createElement(element_tag)
def getTagName(self):
return self.base_element.tagName
def createTextNode(self, node_tag):
return self.document.createTextNode(node_tag)
def appendChild(self, child):
if isinstance(child, XmlElement):
self.base_element.appendChild(child.base_element)
else:
self.base_element.appendChild(child)
@staticmethod
def add_parameter(doc, parent, key, value):
if key and value:
parm = doc.createElement("parameter")
parm.setAttribute("name", str(key))
parm.appendChild(doc.createTextNode(str(value)))
parent.appendChild(parm)
def addParameter(self, key, value):
"""
Add a parameter to the xml element
"""
self.add_parameter(self.document, self, key, value)
def setAttribute(self, name, val):
self.base_element.setAttribute(name, val)
def getAttribute(self, name):
return self.base_element.getAttribute(name)
class NamedXmlElement(XmlElement):
"""
The base class for all "named" xml elements. Named elements are
xml elements in the scenario plan that have an id and a name attribute.
"""
def __init__(self, scen_plan, parent, element_type, element_name):
XmlElement.__init__(self, scen_plan.document, parent, element_type)
self.scenPlan = scen_plan
self.coreSession = scen_plan.coreSession
element_path = ''
self.id = None
if self.parent is not None and isinstance(self.parent, XmlElement) and self.parent.getTagName() != "scenario":
element_path = "%s/" % self.parent.getAttribute("id")
self.id = "%s%s" % (element_path, element_name)
self.setAttribute("name", element_name)
self.setAttribute("id", self.id)
def addPoint(self, core_object):
"""
Add position to an object
"""
(x, y, z) = core_object.position.get()
if x is None or y is None:
return
lat, lon, alt = self.coreSession.location.getgeo(x, y, z)
pt = self.createElement("point")
pt.setAttribute("type", "gps")
pt.setAttribute("lat", "%s" % lat)
pt.setAttribute("lon", "%s" % lon)
if z:
pt.setAttribute("z", "%s" % alt)
self.appendChild(pt)
def createAlias(self, domain, value_str):
"""
Create an alias element for CORE specific information
"""
a = self.createElement("alias")
a.setAttribute("domain", "%s" % domain)
a.appendChild(self.createTextNode(value_str))
return a
class ScenarioPlan(XmlElement):
"""
Container class for ScenarioPlan.
"""
def __init__(self, document, session):
XmlElement.__init__(self, document, parent=document, element_type='scenario')
self.coreSession = session
self.setAttribute('version', '1.0')
self.setAttribute("name", "%s" % session.name)
self.setAttribute('xmlns', 'nmfPlan')
self.setAttribute('xmlns:CORE', 'coreSpecific')
self.setAttribute('compiled', 'true')
self.all_channel_members = {}
self.last_network_id = 0
self.addNetworks()
self.addDevices()
self.addDefaultServices()
self.addSessionConfiguration()
def addNetworks(self):
"""
Add networks in the session to the scenPlan.
"""
for net in self.coreSession.objects.itervalues():
if not isinstance(net, coreobj.PyCoreNet):
continue
if nodeutils.is_node(net, NodeTypes.CONTROL_NET):
continue
# Do not add switches and hubs that belong to another network
if nodeutils.is_node(net, (NodeTypes.SWITCH, NodeTypes.HUB)):
if in_other_network(net):
continue
try:
NetworkElement(self, self, net)
except:
logger.exception("error adding node")
if hasattr(net, "name") and net.name:
logger.warn('Unsupported net name: %s, class: %s, type: %s',
net.name, net.__class__.__name__, net.type)
else:
logger.warn('Unsupported net class: %s', net.__class__.__name__)
def addDevices(self):
"""
Add device elements to the scenario plan.
"""
for node in self.coreSession.objects.itervalues():
if not isinstance(node, nodes.PyCoreNode):
continue
try:
DeviceElement(self, self, node)
except:
logger.exception("error adding device")
if hasattr(node, "name") and node.name:
logger.warn('Unsupported device name: %s, class: %s, type: %s',
node.name, node.__class__.__name__, node.type)
else:
logger.warn('Unsupported device: %s', node.__class__.__name__)
def addDefaultServices(self):
"""
Add default services and node types to the ServicePlan.
"""
defaultservices = self.createElement("CORE:defaultservices")
for type in self.coreSession.services.default_services:
defaults = self.coreSession.services.get_default_services(type)
spn = self.createElement("device")
spn.setAttribute("type", type)
defaultservices.appendChild(spn)
for svc in defaults:
s = self.createElement("service")
spn.appendChild(s)
s.setAttribute("name", str(svc.name))
if defaultservices.hasChildNodes():
self.appendChild(defaultservices)
def addSessionConfiguration(self):
"""
Add CORE-specific session configuration XML elements.
"""
config = self.createElement("CORE:sessionconfig")
# origin: geolocation of cartesian coordinate 0,0,0
refgeo = self.coreSession.location.refgeo
origin = self.createElement("origin")
attrs = ("lat", "lon", "alt")
have_origin = False
for i in xrange(3):
if refgeo[i] is not None:
origin.setAttribute(attrs[i], str(refgeo[i]))
have_origin = True
if have_origin:
if self.coreSession.location.refscale != 1.0: # 100 pixels = refscale m
origin.setAttribute("scale100", str(self.coreSession.location.refscale))
if self.coreSession.location.refxyz != (0.0, 0.0, 0.0):
pt = self.createElement("point")
origin.appendChild(pt)
x, y, z = self.coreSession.location.refxyz
coordstxt = "%s,%s" % (x, y)
if z:
coordstxt += ",%s" % z
coords = self.createTextNode(coordstxt)
pt.appendChild(coords)
config.appendChild(origin)
# options
options = self.createElement("options")
options_config = self.coreSession.options.get_configs()
for _id, default_value in self.coreSession.options.default_values().iteritems():
value = options_config[_id]
if value != default_value:
XmlElement.add_parameter(self.document, options, _id, value)
if options.hasChildNodes():
config.appendChild(options)
# hook scripts
hooks = self.createElement("hooks")
for state in sorted(self.coreSession._hooks.keys()):
for filename, data in self.coreSession._hooks[state]:
hook = self.createElement("hook")
hook.setAttribute("name", filename)
hook.setAttribute("state", str(state))
txt = self.createTextNode(data)
hook.appendChild(txt)
hooks.appendChild(hook)
if hooks.hasChildNodes():
config.appendChild(hooks)
# metadata
meta = self.createElement("metadata")
for k, v in self.coreSession.metadata.get_configs().iteritems():
XmlElement.add_parameter(self.document, meta, k, v)
if meta.hasChildNodes():
config.appendChild(meta)
if config.hasChildNodes():
self.appendChild(config)
class NetworkElement(NamedXmlElement):
def __init__(self, scen_plan, parent, network_object):
"""
Add one PyCoreNet object as one network XML element.
"""
element_name = self.getNetworkName(scen_plan, network_object)
NamedXmlElement.__init__(self, scen_plan, parent, "network", element_name)
self.scenPlan = scen_plan
self.addPoint(network_object)
network_type = None
if nodeutils.is_node(network_object, (NodeTypes.WIRELESS_LAN, NodeTypes.EMANE)):
network_type = NetType.WIRELESS
elif nodeutils.is_node(network_object, (NodeTypes.SWITCH, NodeTypes.HUB,
NodeTypes.PEER_TO_PEER, NodeTypes.TUNNEL)):
network_type = NetType.ETHERNET
else:
network_type = "%s" % network_object.__class__.__name__
type_element = self.createElement("type")
type_element.appendChild(self.createTextNode(network_type))
self.appendChild(type_element)
# Gather all endpoints belonging to this network
self.endpoints = get_endpoints(network_object)
# Special case for a network of switches and hubs
create_alias = True
self.l2devices = []
if nodeutils.is_node(network_object, (NodeTypes.SWITCH, NodeTypes.HUB)):
create_alias = False
self.appendChild(type_element)
self.addL2Devices(network_object)
if create_alias:
a = self.createAlias(Alias.ID, "%d" % int(network_object.objid))
self.appendChild(a)
# XXXX TODO: Move this to channel?
# key used with tunnel node
if hasattr(network_object, 'grekey') and network_object.grekey is not None:
a = self.createAlias("COREGREKEY", "%s" % network_object.grekey)
self.appendChild(a)
self.addNetMembers(network_object)
self.addChannels(network_object)
presentation_element = self.createElement("CORE:presentation")
add_presentation_element = False
if network_object.icon and not network_object.icon.isspace():
presentation_element.setAttribute("icon", network_object.icon)
add_presentation_element = True
if network_object.canvas:
presentation_element.setAttribute("canvas", str(network_object.canvas))
add_presentation_element = True
if add_presentation_element:
self.appendChild(presentation_element)
def getNetworkName(self, scenario_plan, network_object):
"""
Determine the name to use for this network element
:param ScenarioPlan scenario_plan:
:param network_object:
:return:
"""
if nodeutils.is_node(network_object, (NodeTypes.PEER_TO_PEER, NodeTypes.TUNNEL)):
name = "net%s" % scenario_plan.last_network_id
scenario_plan.last_network_id += 1
elif network_object.name:
name = str(network_object.name) # could use net.brname for bridges?
elif nodeutils.is_node(network_object, (NodeTypes.SWITCH, NodeTypes.HUB)):
name = "lan%s" % network_object.objid
else:
name = ''
return name
def addL2Devices(self, network_object):
"""
Add switches and hubs
"""
# Add the netObj as a device
self.l2devices.append(DeviceElement(self.scenPlan, self, network_object))
# Add downstream switches/hubs
l2devs = []
neweps = []
for ep in self.endpoints:
if ep.type and ep.net.objid != network_object.objid:
l2s, eps = get_dowmstream_l2_devices(ep.net)
l2devs.extend(l2s)
neweps.extend(eps)
for l2dev in l2devs:
self.l2devices.append(DeviceElement(self.scenPlan, self, l2dev))
self.endpoints.extend(neweps)
# XXX: Optimize later
def addNetMembers(self, network_object):
"""
Add members to a network XML element.
"""
for ep in self.endpoints:
if ep.type:
MemberElement(self.scenPlan, self, referenced_type=ep.type, referenced_id=ep.id)
if ep.l2devport:
MemberElement(self.scenPlan,
self,
referenced_type=MembType.INTERFACE,
referenced_id="%s/%s" % (self.id, ep.l2devport))
# XXX Revisit this
# Create implied members given the network type
if nodeutils.is_node(network_object, NodeTypes.TUNNEL):
MemberElement(self.scenPlan, self, referenced_type=MembType.TUNNEL,
referenced_id="%s/%s" % (network_object.name, network_object.name))
# XXX: Optimize later
def addChannels(self, network_object):
"""
Add channels to a network XML element
"""
if nodeutils.is_node(network_object, (NodeTypes.WIRELESS_LAN, NodeTypes.EMANE)):
modelconfigs = network_object.session.mobility.get_models(network_object)
modelconfigs += network_object.session.emane.get_models(network_object)
chan = None
for model, conf in modelconfigs:
# Handle mobility parameters below
if model.config_type == RegisterTlvs.MOBILITY.value:
continue
# Create the channel
if chan is None:
name = "wireless"
chan = ChannelElement(self.scenPlan, self, network_object,
channel_type=model.name,
channel_name=name,
channel_domain="CORE")
# Add wireless model parameters
for key, value in conf.iteritems():
if value is not None:
chan.addParameter(key, value)
for model, conf in modelconfigs:
if model.config_type == RegisterTlvs.MOBILITY.value:
# Add wireless mobility parameters
mobility = XmlElement(self.scenPlan, chan, "CORE:mobility")
# Add a type child
type_element = self.createElement("type")
type_element.appendChild(self.createTextNode(model.name))
mobility.appendChild(type_element)
for key, value in conf.iteritems():
if value is not None:
mobility.addParameter(key, value)
# Add members to the channel
if chan is not None:
chan.addChannelMembers(self.endpoints)
self.appendChild(chan.base_element)
elif nodeutils.is_node(network_object, NodeTypes.PEER_TO_PEER):
if len(self.endpoints) < 2:
if len(self.endpoints) == 1:
logger.warn('Pt2Pt network with only 1 endpoint: %s', self.endpoints[0].id)
else:
logger.warn('Pt2Pt network with no endpoints encountered in %s', network_object.name)
return
name = "chan%d" % (0)
chan = ChannelElement(self.scenPlan, self, network_object,
channel_type=NetType.ETHERNET,
channel_name=name)
# Add interface parameters
if self.endpoints[0].params != self.endpoints[1].params:
logger.warn('Pt2Pt Endpoint parameters do not match in %s', network_object.name)
for key, value in self.endpoints[0].params:
# XXX lifted from original addnetem function. revisit this.
# default netem parameters are 0 or None
if value is None or value == 0:
continue
if key == "has_netem" or key == "has_tbf":
continue
chan.addParameter(key, value)
# Add members to the channel
chan.addChannelMembers(self.endpoints)
self.appendChild(chan)
elif nodeutils.is_node(network_object, (NodeTypes.SWITCH, NodeTypes.HUB, NodeTypes.TUNNEL)):
cidx = 0
channels = []
for ep in self.endpoints:
# Create one channel member per ep
if ep.type:
name = "chan%d" % cidx
chan = ChannelElement(self.scenPlan, self, network_object, channel_type=NetType.ETHERNET,
channel_name=name)
# Add interface parameters
for key, value in ep.params:
# XXX lifted from original addnetem function. revisit this.
# default netem parameters are 0 or None
if value is None or value == 0:
continue
if key == "has_netem" or key == "has_tbf":
continue
chan.addParameter(key, value)
# Add members to the channel
chan.addChannelMembers(ep)
channels.append(chan)
cidx += 1
for chan in channels:
self.appendChild(chan)
class DeviceElement(NamedXmlElement):
"""
A device element in the scenario plan.
"""
def __init__(self, scen_plan, parent, device_object):
"""
Add a PyCoreNode object as a device element.
"""
device_type = None
core_device_type = None
if hasattr(device_object, "type") and device_object.type:
core_device_type = device_object.type
if device_object.type in [NodeType.ROUTER, NodeType.MDR]:
device_type = DevType.ROUTER
elif device_object.type == NodeType.HUB:
device_type = DevType.HUB
elif device_object.type == NodeType.SWITCH:
device_type = DevType.SWITCH
# includes known node types (HOST, PC, RJ45)
# Default custom types (defined in ~/.core/nodes.conf) to HOST
else:
device_type = DevType.HOST
if device_type is None:
raise ValueError("unknown device type: %s" % core_device_type)
NamedXmlElement.__init__(self, scen_plan, parent, device_type, device_object.name)
if core_device_type is not None:
type_element = self.createElement("type")
type_element.setAttribute("domain", "CORE")
type_element.appendChild(self.createTextNode("%s" % core_device_type))
self.appendChild(type_element)
self.interfaces = []
self.addInterfaces(device_object)
alias = self.createAlias(Alias.ID, "%s" % device_object.objid)
self.appendChild(alias)
self.addPoint(device_object)
self.addServices(device_object)
presentation_element = self.createElement("CORE:presentation")
add_presentation_element = False
if device_object.icon and not device_object.icon.isspace():
presentation_element.setAttribute("icon", device_object.icon)
add_presentation_element = True
if device_object.canvas:
presentation_element.setAttribute("canvas", str(device_object.canvas))
add_presentation_element = True
if add_presentation_element:
self.appendChild(presentation_element)
def addInterfaces(self, device_object):
"""
Add interfaces to a device element.
"""
idx = 0
for interface_object in device_object.netifs(sort=True):
if interface_object.net and nodeutils.is_node(interface_object.net, NodeTypes.CONTROL_NET):
continue
if isinstance(device_object, nodes.PyCoreNode):
interface_element = InterfaceElement(self.scenPlan, self, device_object, interface_object)
else: # isinstance(node, (nodes.HubNode nodes.SwitchNode)):
interface_element = InterfaceElement(self.scenPlan, self, device_object, interface_object, idx)
idx += 1
netmodel = None
if interface_object.net:
if hasattr(interface_object.net, "model"):
netmodel = interface_object.net.model
if interface_object.mtu and interface_object.mtu != 1500:
interface_element.setAttribute("mtu", "%s" % interface_object.mtu)
# The interfaces returned for Switches and Hubs are the interfaces of the nodes connected to them.
# The addresses are for those interfaces. Don't include them here.
if isinstance(device_object, nodes.PyCoreNode):
# could use ifcObj.params, transport_type
interface_element.addAddresses(interface_object)
# per-interface models
# XXX Remove???
if netmodel and netmodel.name[:6] == "emane_":
cfg = self.coreSession.emane.getifcconfig(device_object.objid, interface_object, netmodel.name)
if cfg:
interface_element.addModels(((netmodel, cfg),))
self.interfaces.append(interface_element)
def addServices(self, device_object):
"""
Add services and their customizations to the ServicePlan.
"""
if not hasattr(device_object, "services"):
return
if len(device_object.services) == 0:
return
defaults = self.coreSession.services.get_default_services(device_object.type)
if device_object.services == defaults:
return
spn = self.createElement("CORE:services")
spn.setAttribute("name", device_object.name)
self.appendChild(spn)
for svc in device_object.services:
s = self.createElement("service")
spn.appendChild(s)
s.setAttribute("name", str(svc.name))
# only record service names if not a customized service
if not svc.custom:
continue
s.setAttribute("custom", str(svc.custom))
xmlutils.add_elements_from_list(self, s, svc.dirs, "directory", "name")
for fn in svc.configs:
if len(fn) == 0:
continue
f = self.createElement("file")
f.setAttribute("name", fn)
# all file names are added to determine when a file has been deleted
s.appendChild(f)
data = svc.config_data.get(fn)
if data is None:
# this includes only customized file contents and skips
# the auto-generated files
continue
txt = self.createTextNode("\n" + data)
f.appendChild(txt)
xmlutils.add_text_elements_from_list(self, s, svc.startup, "command", (("type", "start"),))
xmlutils.add_text_elements_from_list(self, s, svc.shutdown, "command", (("type", "stop"),))
xmlutils.add_text_elements_from_list(self, s, svc.validate, "command", (("type", "validate"),))
class ChannelElement(NamedXmlElement):
"""
A channel element in the scenario plan
"""
def __init__(self, scen_plan, parent, network_object, channel_type, channel_name, channel_domain=None):
NamedXmlElement.__init__(self, scen_plan, parent, "channel", channel_name)
'''
Create a channel element and append a member child referencing this channel element
in the parent element.
'''
# Create a member element for this channel in the parent
MemberElement(self.scenPlan, parent, referenced_type=MembType.CHANNEL, referenced_id=self.id)
# Add a type child
type_element = self.createElement("type")
if channel_domain is not None:
type_element.setAttribute("domain", "%s" % channel_domain)
type_element.appendChild(self.createTextNode(channel_type))
self.appendChild(type_element)
def addChannelMembers(self, endpoints):
"""
Add network channel members referencing interfaces in the channel
"""
if isinstance(endpoints, list):
# A list of endpoints is given. Create one channel member per endpoint
idx = 0
for ep in endpoints:
self.addChannelMember(ep.type, ep.id, idx)
idx += 1
else:
# A single endpoint is given. Create one channel member for the endpoint,
# and if the endpoint is associated with a Layer 2 device port, add the
# port as a second member
ep = endpoints
self.addChannelMember(ep.type, ep.id, 0)
if ep.l2devport is not None:
member_id = "%s/%s" % (self.parent.getAttribute("id"), ep.l2devport)
self.addChannelMember(ep.type, member_id, 1)
def addChannelMember(self, member_interface_type, member_interface_id, member_index):
"""
add a member to a given channel
"""
m = MemberElement(self.scenPlan,
self,
referenced_type=member_interface_type,
referenced_id=member_interface_id,
index=member_index)
self.scenPlan.all_channel_members[member_interface_id] = m
class InterfaceElement(NamedXmlElement):
"""
A network interface element
"""
def __init__(self, scen_plan, parent, device_object, interface_object, interface_index=None):
"""
Create a network interface element with references to channel that this
interface is used.
"""
element_name = None
if interface_index is not None:
element_name = "e%d" % interface_index
else:
element_name = interface_object.name
NamedXmlElement.__init__(self, scen_plan, parent, "interface", element_name)
self.ifcObj = interface_object
self.addChannelReference()
def addChannelReference(self):
"""
Add a reference to the channel that uses this interface
"""
# cm is None when an interface belongs to a switch
# or a hub within a network and the channel is yet to be defined
cm = self.scenPlan.all_channel_members.get(self.id)
if cm is not None:
ch = cm.base_element.parentNode
if ch is not None:
net = ch.parentNode
if net is not None:
MemberElement(self.scenPlan,
self,
referenced_type=MembType.CHANNEL,
referenced_id=ch.getAttribute("id"),
index=int(cm.getAttribute("index")))
MemberElement(self.scenPlan,
self,
referenced_type=MembType.NETWORK,
referenced_id=net.getAttribute("id"))
def addAddresses(self, interface_object):
"""
Add MAC and IP addresses to interface XML elements.
"""
if interface_object.hwaddr:
h = self.createElement("address")
self.appendChild(h)
h.setAttribute("type", "mac")
htxt = self.createTextNode("%s" % interface_object.hwaddr)
h.appendChild(htxt)
for addr in interface_object.addrlist:
a = self.createElement("address")
self.appendChild(a)
(ip, sep, mask) = addr.partition('/')
# mask = int(mask) XXX?
if ipaddress.is_ipv4_address(ip):
a.setAttribute("type", "IPv4")
else:
a.setAttribute("type", "IPv6")
# a.setAttribute("type", )
atxt = self.createTextNode("%s" % addr)
a.appendChild(atxt)
# XXX Remove?
def addModels(self, configs):
"""
Add models from a list of model-class, config values tuples.
"""
for m, conf in configs:
node_element = self.createElement("model")
node_element.setAttribute("name", m.name)
type_string = "wireless"
if m.config_type == RegisterTlvs.MOBILITY.value:
type_string = "mobility"
node_element.setAttribute("type", type_string)
for i, k in enumerate(m.getnames()):
key = self.createElement(k)
value = conf[i]
if value is None:
value = ""
key.appendChild(self.createTextNode("%s" % value))
node_element.appendChild(key)
self.appendChild(node_element)
class MemberElement(XmlElement):
"""
Member elements are references to other elements in the network plan elements of the scenario.
They are used in networks to reference channels, in channels to reference interfaces,
and in interfaces to reference networks/channels. Member elements provided allow bi-directional
traversal of network plan components.
"""
def __init__(self, scene_plan, parent, referenced_type, referenced_id, index=None):
"""
Create a member element
"""
XmlElement.__init__(self, scene_plan.document, parent, "member")
self.setAttribute("type", "%s" % referenced_type)
# See'Understanding the Network Modeling Framework document'
if index is not None:
self.setAttribute("index", "%d" % index)
self.appendChild(self.createTextNode("%s" % referenced_id))
#
# =======================================================================================
# Helpers
# =======================================================================================
def get_endpoint(network_object, interface_object):
"""
Create an Endpoint object given the network and the interface of interest
"""
ep = None
l2devport = None
# skip if either are none
if not network_object or not interface_object:
return ep
# if ifcObj references an interface of a node and is part of this network
if interface_object.net.objid == network_object.objid and hasattr(interface_object,
'node') and interface_object.node:
params = interface_object.getparams()
if nodeutils.is_node(interface_object.net, (NodeTypes.HUB, NodeTypes.SWITCH)):
l2devport = "%s/e%d" % (interface_object.net.name, interface_object.net.getifindex(interface_object))
ep = Endpoint(network_object,
interface_object,
type=MembType.INTERFACE,
id="%s/%s" % (interface_object.node.name, interface_object.name),
l2devport=l2devport,
params=params)
# else if ifcObj references another node and is connected to this network
elif hasattr(interface_object, "othernet"):
if interface_object.othernet.objid == network_object.objid:
# #hack used for upstream parameters for link between switches
# #(see LxBrNet.linknet())
interface_object.swapparams('_params_up')
params = interface_object.getparams()
interface_object.swapparams('_params_up')
owner = interface_object.net
l2devport = "%s/e%d" % (
interface_object.othernet.name, interface_object.othernet.getifindex(interface_object))
# Create the endpoint.
# XXX the interface index might not match what is shown in the gui. For switches and hubs,
# The gui assigns its index but doesn't pass it to the daemon and vice versa.
# The gui stores it's index in the IMN file, which it reads and writes without daemon intervention.
# Fix this!
ep = Endpoint(owner,
interface_object,
type=MembType.INTERFACE,
id="%s/%s/e%d" % (network_object.name, owner.name, owner.getifindex(interface_object)),
l2devport=l2devport,
params=params)
# else this node has an interface that belongs to another network
# i.e. a switch/hub interface connected to another switch/hub and CORE has the other switch/hub
# as the containing network
else:
ep = Endpoint(network_object, interface_object, type=None, id=None, l2devport=None, params=None)
return ep
def get_endpoints(network_object):
"""
Gather all endpoints of the given network
"""
# Get all endpoints
endpoints = []
# XXX TODO: How to represent physical interfaces.
#
# NOTE: The following code works except it would be missing physical (rj45) interfaces from Pt2pt links
# TODO: Fix data in net.netifs to include Pt2Pt physical interfaces
#
# Iterate through all the nodes in the scenario, then iterate through all the interface for each node,
# and check if the interface is connected to this network.
for interface_object in network_object.netifs(sort=True):
try:
ep = get_endpoint(network_object, interface_object)
if ep is not None:
endpoints.append(ep)
except:
logger.debug("error geting endpoints, was skipped before")
return endpoints
def get_dowmstream_l2_devices(network_object):
"""
Helper function for getting a list of all downstream layer 2 devices from the given netObj
"""
l2_device_objects = [network_object]
allendpoints = []
myendpoints = get_endpoints(network_object)
allendpoints.extend(myendpoints)
for ep in myendpoints:
if ep.type and ep.net.objid != network_object.objid:
l2s, eps = get_dowmstream_l2_devices(ep.net)
l2_device_objects.extend(l2s)
allendpoints.extend(eps)
return l2_device_objects, allendpoints
def get_all_network_interfaces(session):
"""
Gather all network interfacecs in the session
"""
netifs = []
for node in session.objects.itervalues():
for netif in node.netifs(sort=True):
if netif not in netifs:
netifs.append(netif)
return netifs
def in_other_network(network_object):
"""
Determine if CORE considers a given network object to be part of another network.
Note: CORE considers layer 2 devices to be their own networks. However, if a l2 device
is connected to another device, it is possible that one of its ports belong to the other
l2 device's network (thus, "othernet").
"""
for netif in network_object.netifs(sort=True):
if hasattr(netif, "othernet"):
if netif.othernet.objid != network_object.objid:
return True
return False

8
daemon/setup.py
View file

@ -14,14 +14,14 @@ _EXAMPLES_DIR = "share/core"
def recursive_files(data_path, files_path):
data_files = []
for path, directories, filenames in os.walk(files_path):
all_files = []
for path, _directories, filenames in os.walk(files_path):
directory = os.path.join(data_path, path)
files = []
for filename in filenames:
files.append(os.path.join(path, filename))
data_files.append((directory, files))
return data_files
all_files.append((directory, files))
return all_files
def glob_files(glob_path):

30
daemon/tests/test_xml.py
View file

@ -8,15 +8,9 @@ from core.enumerations import NodeTypes
from core.mobility import BasicRangeModel
from core.services.utility import SshService
_XML_VERSIONS = [
"0.0",
"1.0"
]
class TestXml:
@pytest.mark.parametrize("version", _XML_VERSIONS)
def test_xml_hooks(self, session, tmpdir, version):
def test_xml_hooks(self, session, tmpdir):
"""
Test save/load hooks in xml.
@ -32,7 +26,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
session.save_xml(file_path, version)
session.save_xml(file_path)
# verify xml file was created and can be parsed
assert xml_file.isfile()
@ -51,8 +45,7 @@ class TestXml:
assert file_name == runtime_hook[0]
assert data == runtime_hook[1]
@pytest.mark.parametrize("version", _XML_VERSIONS)
def test_xml_ptp(self, session, tmpdir, version, ip_prefixes):
def test_xml_ptp(self, session, tmpdir, ip_prefixes):
"""
Test xml client methods for a ptp network.
@ -83,7 +76,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
session.save_xml(file_path, version)
session.save_xml(file_path)
# verify xml file was created and can be parsed
assert xml_file.isfile()
@ -105,8 +98,7 @@ class TestXml:
assert session.get_object(n1_id)
assert session.get_object(n2_id)
@pytest.mark.parametrize("version", _XML_VERSIONS)
def test_xml_ptp_services(self, session, tmpdir, version, ip_prefixes):
def test_xml_ptp_services(self, session, tmpdir, ip_prefixes):
"""
Test xml client methods for a ptp neetwork.
@ -144,7 +136,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
session.save_xml(file_path, version)
session.save_xml(file_path)
# verify xml file was created and can be parsed
assert xml_file.isfile()
@ -170,8 +162,7 @@ class TestXml:
assert session.get_object(n2_id)
assert service.config_data.get(service_file) == file_data
@pytest.mark.parametrize("version", _XML_VERSIONS)
def test_xml_mobility(self, session, tmpdir, version, ip_prefixes):
def test_xml_mobility(self, session, tmpdir, ip_prefixes):
"""
Test xml client methods for mobility.
@ -206,7 +197,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
session.save_xml(file_path, version)
session.save_xml(file_path)
# verify xml file was created and can be parsed
assert xml_file.isfile()
@ -233,8 +224,7 @@ class TestXml:
assert session.get_object(wlan_id)
assert value == "1"
@pytest.mark.parametrize("version", ["1.0"])
def test_xml_emane(self, session, tmpdir, version, ip_prefixes):
def test_xml_emane(self, session, tmpdir, ip_prefixes):
"""
Test xml client methods for emane.
@ -274,7 +264,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
session.save_xml(file_path, version)
session.save_xml(file_path)
# verify xml file was created and can be parsed
assert xml_file.isfile()

16
ns3/corens3/obj.py
View file

@ -102,7 +102,7 @@ class CoreNs3Node(CoreNode, ns.network.Node):
mm = self.GetObject(ns.mobility.MobilityModel.GetTypeId())
if z is None:
z = 0.0
pos = mm.SetPosition(ns.core.Vector(x, y, z))
mm.SetPosition(ns.core.Vector(x, y, z))
except AttributeError:
self.warn("ns-3 mobility model not found, not setting position")
@ -209,10 +209,10 @@ class Ns3LteNet(CoreNs3Net):
Register user equipment with a nodeb.
Optionally install mobility model while we have the ns-3 devs handy.
"""
tmp, nodebdev = self.findns3dev(nodeb)
tmp, dev = self.findns3dev(node)
_tmp, nodebdev = self.findns3dev(nodeb)
_tmp, dev = self.findns3dev(node)
if nodebdev is None or dev is None:
raise KeyError, "ns-3 device for node not found"
raise KeyError("ns-3 device for node not found")
self.lte.RegisterUeToTheEnb(dev, nodebdev)
if mob:
self.lte.AddMobility(dev.GetPhy(), mob)
@ -364,7 +364,7 @@ class Ns3Session(Session):
self.duration = duration
self.nodes = ns.network.NodeContainer()
self.mobhelper = ns.mobility.MobilityHelper()
Session.__init__(self, session_id, persistent=persistent)
Session.__init__(self, session_id)
def run(self, vis=False):
"""
@ -488,7 +488,7 @@ class Ns3Session(Session):
Start a tracing thread using the ASCII output from the ns3
mobility helper.
"""
net.mobility = WayPointMobility(session=self, object_id=net.objid, config=None)
net.mobility = WayPointMobility(session=self, object_id=net.objid)
net.mobility.setendtime()
net.mobility.refresh_ms = 300
net.mobility.empty_queue_stop = False
@ -528,7 +528,7 @@ class Ns3Session(Session):
sleep += 0.001
continue
sleep = 0.001
items = dict(map(lambda x: x.split('='), line.split()))
items = dict(x.split("=") for x in line.split())
logger.info("trace: %s %s %s", items['node'], items['pos'], items['vel'])
x, y, z = map(float, items['pos'].split(':'))
vel = map(float, items['vel'].split(':'))
@ -544,7 +544,7 @@ class Ns3Session(Session):
net.mobility.state = net.mobility.STATE_RUNNING
self.event_loop.add_event(0, net.mobility.runround)
except IOError:
logger.exception("mobilitytrace error opening '%s': %s", filename)
logger.exception("mobilitytrace error opening: %s", filename)
finally:
if f:
f.close()

9
ns3/examples/ns3lte.py
View file

@ -27,12 +27,9 @@ def ltesession(opt):
lte = session.add_object(cls=Ns3LteNet, name="wlan1")
lte.setsubchannels(range(25), range(50, 100))
if opt.verbose:
ascii = ns.network.AsciiTraceHelper()
stream = ascii.CreateFileStream('/tmp/ns3lte.tr')
ascii_helper = ns.network.AsciiTraceHelper()
stream = ascii_helper.CreateFileStream('/tmp/ns3lte.tr')
lte.lte.EnableAsciiAll(stream)
# ns.core.LogComponentEnable("EnbNetDevice", ns.core.LOG_LEVEL_INFO)
# ns.core.LogComponentEnable("UeNetDevice", ns.core.LOG_LEVEL_INFO)
# lte.lte.EnableLogComponents()
prefix = ipaddress.Ipv4Prefix("10.0.0.0/16")
mobb = None
@ -48,7 +45,7 @@ def ltesession(opt):
node.newnetif(lte, ["%s/%s" % (prefix.addr(i), prefix.prefixlen)])
nodes.append(node)
if i == 1:
(tmp, ns3dev) = lte.findns3dev(node)
_tmp, ns3dev = lte.findns3dev(node)
lte.lte.AddMobility(ns3dev.GetPhy(), mob)
if i > 1:
lte.linknodeb(node, nodes[0], mob, mobb)

0
scripts/perf/configuration_hook.sh Normal file → Executable file
View file

0
scripts/perf/datacollect_hook.sh Normal file → Executable file
View file

699
scripts/perf/perflogserver.py
View file

@ -2,25 +2,35 @@
#
# (c)2011-2012 the Boeing Company
#
# perfmon.py - CORE server and node performace metrics logger and alarmer
# server metrics: loadave1, 5, 15, mem, used cpu% of total, cpu1, cpu2, ..., cpun
# node metrics: throughput, mem, cpu total, usr, sys, wait
#
import os, sys, time, re, optparse, signal, commands, pdb
"""
perflogserver.py - CORE server and node performace metrics logger and alarmer
server metrics: loadave1, 5, 15, mem, used cpu% of total, cpu1, cpu2, ..., cpun
node metrics: throughput, mem, cpu total, usr, sys, wait
"""
import commands
import optparse
import os
import signal
import sys
import time
def readfile(fname):
lines=[]
lines = []
try:
f = open(fname, "r")
except:
except IOError:
if options.timestamp == True:
print str(time.time()),
print "ERROR: failed to open file %s\n" % fname
else :
lines = f.readlines()
f.close()
print str(time.time()),
print "ERROR: failed to open file %s\n" % fname
else:
lines = f.readlines()
f.close()
return lines
def numcpus():
lines = readfile("/proc/stat")
n = 0
@ -30,35 +40,36 @@ def numcpus():
n += 1
return n
def handler(signum, frame):
print "stop timestamp:", str(time.time()) + ", cyclecount=", cyclecount, ", caught signal", signum
print "stop timestamp:", str(
time.time()) + ", cyclecount=", cyclecount, ", caught signal", signum
sys.exit(0)
class ServerMetrics(object):
def __init__(self):
self.smetrics = { "serverloadavg1" : 0.0,
"serverloadavg5" : 0.0,
"serverloadavg15" : 0.0,
"serverusedmemory" : 0.0,
"serverusedcputime" : 0.0,
"processorusedcputime" : [] }
self.smetrics = {"serverloadavg1": 0.0,
"serverloadavg5": 0.0,
"serverloadavg15": 0.0,
"serverusedmemory": 0.0,
"serverusedcputime": 0.0,
"processorusedcputime": []}
# set values from val = (nump, ldavg1, ldavg5, adavg15, mem, cpu, p1cpu, p2cpu...)
def setvalues(self, val):
"""
Set values from val = (nump, ldavg1, ldavg5, adavg15, mem, cpu, p1cpu, p2cpu...).
"""
self.smetrics["serverloadavg1"] = val[0]
self.smetrics["serverloadavg5"] = val[1]
self.smetrics["serverloadavg15"] = val[2]
self.smetrics["serverusedmemory"] = val[4]
self.smetrics["serverusedcputime"] = val[5]
#print self.smetrics.keys(), self.smetrics.values()
pcpu = []
for ind in range(5,len(val)):
pcpu.append(val[ind])
# print "[" + ",".join(map(lambda(x):str(round(x, 2)), pcpu)) +"]"
pcpu = []
for ind in range(5, len(val)):
pcpu.append(val[ind])
self.smetrics["processorusedcputime"] = pcpu
#print self.smetrics.keys(), self.smetrics.values()
def setvalue(self, key, val):
self.smetrics[key] = val
@ -70,457 +81,470 @@ class ServerMetrics(object):
return self.smetrics.keys()
def tocsv(self):
rv = "Server"
rv = "Server"
for k in self.smetrics:
# print k, self.smetrics[k]
if isinstance(self.smetrics[k], float):
rv += ", %.2f" % self.smetrics[k]
else:
else:
if isinstance(self.smetrics[k], list):
rv += ", [" + \
", ".join(map(lambda(x):str(round(x, 2)), self.smetrics[k])) \
+ "]"
else:
rv += ", " + str(self.smetrics[k])
return rv
values = ", ".join(str(round(x, 2)) for x in self.smetrics[k])
rv += ", [%s]" % values
else:
rv += ", " + str(self.smetrics[k])
return rv
def readserverthresholds(filename):
if filename is None:
return
return
lines = readfile(filename)
for l in lines:
mval = l.strip().split('=')
#print "read line %s" % mval
if len(mval) > 1 :
thekey = mval[0].strip()
theval = mval[1].strip()
if thekey in serverthresholds.getkeys():
serverthresholds.setvalue(thekey, float(theval))
# print thekey," = %.2f" % float(theval)
def checkserverthreshold(metricval):
# print out an alarm if a ServerMetrics value crosses threshold
mval = l.strip().split('=')
if len(mval) > 1:
thekey = mval[0].strip()
theval = mval[1].strip()
if thekey in serverthresholds.getkeys():
serverthresholds.setvalue(thekey, float(theval))
def checkserverthreshold(metricval):
"""
Print out an alarm if a ServerMetrics value crosses threshold.
"""
for key in serverthresholds.getkeys():
# print "checking threshold of key = ", key
if key == "processorusedcputime":
pcpus = metricval.getvalue(key)
# print key, pcpus, serverthresholds[key]
for ind in range(0, len(pcpus)):
# print ind, pcpus[ind]
if pcpus[ind] > serverthresholds.getvalue(key):
if key == "processorusedcputime":
pcpus = metricval.getvalue(key)
for ind, pcpu in enumerate(pcpus):
if pcpu > serverthresholds.getvalue(key):
alarm = ["server", os.uname()[1], str(ind) + key,
"%.2f" % pcpus[ind], ">", serverthresholds.getvalue(key)]
"%.2f" % pcpus[ind], ">", serverthresholds.getvalue(key)]
if options.timestamp:
print str(time.time()) + ",",
print ", ".join(map(lambda(x):str(x), alarm))
else:
if metricval.getvalue(key) > serverthresholds.getvalue(key):
print str(time.time()) + ",",
print ", ".join(str(x) for x in alarm)
else:
if metricval.getvalue(key) > serverthresholds.getvalue(key):
alarm = ["server", os.uname()[1], key,
"%.2f" % metricval.getvalue(key), ">", serverthresholds.getvalue(key)]
"%.2f" % metricval.getvalue(key), ">", serverthresholds.getvalue(key)]
if options.timestamp:
print str(time.time()) + ",",
print ", ".join(map(lambda(x):str(x), alarm))
print str(time.time()) + ",",
print ", ".join(str(x) for x in alarm)
def collectservercputimes():
# return cpu times in ticks of this server total and each processor 3*(1+#cpu) columns
# (user+nice, sys, idle) from each /proc/stat cpu lines assume columns are:
# cpu# user nice sys idle iowait irq softirq steal guest (man 5 proc)
"""
Return cpu times in ticks of this server total and each processor 3*(1+#cpu) columns
(user+nice, sys, idle) from each /proc/stat cpu lines assume columns are:
cpu# user nice sys idle iowait irq softirq steal guest (man 5 proc)
"""
rval = {}
lines = readfile("/proc/stat")
for i in range(ncpus + 1):
items = lines[i].split()
(user, nice, sys, idle) = map(lambda(x): int(x), items[1:5])
rval[i] = [user+nice, sys, idle]
return rval
def csvservercputimes(cputimes):
# return a csv string of this server total and each processor's cpu times
# (usr, sys, idle) in ticks
rval = ''
for i in range(len(cputimes)):
rval += ', '.join(map(lambda(x):str(x), cputimes[i]))
user, nice, sys, idle = [int(x) for x in items[1:5]]
rval[i] = [user+nice, sys, idle]
return rval
def csvservercputimes(cputimes):
"""
Return a csv string of this server total and each processor's cpu times
(usr, sys, idle) in ticks.
"""
rval = ''
for i in range(len(cputimes)):
rval += ", ".join(str(x) for x in cputimes[i])
return rval
def calcservercputimes(cputimea, cputimeb):
# return cpu used/total % of this server total and each processor (1+#cpu columns)
"""
Return cpu used/total % of this server total and each processor (1+#cpu columns).
"""
p = {}
for n in range(ncpus + 1):
# print cputimeb[n]
p[n] = []
for i in range(len(cputimea[n])):
p[n].append(cputimeb[n][i] - cputimea[n][i])
# print p[n]
total = sum(p[n]) # cpu times total delta
# print total
# cpu times total delta
total = sum(p[n])
if total == 0:
p[n] = 0.0
else:
p[n] = 100 - ((100.0 * p[n][-1]) / total)
return p
def collectservermems():
# return memory (total, free) in KB from proc/meminfo
def collectservermems():
"""
Return memory (total, free) in KB from proc/meminfo.
"""
lines = readfile("/proc/meminfo")
mem = map(lambda(x):x.split(), lines[0:2])
return map(lambda(x):int(x), zip(*mem)[1])
mem = [x.split() for x in lines[0:2]]
return [int(x) for x in zip(*mem)[1]]
def csvservermems(mems):
# return a csv string of this server memory (total, free)
return ", ".join(map(lambda x: str(x), mems))
def calcserverusedmem(mems):
# return int(100*(MemTotal-MemFree)/MemTotal) from /proc/meminfo
def csvservermems(mems):
"""
Return a csv string of this server memory (total, free).
"""
return ", ".join(str(x) for x in mems)
def calcserverusedmem(mems):
"""
Return int(100*(MemTotal-MemFree)/MemTotal) from /proc/meminfo.
"""
return 100 * (mems[0] - mems[1]) / mems[0]
def collectservermetrics(cputimes, mems, thresholdcheck):
# return ServerMetrics object with a dictionary of
# loadavg1,loadavg5,loadavg15, usedmem%, usedcpu% for total, cpu1, cpu2, ...
"""
Return ServerMetrics object with a dictionary of
loadavg1,loadavg5,loadavg15, usedmem%, usedcpu% for total, cpu1, cpu2, ...
"""
metricval = []
ldavgs=os.getloadavg()
ldavgs = os.getloadavg()
for v in ldavgs:
metricval.append(v)
metricval.append(calcserverusedmem(mems))
for i in range(ncpus + 1):
metricval.append(cputimes[i])
# print cputimes[i]
metricval.append(cputimes[i])
srvmetrics = ServerMetrics()
srvmetrics.setvalues(metricval)
# print srvmetrics.tocsv()
if thresholdcheck:
checkserverthreshold(srvmetrics)
checkserverthreshold(srvmetrics)
return srvmetrics
def csvservermetrics(srvmetrics):
# return a csv string of ServerMetrics.tocsv()
# loadavg1,loadavg5,loadavg15, usedmem%, usedcpu% for total, cpu1, cpu2, ...
"""
Return a csv string of ServerMetrics.tocsv()
loadavg1,loadavg5,loadavg15, usedmem%, usedcpu% for total, cpu1, cpu2, ...
"""
rv = ""
if options.timestamp:
rv = str(time.time()) + ", "
rv = str(time.time()) + ", "
rv += srvmetrics.tocsv()
return rv
def csvserverbaseline():
# return a csv string of raw server metrics data: memfree, memtotal, cpuused, cpusystem, cpuidle
"""
Return a csv string of raw server metrics data: memfree, memtotal, cpuused, cpusystem, cpuidle.
"""
return "memory (total, free) = " + csvservermems(collectservermems()) + "\ncputime (used, sys, idl) = " + csvservercputimes(collectservercputimes())
class NodeMetrics(object):
def __init__(self):
self.nmetrics = {"nodethroughput" : 0.0,
"nodeusedmemory" : 0.0,
"nodetotalcpu" : 0.0,
"nodeusercpu" : 0.0,
"nodesystemcpu" : 0.0,
"nodewaitcpu" : 0.0}
self.nmetrics = {"nodethroughput": 0.0,
"nodeusedmemory": 0.0,
"nodetotalcpu": 0.0,
"nodeusercpu": 0.0,
"nodesystemcpu": 0.0,
"nodewaitcpu": 0.0}
# set values from val = (throughput, mem, tcpu, ucpu, scpu, wcpu):
def setvalues(self, val):
self.nmetrics["nodethroughput"] = val[0]
self.nmetrics["nodeusedmemory"] = val[1]
self.nmetrics["nodetotalcpu"] = val[2]
self.nmetrics["nodeusercpu"] = val[3]
self.nmetrics["nodesystemcpu"] = val[4]
self.nmetrics["nodewaitcpu"] = val[5]
"""
Set values from val = (throughput, mem, tcpu, ucpu, scpu, wcpu).
"""
self.nmetrics["nodethroughput"] = val[0]
self.nmetrics["nodeusedmemory"] = val[1]
self.nmetrics["nodetotalcpu"] = val[2]
self.nmetrics["nodeusercpu"] = val[3]
self.nmetrics["nodesystemcpu"] = val[4]
self.nmetrics["nodewaitcpu"] = val[5]
def setvalue(self, key, val):
self.nmetrics[key] = val
self.nmetrics[key] = val
def getvalue(self, key):
return self.nmetrics[key]
return self.nmetrics[key]
def getkeys(self):
return self.nmetrics.keys()
return self.nmetrics.keys()
def tocsv(self):
return ", ".join(map(lambda(x):str(x), self.nmetrics.values()))
return ", ".join(str(x) for x in self.nmetrics.values())
class LogSession(object):
def __init__(self):
self.nodethresholds = NodeMetrics()
# set node threshold default values:
# nodethroughput=20.0, nodeusedmemory=15.0, nodetotalcpu=90.0,
# nodeusercpu=30.0, nodewaitcpu=50.0, nodesystemcpu=20.0}
self.nodethresholds.setvalues([20.0, 15.0, 90.0, 30.0, 50.0, 20.0])
if options.configfile is not None:
self.readnodethresholds(options.configfile)
# set node threshold default values:
# nodethroughput=20.0, nodeusedmemory=15.0, nodetotalcpu=90.0,
# nodeusercpu=30.0, nodewaitcpu=50.0, nodesystemcpu=20.0}
self.nodethresholds.setvalues([20.0, 15.0, 90.0, 30.0, 50.0, 20.0])
if options.configfile is not None:
self.readnodethresholds(options.configfile)
self.pids = {}
self.nodemetricsA = {}
self.nodemetricsB = {}
self.nodemetricsC = {}
def getpids(self):
# return dict of all CORE session pids in a dict using node name as the keys
# parent pid (vnoded) is the first value
def getpids(self):
"""
Return dict of all CORE session pids in a dict using node name as the keys
parent pid (vnoded) is the first value.
"""
self.pids = {}
nodes = commands.getstatusoutput("ls /tmp/pycore.%s/*pid" % options.session)
nodes = commands.getstatusoutput(
"ls /tmp/pycore.%s/*pid" % options.session)
if nodes[0] != 0:
# if options.timestamp == True:
# print str(time.time()),
# print "CORE session %s has not created nodes" % options.session
return
return
nodes = nodes[1].split('\n')
for nod in nodes:
nodename = nod.split('/')[-1].strip(".pid")
self.pids[nodename] = commands.getoutput("cat %s" % nod)
nodename = nod.split('/')[-1].strip(".pid")
self.pids[nodename] = commands.getoutput("cat %s" % nod)
# do not expect failure of this command
procs = commands.getoutput('ps -eo ppid,pid,comm').split('\n')
# build self.pids dict with key=nodename and val="ppid,pid,cmd"
for nname in self.pids:
# print nname, self.pids[nname]
if self.pids[nname] == "":
if options.timestamp == True:
print str(time.time()),
print "ERROR: null vnoded pid of node: %s" % nname
else:
childprocs = []
ppid = self.pids[nname]
if options.timestamp == True:
print str(time.time()),
print "ERROR: null vnoded pid of node: %s" % nname
else:
childprocs = []
ppid = self.pids[nname]
for proc in procs:
val=proc.split()
if ppid == val[1]:
childprocs.append([val[1], val[2]] )
if ppid == val[0]:
childprocs.append([val[1], val[2]])
val = proc.split()
if ppid == val[1]:
childprocs.append([val[1], val[2]])
if ppid == val[0]:
childprocs.append([val[1], val[2]])
self.pids[nname] = childprocs
# print nname, self.pids[nname]
return self.pids
def printsesspids(self):
if self.pids == {}:
return {}
for pp in self.pids:
if self.pids[pp] != []:
if self.pids[pp] != []:
for ap in range(len(self.pids[pp]) - 1):
print ", " + self.pids[pp][ap][0], # ap pid
print ", " + self.pids[pp][ap][1], # ap cmd
procmetrics = map(lambda(x):str(x),self.pids[pp][ap][-1])
print ", " + ", ".join(procmetrics),
nodemetrics = map(lambda(x):str(x), self.pids[pp][-1])
print ", " + ", ".join(nodemetrics)
# ap pid
print ", " + self.pids[pp][ap][0],
# ap cmd
print ", " + self.pids[pp][ap][1],
procmetrics = [str(x) for x in self.pids[pp][ap][-1]]
print ", " + ", ".join(procmetrics),
nodemetrics = [str(x) for x in self.pids[pp][-1]]
print ", " + ", ".join(nodemetrics)
def getprocessmetrics(self, pid):
# return [cpu#, vsize(kb), ttime, utime, stime, wtime]
# from a /proc/pid/stat (a single line file) assume columns are:
# pid(0) comm(1) state ppid pgrp sess tty_nr tpgid flags
# minflt cmiflt majflt cmajflt # utime(12) stime cutime cstime
# priority nice num_threads itrealvalue starttime vsize(22) rss rsslim
# startcode endcode startstack kstkesp signal blocked sigignore sigcatch
# wchan nswap cnswap exit_signal processor(38) rt_priority
# policy ioblock guest_time cguest_time (man 5 proc)
#rval = ProcessMetrics()
#rval.__init__()
"""
Return [cpu#, vsize(kb), ttime, utime, stime, wtime]
from a /proc/pid/stat (a single line file) assume columns are:
pid(0) comm(1) state ppid pgrp sess tty_nr tpgid flags
minflt cmiflt majflt cmajflt # utime(12) stime cutime cstime
priority nice num_threads itrealvalue starttime vsize(22) rss rsslim
startcode endcode startstack kstkesp signal blocked sigignore sigcatch
wchan nswap cnswap exit_signal processor(38) rt_priority
policy ioblock guest_time cguest_time (man 5 proc)
"""
rval = {}
lines = readfile("/proc/" + pid + "/stat")
if lines == []:
return rval
return rval
items = lines[0].split()
(utime, stime, cutime, cstime) = map(lambda(x):int(x), items[13:17])
# print ">???", utime, stime, cutime, cstime
utime, stime, cutime, cstime = [int(x) for x in items[13:17]]
rval = (items[38], # last run processor
int(items[22])/1000, # process virtual mem in kb
utime + stime + cutime + cstime,# totoal time
int(items[22])/1000, # process virtual mem in kb
utime + stime + cutime + cstime, # totoal time
utime, # user time
stime, # system time
cutime + cstime) # wait time
# print "pid --- processmetrics", rval
return rval
def getnodethroughput(self, pid):
# return node throughput of total receive and transmit packets in kb
"""
Return node throughput of total receive and transmit packets in kb.
"""
lines = readfile("/proc/" + pid + "/net/dev")
if lines == []:
return -0.00
ifs = map(lambda(x): x.split(), lines[2:])
return -0.00
ifs = [x.split() for x in lines[2:]]
ifm = zip(*ifs)
rv = sum(map(lambda(x):int(x), ifm[1])) # received bytes
tr = sum(map(lambda(x):int(x), ifm[9])) # transmited bytes
#print 'node thruput :', rv, tr, (rv + tr)/1000
rv = sum(int(x) for x in ifm[1]) # received bytes
tr = sum(int(x) for x in ifm[9]) # transmited bytes
return (rv + tr)/1000
def getnodemetrics(self, mindex):
# return NodeMetrics with indexed by nodename, values are rows of
# [ [ppid, vnoded, [cpu#, vmem(kb), ttime, utime, stime, wtime]],
# [cpid, cmd, [cpu#, vmem(kb), ttime, utime, stime, wtime]], ... ,
# [thrput, vmem(kb), ttime, utime, stime, wtime]]
if mindex == 'a':
metricref = self.nodemetricsA
else:
metricref = self.nodemetricsB
"""
Return NodeMetrics with indexed by nodename, values are rows of
[ [ppid, vnoded, [cpu#, vmem(kb), ttime, utime, stime, wtime]],
[cpid, cmd, [cpu#, vmem(kb), ttime, utime, stime, wtime]], ... ,
[thrput, vmem(kb), ttime, utime, stime, wtime]]
"""
if mindex == 'a':
metricref = self.nodemetricsA
else:
metricref = self.nodemetricsB
self.getpids()
# print " inside getnodemetrics()", self.pids
if self.pids == {}:
return {}
return {}
for nod in self.pids:
nmetric = NodeMetrics()
nmetric.__init__()
nodeapps = {}
for ap in range(len(self.pids[nod])): # get each process metrics
procm = self.getprocessmetrics(self.pids[nod][ap][0])
if procm == []:
if options.timestamp == True:
print str(time.time()),
print "WARNING: transient process", self.pids[nod][ap][1], \
"/", self.pids[nod][ap][0], "on node %s" % nod
else:
nodeapps[ap] = procm
self.pids[nod][ap].append(nodeapps[ap])
processm = zip(*nodeapps.values()) # get overall node metrics
# print processm
if len(processm) > 0:
# if nod == 'n6':
# print nod, self.getnodethroughput(self.pids[nod][0][0])
nmetric.setvalues(( self.getnodethroughput(self.pids[nod][0][0]),
sum(map(lambda(x):int(x), processm[1])), # vsize(kb)
sum(map(lambda(x):int(x), processm[2])), # ttime
sum(map(lambda(x):int(x), processm[3])), # utime
sum(map(lambda(x):int(x), processm[4])), # stime
sum(map(lambda(x):int(x), processm[5])))) # wtime
for ap in range(len(self.pids[nod])): # get each process metrics
procm = self.getprocessmetrics(self.pids[nod][ap][0])
if procm == []:
if options.timestamp == True:
print str(time.time()),
print "WARNING: transient process", self.pids[nod][ap][1], \
"/", self.pids[nod][ap][0], "on node %s" % nod
else:
nodeapps[ap] = procm
self.pids[nod][ap].append(nodeapps[ap])
processm = zip(*nodeapps.values()) # get overall node metrics
if len(processm) > 0:
nmetric.setvalues((self.getnodethroughput(self.pids[nod][0][0]),
# vsize(kb)
sum(int(x) for x in processm[1]),
# ttime
sum(int(x) for x in processm[2]),
# utime
sum(int(x) for x in processm[3]),
# stime
sum(int(x) for x in processm[4]),
sum(int(x) for x in processm[5]))) # wtime
metricref[nod] = nmetric
# print nod, self.pids[nod][0][0], metricref[nod].tocsv()
return metricref
def setnodemetricsC(self, key, val):
self.nodemetricsC[key] = val
def setnodemetricsC(self, key, val):
self.nodemetricsC[key] = val
def printnodemetrics(self, mindex):
def printnodemetrics(self, mindex):
if mindex == 'c':
mm = self.nodemetricsC
else:
if mindex == 'a':
mm = self.nodemetricsA
else:
mm = self.nodemetricsB
mm = self.nodemetricsC
else:
if mindex == 'a':
mm = self.nodemetricsA
else:
mm = self.nodemetricsB
for k in self.nodemetricsC:
for k in self.nodemetricsC:
if options.timestamp:
print str(time.time()) + ",",
print k, ",", mm[k].tocsv()
print k, ",", mm[k].tocsv()
def readnodethresholds(self, filename):
if filename is None:
return
lines = readfile(filename)
def readnodethresholds(self, filename):
if filename is None:
return
lines = readfile(filename)
for l in lines:
mval = l.strip().split('=')
# print "read line %s" % mval
if len(mval) > 1 :
thekey = mval[0].strip()
theval = mval[1].strip()
if thekey in self.nodethresholds.getkeys():
self.nodethresholds.setvalue(thekey, float(theval))
#print thekey," = %.2f" % float(theval)
mval = l.strip().split('=')
if len(mval) > 1:
thekey = mval[0].strip()
theval = mval[1].strip()
if thekey in self.nodethresholds.getkeys():
self.nodethresholds.setvalue(thekey, float(theval))
def checknodethresholds(self, nname):
# print "check node thresholds", nname
calcm = self.nodemetricsC[nname]
calcm = self.nodemetricsC[nname]
for keyname in self.nodethresholds.getkeys():
# print "HIII", keyname, calcm.getvalue(keyname), self.nodethresholds.getvalue(keyname)
if float(calcm.getvalue(keyname)) > float(self.nodethresholds.getvalue(keyname)):
# print calculatednodem.getvalue(m)
alarm = ["node", nname + "/" + self.pids[nname][0][0], keyname,\
calcm.getvalue(keyname), ">", self.nodethresholds.getvalue(keyname)]
if float(calcm.getvalue(keyname)) > float(self.nodethresholds.getvalue(keyname)):
alarm = ["node", nname + "/" + self.pids[nname][0][0], keyname,
calcm.getvalue(keyname), ">", self.nodethresholds.getvalue(keyname)]
if options.timestamp:
print str(time.time()) + ",",
print ", ".join(map(lambda(x):str(x), alarm))
print ", ".join(str(x) for x in alarm)
def calcnodemetrics(self, cputimea, cputimeb, mems):
# return a dict of nodemetrics indexed by node name
# nodemetrics[nodename][-1] = node/host%
"""
Return a dict of nodemetrics indexed by node name
nodemetrics[nodename][-1] = node/host%.
"""
p = []
for i in range(len(cputimeb[0])):
p.append(cputimeb[0][i] - cputimea[0][i])
hostusedcpu = p[0] + p[1]
hostusedmem = mems[0] - mems[1]
if hostusedcpu == 0:
print "WARNING: host used cpu = 0, ", p[0], p[1]
hostusedcpu = 1
print "WARNING: host used cpu = 0, ", p[0], p[1]
hostusedcpu = 1
if hostusedmem == 0:
print "WARNING: host used mem = 0, ", mems[0], mems[1]
hostusedmem = 1
print "WARNING: host used mem = 0, ", mems[0], mems[1]
hostusedmem = 1
nodesa = self.nodemetricsA
nodesb = self.nodemetricsB
nodesa = self.nodemetricsA
nodesb = self.nodemetricsB
for nod in nodesb:
calcm = self.nodemetricsC
calcm = self.nodemetricsC
calcm = NodeMetrics()
calcm.__init__()
if (nod in nodesa):
try:
if (nodesb[nod] == []) | (nodesa[nod] == []) | \
( False == isinstance(nodesb[nod], NodeMetrics)) | \
( False == isinstance(nodesa[nod], NodeMetrics)):
if options.timestamp == True:
print str(time.time()),
print "Warning: nodes %s is not fully instanciated" % nod
else:
# calc throughput kbps
#print "node b : ", nodesb[nod].tocsv()
#print "node a : ", nodesa[nod].tocsv()
#if nod == 'n6':
#print nodesb[nod].getvalue("nodethroughput"), nodesa[nod].getvalue("nodethroughput")
calcm.setvalue("nodethroughput", "%.2f" % (8 * (nodesb[nod].getvalue("nodethroughput") \
- nodesa[nod].getvalue("nodethroughput")) / options.interval))
# calc mem node used / host used
calcm.setvalue("nodeusedmemory", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodeusedmemory") / hostusedmem)))
# calc total cpu time node / host
calcm.setvalue("nodetotalcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodetotalcpu")\
- nodesa[nod].getvalue("nodetotalcpu")) /hostusedcpu))
# calc user cpu time node / host
calcm.setvalue("nodeusercpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodeusercpu")\
- nodesa[nod].getvalue("nodeusercpu")) /hostusedcpu))
# calc system cpu time node / host
calcm.setvalue("nodesystemcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodesystemcpu")\
- nodesa[nod].getvalue("nodesystemcpu")) /hostusedcpu))
# calc waitcpu time node / host
calcm.setvalue("nodewaitcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodewaitcpu")\
- nodesa[nod].getvalue("nodewaitcpu")) /hostusedcpu))
calcm.__init__()
if (nod in nodesa):
try:
if (nodesb[nod] == []) | (nodesa[nod] == []) | \
(False == isinstance(nodesb[nod], NodeMetrics)) | \
(False == isinstance(nodesa[nod], NodeMetrics)):
if options.timestamp == True:
print str(time.time()),
print "Warning: nodes %s is not fully instanciated" % nod
else:
# calc throughput kbps
calcm.setvalue("nodethroughput", "%.2f" % (8 * (nodesb[nod].getvalue("nodethroughput")
- nodesa[nod].getvalue("nodethroughput")) / options.interval))
# calc mem node used / host used
calcm.setvalue("nodeusedmemory", "%.2f" % (
100.0 * (nodesb[nod].getvalue("nodeusedmemory") / hostusedmem)))
#print nod, calcm.tocsv()
#print '=========================='
logsession.nodemetricsC[nod] = calcm
# logsession.printnodemetrics('c')
# calc total cpu time node / host
calcm.setvalue("nodetotalcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodetotalcpu")
- nodesa[nod].getvalue("nodetotalcpu")) / hostusedcpu))
# calc user cpu time node / host
calcm.setvalue("nodeusercpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodeusercpu")
- nodesa[nod].getvalue("nodeusercpu")) / hostusedcpu))
# calc system cpu time node / host
calcm.setvalue("nodesystemcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodesystemcpu")
- nodesa[nod].getvalue("nodesystemcpu")) / hostusedcpu))
# calc waitcpu time node / host
calcm.setvalue("nodewaitcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodewaitcpu")
- nodesa[nod].getvalue("nodewaitcpu")) / hostusedcpu))
logsession.nodemetricsC[nod] = calcm
if options.alarm is not None:
logsession.checknodethresholds(nod)
except IndexError:
pass
else:
print "Warning: transient node %s " % nod
logsession.checknodethresholds(nod)
except IndexError:
pass
else:
print "Warning: transient node %s " % nod
return nodesb
def main():
usagestr = "%prog [-h] [options] [args]\n\nLog server and optional CORE session metrics to stdout."
parser = optparse.OptionParser(usage = usagestr)
parser.set_defaults(interval=2, timestamp=False,
configfile = "/etc/core/perflogserver.conf",\
alarm = True, session = None)
parser.add_option("-i", "--interval", dest = "interval", type = int,
help = "seconds to wait between samples; default=%s" %
parser = optparse.OptionParser(usage=usagestr)
parser.set_defaults(interval=2, timestamp=False,
configfile="/etc/core/perflogserver.conf",
alarm=True, session=None)
parser.add_option("-i", "--interval", dest="interval", type=int,
help="seconds to wait between samples; default=%s" %
parser.defaults["interval"])
parser.add_option("-t", "--timestamp", action = "store_true",
dest = "timestamp",
help = "include timestamp on each line")
parser.add_option("-c", "--configfile", dest = "configfile",
type = "string",
help = "read threshold values from the specified file;\
parser.add_option("-t", "--timestamp", action="store_true",
dest="timestamp",
help="include timestamp on each line")
parser.add_option("-c", "--configfile", dest="configfile",
type="string",
help="read threshold values from the specified file;\
default=%s" % parser.defaults["configfile"])
parser.add_option("-a", "--alarm", action = "store_true",
dest = "alarm",
help = "generate alarms based threshold check on each cycle")
parser.add_option("-s", "--session", dest = "session", type = int,
help = "CORE session id; default=%s" %
parser.add_option("-a", "--alarm", action="store_true",
dest="alarm",
help="generate alarms based threshold check on each cycle")
parser.add_option("-s", "--session", dest="session", type=int,
help="CORE session id; default=%s" %
parser.defaults["session"])
global options
global ncpus
@ -528,8 +552,7 @@ def main():
global logsession
global cyclecount
(options, args) = parser.parse_args()
# print options
options, _args = parser.parse_args()
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
@ -538,51 +561,47 @@ def main():
# server threshold dictionary - a ServerMetrics instant with default values
serverthresholds = ServerMetrics()
# set to server threshold default values: serverloadavg1=3.5,
# serverloadavg5=3.5, serverloadavg15=3.5, serverusedmemory=80.0,
# set to server threshold default values: serverloadavg1=3.5,
# serverloadavg5=3.5, serverloadavg15=3.5, serverusedmemory=80.0,
# serverusedcputime=80.0, processorusedcputime=90.0
serverthresholds.setvalues([3.5, 3.5, 3.5, 80.0, 80.0, 90.0])
if options.alarm is True:
# read server threshold values from configuration file
readserverthresholds(options.configfile)
readserverthresholds(options.configfile)
if options.session is not None:
logsession = LogSession()
# print logsession
# mark host log baseline
print "server: ", ", ".join(map(lambda(x):str(x), os.uname())), ",", ncpus, "CPU cores"
print "server: ", ", ".join(str(x) for x in os.uname()), ",", ncpus, "CPU cores"
print "start timestamp:", time.time(), ", baseline data: "
print csvserverbaseline()
print "server metrics: ", ", ".join(map(lambda(x):str(x), serverthresholds.getkeys()))
print "server metrics: ", ", ".join(str(x) for x in serverthresholds.getkeys())
if options.session is not None:
print "node metrics: nodename, ", ", ".join(map(lambda(x):str(x), logsession.nodethresholds.getkeys()))
print "node metrics: nodename, ", ", ".join(str(x) for x in logsession.nodethresholds.getkeys())
cyclecount = 0
while True:
cputimea = collectservercputimes()
if options.session is not None:
nodesa = logsession.getnodemetrics('a')
# print "nodes a:", nodesa
nodesa = logsession.getnodemetrics("a")
time.sleep(options.interval)
cputimeb = collectservercputimes()
mems = collectservermems()
mems = collectservermems()
calccputime = calcservercputimes(cputimea, cputimeb)
m = csvservermetrics(collectservermetrics(calccputime, mems, options.alarm))
print m
m = csvservermetrics(collectservermetrics(
calccputime, mems, options.alarm))
print m
if options.session is not None:
nodesb = logsession.getnodemetrics('b')
# print "nodes b:", nodesb
if nodesb != {}:
logsession.calcnodemetrics(cputimea, cputimeb, mems)
logsession.printnodemetrics('c')
nodesb = logsession.getnodemetrics('b')
if nodesb != {}:
logsession.calcnodemetrics(cputimea, cputimeb, mems)
logsession.printnodemetrics('c')
sys.stdout.flush()
sys.stdout.flush()
cyclecount = cyclecount + 1
if __name__ == "__main__":
main()

4
scripts/perf/perflogstart.sh Normal file → Executable file
View file

@ -4,7 +4,7 @@
#
# start core performance logging and collect output to file
# /tmp/pycore.nnnn/perf<sessionid>.log
HOOKS_DIR=`dirname $0`
HOOKS_DIR=$(dirname $0)
sid=` pwd | awk -F / {'print $3'} | awk -F . {'print $2'} `
sid=$(pwd | awk -F / {'print $3'} | awk -F . {'print $2'})
python $HOOKS_DIR/perflogserver.py -t -a -c /etc/core/perflogserver.conf -s $sid > perf$sid.log &

2
scripts/perf/perflogstop.sh Normal file → Executable file
View file

@ -4,7 +4,7 @@
#
# terminate core perfromance logging process
perfproc=` ps aux | grep perflogserver | grep python | awk {'print $2'} `
perfproc=$(ps aux | grep perflogserver | grep python | awk {'print $2'})
if [ ! $perfproc = "" ]; then
echo "terminating core performance log process $perfproc"
kill -9 $perfproc

12
scripts/perf/sessiondatacollect.sh Normal file → Executable file
View file

@ -6,9 +6,9 @@
# Collection such runtime tar balls from all distributed servers to folder
# /tmp/<sessionid>-<date>-<time> for example: /tmp/56779-11Oct14-09:33:13
currentdir=` pwd `
currentdir=$(pwd)
sid=${currentdir##*.}
ts=` date +%y%h%d-%T `
ts=$(date +%y%h%d-%T)
logfile=/tmp/corelog.tgz
echo Collect data from localhost:$currentdir to $logfile
cd ..
@ -18,7 +18,7 @@ if [ ! $? = 0 ]; then
fi
cd $currentdir
m=` grep master $currentdir/servers `
m=$(grep master $currentdir/servers)
if [ ! $sid = ${m##*=} ]; then
# quite if this is not a master server
echo not a master server
@ -34,13 +34,13 @@ fi
cp $logfile $logdir/localhost-${currentdir##*.}.tgz
# harvest CORE data from distributed servers
hs=` grep -v master= $currentdir/servers | awk {'print $2'} `
hs=$(grep -v master= $currentdir/servers | awk {'print $2'})
echo hosts are localhost $hs
for h in $hs; do
echo checking host $h ...
out=` ping -c 1 -w 1 $h | grep " 0 received," `
out=$(ping -c 1 -w 1 $h | grep " 0 received,")
if [ " $out " = " " ]; then
slavesid=` ssh $h tar -tzf $logfile | awk -F / {'print $1'} `
slavesid=$(ssh $h tar -tzf $logfile | awk -F / {'print $1'})
if [ $? = 0 ]; then
destlogfile=$logdir/$h-${slavesid##*.}.tgz
echo Collect data from $h:$logfile to $destlogfile

0
scripts/perf/timesyncstart.sh Normal file → Executable file
View file

4
scripts/perf/timesyncstop.sh Normal file → Executable file
View file

@ -18,7 +18,7 @@ if [ -e /usr/sbin/ntpd ]; then
# It is not quick enough for our need here
/etc/init.d/ntp status
if [ $? = 0 ]; then
svs=`ntpq -c peers | awk {'print $1'} | grep ^*`
svs=$(ntpq -c peers | awk {'print $1'} | grep ^*)
if [ "$dbg" = "debug" ]; then
ntpq -c peers
echo "get time servers for later need: $svs"
@ -31,7 +31,7 @@ if [ -e /usr/sbin/ntpd ]; then
# if there are time servers configured in file /etc/ntp.conf,
# adjust time to the first server
if [ "$svs" = "" ]; then
svs=`grep ^server /etc/ntp.conf | awk {'print $2'} `
svs=$(grep ^server /etc/ntp.conf | awk {'print $2'})
if [ "$dbg" = "debug" ]; then
echo "$svs"
fi