diff --git a/daemon/core/misc/xmlparser.py b/daemon/core/misc/xmlparser.py index fb07fb48..529dbf4b 100644 --- a/daemon/core/misc/xmlparser.py +++ b/daemon/core/misc/xmlparser.py @@ -3,10 +3,13 @@ # See the LICENSE file included in this distribution. from xml.dom.minidom import parse -from xmlutils import getoneelement +from xmlutils import getFirstChildByTagName from xmlparser0 import CoreDocumentParser0 +from xmlparser1 import CoreDocumentParser1 class CoreVersionParser(object): + DEFAULT_SCENARIO_VERSION = '1.0' + '''\ Helper class to check the version of Network Plan document. This simply looks for a "Scenario" element; when present, this @@ -19,9 +22,14 @@ class CoreVersionParser(object): self.dom = options['dom'] else: self.dom = parse(filename) - self.scenario = getoneelement(self.dom, 'Scenario') - if self.scenario is not None: - self.version = 0.0 + scenario = getFirstChildByTagName(self.dom, 'scenario') + if scenario: + version = scenario.getAttribute('version') + if not version: + version = self.DEFAULT_SCENARIO_VERSION + self.version = version + elif getFirstChildByTagName(self.dom, 'Scenario'): + self.version = '0.0' else: self.version = 'unknown' @@ -29,8 +37,10 @@ def core_document_parser(session, filename, options): vp = CoreVersionParser(filename, options) if 'dom' not in options: options['dom'] = vp.dom - if vp.version == 0.0: + if vp.version == '0.0': doc = CoreDocumentParser0(session, filename, options) + elif vp.version == '1.0': + doc = CoreDocumentParser1(session, filename, options) else: raise ValueError, 'unsupported document version: %s' % vp.version return doc diff --git a/daemon/core/misc/xmlparser1.py b/daemon/core/misc/xmlparser1.py new file mode 100644 index 00000000..8b0a22c1 --- /dev/null +++ b/daemon/core/misc/xmlparser1.py @@ -0,0 +1,942 @@ +# +# CORE +# Copyright (c) 2015 the Boeing Company. +# See the LICENSE file included in this distribution. +# + +import sys +import random +from core.netns import nodes +from core import constants +from core.misc.ipaddr import MacAddr +from xml.dom.minidom import parse +from xmlutils import * + +class CoreDocumentParser1(object): + + layer2_device_types = 'hub', 'switch' + layer3_device_types = 'host', 'router' + device_types = layer2_device_types + layer3_device_types + + # TODO: support CORE interface classes: + # RJ45Node + # TunnelNode + + def __init__(self, session, filename, options): + self.session = session + self.verbose = self.session.getcfgitembool('verbose', False) + self.filename = filename + if 'dom' in options: + # this prevents parsing twice when detecting file versions + self.dom = options['dom'] + else: + self.dom = parse(filename) + self.start = options['start'] + self.nodecls = options['nodecls'] + self.scenario = self.get_scenario(self.dom) + self.location_refgeo_set = False + self.location_refxyz_set = False + # saved link parameters saved when parsing networks and applied later + self.link_params = {} + # map from id-string to objid, for files having node names but + # not node numbers + self.objidmap = {} + self.objids = set() + self.default_services = {} + if self.scenario: + self.parse_scenario() + + def info(self, msg): + s = 'XML parsing \'%s\': %s' % (self.filename, msg) + if self.session: + self.session.info(s) + else: + sys.stdout.write(s + '\n') + + def warn(self, msg): + s = 'WARNING XML parsing \'%s\': %s' % (self.filename, msg) + if self.session: + self.session.warn(s) + else: + sys.stderr.write(s + '\n') + + @staticmethod + def get_scenario(dom): + scenario = getFirstChildByTagName(dom, 'scenario') + if not scenario: + raise ValueError, 'no scenario element found' + version = scenario.getAttribute('version') + if version and version != '1.0': + raise ValueError, \ + 'unsupported scenario version found: \'%s\'' % version + return scenario + + def parse_scenario(self): + self.parse_default_services() + self.parse_session_config() + self.parse_network_plan() + + def assign_id(self, idstr, idval): + if idstr in self.objidmap: + assert self.objidmap[idstr] == idval and idval in self.objids + return + self.objidmap[idstr] = idval + self.objids.add(idval) + + def rand_id(self): + while True: + x = random.randint(0, 0xffff) + if x not in self.objids: + return x + + def get_id(self, idstr): + '''\ + Get a, possibly new, object id (node number) corresponding to + the given XML string id. + ''' + if not idstr: + idn = self.rand_id() + self.objids.add(idn) + return idn + elif idstr in self.objidmap: + return self.objidmap[idstr] + else: + try: + idn = int(idstr) + except ValueError: + idn = self.rand_id() + self.assign_id(idstr, idn) + return idn + + def get_common_attributes(self, node): + '''\ + Return id, name attributes for the given XML element. These + attributes are common to nodes and networks. + ''' + idstr = node.getAttribute('id') + # use an explicit set COREID if it exists + coreid = self.find_core_id(node) + if coreid: + idn = int(coreid) + if idstr: + self.assign_id(idstr, idn) + else: + idn = self.get_id(idstr) + # TODO: consider supporting unicode; for now convert to an + # ascii string + namestr = str(node.getAttribute('name')) + return idn, namestr + + def iter_network_member_devices(self, element): + # element can be a network or a channel + for interface in iterChildrenWithAttribute(element, 'member', + 'type', 'interface'): + if_id = getChildTextTrim(interface) + assert if_id # XXX for testing + if not if_id: + continue + device, if_name = self.find_device_with_interface(if_id) + assert device, 'no device for if_id: %s' % if_id # XXX for testing + if device: + yield device, if_name + + def network_class(self, network, network_type): + '''\ + Return the corresponding CORE network class for the given + network/network_type. + ''' + if network_type == 'ethernet': + return nodes.PtpNet + elif network_type == 'satcom': + return nodes.PtpNet + elif network_type == 'wireless': + channel = getFirstChildByTagName(network, 'channel') + if channel: + # use an explicit CORE type if it exists + coretype = getFirstChildTextTrimWithAttribute(channel, 'type', + 'domain', 'CORE') + if coretype: + if coretype == 'basic_range': + return nodes.WlanNode + elif coretype.startswith('emane'): + return nodes.EmaneNode + else: + self.warn('unknown network type: \'%s\'' % coretype) + return xmltypetonodeclass(self.session, coretype) + return nodes.WlanNode + self.warn('unknown network type: \'%s\'' % network_type) + return None + + def create_core_object(self, objcls, objid, objname, element, node_type): + obj = self.session.addobj(cls = objcls, objid = objid, + name = objname, start = self.start) + if self.verbose: + self.info('added object objid=%s name=%s cls=%s' % \ + (objid, objname, objcls)) + self.set_object_position(obj, element) + self.set_object_presentation(obj, element, node_type) + return obj + + def get_core_object(self, idstr): + if idstr and idstr in self.objidmap: + objid = self.objidmap[idstr] + return self.session.obj(objid) + return None + + def parse_network_plan(self): + # parse the scenario in the following order: + # 1. layer-2 devices + # 2. other networks (ptp/wlan) + # 3. layer-3 devices + self.parse_layer2_devices() + self.parse_networks() + self.parse_layer3_devices() + + def set_ethernet_link_parameters(self, channel, link_params, + mobility_model_name, mobility_params): + # save link parameters for later use, indexed by the tuple + # (device_id, interface_name) + for dev, if_name in self.iter_network_member_devices(channel): + if self.device_type(dev) in self.device_types: + dev_id = dev.getAttribute('id') + key = (dev_id, if_name) + self.link_params[key] = link_params + if mobility_model_name or mobility_params: + raise NotImplementedError + + def set_wireless_link_parameters(self, channel, link_params, + mobility_model_name, mobility_params): + network = self.find_channel_network(channel) + network_id = network.getAttribute('id') + if network_id in self.objidmap: + nodenum = self.objidmap[network_id] + else: + self.warn('unknown network: %s' % network.toxml('utf-8')) + assert False # XXX for testing + return + model_name = getFirstChildTextTrimWithAttribute(channel, 'type', + 'domain', 'CORE') + if not model_name: + model_name = 'basic_range' + if model_name == 'basic_range': + mgr = self.session.mobility + elif model_name.startswith('emane'): + mgr = self.session.emane + elif model_name.startswith('xen'): + mgr = self.session.xen + else: + # TODO: any other config managers? + raise NotImplementedError + mgr.setconfig_keyvalues(nodenum, model_name, link_params.items()) + if mobility_model_name and mobility_params: + mgr.setconfig_keyvalues(nodenum, mobility_model_name, + mobility_params.items()) + + def link_layer2_devices(self, device1, ifname1, device2, ifname2): + '''\ + Link two layer-2 devices together. + ''' + devid1 = device1.getAttribute('id') + dev1 = self.get_core_object(devid1) + devid2 = device2.getAttribute('id') + dev2 = self.get_core_object(devid2) + assert dev1 and dev2 # XXX for testing + if dev1 and dev2: + # TODO: review this + if isinstance(dev2, nodes.RJ45Node): + # RJ45 nodes have different linknet() + netif = dev2.linknet(dev1) + else: + netif = dev1.linknet(dev2) + self.set_wired_link_parameters(dev1, netif, devid1, ifname1) + + @classmethod + def parse_xml_value(cls, valtext): + if not valtext: + return None + try: + if not valtext.translate(None, '0123456789'): + val = int(valtext) + else: + val = float(valtext) + except ValueError: + val = str(valtext) + return val + + @classmethod + def parse_parameter_children(cls, parent): + params = {} + for parameter in iterChildrenWithName(parent, 'parameter'): + param_name = parameter.getAttribute('name') + assert param_name # XXX for testing + if not param_name: + continue + # TODO: consider supporting unicode; for now convert + # to an ascii string + param_name = str(param_name) + param_val = cls.parse_xml_value(getChildTextTrim(parameter)) + # TODO: check if the name already exists? + if param_name and param_val: + params[param_name] = param_val + return params + + def parse_network_channel(self, channel): + element = self.search_for_element(channel, 'type', + lambda x: not x.hasAttributes()) + channel_type = getChildTextTrim(element) + link_params = self.parse_parameter_children(channel) + + mobility = getFirstChildByTagName(channel, 'CORE:mobility') + if mobility: + mobility_model_name = \ + getFirstChildTextTrimByTagName(mobility, 'type') + mobility_params = self.parse_parameter_children(mobility) + else: + mobility_model_name = None + mobility_params = None + if channel_type == 'wireless': + self.set_wireless_link_parameters(channel, link_params, + mobility_model_name, + mobility_params) + elif channel_type == 'ethernet': + # TODO: maybe this can be done in the loop below to avoid + # iterating through channel members multiple times + self.set_ethernet_link_parameters(channel, link_params, + mobility_model_name, + mobility_params) + else: + raise NotImplementedError + layer2_device = [] + for dev, if_name in self.iter_network_member_devices(channel): + if self.device_type(dev) in self.layer2_device_types: + layer2_device.append((dev, if_name)) + assert len(layer2_device) <= 2 + if len(layer2_device) == 2: + self.link_layer2_devices(layer2_device[0][0], layer2_device[0][1], + layer2_device[1][0], layer2_device[1][1]) + + def parse_network(self, network): + '''\ + Each network element should have an 'id' and 'name' attribute + and include the following child elements: + + type (one) + member (zero or more with type="interface" or type="channel") + channel (zero or more) + ''' + layer2_members = set() + layer3_members = 0 + for dev, if_name in self.iter_network_member_devices(network): + if not dev: + continue + devtype = self.device_type(dev) + if devtype in self.layer2_device_types: + layer2_members.add(dev) + elif devtype in self.layer3_device_types: + layer3_members += 1 + else: + raise NotImplementedError + if len(layer2_members) == 0: + net_type = getFirstChildTextTrimByTagName(network, 'type') + if not net_type: + msg = 'no network type found for network: \'%s\'' % \ + network.toxml('utf-8') + self.warn(msg) + assert False # XXX for testing + return + net_cls = self.network_class(network, net_type) + objid, net_name = self.get_common_attributes(network) + if self.verbose: + self.info('parsing network: %s %s' % (net_name, objid)) + if objid in self.session._objs: + return + n = self.create_core_object(net_cls, objid, net_name, + network, None) + # handle channel parameters + for channel in iterChildrenWithName(network, 'channel'): + self.parse_network_channel(channel) + + def parse_networks(self): + '''\ + Parse all 'network' elements. + ''' + for network in iterDescendantsWithName(self.scenario, 'network'): + self.parse_network(network) + + def parse_addresses(self, interface): + mac = [] + ipv4 = [] + ipv6= [] + hostname = [] + for address in iterChildrenWithName(interface, 'address'): + addr_type = address.getAttribute('type') + if not addr_type: + msg = 'no type attribute found for address ' \ + 'in interface: \'%s\'' % interface.toxml('utf-8') + self.warn(msg) + assert False # XXX for testing + continue + addr_text = getChildTextTrim(address) + if not addr_text: + msg = 'no text found for address ' \ + 'in interface: \'%s\'' % interface.toxml('utf-8') + self.warn(msg) + assert False # XXX for testing + continue + if addr_type == 'mac': + mac.append(addr_text) + elif addr_type == 'IPv4': + ipv4.append(addr_text) + elif addr_type == 'IPv6': + ipv6.append(addr_text) + elif addr_type == 'hostname': + hostname.append(addr_text) + else: + msg = 'skipping unknown address type \'%s\' in ' \ + 'interface: \'%s\'' % (addr_type, interface.toxml('utf-8')) + self.warn(msg) + assert False # XXX for testing + continue + return mac, ipv4, ipv6, hostname + + def parse_interface(self, node, device_id, interface): + '''\ + Each interface can have multiple 'address' elements. + ''' + if_name = interface.getAttribute('name') + network = self.find_interface_network_object(interface) + if not network: + msg = 'skipping node \'%s\' interface \'%s\': ' \ + 'unknown network' % (node.name, if_name) + self.warn(msg) + assert False # XXX for testing + return + mac, ipv4, ipv6, hostname = self.parse_addresses(interface) + if mac: + hwaddr = MacAddr.fromstring(mac[0]) + else: + hwaddr = None + ifindex = node.newnetif(network, addrlist = ipv4 + ipv6, + hwaddr = hwaddr, ifindex = None, + ifname = if_name) + # TODO: 'hostname' addresses are unused + if self.verbose: + msg = 'node \'%s\' interface \'%s\' connected ' \ + 'to network \'%s\'' % (node.name, if_name, network.name) + self.info(msg) + # set link parameters for wired links + if isinstance(network, + (nodes.HubNode, nodes.PtpNet, nodes.SwitchNode)): + netif = node.netif(ifindex) + self.set_wired_link_parameters(network, netif, device_id) + + def set_wired_link_parameters(self, network, netif, + device_id, netif_name = None): + if netif_name is None: + netif_name = netif.name + key = (device_id, netif_name) + if key in self.link_params: + link_params = self.link_params[key] + if self.start: + bw = link_params.get('bw') + delay = link_params.get('delay') + loss = link_params.get('loss') + duplicate = link_params.get('duplicate') + jitter = link_params.get('jitter') + network.linkconfig(netif, bw = bw, delay = delay, loss = loss, + duplicate = duplicate, jitter = jitter) + else: + for k, v in link_params.iteritems(): + netif.setparam(k, v) + + @staticmethod + def search_for_element(node, tagName, match = None): + '''\ + Search the given node and all ancestors for an element named + tagName that satisfies the given matching function. + ''' + while True: + for child in iterChildren(node, Node.ELEMENT_NODE): + if child.tagName == tagName and \ + (match is None or match(child)): + return child + node = node.parentNode + if not node: + break + return None + + @classmethod + def find_core_id(cls, node): + def match(x): + domain = x.getAttribute('domain') + return domain == 'COREID' + alias = cls.search_for_element(node, 'alias', match) + if alias: + return getChildTextTrim(alias) + return None + + @classmethod + def find_point(cls, node): + return cls.search_for_element(node, 'point') + + @staticmethod + def find_channel_network(channel): + p = channel.parentNode + if p and p.tagName == 'network': + return p + return None + + def find_interface_network_object(self, interface): + network_id = getFirstChildTextTrimWithAttribute(interface, 'member', + 'type', 'network') + if not network_id: + # support legacy notation: value tag to the parent element, when value is not None. @@ -108,7 +108,7 @@ def addtextparamtoparent(dom, parent, name, value): txt = dom.createTextNode(value) p.appendChild(txt) return p - + def addparamlisttoparent(dom, parent, name, values): ''' XML helper to return a parameter list and optionally add it to the parent element: @@ -128,13 +128,76 @@ def addparamlisttoparent(dom, parent, name, values): item.setAttribute("value", str(v)) p.appendChild(item) return p - + def getoneelement(dom, name): e = dom.getElementsByTagName(name) if len(e) == 0: return None return e[0] +def iterDescendants(dom, max_depth = 0): + '''\ + Iterate over all descendant element nodes in breadth first order. + Only consider nodes up to max_depth deep when max_depth is greater + than zero. + ''' + nodes = [dom] + depth = 0 + current_depth_nodes = 1 + next_depth_nodes = 0 + while nodes: + n = nodes.pop(0) + for child in n.childNodes: + if child.nodeType == Node.ELEMENT_NODE: + yield child + nodes.append(child) + next_depth_nodes += 1 + current_depth_nodes -= 1 + if current_depth_nodes == 0: + depth += 1 + if max_depth > 0 and depth == max_depth: + return + current_depth_nodes = next_depth_nodes + next_depth_nodes = 0 + +def iterMatchingDescendants(dom, matchFunction, max_depth = 0): + '''\ + Iterate over descendant elements where matchFunction(descendant) + returns true. Only consider nodes up to max_depth deep when + max_depth is greater than zero. + ''' + for d in iterDescendants(dom, max_depth): + if matchFunction(d): + yield d + +def iterDescendantsWithName(dom, tagName, max_depth = 0): + '''\ + Iterate over descendant elements whose name is contained in + tagName (or is named tagName if tagName is a string). Only + consider nodes up to max_depth deep when max_depth is greater than + zero. + ''' + if isinstance(tagName, basestring): + tagName = (tagName,) + def match(d): + return d.tagName in tagName + return iterMatchingDescendants(dom, match, max_depth) + +def iterDescendantsWithAttribute(dom, tagName, attrName, attrValue, + max_depth = 0): + '''\ + Iterate over descendant elements whose name is contained in + tagName (or is named tagName if tagName is a string) and have an + attribute named attrName with value attrValue. Only consider + nodes up to max_depth deep when max_depth is greater than zero. + ''' + if isinstance(tagName, basestring): + tagName = (tagName,) + def match(d): + return d.tagName in tagName and \ + d.getAttribute(attrName) == attrValue + return iterMatchingDescendants(dom, match, max_depth) + def iterChildren(dom, nodeType): '''\ Iterate over all child elements of the given type. @@ -151,9 +214,15 @@ def gettextchild(dom): return str(child.nodeValue) return None +def getChildTextTrim(dom): + text = gettextchild(dom) + if text: + text = text.strip() + return text + def getparamssetattrs(dom, param_names, target): ''' XML helper to get tags and set - the attribute in the target object. String type is used. Target object + the attribute in the target object. String type is used. Target object attribute is unchanged if the XML attribute is not present. ''' params = dom.getElementsByTagName("param") @@ -172,3 +241,63 @@ def xmltypetonodeclass(session, type): return eval("nodes.%s" % type) else: return None + +def iterChildrenWithName(dom, tagName): + return iterDescendantsWithName(dom, tagName, 1) + +def iterChildrenWithAttribute(dom, tagName, attrName, attrValue): + return iterDescendantsWithAttribute(dom, tagName, attrName, attrValue, 1) + +def getFirstChildByTagName(dom, tagName): + '''\ + Return the first child element whose name is contained in tagName + (or is named tagName if tagName is a string). + ''' + for child in iterChildrenWithName(dom, tagName): + return child + return None + +def getFirstChildTextByTagName(dom, tagName): + '''\ + Return the corresponding text of the first child element whose + name is contained in tagName (or is named tagName if tagName is a + string). + ''' + child = getFirstChildByTagName(dom, tagName) + if child: + return gettextchild(child) + return None + +def getFirstChildTextTrimByTagName(dom, tagName): + text = getFirstChildTextByTagName(dom, tagName) + if text: + text = text.strip() + return text + +def getFirstChildWithAttribute(dom, tagName, attrName, attrValue): + '''\ + Return the first child element whose name is contained in tagName + (or is named tagName if tagName is a string) that has an attribute + named attrName with value attrValue. + ''' + for child in \ + iterChildrenWithAttribute(dom, tagName, attrName, attrValue): + return child + return None + +def getFirstChildTextWithAttribute(dom, tagName, attrName, attrValue): + '''\ + Return the corresponding text of the first child element whose + name is contained in tagName (or is named tagName if tagName is a + string) that has an attribute named attrName with value attrValue. + ''' + child = getFirstChildWithAttribute(dom, tagName, attrName, attrValue) + if child: + return gettextchild(child) + return None + +def getFirstChildTextTrimWithAttribute(dom, tagName, attrName, attrValue): + text = getFirstChildTextWithAttribute(dom, tagName, attrName, attrValue) + if text: + text = text.strip() + return text