initial commit after bringing over cleaned up code and testing some examples

This commit is contained in:
Blake J. Harnden 2017-04-25 08:45:34 -07:00
parent c4858e6e0d
commit 00f4ebf5a9
93 changed files with 15189 additions and 13083 deletions

View file

@ -7,76 +7,62 @@
# authors: Tom Goff <thomas.goff@boeing.com>
# Jeff Ahrenholz <jeffrey.m.ahrenholz@boeing.com>
#
'''
core-daemon: the CORE daemon is a server process that receives CORE API
"""
core-daemon: the CORE daemon is a server process that receives CORE API
messages and instantiates emulated nodes and networks within the kernel. Various
message handlers are defined and some support for sending messages.
'''
"""
import os, optparse, ConfigParser, gc, shlex, socket, shutil
import ConfigParser
import atexit
import signal
import importlib
import logging
import optparse
import os
import signal
import socket
import sys
import threading
import time
try:
from core import pycore
except ImportError:
# hack for Fedora autoconf that uses the following pythondir:
if "/usr/lib/python2.6/site-packages" in sys.path:
sys.path.append("/usr/local/lib/python2.6/site-packages")
if "/usr/lib64/python2.6/site-packages" in sys.path:
sys.path.append("/usr/local/lib64/python2.6/site-packages")
if "/usr/lib/python2.7/site-packages" in sys.path:
sys.path.append("/usr/local/lib/python2.7/site-packages")
if "/usr/lib64/python2.7/site-packages" in sys.path:
sys.path.append("/usr/local/lib64/python2.7/site-packages")
from core import pycore
from core.coreserver import *
from core.constants import *
from core import constants
from core import corehandlers
from core import coreserver
from core import enumerations
from core.api import coreapi
from core.misc.utils import daemonize, closeonexec
from core.enumerations import MessageFlags
from core.enumerations import RegisterTlvs
from core.misc import log
from core.misc import nodemaps
from core.misc import nodeutils
from core.misc.utils import closeonexec
from core.misc.utils import daemonize
from core.services import bird
from core.services import dockersvc
from core.services import nrl
from core.services import quagga
from core.services import security
from core.services import startup
from core.services import ucarp
from core.services import utility
from core.services import xorp
DEFAULT_MAXFD = 1024
# garbage collection debugging
# gc.set_debug(gc.DEBUG_STATS | gc.DEBUG_LEAK)
coreapi.add_node_class("CORE_NODE_DEF",
coreapi.CORE_NODE_DEF, pycore.nodes.CoreNode)
coreapi.add_node_class("CORE_NODE_PHYS",
coreapi.CORE_NODE_PHYS, pycore.pnodes.PhysicalNode)
try:
coreapi.add_node_class("CORE_NODE_XEN",
coreapi.CORE_NODE_XEN, pycore.xen.XenNode)
except Exception:
#print "XenNode class unavailable."
pass
coreapi.add_node_class("CORE_NODE_TBD",
coreapi.CORE_NODE_TBD, None)
coreapi.add_node_class("CORE_NODE_SWITCH",
coreapi.CORE_NODE_SWITCH, pycore.nodes.SwitchNode)
coreapi.add_node_class("CORE_NODE_HUB",
coreapi.CORE_NODE_HUB, pycore.nodes.HubNode)
coreapi.add_node_class("CORE_NODE_WLAN",
coreapi.CORE_NODE_WLAN, pycore.nodes.WlanNode)
coreapi.add_node_class("CORE_NODE_RJ45",
coreapi.CORE_NODE_RJ45, pycore.nodes.RJ45Node)
coreapi.add_node_class("CORE_NODE_TUNNEL",
coreapi.CORE_NODE_TUNNEL, pycore.nodes.TunnelNode)
coreapi.add_node_class("CORE_NODE_EMANE",
coreapi.CORE_NODE_EMANE, pycore.nodes.EmaneNode)
#
# UDP server startup
#
#
def startudp(mainserver, server_address):
''' Start a thread running a UDP server on the same host,port for
""" Start a thread running a UDP server on the same host,port for
connectionless requests.
'''
mainserver.udpserver = CoreUdpServer(server_address,
CoreDatagramRequestHandler, mainserver)
mainserver.udpthread = threading.Thread(target = mainserver.udpserver.start)
"""
mainserver.udpserver = coreserver.CoreUdpServer(
server_address,
corehandlers.CoreDatagramRequestHandler,
mainserver)
mainserver.udpthread = threading.Thread(target=mainserver.udpserver.start)
mainserver.udpthread.daemon = True
mainserver.udpthread.start()
return mainserver.udpserver
@ -84,76 +70,76 @@ def startudp(mainserver, server_address):
#
# Auxiliary server startup
#
#
def startaux(mainserver, aux_address, aux_handler):
''' Start a thread running an auxiliary TCP server on the given address.
""" Start a thread running an auxiliary TCP server on the given address.
This server will communicate with client requests using a handler
using the aux_handler class. The aux_handler can provide an alternative
using the aux_handler class. The aux_handler can provide an alternative
API to CORE.
'''
handlermodname,dot,handlerclassname = aux_handler.rpartition('.')
"""
handlermodname, dot, handlerclassname = aux_handler.rpartition(".")
handlermod = importlib.import_module(handlermodname)
handlerclass = getattr(handlermod, handlerclassname)
mainserver.auxserver = CoreAuxServer(aux_address,
handlerclass,
mainserver)
mainserver.auxthread = threading.Thread(target = mainserver.auxserver.start)
mainserver.auxserver = coreserver.CoreAuxServer(aux_address, handlerclass, mainserver)
mainserver.auxthread = threading.Thread(target=mainserver.auxserver.start)
mainserver.auxthread.daemon = True
mainserver.auxthread.start()
return mainserver.auxserver
def banner():
''' Output the program banner printed to the terminal or log file.
'''
sys.stdout.write("CORE daemon v.%s started %s\n" % \
(COREDPY_VERSION, time.ctime()))
""" Output the program banner printed to the terminal or log file.
"""
sys.stdout.write("CORE daemon v.%s started %s\n" % (constants.COREDPY_VERSION, time.ctime()))
sys.stdout.flush()
def cored(cfg = None):
''' Start the CoreServer object and enter the server loop.
'''
host = cfg['listenaddr']
port = int(cfg['port'])
if host == '' or host is None:
def cored(cfg=None):
""" Start the CoreServer object and enter the server loop.
"""
host = cfg["listenaddr"]
port = int(cfg["port"])
if host == "" or host is None:
host = "localhost"
try:
server = CoreServer((host, port), CoreRequestHandler, cfg)
server = coreserver.CoreServer((host, port), corehandlers.CoreRequestHandler, cfg)
except Exception, e:
sys.stderr.write("error starting main server on: %s:%s\n\t%s\n" % \
(host, port, e))
sys.stderr.write("error starting main server on: %s:%s\n\t%s\n" % (host, port, e))
sys.stderr.flush()
sys.exit(1)
closeonexec(server.fileno())
sys.stdout.write("main server started, listening on: %s:%s\n" % (host, port))
sys.stdout.flush()
udpserver = startudp(server, (host,port))
udpserver = startudp(server, (host, port))
closeonexec(udpserver.fileno())
auxreqhandler = cfg['aux_request_handler']
auxreqhandler = cfg["aux_request_handler"]
if auxreqhandler:
try:
handler, auxport = auxreqhandler.rsplit(':')
auxserver = startaux(server, (host,int(auxport)), handler)
handler, auxport = auxreqhandler.rsplit(":")
auxserver = startaux(server, (host, int(auxport)), handler)
closeonexec(auxserver.fileno())
except Exception as e:
raise ValueError, "invalid auxreqhandler:(%s)\nError: %s" % (auxreqhandler, e)
server.serve_forever()
def cleanup():
while CoreServer.servers:
server = CoreServer.servers.pop()
while coreserver.CoreServer.servers:
server = coreserver.CoreServer.servers.pop()
server.shutdown()
atexit.register(cleanup)
def sighandler(signum, stackframe):
print >> sys.stderr, "terminated by signal:", signum
sys.exit(signum)
signal.signal(signal.SIGHUP, sighandler)
signal.signal(signal.SIGINT, sighandler)
signal.signal(signal.SIGPIPE, sighandler)
@ -161,7 +147,8 @@ signal.signal(signal.SIGTERM, sighandler)
signal.signal(signal.SIGUSR1, sighandler)
signal.signal(signal.SIGUSR2, sighandler)
def logrotate(stdout, stderr, stdoutmode = 0644, stderrmode = 0644):
def logrotate(stdout, stderr, stdoutmode=0644, stderrmode=0644):
def reopen(fileno, filename, mode):
err = 0
fd = -1
@ -175,6 +162,7 @@ def logrotate(stdout, stderr, stdoutmode = 0644, stderrmode = 0644):
if fd >= 0:
os.close(fd)
return err
if stdout:
err = reopen(1, stdout, stdoutmode)
if stderr:
@ -186,56 +174,57 @@ def logrotate(stdout, stderr, stdoutmode = 0644, stderrmode = 0644):
else:
reopen(2, stderr, stderrmode)
def getMergedConfig(filename):
''' Return a configuration after merging config file and command-line
def get_merged_config(filename):
""" Return a configuration after merging config file and command-line
arguments.
'''
"""
# these are the defaults used in the config file
defaults = { 'port' : '%d' % coreapi.CORE_API_PORT,
'listenaddr' : 'localhost',
'pidfile' : '%s/run/core-daemon.pid' % CORE_STATE_DIR,
'logfile' : '%s/log/core-daemon.log' % CORE_STATE_DIR,
'xmlfilever' : '1.0',
'numthreads' : '1',
'verbose' : 'False',
'daemonize' : 'False',
'debug' : 'False',
'execfile' : None,
'aux_request_handler' : None,
}
defaults = {"port": "%d" % enumerations.CORE_API_PORT,
"listenaddr": "localhost",
"pidfile": "%s/run/core-daemon.pid" % constants.CORE_STATE_DIR,
"logfile": "%s/log/core-daemon.log" % constants.CORE_STATE_DIR,
"xmlfilever": "1.0",
"numthreads": "1",
"verbose": "False",
"daemonize": "False",
"debug": "False",
"execfile": None,
"aux_request_handler": None,
}
usagestr = "usage: %prog [-h] [options] [args]\n\n" + \
"CORE daemon v.%s instantiates Linux network namespace " \
"nodes." % COREDPY_VERSION
parser = optparse.OptionParser(usage = usagestr)
parser.add_option("-f", "--configfile", dest = "configfile",
type = "string",
help = "read config from specified file; default = %s" %
filename)
parser.add_option("-d", "--daemonize", dest = "daemonize",
"nodes." % constants.COREDPY_VERSION
parser = optparse.OptionParser(usage=usagestr)
parser.add_option("-f", "--configfile", dest="configfile",
type="string",
help="read config from specified file; default = %s" %
filename)
parser.add_option("-d", "--daemonize", dest="daemonize",
action="store_true",
help = "run in background as daemon; default=%s" % \
defaults["daemonize"])
parser.add_option("-e", "--execute", dest = "execfile", type = "string",
help = "execute a Python/XML-based session")
parser.add_option("-l", "--logfile", dest = "logfile", type = "string",
help = "log output to specified file; default = %s" %
defaults["logfile"])
parser.add_option("-p", "--port", dest = "port", type = int,
help = "port number to listen on; default = %s" % \
defaults["port"])
parser.add_option("-i", "--pidfile", dest = "pidfile",
help = "filename to write pid to; default = %s" % \
defaults["pidfile"])
parser.add_option("-t", "--numthreads", dest = "numthreads", type = int,
help = "number of server threads; default = %s" % \
defaults["numthreads"])
parser.add_option("-v", "--verbose", dest = "verbose", action="store_true",
help = "enable verbose logging; default = %s" % \
defaults["verbose"])
parser.add_option("-g", "--debug", dest = "debug", action="store_true",
help = "enable debug logging; default = %s" % \
defaults["debug"])
help="run in background as daemon; default=%s" % \
defaults["daemonize"])
parser.add_option("-e", "--execute", dest="execfile", type="string",
help="execute a Python/XML-based session")
parser.add_option("-l", "--logfile", dest="logfile", type="string",
help="log output to specified file; default = %s" %
defaults["logfile"])
parser.add_option("-p", "--port", dest="port", type=int,
help="port number to listen on; default = %s" % \
defaults["port"])
parser.add_option("-i", "--pidfile", dest="pidfile",
help="filename to write pid to; default = %s" % \
defaults["pidfile"])
parser.add_option("-t", "--numthreads", dest="numthreads", type=int,
help="number of server threads; default = %s" % \
defaults["numthreads"])
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="enable verbose logging; default = %s" % \
defaults["verbose"])
parser.add_option("-g", "--debug", dest="debug", action="store_true",
help="enable debug logging; default = %s" % \
defaults["debug"])
# parse command line options
(options, args) = parser.parse_args()
@ -253,17 +242,17 @@ def getMergedConfig(filename):
# gracefully support legacy configs (cored.py/cored now core-daemon)
if cfg.has_section("cored.py"):
for name, val in cfg.items("cored.py"):
if name == 'pidfile' or name == 'logfile':
bn = os.path.basename(val).replace('coredpy', 'core-daemon')
if name == "pidfile" or name == "logfile":
bn = os.path.basename(val).replace("coredpy", "core-daemon")
val = os.path.join(os.path.dirname(val), bn)
cfg.set(section, name, val)
if cfg.has_section("cored"):
for name, val in cfg.items("cored"):
if name == 'pidfile' or name == 'logfile':
bn = os.path.basename(val).replace('cored', 'core-daemon')
if name == "pidfile" or name == "logfile":
bn = os.path.basename(val).replace("cored", "core-daemon")
val = os.path.join(os.path.dirname(val), bn)
cfg.set(section, name, val)
# merge command line with config file
for opt in options.__dict__:
val = options.__dict__[opt]
@ -272,41 +261,43 @@ def getMergedConfig(filename):
return dict(cfg.items(section)), args
def exec_file(cfg):
''' Send a Register Message to execute a new session based on XML or Python
""" Send a Register Message to execute a new session based on XML or Python
script file.
'''
filename = cfg['execfile']
sys.stdout.write("Telling daemon to execute file: '%s'...\n" % filename)
"""
filename = cfg["execfile"]
sys.stdout.write("Telling daemon to execute file: %s...\n" % filename)
sys.stdout.flush()
tlvdata = coreapi.CoreRegTlv.pack(coreapi.CORE_TLV_REG_EXECSRV, filename)
msg = coreapi.CoreRegMessage.pack(coreapi.CORE_API_ADD_FLAG, tlvdata)
tlvdata = coreapi.CoreRegisterTlv.pack(RegisterTlvs.EXECUTE_SERVER.value, filename)
msg = coreapi.CoreRegMessage.pack(MessageFlags.ADD.value, tlvdata)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.connect(("localhost", int(cfg['port']))) # TODO: connect address option
sock.connect(("localhost", int(cfg["port"]))) # TODO: connect address option
sock.sendall(msg)
return 0
def main():
''' Main program startup.
'''
# get a configuration merged from config file and command-line arguments
cfg, args = getMergedConfig("%s/core.conf" % CORE_CONF_DIR)
for a in args:
sys.stderr.write("ignoring command line argument: '%s'\n" % a)
if cfg['daemonize'] == 'True':
daemonize(rootdir = None, umask = 0, close_fds = False,
stdin = os.devnull,
stdout = cfg['logfile'], stderr = cfg['logfile'],
pidfilename = cfg['pidfile'],
defaultmaxfd = DEFAULT_MAXFD)
def main():
""" Main program startup.
"""
# get a configuration merged from config file and command-line arguments
cfg, args = get_merged_config("%s/core.conf" % constants.CORE_CONF_DIR)
for a in args:
sys.stderr.write("ignoring command line argument: %s\n" % a)
if cfg["daemonize"] == "True":
daemonize(rootdir=None, umask=0, close_fds=False,
stdin=os.devnull,
stdout=cfg["logfile"], stderr=cfg["logfile"],
pidfilename=cfg["pidfile"],
defaultmaxfd=DEFAULT_MAXFD)
signal.signal(signal.SIGUSR1, lambda signum, stackframe:
logrotate(stdout = cfg['logfile'],
stderr = cfg['logfile']))
logrotate(stdout=cfg["logfile"],
stderr=cfg["logfile"]))
banner()
if cfg['execfile']:
cfg['execfile'] = os.path.abspath(cfg['execfile'])
if cfg["execfile"]:
cfg["execfile"] = os.path.abspath(cfg["execfile"])
sys.exit(exec_file(cfg))
try:
cored(cfg)
@ -317,4 +308,23 @@ def main():
if __name__ == "__main__":
log.setup(level=logging.INFO)
# configure nodes to use
node_map = nodemaps.CLASSIC_NODES
if len(sys.argv) == 2 and sys.argv[1] == "ovs":
node_map = nodemaps.OVS_NODES
nodeutils.set_node_map(node_map)
# load default services
quagga.load_services()
nrl.load_services()
xorp.load_services()
bird.load_services()
utility.load_services()
security.load_services()
ucarp.load_services()
dockersvc.load_services()
startup.load_services()
main()

View file

@ -6,29 +6,31 @@
#
# author: Jeff Ahrenholz <jeffrey.m.ahrenholz@boeing.com>
#
'''
"""
core-manage: Helper tool to add, remove, or check for services, models, and
node types in a CORE installation.
'''
"""
import os
import sys
import ast
import optparse
import os
import re
import sys
from core import pycore
from core import services
from core.constants import CORE_CONF_DIR
class FileUpdater(object):
''' Helper class for changing configuration files.
'''
"""
Helper class for changing configuration files.
"""
actions = ("add", "remove", "check")
targets = ("service", "model", "nodetype")
def __init__(self, action, target, data, options):
'''
'''
"""
"""
self.action = action
self.target = target
self.data = data
@ -37,13 +39,13 @@ class FileUpdater(object):
self.search, self.filename = self.get_filename(target)
def process(self):
''' Invoke update_file() using a helper method depending on target.
'''
""" Invoke update_file() using a helper method depending on target.
"""
if self.verbose:
txt = "Updating"
if self.action == "check":
txt = "Checking"
sys.stdout.write("%s file: '%s'\n" % (txt, self.filename))
sys.stdout.write("%s file: %s\n" % (txt, self.filename))
if self.target == "service":
r = self.update_file(fn=self.update_services)
@ -64,41 +66,40 @@ class FileUpdater(object):
return r
def update_services(self, line):
''' Modify the __init__.py file having this format:
""" Modify the __init__.py file having this format:
__all__ = ["quagga", "nrl", "xorp", "bird", ]
Returns True or False when "check" is the action, a modified line
otherwise.
'''
line = line.strip('\n')
key, valstr = line.split('= ')
"""
line = line.strip("\n")
key, valstr = line.split("= ")
vals = ast.literal_eval(valstr)
r = self.update_keyvals(key, vals)
if self.action == "check":
return r
valstr = '%s' % r
return '= '.join([key, valstr]) + '\n'
valstr = "%s" % r
return "= ".join([key, valstr]) + "\n"
def update_emane_models(self, line):
''' Modify the core.conf file having this format:
""" Modify the core.conf file having this format:
emane_models = RfPipe, Ieee80211abg, CommEffect, Bypass
Returns True or False when "check" is the action, a modified line
otherwise.
'''
line = line.strip('\n')
key, valstr = line.split('= ')
vals = valstr.split(', ')
"""
line = line.strip("\n")
key, valstr = line.split("= ")
vals = valstr.split(", ")
r = self.update_keyvals(key, vals)
if self.action == "check":
return r
valstr = ', '.join(r)
return '= '.join([key, valstr]) + '\n'
valstr = ", ".join(r)
return "= ".join([key, valstr]) + "\n"
def update_keyvals(self, key, vals):
''' Perform self.action on (key, vals).
""" Perform self.action on (key, vals).
Returns True or False when "check" is the action, a modified line
otherwise.
'''
"""
if self.action == "check":
if self.data in vals:
return True
@ -115,11 +116,10 @@ class FileUpdater(object):
return vals
def get_filename(self, target):
''' Return search string and filename based on target.
'''
""" Return search string and filename based on target.
"""
if target == "service":
pypath = os.path.dirname(pycore.__file__)
filename = os.path.join(pypath, "services", "__init__.py")
filename = os.path.abspath(services.__file__)
search = "__all__ ="
elif target == "model":
filename = os.path.join(CORE_CONF_DIR, "core.conf")
@ -132,21 +132,21 @@ class FileUpdater(object):
else:
raise ValueError, "unknown target"
if not os.path.exists(filename):
raise ValueError, "file '%s' does not exist" % filename
raise ValueError, "file %s does not exist" % filename
return search, filename
def update_file(self, fn=None):
''' Open a file and search for self.search, invoking the supplied
""" Open a file and search for self.search, invoking the supplied
function on the matching line. Write file changes if necessary.
Returns True if the file has changed (or action is "check" and the
search string is found), False otherwise.
'''
"""
changed = False
output = "" # this accumulates output, assumes input is small
output = "" # this accumulates output, assumes input is small
with open(self.filename, "r") as f:
for line in f:
if line[:len(self.search)] == self.search:
r = fn(line) # line may be modified by fn() here
r = fn(line) # line may be modified by fn() here
if self.action == "check":
return r
else:
@ -157,17 +157,17 @@ class FileUpdater(object):
if changed:
with open(self.filename, "w") as f:
f.write(output)
return changed
def update_nodes_conf(self):
''' Add/remove/check entries from nodes.conf. This file
""" Add/remove/check entries from nodes.conf. This file
contains a Tcl-formatted array of node types. The array index must be
properly set for new entries. Uses self.{action, filename, search,
data} variables as input and returns the same value as update_file().
'''
"""
changed = False
output = "" # this accumulates output, assumes input is small
output = "" # this accumulates output, assumes input is small
with open(self.filename, "r") as f:
for line in f:
# make sure data is not added twice
@ -181,14 +181,15 @@ class FileUpdater(object):
continue
else:
output += line
if self.action == "add":
index = int(re.match('^\d+', line).group(0))
output += str(index + 1) + ' ' + self.data + '\n'
index = int(re.match("^\d+", line).group(0))
output += str(index + 1) + " " + self.data + "\n"
changed = True
if changed:
with open(self.filename, "w") as f:
f.write(output)
return changed
@ -200,21 +201,21 @@ def main():
usagestr += "\n %prog -v check model RfPipe"
usagestr += "\n %prog --userpath=\"$HOME/.core\" add nodetype \"{ftp ftp.gif ftp.gif {DefaultRoute FTP} netns {FTP server} }\" \n"
usagestr += "\nArguments:\n <action> should be one of: %s" % \
', '.join(FileUpdater.actions)
", ".join(FileUpdater.actions)
usagestr += "\n <target> should be one of: %s" % \
', '.join(FileUpdater.targets)
", ".join(FileUpdater.targets)
usagestr += "\n <string> is the text to %s" % \
', '.join(FileUpdater.actions)
parser = optparse.OptionParser(usage = usagestr)
parser.set_defaults(userpath = None, verbose = False,)
", ".join(FileUpdater.actions)
parser = optparse.OptionParser(usage=usagestr)
parser.set_defaults(userpath=None, verbose=False, )
parser.add_option("--userpath", dest = "userpath", type = "string",
help = "use the specified user path (e.g. \"$HOME/.core" \
"\") to access nodes.conf")
parser.add_option("-v", "--verbose", dest = "verbose", action="store_true",
help = "be verbose when performing action")
parser.add_option("--userpath", dest="userpath", type="string",
help="use the specified user path (e.g. \"$HOME/.core" \
"\") to access nodes.conf")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="be verbose when performing action")
def usage(msg = None, err = 0):
def usage(msg=None, err=0):
sys.stdout.write("\n")
if msg:
sys.stdout.write(msg + "\n\n")
@ -228,11 +229,11 @@ def main():
action = args[0]
if action not in FileUpdater.actions:
usage("invalid action '%s'" % action, 1)
usage("invalid action %s" % action, 1)
target = args[1]
if target not in FileUpdater.targets:
usage("invalid target '%s'" % target, 1)
usage("invalid target %s" % target, 1)
if target == "nodetype" and not options.userpath:
usage("user path option required for this target (%s)" % target)
@ -249,5 +250,6 @@ def main():
sys.exit(1)
sys.exit(0)
if __name__ == "__main__":
main()