daemon: removing python tlv related code and tests related to it

This commit is contained in:
Blake Harnden 2022-03-08 14:18:47 -08:00
parent dc7bb37252
commit c8a589ef76
29 changed files with 29 additions and 5439 deletions

View file

@ -8,19 +8,15 @@ message handlers are defined and some support for sending messages.
import argparse
import logging
import os
import sys
import threading
import time
from configparser import ConfigParser
from pathlib import Path
from core import constants
from core.api.grpc.server import CoreGrpcServer
from core.api.tlv.corehandlers import CoreHandler, CoreUdpHandler
from core.api.tlv.coreserver import CoreServer, CoreUdpServer
from core.api.tlv.enumerations import CORE_API_PORT
from core.constants import CORE_CONF_DIR, COREDPY_VERSION
from core.utils import close_onexec, load_logging_config
from core.emulator.coreemu import CoreEmu
from core.utils import load_logging_config
logger = logging.getLogger(__name__)
@ -34,20 +30,6 @@ def banner():
logger.info("CORE daemon v.%s started %s", constants.COREDPY_VERSION, time.ctime())
def start_udp(mainserver, server_address):
"""
Start a thread running a UDP server on the same host,port for
connectionless requests.
:param CoreServer mainserver: main core tcp server to piggy back off of
:param server_address:
:return: CoreUdpServer
"""
mainserver.udpserver = CoreUdpServer(server_address, CoreUdpHandler, mainserver)
mainserver.udpthread = threading.Thread(target=mainserver.udpserver.start, daemon=True)
mainserver.udpthread.start()
def cored(cfg):
"""
Start the CoreServer object and enter the server loop.
@ -55,34 +37,13 @@ def cored(cfg):
:param dict cfg: core configuration
:return: nothing
"""
host = cfg["listenaddr"]
port = int(cfg["port"])
if host == "" or host is None:
host = "localhost"
try:
address = (host, port)
server = CoreServer(address, CoreHandler, cfg)
except:
logger.exception("error starting main server on: %s:%s", host, port)
sys.exit(1)
# initialize grpc api
grpc_server = CoreGrpcServer(server.coreemu)
coreemu = CoreEmu(cfg)
grpc_server = CoreGrpcServer(coreemu)
address_config = cfg["grpcaddress"]
port_config = cfg["grpcport"]
grpc_address = f"{address_config}:{port_config}"
grpc_thread = threading.Thread(target=grpc_server.listen, args=(grpc_address,), daemon=True)
grpc_thread.start()
# start udp server
start_udp(server, address)
# close handlers
close_onexec(server.fileno())
logger.info("CORE TLV API TCP/UDP listening on: %s:%s", host, port)
server.serve_forever()
grpc_server.listen(grpc_address)
def get_merged_config(filename):
@ -98,49 +59,38 @@ def get_merged_config(filename):
default_grpc_port = "50051"
default_address = "localhost"
defaults = {
"port": str(CORE_API_PORT),
"listenaddr": default_address,
"grpcport": default_grpc_port,
"grpcaddress": default_address,
"logfile": default_log
}
parser = argparse.ArgumentParser(
description=f"CORE daemon v.{COREDPY_VERSION} instantiates Linux network namespace nodes.")
parser.add_argument("-f", "--configfile", dest="configfile",
help=f"read config from specified file; default = {filename}")
parser.add_argument("-p", "--port", dest="port", type=int,
help=f"port number to listen on; default = {CORE_API_PORT}")
parser.add_argument("--ovs", action="store_true", help="enable experimental ovs mode, default is false")
parser.add_argument("--grpc-port", dest="grpcport",
help=f"grpc port to listen on; default {default_grpc_port}")
parser.add_argument("--grpc-address", dest="grpcaddress",
help=f"grpc address to listen on; default {default_address}")
parser.add_argument("-l", "--logfile", help=f"core logging configuration; default {default_log}")
# parse command line options
args = parser.parse_args()
# convert ovs to internal format
args.ovs = "1" if args.ovs else "0"
# read the config file
if args.configfile is not None:
filename = args.configfile
del args.configfile
cfg = ConfigParser(defaults)
cfg.read(filename)
section = "core-daemon"
if not cfg.has_section(section):
cfg.add_section(section)
# merge argparse with configparser
for opt in vars(args):
val = getattr(args, opt)
if val is not None:
cfg.set(section, opt, str(val))
return dict(cfg.items(section))

View file

@ -1,70 +0,0 @@
#!/usr/bin/env python3
import argparse
import re
import sys
from pathlib import Path
from core import utils
from core.api.grpc.client import CoreGrpcClient
from core.errors import CoreCommandError
if __name__ == "__main__":
# parse flags
parser = argparse.ArgumentParser(description="Converts CORE imn files to xml")
parser.add_argument("-f", "--file", dest="file", help="imn file to convert")
parser.add_argument(
"-d", "--dest", dest="dest", default=None, help="destination for xml file, defaults to same location as imn"
)
args = parser.parse_args()
# validate provided file exists
imn_file = Path(args.file)
if not imn_file.exists():
print(f"{args.file} does not exist")
sys.exit(1)
# validate destination
if args.dest is not None:
dest = Path(args.dest)
if not dest.exists() or not dest.is_dir():
print(f"{dest.resolve()} does not exist or is not a directory")
sys.exit(1)
xml_file = Path(dest, imn_file.with_suffix(".xml").name)
else:
xml_file = Path(imn_file.with_suffix(".xml").name)
# validate xml file
if xml_file.exists():
print(f"{xml_file.resolve()} already exists")
sys.exit(1)
# run provided imn using core-gui batch mode
try:
print(f"running {imn_file.resolve()} in batch mode")
output = utils.cmd(f"core-gui --batch {imn_file.resolve()}")
last_line = output.split("\n")[-1].strip()
# check for active session
if last_line == "Another session is active.":
print("need to restart core-daemon or shutdown previous batch session")
sys.exit(1)
# parse session id
m = re.search(r"Session id is (\d+)\.", last_line)
if not m:
print(f"failed to find session id: {output}")
sys.exit(1)
session_id = int(m.group(1))
print(f"created session {session_id}")
# save xml and delete session
client = CoreGrpcClient()
with client.context_connect():
print(f"saving xml {xml_file.resolve()}")
client.save_xml(session_id, str(xml_file))
print(f"deleting session {session_id}")
client.delete_session(session_id)
except CoreCommandError as e:
print(f"core-gui batch failed for {imn_file.resolve()}: {e}")
sys.exit(1)

View file

@ -1,247 +0,0 @@
#!/usr/bin/env python3
"""
core-manage: Helper tool to add, remove, or check for services, models, and
node types in a CORE installation.
"""
import ast
import optparse
import os
import re
import sys
from core import services
from core.constants import CORE_CONF_DIR
class FileUpdater:
"""
Helper class for changing configuration files.
"""
actions = ("add", "remove", "check")
targets = ("service", "model", "nodetype")
def __init__(self, action, target, data, options):
"""
"""
self.action = action
self.target = target
self.data = data
self.options = options
self.verbose = options.verbose
self.search, self.filename = self.get_filename(target)
def process(self):
""" Invoke update_file() using a helper method depending on target.
"""
if self.verbose:
txt = "Updating"
if self.action == "check":
txt = "Checking"
sys.stdout.write(f"{txt} file: {self.filename}\n")
if self.target == "service":
r = self.update_file(fn=self.update_services)
elif self.target == "model":
r = self.update_file(fn=self.update_emane_models)
elif self.target == "nodetype":
r = self.update_nodes_conf()
if self.verbose:
txt = ""
if not r:
txt = "NOT "
if self.action == "check":
sys.stdout.write(f"String {txt} found.\n")
else:
sys.stdout.write(f"File {txt} updated.\n")
return r
def update_services(self, line):
""" Modify the __init__.py file having this format:
__all__ = ["quagga", "nrl", "xorp", "bird", ]
Returns True or False when "check" is the action, a modified line
otherwise.
"""
line = line.strip("\n")
key, valstr = line.split("= ")
vals = ast.literal_eval(valstr)
r = self.update_keyvals(key, vals)
if self.action == "check":
return r
valstr = str(r)
return "= ".join([key, valstr]) + "\n"
def update_emane_models(self, line):
""" Modify the core.conf file having this format:
emane_models = RfPipe, Ieee80211abg, CommEffect, Bypass
Returns True or False when "check" is the action, a modified line
otherwise.
"""
line = line.strip("\n")
key, valstr = line.split("= ")
vals = valstr.split(", ")
r = self.update_keyvals(key, vals)
if self.action == "check":
return r
valstr = ", ".join(r)
return "= ".join([key, valstr]) + "\n"
def update_keyvals(self, key, vals):
""" Perform self.action on (key, vals).
Returns True or False when "check" is the action, a modified line
otherwise.
"""
if self.action == "check":
if self.data in vals:
return True
else:
return False
elif self.action == "add":
if self.data not in vals:
vals.append(self.data)
elif self.action == "remove":
try:
vals.remove(self.data)
except ValueError:
pass
return vals
def get_filename(self, target):
""" Return search string and filename based on target.
"""
if target == "service":
filename = os.path.abspath(services.__file__)
search = "__all__ ="
elif target == "model":
filename = os.path.join(CORE_CONF_DIR, "core.conf")
search = "emane_models ="
elif target == "nodetype":
if self.options.userpath is None:
raise ValueError("missing user path")
filename = os.path.join(self.options.userpath, "nodes.conf")
search = self.data
else:
raise ValueError("unknown target")
if not os.path.exists(filename):
raise ValueError(f"file {filename} does not exist")
return search, filename
def update_file(self, fn=None):
""" Open a file and search for self.search, invoking the supplied
function on the matching line. Write file changes if necessary.
Returns True if the file has changed (or action is "check" and the
search string is found), False otherwise.
"""
changed = False
output = "" # this accumulates output, assumes input is small
with open(self.filename, "r") as f:
for line in f:
if line[:len(self.search)] == self.search:
r = fn(line) # line may be modified by fn() here
if self.action == "check":
return r
else:
if line != r:
changed = True
line = r
output += line
if changed:
with open(self.filename, "w") as f:
f.write(output)
return changed
def update_nodes_conf(self):
""" Add/remove/check entries from nodes.conf. This file
contains a Tcl-formatted array of node types. The array index must be
properly set for new entries. Uses self.{action, filename, search,
data} variables as input and returns the same value as update_file().
"""
changed = False
output = "" # this accumulates output, assumes input is small
with open(self.filename, "r") as f:
for line in f:
# make sure data is not added twice
if line.find(self.search) >= 0:
if self.action == "check":
return True
elif self.action == "add":
return False
elif self.action == "remove":
changed = True
continue
else:
output += line
if self.action == "add":
index = int(re.match("^\d+", line).group(0))
output += str(index + 1) + " " + self.data + "\n"
changed = True
if changed:
with open(self.filename, "w") as f:
f.write(output)
return changed
def main():
actions = ", ".join(FileUpdater.actions)
targets = ", ".join(FileUpdater.targets)
usagestr = "usage: %prog [-h] [options] <action> <target> <string>\n"
usagestr += "\nHelper tool to add, remove, or check for "
usagestr += "services, models, and node types\nin a CORE installation.\n"
usagestr += "\nExamples:\n %prog add service newrouting"
usagestr += "\n %prog -v check model RfPipe"
usagestr += "\n %prog --userpath=\"$HOME/.core\" add nodetype \"{ftp ftp.gif ftp.gif {DefaultRoute FTP} netns {FTP server} }\" \n"
usagestr += f"\nArguments:\n <action> should be one of: {actions}"
usagestr += f"\n <target> should be one of: {targets}"
usagestr += f"\n <string> is the text to {actions}"
parser = optparse.OptionParser(usage=usagestr)
parser.set_defaults(userpath=None, verbose=False, )
parser.add_option("--userpath", dest="userpath", type="string",
help="use the specified user path (e.g. \"$HOME/.core" \
"\") to access nodes.conf")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="be verbose when performing action")
def usage(msg=None, err=0):
sys.stdout.write("\n")
if msg:
sys.stdout.write(msg + "\n\n")
parser.print_help()
sys.exit(err)
(options, args) = parser.parse_args()
if len(args) != 3:
usage("Missing required arguments!", 1)
action = args[0]
if action not in FileUpdater.actions:
usage(f"invalid action {action}", 1)
target = args[1]
if target not in FileUpdater.targets:
usage(f"invalid target {target}", 1)
if target == "nodetype" and not options.userpath:
usage(f"user path option required for this target ({target})")
data = args[2]
try:
up = FileUpdater(action, target, data, options)
r = up.process()
except Exception as e:
sys.stderr.write(f"Exception: {e}\n")
sys.exit(1)
if not r:
sys.exit(1)
sys.exit(0)
if __name__ == "__main__":
main()

View file

@ -1,279 +0,0 @@
#!/usr/bin/env python3
"""
coresendmsg: utility for generating CORE messages
"""
import optparse
import os
import socket
import sys
from core.api.tlv import coreapi
from core.api.tlv.enumerations import CORE_API_PORT, MessageTypes, SessionTlvs
from core.emulator.enumerations import MessageFlags
def print_available_tlvs(t, tlv_class):
"""
Print a TLV list.
"""
print(f"TLVs available for {t} message:")
for tlv in sorted([tlv for tlv in tlv_class.tlv_type_map], key=lambda x: x.name):
print(tlv.name.lower())
def print_examples(name):
"""
Print example usage of this script.
"""
examples = [
("node number=3 x_position=125 y_position=525",
"move node number 3 to x,y=(125,525)"),
("node number=4 icon=/usr/local/share/core/icons/normal/router_red.gif",
"change node number 4\"s icon to red"),
("node flags=add number=5 type=0 name=\"n5\" x_position=500 y_position=500",
"add a new router node n5"),
("link n1_number=2 n2_number=3 delay=15000",
"set a 15ms delay on the link between n2 and n3"),
("link n1_number=2 n2_number=3 gui_attributes=\"color=blue\"",
"change the color of the link between n2 and n3"),
("link flags=add n1_number=4 n2_number=5 interface1_ip4=\"10.0.3.2\" "
"interface1_ip4_mask=24 interface2_ip4=\"10.0.3.1\" interface2_ip4_mask=24",
"link node n5 with n4 using the given interface addresses"),
("execute flags=string,text node=1 number=1000 command=\"uname -a\" -l",
"run a command on node 1 and wait for the result"),
("execute node=2 number=1001 command=\"killall ospfd\"",
"run a command on node 2 and ignore the result"),
("file flags=add node=1 name=\"/var/log/test.log\" data=\"hello world.\"",
"write a test.log file on node 1 with the given contents"),
("file flags=add node=2 name=\"test.log\" source_name=\"./test.log\"",
"move a test.log file from host to node 2"),
]
print(f"Example {name} invocations:")
for cmd, descr in examples:
print(f" {name} {cmd}\n\t\t{descr}")
def receive_message(sock):
"""
Retrieve a message from a socket and return the CoreMessage object or
None upon disconnect. Socket data beyond the first message is dropped.
"""
try:
# large receive buffer used for UDP sockets, instead of just receiving
# the 4-byte header
data = sock.recv(4096)
msghdr = data[:coreapi.CoreMessage.header_len]
except KeyboardInterrupt:
print("CTRL+C pressed")
sys.exit(1)
if len(msghdr) == 0:
return None
msgdata = None
msgtype, msgflags, msglen = coreapi.CoreMessage.unpack_header(msghdr)
if msglen:
msgdata = data[coreapi.CoreMessage.header_len:]
try:
msgcls = coreapi.CLASS_MAP[msgtype]
except KeyError:
msg = coreapi.CoreMessage(msgflags, msghdr, msgdata)
msg.message_type = msgtype
print(f"unimplemented CORE message type: {msg.type_str()}")
return msg
if len(data) > msglen + coreapi.CoreMessage.header_len:
data_size = len(data) - (msglen + coreapi.CoreMessage.header_len)
print(f"received a message of type {msgtype}, dropping {data_size} bytes of extra data")
return msgcls(msgflags, msghdr, msgdata)
def connect_to_session(sock, requested):
"""
Use Session Messages to retrieve the current list of sessions and
connect to the first one.
"""
# request the session list
tlvdata = coreapi.CoreSessionTlv.pack(SessionTlvs.NUMBER.value, "")
flags = MessageFlags.STRING.value
smsg = coreapi.CoreSessionMessage.pack(flags, tlvdata)
sock.sendall(smsg)
print("waiting for session list...")
smsgreply = receive_message(sock)
if smsgreply is None:
print("disconnected")
return False
sessstr = smsgreply.get_tlv(SessionTlvs.NUMBER.value)
if sessstr is None:
print("missing session numbers")
return False
# join the first session (that is not our own connection)
tmp, localport = sock.getsockname()
sessions = sessstr.split("|")
sessions.remove(str(localport))
if len(sessions) == 0:
print("no sessions to join")
return False
if not requested:
session = sessions[0]
elif requested in sessions:
session = requested
else:
print("requested session not found!")
return False
print(f"joining session: {session}")
tlvdata = coreapi.CoreSessionTlv.pack(SessionTlvs.NUMBER.value, session)
flags = MessageFlags.ADD.value
smsg = coreapi.CoreSessionMessage.pack(flags, tlvdata)
sock.sendall(smsg)
return True
def receive_response(sock, opt):
"""
Receive and print a CORE message from the given socket.
"""
print("waiting for response...")
msg = receive_message(sock)
if msg is None:
print(f"disconnected from {opt.address}:{opt.port}")
sys.exit(0)
print(f"received message: {msg}")
def main():
"""
Parse command-line arguments to build and send a CORE message.
"""
types = [message_type.name.lower() for message_type in MessageTypes]
flags = [flag.name.lower() for flag in MessageFlags]
types_usage = " ".join(types)
flags_usage = " ".join(flags)
usagestr = (
"usage: %prog [-h|-H] [options] [message-type] [flags=flags] "
"[message-TLVs]\n\n"
f"Supported message types:\n {types_usage}\n"
f"Supported message flags (flags=f1,f2,...):\n {flags_usage}"
)
parser = optparse.OptionParser(usage=usagestr)
default_address = "localhost"
default_session = None
default_tcp = False
parser.set_defaults(
port=CORE_API_PORT,
address=default_address,
session=default_session,
listen=False,
examples=False,
tlvs=False,
tcp=default_tcp
)
parser.add_option("-H", dest="examples", action="store_true",
help="show example usage help message and exit")
parser.add_option("-p", "--port", dest="port", type=int,
help=f"TCP port to connect to, default: {CORE_API_PORT}")
parser.add_option("-a", "--address", dest="address", type=str,
help=f"Address to connect to, default: {default_address}")
parser.add_option("-s", "--session", dest="session", type=str,
help=f"Session to join, default: {default_session}")
parser.add_option("-l", "--listen", dest="listen", action="store_true",
help="Listen for a response message and print it.")
parser.add_option("-t", "--list-tlvs", dest="tlvs", action="store_true",
help="List TLVs for the specified message type.")
parser.add_option("--tcp", dest="tcp", action="store_true",
help=f"Use TCP instead of UDP and connect to a session default: {default_tcp}")
def usage(msg=None, err=0):
print()
if msg:
print(f"{msg}\n")
parser.print_help()
sys.exit(err)
# parse command line opt
opt, args = parser.parse_args()
if opt.examples:
print_examples(os.path.basename(sys.argv[0]))
sys.exit(0)
if len(args) == 0:
usage("Please specify a message type to send.")
# given a message type t, determine the message and TLV classes
t = args.pop(0)
t = t.lower()
if t not in types:
usage(f"Unknown message type requested: {t}")
message_type = MessageTypes[t.upper()]
msg_cls = coreapi.CLASS_MAP[message_type.value]
tlv_cls = msg_cls.tlv_class
# list TLV types for this message type
if opt.tlvs:
print_available_tlvs(t, tlv_cls)
sys.exit(0)
# build a message consisting of TLVs from "type=value" arguments
flagstr = ""
tlvdata = b""
for a in args:
typevalue = a.split("=")
if len(typevalue) < 2:
usage(f"Use \"type=value\" syntax instead of \"{a}\".")
tlv_typestr = typevalue[0].lower()
tlv_valstr = "=".join(typevalue[1:])
if tlv_typestr == "flags":
flagstr = tlv_valstr
continue
try:
tlv_type = tlv_cls.tlv_type_map[tlv_typestr.upper()]
tlvdata += tlv_cls.pack_string(tlv_type.value, tlv_valstr)
except KeyError:
usage(f"Unknown TLV: \"{tlv_typestr}\"")
flags = 0
for f in flagstr.split(","):
if f == "":
continue
try:
flag_enum = MessageFlags[f.upper()]
n = flag_enum.value
flags |= n
except KeyError:
usage(f"Invalid flag \"{f}\".")
msg = msg_cls.pack(flags, tlvdata)
if opt.tcp:
protocol = socket.SOCK_STREAM
else:
protocol = socket.SOCK_DGRAM
sock = socket.socket(socket.AF_INET, protocol)
sock.setblocking(True)
try:
sock.connect((opt.address, opt.port))
except Exception as e:
print(f"Error connecting to {opt.address}:{opt.port}:\n\t{e}")
sys.exit(1)
if opt.tcp and not connect_to_session(sock, opt.session):
print("warning: continuing without joining a session!")
sock.sendall(msg)
if opt.listen:
receive_response(sock, opt)
if opt.tcp:
sock.shutdown(socket.SHUT_RDWR)
sock.close()
sys.exit(0)
if __name__ == "__main__":
main()