From 314eee33f6900a0fa185d0622eb4a3b9dd69142d Mon Sep 17 00:00:00 2001 From: Gabriel Somlo Date: Tue, 15 Mar 2022 14:09:15 -0400 Subject: [PATCH 001/157] frrboot: add fedora sbin path to default Add the Fedora default location for FRR binaries (/usr/libexec/frr/*.) to CORE's frrboot default "sbin" search path Signed-off-by: Gabriel Somlo --- daemon/core/configservices/frrservices/services.py | 2 +- daemon/core/services/frr.py | 2 +- daemon/data/core.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/daemon/core/configservices/frrservices/services.py b/daemon/core/configservices/frrservices/services.py index dd2d1f9d..5e6c9948 100644 --- a/daemon/core/configservices/frrservices/services.py +++ b/daemon/core/configservices/frrservices/services.py @@ -89,7 +89,7 @@ class FRRZebra(ConfigService): "frr_bin_search", default="/usr/local/bin /usr/bin /usr/lib/frr" ).strip('"') frr_sbin_search = self.node.session.options.get( - "frr_sbin_search", default="/usr/local/sbin /usr/sbin /usr/lib/frr" + "frr_sbin_search", default="/usr/local/sbin /usr/sbin /usr/lib/frr /usr/libexec/frr" ).strip('"') services = [] diff --git a/daemon/core/services/frr.py b/daemon/core/services/frr.py index 87145d37..c8ad1048 100644 --- a/daemon/core/services/frr.py +++ b/daemon/core/services/frr.py @@ -142,7 +142,7 @@ class FRRZebra(CoreService): "frr_bin_search", '"/usr/local/bin /usr/bin /usr/lib/frr"' ) frr_sbin_search = node.session.options.get( - "frr_sbin_search", '"/usr/local/sbin /usr/sbin /usr/lib/frr"' + "frr_sbin_search", '"/usr/local/sbin /usr/sbin /usr/lib/frr /usr/libexec/frr"' ) cfg = """\ #!/bin/sh diff --git a/daemon/data/core.conf b/daemon/data/core.conf index 874ba567..1923250d 100644 --- a/daemon/data/core.conf +++ b/daemon/data/core.conf @@ -5,7 +5,7 @@ grpcport = 50051 quagga_bin_search = "/usr/local/bin /usr/bin /usr/lib/quagga" quagga_sbin_search = "/usr/local/sbin /usr/sbin /usr/lib/quagga" frr_bin_search = "/usr/local/bin /usr/bin /usr/lib/frr" -frr_sbin_search = "/usr/local/sbin /usr/sbin /usr/lib/frr" +frr_sbin_search = "/usr/local/sbin /usr/sbin /usr/lib/frr /usr/libexec/frr" # uncomment the following line to load custom services from the specified dir # this may be a comma-separated list, and directory names should be unique From 1d718aeda2460cd43fb9e6802f0625f317138da8 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 8 Jun 2022 14:00:20 -0700 Subject: [PATCH 002/157] update configservice example file to a proper name --- daemon/examples/configservices/{testing.py => switch.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename daemon/examples/configservices/{testing.py => switch.py} (100%) diff --git a/daemon/examples/configservices/testing.py b/daemon/examples/configservices/switch.py similarity index 100% rename from daemon/examples/configservices/testing.py rename to daemon/examples/configservices/switch.py From 1aa9d4bccf9bdb11c0a0e6fcf15aa1575e10f1cc Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 8 Jun 2022 14:06:54 -0700 Subject: [PATCH 003/157] examples: restore docker switch example to previous state --- daemon/examples/docker/switch.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/daemon/examples/docker/switch.py b/daemon/examples/docker/switch.py index a3f8e31e..3f696c56 100644 --- a/daemon/examples/docker/switch.py +++ b/daemon/examples/docker/switch.py @@ -3,16 +3,17 @@ import logging from core.emulator.coreemu import CoreEmu from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes +from core.nodes.base import CoreNode from core.nodes.docker import DockerNode from core.nodes.network import SwitchNode if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) + # create core session coreemu = CoreEmu() session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) - try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") @@ -21,9 +22,7 @@ if __name__ == "__main__": # node one options = DockerNode.create_options() - options.image = "core" - options.binds.append(("/tmp/testbind", "/tmp/bind")) - options.volumes.append(("var.log", "/tmp/var.log", True, True)) + options.image = "ubuntu" node1 = session.add_node(DockerNode, options=options) interface1_data = prefixes.create_iface(node1) @@ -32,13 +31,13 @@ if __name__ == "__main__": interface2_data = prefixes.create_iface(node2) # node three - # node_three = session.add_node(CoreNode) - # interface_three = prefixes.create_iface(node_three) + node_three = session.add_node(CoreNode) + interface_three = prefixes.create_iface(node_three) # add links session.add_link(node1.id, switch.id, interface1_data) session.add_link(node2.id, switch.id, interface2_data) - # session.add_link(node_three.id, switch.id, interface_three) + session.add_link(node_three.id, switch.id, interface_three) # instantiate session.instantiate() From 469f8f087a532fe897d44df38a6b5cea63be9a62 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 8 Jun 2022 14:17:13 -0700 Subject: [PATCH 004/157] grpc: removed removed api call from client code get_node_links, this information is included in get node --- daemon/core/api/grpc/client.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index 9da2cb14..4ea3332a 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -573,23 +573,6 @@ class CoreGrpcClient: response = self.stub.GetNodeTerminal(request) return response.terminal - def get_node_links(self, session_id: int, node_id: int) -> List[wrappers.Link]: - """ - Get current links for a node. - - :param session_id: session id - :param node_id: node id - :return: list of links - :raises grpc.RpcError: when session or node doesn't exist - """ - request = core_pb2.GetNodeLinksRequest(session_id=session_id, node_id=node_id) - response = self.stub.GetNodeLinks(request) - links = [] - for link_proto in response.links: - link = wrappers.Link.from_proto(link_proto) - links.append(link) - return links - def add_link( self, session_id: int, link: wrappers.Link, source: str = None ) -> Tuple[bool, wrappers.Interface, wrappers.Interface]: From 9991942e7bebc3a41e4757216a2646f1170e66ed Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 10 Jun 2022 09:20:49 -0700 Subject: [PATCH 005/157] daemon: cleanup code for lxd based nodes and properly implement command exectution --- daemon/core/nodes/lxd.py | 95 ++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 57 deletions(-) diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index b99b4576..8bf02ae4 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -1,15 +1,16 @@ import json import logging +import shlex import time from dataclasses import dataclass, field from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Tuple -from core import utils from core.emulator.data import InterfaceData, LinkOptions from core.emulator.distributed import DistributedServer from core.errors import CoreCommandError +from core.executables import BASH from core.nodes.base import CoreNode, CoreNodeOptions from core.nodes.interface import CoreInterface @@ -19,54 +20,6 @@ if TYPE_CHECKING: from core.emulator.session import Session -class LxdClient: - def __init__(self, name: str, image: str, run: Callable[..., str]) -> None: - self.name: str = name - self.image: str = image - self.run: Callable[..., str] = run - self.pid: Optional[int] = None - - def create_container(self) -> int: - self.run(f"lxc launch {self.image} {self.name}") - data = self.get_info() - self.pid = data["state"]["pid"] - return self.pid - - def get_info(self) -> Dict: - args = f"lxc list {self.name} --format json" - output = self.run(args) - data = json.loads(output) - if not data: - raise CoreCommandError(1, args, f"LXC({self.name}) not present") - return data[0] - - def is_alive(self) -> bool: - try: - data = self.get_info() - return data["state"]["status"] == "Running" - except CoreCommandError: - return False - - def stop_container(self) -> None: - self.run(f"lxc delete --force {self.name}") - - def create_cmd(self, cmd: str) -> str: - return f"lxc exec -nT {self.name} -- {cmd}" - - def create_ns_cmd(self, cmd: str) -> str: - return f"nsenter -t {self.pid} -m -u -i -p -n {cmd}" - - def check_cmd(self, cmd: str, wait: bool = True, shell: bool = False) -> str: - args = self.create_cmd(cmd) - return utils.cmd(args, wait=wait, shell=shell) - - def copy_file(self, src_path: Path, dst_path: Path) -> None: - if not str(dst_path).startswith("/"): - dst_path = Path("/root/") / dst_path - args = f"lxc file push {src_path} {self.name}/{dst_path}" - self.run(args) - - @dataclass class LxcOptions(CoreNodeOptions): image: str = "ubuntu" @@ -104,19 +57,42 @@ class LxcNode(CoreNode): options = options or LxcOptions() super().__init__(session, _id, name, server, options) self.image: str = options.image - self.client: Optional[LxdClient] = None @classmethod def create_options(cls) -> LxcOptions: return LxcOptions() + def _create_cmd(self, args: str, shell: bool = False) -> str: + """ + Create command used to run commands within the context of a node. + + :param args: command arguments + :param shell: True to run shell like, False otherwise + :return: node command + """ + if shell: + args = f"{BASH} -c {shlex.quote(args)}" + return f"nsenter -t {self.pid} -m -u -i -p -n {args}" + + def _get_info(self) -> Dict: + args = f"lxc list {self.name} --format json" + output = self.host_cmd(args) + data = json.loads(output) + if not data: + raise CoreCommandError(1, args, f"LXC({self.name}) not present") + return data[0] + def alive(self) -> bool: """ Check if the node is alive. :return: True if node is alive, False otherwise """ - return self.client.is_alive() + try: + data = self._get_info() + return data["state"]["status"] == "Running" + except CoreCommandError: + return False def startup(self) -> None: """ @@ -128,8 +104,9 @@ class LxcNode(CoreNode): if self.up: raise ValueError("starting a node that is already up") self.makenodedir() - self.client = LxdClient(self.name, self.image, self.host_cmd) - self.pid = self.client.create_container() + self.host_cmd(f"lxc launch {self.image} {self.name}") + data = self._get_info() + self.pid = data["state"]["pid"] self.up = True def shutdown(self) -> None: @@ -143,7 +120,7 @@ class LxcNode(CoreNode): return with self.lock: self.ifaces.clear() - self.client.stop_container() + self.host_cmd(f"lxc delete --force {self.name}") self.up = False def termcmdstring(self, sh: str = "/bin/sh") -> str: @@ -197,7 +174,9 @@ class LxcNode(CoreNode): self.cmd(f"mkdir -m {0o755:o} -p {directory}") if self.server is not None: self.server.remote_put(temp_path, temp_path) - self.client.copy_file(temp_path, file_path) + if not str(file_path).startswith("/"): + file_path = Path("/root/") / file_path + self.host_cmd(f"lxc file push {temp_path} {self.name}/{file_path}") self.cmd(f"chmod {mode:o} {file_path}") if self.server is not None: self.host_cmd(f"rm -f {temp_path}") @@ -223,7 +202,9 @@ class LxcNode(CoreNode): temp_path = Path(temp.name) src_path = temp_path self.server.remote_put(src_path, temp_path) - self.client.copy_file(src_path, dst_path) + if not str(dst_path).startswith("/"): + dst_path = Path("/root/") / dst_path + self.host_cmd(f"lxc file push {src_path} {self.name}/{dst_path}") if mode is not None: self.cmd(f"chmod {mode:o} {dst_path}") From 9c265ab28379f72147292311a7493bc761d8719b Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 10 Jun 2022 10:01:48 -0700 Subject: [PATCH 006/157] daemon: updates to change hostname settings to replace _ to - due to _ being an invalid character --- daemon/core/nodes/docker.py | 3 ++- daemon/core/nodes/netclient.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index a97ba08c..d454ea42 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -142,8 +142,9 @@ class DockerNode(CoreNode): volumes += ( f"--mount type=volume," f"source={volume.src},target={volume.dst} " ) + hostname = self.name.replace("_", "-") self.host_cmd( - f"{DOCKER} run -td --init --net=none --hostname {self.name} " + f"{DOCKER} run -td --init --net=none --hostname {hostname} " f"--name {self.name} --sysctl net.ipv6.conf.all.disable_ipv6=0 " f"{binds} {volumes} " f"--privileged {self.image} tail -f /dev/null" diff --git a/daemon/core/nodes/netclient.py b/daemon/core/nodes/netclient.py index 09cf94ec..e0a409f4 100644 --- a/daemon/core/nodes/netclient.py +++ b/daemon/core/nodes/netclient.py @@ -28,6 +28,7 @@ class LinuxNetClient: :param name: name for hostname :return: nothing """ + name = name.replace("_", "-") self.run(f"hostname {name}") def create_route(self, route: str, device: str) -> None: From 60a48c7084d249cdb8061bdd13a3f7c9e50f677e Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 10 Jun 2022 12:12:25 -0700 Subject: [PATCH 007/157] daemon: update node commands to make use of shlex.quote for shell=True commands --- daemon/core/nodes/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 404e9ab2..5866b5e2 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -3,6 +3,7 @@ Defines the base logic for nodes used within core. """ import abc import logging +import shlex import shutil import threading from dataclasses import dataclass, field @@ -702,7 +703,7 @@ class CoreNode(CoreNodeBase): :return: node command """ if shell: - args = f'{BASH} -c "{args}"' + args = f"{BASH} -c {shlex.quote(args)}" return f"{VCMD} -c {self.ctrlchnlname} -- {args}" def cmd(self, args: str, wait: bool = True, shell: bool = False) -> str: From 9c69881aad57b0c2cf5c7fb2741d7b9ddc6ce298 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 10 Jun 2022 14:23:06 -0700 Subject: [PATCH 008/157] daemon: updates to expose node.create_cmd and not be private, added utility functions for running multiple commands on multiple nodes more efficiently --- daemon/core/nodes/base.py | 6 +- daemon/core/nodes/docker.py | 2 +- daemon/core/nodes/lxd.py | 2 +- daemon/core/nodes/physical.py | 2 +- daemon/core/utils.py | 117 +++++++++++++++++++++++++++++++++- 5 files changed, 121 insertions(+), 8 deletions(-) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 5866b5e2..7edbc198 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -694,7 +694,7 @@ class CoreNode(CoreNodeBase): finally: self.rmnodedir() - def _create_cmd(self, args: str, shell: bool = False) -> str: + def create_cmd(self, args: str, shell: bool = False) -> str: """ Create command used to run commands within the context of a node. @@ -717,7 +717,7 @@ class CoreNode(CoreNodeBase): :return: combined stdout and stderr :raises CoreCommandError: when a non-zero exit status occurs """ - args = self._create_cmd(args, shell) + args = self.create_cmd(args, shell) if self.server is None: return utils.cmd(args, wait=wait, shell=shell) else: @@ -743,7 +743,7 @@ class CoreNode(CoreNodeBase): :param sh: shell to execute command in :return: str """ - terminal = self._create_cmd(sh) + terminal = self.create_cmd(sh) if self.server is None: return terminal else: diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index d454ea42..41625c78 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -89,7 +89,7 @@ class DockerNode(CoreNode): """ return DockerOptions() - def _create_cmd(self, args: str, shell: bool = False) -> str: + def create_cmd(self, args: str, shell: bool = False) -> str: """ Create command used to run commands within the context of a node. diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 8bf02ae4..33b8d466 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -62,7 +62,7 @@ class LxcNode(CoreNode): def create_options(cls) -> LxcOptions: return LxcOptions() - def _create_cmd(self, args: str, shell: bool = False) -> str: + def create_cmd(self, args: str, shell: bool = False) -> str: """ Create command used to run commands within the context of a node. diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index 02c200db..8ab13f20 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -249,7 +249,7 @@ class PhysicalNode(CoreNode): iface.shutdown() self.rmnodedir() - def _create_cmd(self, args: str, shell: bool = False) -> str: + def create_cmd(self, args: str, shell: bool = False) -> str: if shell: args = f'{BASH} -c "{args}"' return args diff --git a/daemon/core/utils.py b/daemon/core/utils.py index d2308f30..244590f8 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -216,8 +216,7 @@ def cmd( shell: bool = False, ) -> str: """ - Execute a command on the host and return a tuple containing the exit status and - result string. stderr output is folded into the stdout result string. + Execute a command on the host and returns the combined stderr stdout output. :param args: command arguments :param env: environment to run command with @@ -250,6 +249,25 @@ def cmd( raise CoreCommandError(1, input_args, "", e.strerror) +def run_cmds(args: List[str], wait: bool = True, shell: bool = False) -> List[str]: + """ + Execute a series of commands on the host and returns a list of the combined stderr + stdout output. + + :param args: command arguments + :param wait: True to wait for status, False otherwise + :param shell: True to use shell, False otherwise + :return: combined stdout and stderr + :raises CoreCommandError: when there is a non-zero exit status or the file to + execute is not found + """ + outputs = [] + for arg in args: + output = cmd(arg, wait=wait, shell=shell) + outputs.append(output) + return outputs + + def file_munge(pathname: str, header: str, text: str) -> None: """ Insert text at the end of a file, surrounded by header comments. @@ -407,6 +425,101 @@ def load_logging_config(config_path: Path) -> None: logging.config.dictConfig(log_config) +def run_cmds_threaded( + nodes: List["CoreNode"], + cmds: List[str], + wait: bool = True, + shell: bool = False, + workers: int = None, +) -> Tuple[Dict[int, List[str]], List[Exception]]: + """ + Run a set of commands in order across a provided set of nodes. Each node will + run the commands within the context of a threadpool. + + :param nodes: nodes to run commands in + :param cmds: commands to run in nodes + :param wait: True to wait for status, False otherwise + :param shell: True to run shell like, False otherwise + :param workers: number of workers for threadpool, uses library default otherwise + :return: tuple including dict of node id to list of command output and a list of + exceptions if any + """ + + def _node_cmds( + _target: "CoreNode", _cmds: List[str], _wait: bool, _shell: bool + ) -> List[str]: + outputs = [] + for _cmd in _cmds: + output = _target.cmd(_cmd, wait=_wait, shell=_shell) + outputs.append(output) + return outputs + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [] + node_mappings = {} + for node in nodes: + future = executor.submit(_node_cmds, node, cmds, wait, shell) + node_mappings[future] = node + futures.append(future) + outputs = {} + exceptions = [] + for future in concurrent.futures.as_completed(futures): + try: + result = future.result() + node = node_mappings[future] + outputs[node.id] = result + except Exception as e: + logger.exception("thread pool exception") + exceptions.append(e) + return outputs, exceptions + + +def run_cmds_mp( + nodes: List["CoreNode"], + cmds: List[str], + wait: bool = True, + shell: bool = False, + workers: int = None, +) -> Tuple[Dict[int, List[str]], List[Exception]]: + """ + Run a set of commands in order across a provided set of nodes. Each node will + run the commands within the context of a process pool. This will not work + for distributed nodes and throws an exception when encountered. + + :param nodes: nodes to run commands in + :param cmds: commands to run in nodes + :param wait: True to wait for status, False otherwise + :param shell: True to run shell like, False otherwise + :param workers: number of workers for threadpool, uses library default otherwise + :return: tuple including dict of node id to list of command output and a list of + exceptions if any + :raises CoreError: when a distributed node is provided as input + """ + with concurrent.futures.ProcessPoolExecutor(max_workers=workers) as executor: + futures = [] + node_mapping = {} + for node in nodes: + node_cmds = [node.create_cmd(x) for x in cmds] + if node.server: + raise CoreError( + f"{node.name} uses a distributed server and not supported" + ) + future = executor.submit(run_cmds, node_cmds, wait=wait, shell=shell) + node_mapping[future] = node + futures.append(future) + exceptions = [] + outputs = {} + for future in concurrent.futures.as_completed(futures): + try: + result = future.result() + node = node_mapping[future] + outputs[node.id] = result + except Exception as e: + logger.exception("thread pool exception") + exceptions.append(e) + return outputs, exceptions + + def threadpool( funcs: List[Tuple[Callable, Iterable[Any], Dict[Any, Any]]], workers: int = 10 ) -> Tuple[List[Any], List[Exception]]: From 3c28ea373a701e46d6de39644b872e88289c7922 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 10 Jun 2022 14:53:49 -0700 Subject: [PATCH 009/157] daemon: adjustments to fix terminal command string generation for docker/lxd nodes to account for being on a distributed server --- daemon/core/nodes/docker.py | 6 +++++- daemon/core/nodes/lxd.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 41625c78..45d2b892 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -188,7 +188,11 @@ class DockerNode(CoreNode): :param sh: shell to execute command in :return: str """ - return f"{DOCKER} exec -it {self.name} {sh}" + terminal = f"{DOCKER} exec -it {self.name} {sh}" + if self.server is None: + return terminal + else: + return f"ssh -X -f {self.server.host} xterm -e {terminal}" def create_dir(self, dir_path: Path) -> None: """ diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 33b8d466..01bd2db7 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -130,7 +130,11 @@ class LxcNode(CoreNode): :param sh: shell to execute command in :return: str """ - return f"lxc exec {self.name} -- {sh}" + terminal = f"lxc exec {self.name} -- {sh}" + if self.server is None: + return terminal + else: + return f"ssh -X -f {self.server.host} xterm -e {terminal}" def create_dir(self, dir_path: Path) -> None: """ From fe1593b51feb04c02d7d2a872fa64e6fc5b17fc9 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:38:52 -0700 Subject: [PATCH 010/157] update dockerfiles to fix default branch name in the oracle linux dockerfile and update pip in the centos dockerfile to get back to working order --- Dockerfile.centos | 2 +- Dockerfile.oracle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.centos b/Dockerfile.centos index 509e6dfe..2096556c 100644 --- a/Dockerfile.centos +++ b/Dockerfile.centos @@ -17,7 +17,7 @@ WORKDIR /root RUN git clone https://github.com/coreemu/core WORKDIR /root/core RUN git checkout ${BRANCH} -RUN ./setup.sh +RUN ./setup.sh && python3 -m pip install --upgrade pip RUN . /root/.bashrc && inv install -v -p ${PREFIX} # install emane packages and python bindings WORKDIR /root diff --git a/Dockerfile.oracle b/Dockerfile.oracle index 28a488b0..455e5874 100644 --- a/Dockerfile.oracle +++ b/Dockerfile.oracle @@ -4,7 +4,7 @@ LABEL Description="CORE Docker Oracle Linux Image" # define variables ARG PREFIX=/usr -ARG BRANCH=docker-updates +ARG BRANCH=master # define environment ENV DEBIAN_FRONTEND=noninteractive From d63722c0ed179da266135cc98f522e7b1f3e9c29 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 21 Jul 2022 15:35:31 -0700 Subject: [PATCH 011/157] install: add option to avoid installing python system dependencies, in case the python version being used is from source --- tasks.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tasks.py b/tasks.py index 63231de5..0cf4613f 100644 --- a/tasks.py +++ b/tasks.py @@ -166,21 +166,27 @@ def check_existing_core(c: Context, hide: bool) -> None: raise SystemError("core scripts found, please remove old installation") -def install_system(c: Context, os_info: OsInfo, hide: bool) -> None: +def install_system(c: Context, os_info: OsInfo, hide: bool, no_python: bool) -> None: python_dep = get_env_python_dep() if os_info.like == OsLike.DEBIAN: c.run( "sudo apt install -y automake pkg-config gcc libev-dev nftables " - f"iproute2 ethtool tk {python_dep}-tk bash", + f"iproute2 ethtool tk bash", hide=hide ) + if not no_python: + c.run(f"sudo apt install -y {python_dep}-tk", hide=hide) elif os_info.like == OsLike.REDHAT: c.run( "sudo yum install -y automake pkgconf-pkg-config gcc gcc-c++ " - f"libev-devel nftables iproute {python_dep}-devel {python_dep}-tkinter " - "tk ethtool make bash", - hide=hide + f"libev-devel nftables iproute tk ethtool make bash", + hide=hide, ) + if not no_python: + c.run( + f"sudo yum install -y {python_dep}-devel {python_dep}-tkinter ", + hide=hide, + ) # centos 8+ does not support netem by default if os_info.name == OsName.CENTOS and os_info.version >= 8: c.run("sudo yum install -y kernel-modules-extra", hide=hide) @@ -336,6 +342,7 @@ def install_core_files(c, local=False, verbose=False, prefix=DEFAULT_PREFIX): "install-type": "used to force an install type, " "can be one of the following (redhat, debian)", "ospf": "disable ospf installation", + "no-python": "avoid installing python system dependencies", }, ) def install( @@ -346,6 +353,7 @@ def install( prefix=DEFAULT_PREFIX, install_type=None, ospf=True, + no_python=False, ): """ install core, poetry, scripts, service, and ospf mdr @@ -360,7 +368,7 @@ def install( with p.start("checking for old installations"): check_existing_core(c, hide) with p.start("installing system dependencies"): - install_system(c, os_info, hide) + install_system(c, os_info, hide, no_python) with p.start("installing system grpcio-tools"): install_grpcio(c, hide) with p.start("building core"): From fcf6f3030275f7320335d2d7e88c64de6e426988 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 27 Jul 2022 16:00:10 -0700 Subject: [PATCH 012/157] install: updates to support building deb/rpm packages that contain python wheels and install core from a single file, updates to core to install scripts by way of python directly --- Makefile.am | 79 ++++++++++-- daemon/Makefile.am | 3 - .../grpc => core/scripts}/__init__.py | 0 daemon/core/scripts/cleanup.py | 103 ++++++++++++++++ .../{scripts/core-cli => core/scripts/cli.py} | 98 ++++++++++----- .../core-daemon => core/scripts/daemon.py} | 38 ++++-- .../{scripts/core-gui => core/scripts/gui.py} | 24 ++-- .../core-player => core/scripts/player.py} | 7 +- .../scripts/routemonitor.py} | 19 ++- .../scripts/serviceupdate.py} | 48 +++++--- daemon/pyproject.toml | 8 ++ daemon/scripts/core-cleanup | 71 ----------- package/Dockerfile.centos | 15 +++ package/Dockerfile.ubuntu | 15 +++ package/after-install.sh | 16 +++ package/after-remove.sh | 13 ++ package/core-daemon | 112 ++++++++++++++++++ package/core-daemon.service | 12 ++ {daemon/data => package/etc}/core.conf | 0 {daemon/data => package/etc}/logging.conf | 0 .../examples/configservices/switch.py | 0 .../examples/controlnet_updown | 0 .../examples/docker/docker2core.py | 0 .../examples/docker/docker2docker.py | 0 {daemon => package}/examples/docker/switch.py | 0 .../examples/grpc}/__init__.py | 0 .../examples/grpc/distributed_switch.py | 0 .../examples/grpc/emane80211.py | 0 .../examples/grpc/peertopeer.py | 0 {daemon => package}/examples/grpc/switch.py | 0 {daemon => package}/examples/grpc/wireless.py | 0 {daemon => package}/examples/grpc/wlan.py | 0 {daemon => package}/examples/lxd/lxd2core.py | 0 {daemon => package}/examples/lxd/lxd2lxd.py | 0 {daemon => package}/examples/lxd/switch.py | 0 package/examples/myemane/__init__.py | 0 .../examples/myemane/examplemodel.py | 0 .../examples/myservices/__init__.py | 0 .../examples/myservices/exampleservice.py | 0 .../examples/python/distributed_emane.py | 0 .../examples/python/distributed_lxd.py | 0 .../examples/python/distributed_ptp.py | 0 .../examples/python/distributed_switch.py | 0 .../examples/python/emane80211.py | 0 .../examples/python/peertopeer.py | 0 {daemon => package}/examples/python/switch.py | 0 .../examples/python/wireless.py | 0 {daemon => package}/examples/python/wlan.py | 0 .../examples/services/sampleFirewall | 0 .../examples/services/sampleIPsec | 0 .../examples/services/sampleVPNClient | 0 .../examples/services/sampleVPNServer | 0 .../examples/tdma/schedule.xml | 0 tasks.py | 34 +----- 54 files changed, 528 insertions(+), 187 deletions(-) rename daemon/{examples/grpc => core/scripts}/__init__.py (100%) create mode 100755 daemon/core/scripts/cleanup.py rename daemon/{scripts/core-cli => core/scripts/cli.py} (88%) rename daemon/{scripts/core-daemon => core/scripts/daemon.py} (78%) rename daemon/{scripts/core-gui => core/scripts/gui.py} (60%) rename daemon/{scripts/core-player => core/scripts/player.py} (90%) rename daemon/{scripts/core-route-monitor => core/scripts/routemonitor.py} (97%) rename daemon/{scripts/core-service-update => core/scripts/serviceupdate.py} (50%) delete mode 100755 daemon/scripts/core-cleanup create mode 100644 package/Dockerfile.centos create mode 100644 package/Dockerfile.ubuntu create mode 100644 package/after-install.sh create mode 100644 package/after-remove.sh create mode 100644 package/core-daemon create mode 100644 package/core-daemon.service rename {daemon/data => package/etc}/core.conf (100%) rename {daemon/data => package/etc}/logging.conf (100%) rename {daemon => package}/examples/configservices/switch.py (100%) rename {daemon => package}/examples/controlnet_updown (100%) rename {daemon => package}/examples/docker/docker2core.py (100%) rename {daemon => package}/examples/docker/docker2docker.py (100%) rename {daemon => package}/examples/docker/switch.py (100%) rename {daemon/examples/myemane => package/examples/grpc}/__init__.py (100%) rename {daemon => package}/examples/grpc/distributed_switch.py (100%) rename {daemon => package}/examples/grpc/emane80211.py (100%) rename {daemon => package}/examples/grpc/peertopeer.py (100%) rename {daemon => package}/examples/grpc/switch.py (100%) rename {daemon => package}/examples/grpc/wireless.py (100%) rename {daemon => package}/examples/grpc/wlan.py (100%) rename {daemon => package}/examples/lxd/lxd2core.py (100%) rename {daemon => package}/examples/lxd/lxd2lxd.py (100%) rename {daemon => package}/examples/lxd/switch.py (100%) create mode 100644 package/examples/myemane/__init__.py rename {daemon => package}/examples/myemane/examplemodel.py (100%) rename {daemon => package}/examples/myservices/__init__.py (100%) rename {daemon => package}/examples/myservices/exampleservice.py (100%) rename {daemon => package}/examples/python/distributed_emane.py (100%) rename {daemon => package}/examples/python/distributed_lxd.py (100%) rename {daemon => package}/examples/python/distributed_ptp.py (100%) rename {daemon => package}/examples/python/distributed_switch.py (100%) rename {daemon => package}/examples/python/emane80211.py (100%) rename {daemon => package}/examples/python/peertopeer.py (100%) rename {daemon => package}/examples/python/switch.py (100%) rename {daemon => package}/examples/python/wireless.py (100%) rename {daemon => package}/examples/python/wlan.py (100%) rename {daemon => package}/examples/services/sampleFirewall (100%) rename {daemon => package}/examples/services/sampleIPsec (100%) rename {daemon => package}/examples/services/sampleVPNClient (100%) rename {daemon => package}/examples/services/sampleVPNServer (100%) rename {daemon => package}/examples/tdma/schedule.xml (100%) diff --git a/Makefile.am b/Makefile.am index 83645c1b..2b5f29e2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -21,6 +21,7 @@ ACLOCAL_AMFLAGS = -I config # extra files to include with distribution tarball EXTRA_DIST = bootstrap.sh \ + package \ LICENSE \ README.md \ ASSIGNMENT_OF_COPYRIGHT.pdf \ @@ -47,7 +48,7 @@ fpm -s dir -t deb -n core-distributed \ --description "Common Open Research Emulator Distributed Package" \ --url https://github.com/coreemu/core \ --vendor "$(PACKAGE_VENDOR)" \ - -p core_distributed_VERSION_ARCH.deb \ + -p core-distributed_VERSION_ARCH.deb \ -v $(PACKAGE_VERSION) \ -d "ethtool" \ -d "procps" \ @@ -58,7 +59,8 @@ fpm -s dir -t deb -n core-distributed \ -d "libev4" \ -d "openssh-server" \ -d "xterm" \ - -C $(DESTDIR) + netns/vnoded=/usr/bin/ \ + netns/vcmd=/usr/bin/ endef define fpm-distributed-rpm = @@ -68,7 +70,7 @@ fpm -s dir -t rpm -n core-distributed \ --description "Common Open Research Emulator Distributed Package" \ --url https://github.com/coreemu/core \ --vendor "$(PACKAGE_VENDOR)" \ - -p core_distributed_VERSION_ARCH.rpm \ + -p core-distributed_VERSION_ARCH.rpm \ -v $(PACKAGE_VERSION) \ -d "ethtool" \ -d "procps-ng" \ @@ -79,12 +81,75 @@ fpm -s dir -t rpm -n core-distributed \ -d "net-tools" \ -d "openssh-server" \ -d "xterm" \ - -C $(DESTDIR) + netns/vnoded=/usr/bin/ \ + netns/vcmd=/usr/bin/ endef -.PHONY: fpm-distributed -fpm-distributed: clean-local-fpm - $(MAKE) -C netns install DESTDIR=$(DESTDIR) +define fpm-rpm = +fpm -s dir -t rpm -n core \ + -m "$(PACKAGE_MAINTAINERS)" \ + --license "BSD" \ + --description "core vnoded/vcmd and system dependencies" \ + --url https://github.com/coreemu/core \ + --vendor "$(PACKAGE_VENDOR)" \ + -p core_VERSION_ARCH.rpm \ + -v $(PACKAGE_VERSION) \ + --rpm-init package/core-daemon \ + --after-install package/after-install.sh \ + --after-remove package/after-remove.sh \ + -d "ethtool" \ + -d "tk" \ + -d "procps-ng" \ + -d "bash >= 3.0" \ + -d "ebtables" \ + -d "iproute" \ + -d "libev" \ + -d "net-tools" \ + -d "nftables" \ + netns/vnoded=/usr/bin/ \ + netns/vcmd=/usr/bin/ \ + package/etc/core.conf=/etc/core/ \ + package/etc/logging.conf=/etc/core/ \ + package/examples=/opt/core/ \ + daemon/dist/core-$(PACKAGE_VERSION)-py3-none-any.whl=/opt/core/ +endef + +define fpm-deb = +fpm -s dir -t deb -n core \ + -m "$(PACKAGE_MAINTAINERS)" \ + --license "BSD" \ + --description "core vnoded/vcmd and system dependencies" \ + --url https://github.com/coreemu/core \ + --vendor "$(PACKAGE_VENDOR)" \ + -p core_VERSION_ARCH.deb \ + -v $(PACKAGE_VERSION) \ + --deb-systemd package/core-daemon.service \ + --deb-no-default-config-files \ + --after-install package/after-install.sh \ + --after-remove package/after-remove.sh \ + -d "ethtool" \ + -d "tk" \ + -d "libtk-img" \ + -d "procps" \ + -d "libc6 >= 2.14" \ + -d "bash >= 3.0" \ + -d "ebtables" \ + -d "iproute2" \ + -d "libev4" \ + -d "nftables" \ + netns/vnoded=/usr/bin/ \ + netns/vcmd=/usr/bin/ \ + package/etc/core.conf=/etc/core/ \ + package/etc/logging.conf=/etc/core/ \ + package/examples=/opt/core/ \ + daemon/dist/core-$(PACKAGE_VERSION)-py3-none-any.whl=/opt/core/ +endef + +.PHONY: fpm +fpm: clean-local-fpm + cd daemon && poetry build -f wheel + $(call fpm-deb) + $(call fpm-rpm) $(call fpm-distributed-deb) $(call fpm-distributed-rpm) diff --git a/daemon/Makefile.am b/daemon/Makefile.am index 79459b80..2585ea1a 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -21,10 +21,7 @@ DISTCLEANFILES = Makefile.in # files to include with distribution tarball EXTRA_DIST = core \ - data \ doc/conf.py.in \ - examples \ - scripts \ tests \ setup.cfg \ poetry.lock \ diff --git a/daemon/examples/grpc/__init__.py b/daemon/core/scripts/__init__.py similarity index 100% rename from daemon/examples/grpc/__init__.py rename to daemon/core/scripts/__init__.py diff --git a/daemon/core/scripts/cleanup.py b/daemon/core/scripts/cleanup.py new file mode 100755 index 00000000..080cd62b --- /dev/null +++ b/daemon/core/scripts/cleanup.py @@ -0,0 +1,103 @@ +import argparse +import os +import subprocess +import sys +import time + + +def check_root() -> None: + if os.geteuid() != 0: + print("permission denied, run this script as root") + sys.exit(1) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="helps cleanup lingering core processes and files", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-d", "--daemon", action="store_true", help="also kill core-daemon" + ) + return parser.parse_args() + + +def cleanup_daemon() -> None: + print("killing core-daemon process ... ", end="") + result = subprocess.call("pkill -9 core-daemon", shell=True) + if result: + print("not found") + else: + print("done") + + +def cleanup_nodes() -> None: + print("killing vnoded processes ... ", end="") + result = subprocess.call("pkill -KILL vnoded", shell=True) + if result: + print("none found") + else: + time.sleep(1) + print("done") + + +def cleanup_emane() -> None: + print("killing emane processes ... ", end="") + result = subprocess.call("pkill emane", shell=True) + if result: + print("none found") + else: + print("done") + + +def cleanup_sessions() -> None: + print("removing session directories ... ", end="") + result = subprocess.call("rm -rf /tmp/pycore*", shell=True) + if result: + print("none found") + else: + print("done") + + +def cleanup_interfaces() -> None: + print("cleaning up devices") + output = subprocess.check_output("ip -o -br link show", shell=True) + lines = output.decode().strip().split("\n") + for line in lines: + values = line.split() + name = values[0] + if ( + name.startswith("veth") + or name.startswith("gt.") + or name.startswith("b.") + or name.startswith("ctrl") + ): + result = subprocess.call(f"ip link delete {name}", shell=True) + if result: + print(f"failed to remove {name}") + else: + print(f"removed {name}") + if name.startswith("b."): + result = subprocess.call( + f"nft delete table bridge {name}", + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=True, + ) + if not result: + print(f"cleared nft rules for {name}") + + +def main() -> None: + check_root() + args = parse_args() + if args.daemon: + cleanup_daemon() + cleanup_nodes() + cleanup_emane() + cleanup_interfaces() + cleanup_sessions() + + +if __name__ == "__main__": + main() diff --git a/daemon/scripts/core-cli b/daemon/core/scripts/cli.py similarity index 88% rename from daemon/scripts/core-cli rename to daemon/core/scripts/cli.py index b9de2615..31ad086e 100755 --- a/daemon/scripts/core-cli +++ b/daemon/core/scripts/cli.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import json import sys from argparse import ( @@ -32,7 +31,9 @@ NODE_TYPES = [x.name for x in NodeType if x != NodeType.PEER_TO_PEER] def protobuf_to_json(message: Any) -> Dict[str, Any]: - return MessageToDict(message, including_default_value_fields=True, preserving_proto_field_name=True) + return MessageToDict( + message, including_default_value_fields=True, preserving_proto_field_name=True + ) def print_json(data: Any) -> None: @@ -122,18 +123,15 @@ def get_current_session(core: CoreGrpcClient, session_id: Optional[int]) -> int: return sessions[0].id -def create_iface(iface_id: int, mac: str, ip4_net: IPNetwork, ip6_net: IPNetwork) -> Interface: +def create_iface( + iface_id: int, mac: str, ip4_net: IPNetwork, ip6_net: IPNetwork +) -> Interface: ip4 = str(ip4_net.ip) if ip4_net else None ip4_mask = ip4_net.prefixlen if ip4_net else None ip6 = str(ip6_net.ip) if ip6_net else None ip6_mask = ip6_net.prefixlen if ip6_net else None return Interface( - id=iface_id, - mac=mac, - ip4=ip4, - ip4_mask=ip4_mask, - ip6=ip6, - ip6_mask=ip6_mask, + id=iface_id, mac=mac, ip4=ip4, ip4_mask=ip4_mask, ip6=ip6, ip6_mask=ip6_mask ) @@ -216,12 +214,14 @@ def query_session(core: CoreGrpcClient, args: Namespace) -> None: for node in session.nodes.values(): xy_pos = f"{int(node.position.x)},{int(node.position.y)}" geo_pos = f"{node.geo.lon:.7f},{node.geo.lat:.7f},{node.geo.alt:f}" - print(f"{node.id:<7} | {node.name[:7]:<7} | {node.type.name[:7]:<7} | {xy_pos:<9} | {geo_pos}") + print( + f"{node.id:<7} | {node.name[:7]:<7} | {node.type.name[:7]:<7} | {xy_pos:<9} | {geo_pos}" + ) print("\nLinks") for link in session.links: n1 = session.nodes[link.node1_id].name n2 = session.nodes[link.node2_id].name - print(f"Node | ", end="") + print("Node | ", end="") print_iface_header() print(f"{n1:<6} | ", end="") if link.iface1: @@ -248,7 +248,9 @@ def query_node(core: CoreGrpcClient, args: Namespace) -> None: print("ID | Name | Type | XY | Geo") xy_pos = f"{int(node.position.x)},{int(node.position.y)}" geo_pos = f"{node.geo.lon:.7f},{node.geo.lat:.7f},{node.geo.alt:f}" - print(f"{node.id:<7} | {node.name[:7]:<7} | {node.type.name[:7]:<7} | {xy_pos:<9} | {geo_pos}") + print( + f"{node.id:<7} | {node.name[:7]:<7} | {node.type.name[:7]:<7} | {xy_pos:<9} | {geo_pos}" + ) if ifaces: print("Interfaces") print("Connected To | ", end="") @@ -348,10 +350,14 @@ def add_link(core: CoreGrpcClient, args: Namespace) -> None: session_id = get_current_session(core, args.session) iface1 = None if args.iface1_id is not None: - iface1 = create_iface(args.iface1_id, args.iface1_mac, args.iface1_ip4, args.iface1_ip6) + iface1 = create_iface( + args.iface1_id, args.iface1_mac, args.iface1_ip4, args.iface1_ip6 + ) iface2 = None if args.iface2_id is not None: - iface2 = create_iface(args.iface2_id, args.iface2_mac, args.iface2_ip4, args.iface2_ip6) + iface2 = create_iface( + args.iface2_id, args.iface2_mac, args.iface2_ip4, args.iface2_ip6 + ) options = LinkOptions( bandwidth=args.bandwidth, loss=args.loss, @@ -432,13 +438,17 @@ def setup_node_parser(parent) -> None: add_parser.add_argument( "-t", "--type", choices=NODE_TYPES, default="DEFAULT", help="type of node" ) - add_parser.add_argument("-m", "--model", help="used to determine services, optional") + add_parser.add_argument( + "-m", "--model", help="used to determine services, optional" + ) group = add_parser.add_mutually_exclusive_group(required=True) group.add_argument("-p", "--pos", type=position_type, help="x,y position") group.add_argument("-g", "--geo", type=geo_type, help="lon,lat,alt position") add_parser.add_argument("-ic", "--icon", help="icon to use, optional") add_parser.add_argument("-im", "--image", help="container image, optional") - add_parser.add_argument("-e", "--emane", help="emane model, only required for emane nodes") + add_parser.add_argument( + "-e", "--emane", help="emane model, only required for emane nodes" + ) add_parser.set_defaults(func=add_node) edit_parser = subparsers.add_parser("edit", help="edit a node") @@ -449,7 +459,9 @@ def setup_node_parser(parent) -> None: move_parser = subparsers.add_parser("move", help="move a node") move_parser.formatter_class = ArgumentDefaultsHelpFormatter - move_parser.add_argument("-i", "--id", type=int, help="id to use, optional", required=True) + move_parser.add_argument( + "-i", "--id", type=int, help="id to use, optional", required=True + ) group = move_parser.add_mutually_exclusive_group(required=True) group.add_argument("-p", "--pos", type=position_type, help="x,y position") group.add_argument("-g", "--geo", type=geo_type, help="lon,lat,alt position") @@ -474,19 +486,33 @@ def setup_link_parser(parent) -> None: add_parser.add_argument("-n1", "--node1", type=int, help="node1 id", required=True) add_parser.add_argument("-n2", "--node2", type=int, help="node2 id", required=True) add_parser.add_argument("-i1-i", "--iface1-id", type=int, help="node1 interface id") - add_parser.add_argument("-i1-m", "--iface1-mac", type=mac_type, help="node1 interface mac") - add_parser.add_argument("-i1-4", "--iface1-ip4", type=ip4_type, help="node1 interface ip4") - add_parser.add_argument("-i1-6", "--iface1-ip6", type=ip6_type, help="node1 interface ip6") + add_parser.add_argument( + "-i1-m", "--iface1-mac", type=mac_type, help="node1 interface mac" + ) + add_parser.add_argument( + "-i1-4", "--iface1-ip4", type=ip4_type, help="node1 interface ip4" + ) + add_parser.add_argument( + "-i1-6", "--iface1-ip6", type=ip6_type, help="node1 interface ip6" + ) add_parser.add_argument("-i2-i", "--iface2-id", type=int, help="node2 interface id") - add_parser.add_argument("-i2-m", "--iface2-mac", type=mac_type, help="node2 interface mac") - add_parser.add_argument("-i2-4", "--iface2-ip4", type=ip4_type, help="node2 interface ip4") - add_parser.add_argument("-i2-6", "--iface2-ip6", type=ip6_type, help="node2 interface ip6") + add_parser.add_argument( + "-i2-m", "--iface2-mac", type=mac_type, help="node2 interface mac" + ) + add_parser.add_argument( + "-i2-4", "--iface2-ip4", type=ip4_type, help="node2 interface ip4" + ) + add_parser.add_argument( + "-i2-6", "--iface2-ip6", type=ip6_type, help="node2 interface ip6" + ) add_parser.add_argument("-b", "--bandwidth", type=int, help="bandwidth (bps)") add_parser.add_argument("-l", "--loss", type=float, help="loss (%%)") add_parser.add_argument("-j", "--jitter", type=int, help="jitter (us)") add_parser.add_argument("-de", "--delay", type=int, help="delay (us)") add_parser.add_argument("-du", "--duplicate", type=int, help="duplicate (%%)") - add_parser.add_argument("-u", "--uni", action="store_true", help="is link unidirectional?") + add_parser.add_argument( + "-u", "--uni", action="store_true", help="is link unidirectional?" + ) add_parser.set_defaults(func=add_link) edit_parser = subparsers.add_parser("edit", help="edit a link") @@ -507,8 +533,12 @@ def setup_link_parser(parent) -> None: delete_parser = subparsers.add_parser("delete", help="delete a link") delete_parser.formatter_class = ArgumentDefaultsHelpFormatter - delete_parser.add_argument("-n1", "--node1", type=int, help="node1 id", required=True) - delete_parser.add_argument("-n2", "--node2", type=int, help="node1 id", required=True) + delete_parser.add_argument( + "-n1", "--node1", type=int, help="node1 id", required=True + ) + delete_parser.add_argument( + "-n2", "--node2", type=int, help="node1 id", required=True + ) delete_parser.add_argument("-i1", "--iface1", type=int, help="node1 interface id") delete_parser.add_argument("-i2", "--iface2", type=int, help="node2 interface id") delete_parser.set_defaults(func=delete_link) @@ -526,20 +556,28 @@ def setup_query_parser(parent) -> None: session_parser = subparsers.add_parser("session", help="query session") session_parser.formatter_class = ArgumentDefaultsHelpFormatter - session_parser.add_argument("-i", "--id", type=int, help="session to query", required=True) + session_parser.add_argument( + "-i", "--id", type=int, help="session to query", required=True + ) session_parser.set_defaults(func=query_session) node_parser = subparsers.add_parser("node", help="query node") node_parser.formatter_class = ArgumentDefaultsHelpFormatter - node_parser.add_argument("-i", "--id", type=int, help="session to query", required=True) - node_parser.add_argument("-n", "--node", type=int, help="node to query", required=True) + node_parser.add_argument( + "-i", "--id", type=int, help="session to query", required=True + ) + node_parser.add_argument( + "-n", "--node", type=int, help="node to query", required=True + ) node_parser.set_defaults(func=query_node) def setup_xml_parser(parent) -> None: parser = parent.add_parser("xml", help="open session xml") parser.formatter_class = ArgumentDefaultsHelpFormatter - parser.add_argument("-f", "--file", type=file_type, help="xml file to open", required=True) + parser.add_argument( + "-f", "--file", type=file_type, help="xml file to open", required=True + ) parser.add_argument("-s", "--start", action="store_true", help="start the session?") parser.set_defaults(func=open_xml) diff --git a/daemon/scripts/core-daemon b/daemon/core/scripts/daemon.py similarity index 78% rename from daemon/scripts/core-daemon rename to daemon/core/scripts/daemon.py index a2ae3343..6b9caa54 100755 --- a/daemon/scripts/core-daemon +++ b/daemon/core/scripts/daemon.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ core-daemon: the CORE daemon is a server process that receives CORE API messages and instantiates emulated nodes and networks within the kernel. Various @@ -61,18 +60,35 @@ def get_merged_config(filename): defaults = { "grpcport": default_grpc_port, "grpcaddress": default_address, - "logfile": default_log + "logfile": default_log, } parser = argparse.ArgumentParser( - description=f"CORE daemon v.{COREDPY_VERSION} instantiates Linux network namespace nodes.") - parser.add_argument("-f", "--configfile", dest="configfile", - help=f"read config from specified file; default = {filename}") - parser.add_argument("--ovs", action="store_true", help="enable experimental ovs mode, default is false") - parser.add_argument("--grpc-port", dest="grpcport", - help=f"grpc port to listen on; default {default_grpc_port}") - parser.add_argument("--grpc-address", dest="grpcaddress", - help=f"grpc address to listen on; default {default_address}") - parser.add_argument("-l", "--logfile", help=f"core logging configuration; default {default_log}") + description=f"CORE daemon v.{COREDPY_VERSION} instantiates Linux network namespace nodes." + ) + parser.add_argument( + "-f", + "--configfile", + dest="configfile", + help=f"read config from specified file; default = {filename}", + ) + parser.add_argument( + "--ovs", + action="store_true", + help="enable experimental ovs mode, default is false", + ) + parser.add_argument( + "--grpc-port", + dest="grpcport", + help=f"grpc port to listen on; default {default_grpc_port}", + ) + parser.add_argument( + "--grpc-address", + dest="grpcaddress", + help=f"grpc address to listen on; default {default_address}", + ) + parser.add_argument( + "-l", "--logfile", help=f"core logging configuration; default {default_log}" + ) # parse command line options args = parser.parse_args() # convert ovs to internal format diff --git a/daemon/scripts/core-gui b/daemon/core/scripts/gui.py similarity index 60% rename from daemon/scripts/core-gui rename to daemon/core/scripts/gui.py index ff7795a3..9c0560b2 100755 --- a/daemon/scripts/core-gui +++ b/daemon/core/scripts/gui.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import argparse import logging from logging.handlers import TimedRotatingFileHandler @@ -9,12 +8,19 @@ from core.gui.app import Application def main() -> None: # parse flags - parser = argparse.ArgumentParser(description=f"CORE Python GUI") - parser.add_argument("-l", "--level", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="INFO", - help="logging level") + parser = argparse.ArgumentParser(description="CORE Python GUI") + parser.add_argument( + "-l", + "--level", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + default="INFO", + help="logging level", + ) parser.add_argument("-p", "--proxy", action="store_true", help="enable proxy") parser.add_argument("-s", "--session", type=int, help="session id to join") - parser.add_argument("--create-dir", action="store_true", help="create gui directory and exit") + parser.add_argument( + "--create-dir", action="store_true", help="create gui directory and exit" + ) args = parser.parse_args() # check home directory exists and create if necessary @@ -25,9 +31,13 @@ def main() -> None: # setup logging log_format = "%(asctime)s - %(levelname)s - %(module)s:%(funcName)s - %(message)s" stream_handler = logging.StreamHandler() - file_handler = TimedRotatingFileHandler(filename=appconfig.LOG_PATH, when="D", backupCount=5) + file_handler = TimedRotatingFileHandler( + filename=appconfig.LOG_PATH, when="D", backupCount=5 + ) log_level = logging.getLevelName(args.level) - logging.basicConfig(level=log_level, format=log_format, handlers=[stream_handler, file_handler]) + logging.basicConfig( + level=log_level, format=log_format, handlers=[stream_handler, file_handler] + ) logging.getLogger("PIL").setLevel(logging.ERROR) # start app diff --git a/daemon/scripts/core-player b/daemon/core/scripts/player.py similarity index 90% rename from daemon/scripts/core-player rename to daemon/core/scripts/player.py index c7942b37..07728939 100755 --- a/daemon/scripts/core-player +++ b/daemon/core/scripts/player.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import argparse import logging import sys @@ -31,7 +29,10 @@ def parse_args() -> argparse.Namespace: "-f", "--file", required=True, type=path_type, help="core file to play" ) parser.add_argument( - "-s", "--session", type=int, help="session to play to, first found session otherwise" + "-s", + "--session", + type=int, + help="session to play to, first found session otherwise", ) return parser.parse_args() diff --git a/daemon/scripts/core-route-monitor b/daemon/core/scripts/routemonitor.py similarity index 97% rename from daemon/scripts/core-route-monitor rename to daemon/core/scripts/routemonitor.py index bc61f6fa..2ebfdfad 100755 --- a/daemon/scripts/core-route-monitor +++ b/daemon/core/scripts/routemonitor.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import argparse import enum import select @@ -60,15 +59,15 @@ class SdtClient: class RouterMonitor: def __init__( - self, - session: int, - src: str, - dst: str, - pkt: str, - rate: int, - dead: int, - sdt_host: str, - sdt_port: int, + self, + session: int, + src: str, + dst: str, + pkt: str, + rate: int, + dead: int, + sdt_host: str, + sdt_port: int, ) -> None: self.queue = Queue() self.core = CoreGrpcClient() diff --git a/daemon/scripts/core-service-update b/daemon/core/scripts/serviceupdate.py similarity index 50% rename from daemon/scripts/core-service-update rename to daemon/core/scripts/serviceupdate.py index d0ca863f..50ada96d 100755 --- a/daemon/scripts/core-service-update +++ b/daemon/core/scripts/serviceupdate.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import argparse import re from io import TextIOWrapper @@ -6,9 +5,15 @@ from io import TextIOWrapper def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( - description=f"Helps transition older CORE services to work with newer versions") - parser.add_argument("-f", "--file", dest="file", type=argparse.FileType("r"), - help=f"service file to update") + description="Helps transition older CORE services to work with newer versions" + ) + parser.add_argument( + "-f", + "--file", + dest="file", + type=argparse.FileType("r"), + help="service file to update", + ) return parser.parse_args() @@ -20,17 +25,32 @@ def update_service(service_file: TextIOWrapper) -> None: # rename dirs to directories line = re.sub(r"^(\s+)dirs", r"\1directories", line) # fix import states for service - line = re.sub(r"^.+import.+CoreService.+$", - r"from core.services.coreservices import CoreService", line) + line = re.sub( + r"^.+import.+CoreService.+$", + r"from core.services.coreservices import CoreService", + line, + ) # fix method signatures - line = re.sub(r"def generateconfig\(cls, node, filename, services\)", - r"def generate_config(cls, node, filename)", line) - line = re.sub(r"def getvalidate\(cls, node, services\)", - r"def get_validate(cls, node)", line) - line = re.sub(r"def getstartup\(cls, node, services\)", - r"def get_startup(cls, node)", line) - line = re.sub(r"def getconfigfilenames\(cls, nodenum, services\)", - r"def get_configs(cls, node)", line) + line = re.sub( + r"def generateconfig\(cls, node, filename, services\)", + r"def generate_config(cls, node, filename)", + line, + ) + line = re.sub( + r"def getvalidate\(cls, node, services\)", + r"def get_validate(cls, node)", + line, + ) + line = re.sub( + r"def getstartup\(cls, node, services\)", + r"def get_startup(cls, node)", + line, + ) + line = re.sub( + r"def getconfigfilenames\(cls, nodenum, services\)", + r"def get_configs(cls, node)", + line, + ) # remove unwanted lines if re.search(r"addservice\(", line): continue diff --git a/daemon/pyproject.toml b/daemon/pyproject.toml index 91b5dbad..70d684dd 100644 --- a/daemon/pyproject.toml +++ b/daemon/pyproject.toml @@ -14,6 +14,14 @@ include = [ ] exclude = ["core/constants.py.in"] +[tool.poetry.scripts] +core-daemon = "core.scripts.daemon:main" +core-cli = "core.scripts.cli:main" +core-gui = "core.scripts.gui:main" +core-player = "core.scripts.player:main" +core-route-monitor = "core.scripts.routemonitor:main" +core-service-update = "core.scripts.serviceupdate:main" +core-cleanup = "core.scripts.cleanup:main" [tool.poetry.dependencies] python = "^3.6" diff --git a/daemon/scripts/core-cleanup b/daemon/scripts/core-cleanup deleted file mode 100755 index ced76634..00000000 --- a/daemon/scripts/core-cleanup +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh - -if [ "z$1" = "z-h" -o "z$1" = "z--help" ]; then - echo "usage: $0 [-d [-l]]" - echo -n " Clean up all CORE namespaces processes, bridges, interfaces, " - echo "and session\n directories. Options:" - echo " -h show this help message and exit" - echo " -d also kill the Python daemon" - echo " -l remove the core-daemon.log file" - exit 0 -fi - -if [ `id -u` != 0 ]; then - echo "Permission denied. Re-run this script as root." - exit 1 -fi - -PATH="/sbin:/bin:/usr/sbin:/usr/bin" -export PATH - -if [ "z$1" = "z-d" ]; then - pypids=`pidof python3 python` - for p in $pypids; do - grep -q core-daemon /proc/$p/cmdline - if [ $? = 0 ]; then - echo "cleaning up core-daemon process: $p" - kill -9 $p - fi - done -fi - -if [ "z$2" = "z-l" ]; then - rm -f /var/log/core-daemon.log -fi - -kaopts="-v" -killall --help 2>&1 | grep -q namespace -if [ $? = 0 ]; then - kaopts="$kaopts --ns 0" -fi - -vnodedpids=`pidof vnoded` -if [ "z$vnodedpids" != "z" ]; then - echo "cleaning up old vnoded processes: $vnodedpids" - killall $kaopts -KILL vnoded - # pause for 1 second for interfaces to disappear - sleep 1 -fi -killall -q emane -killall -q emanetransportd -killall -q emaneeventservice - -if [ -d /sys/class/net ]; then - ifcommand="ls -1 /sys/class/net" -else - ifcommand="ip -o link show | sed -r -e 's/[0-9]+: ([^[:space:]]+): .*/\1/'" -fi - -eval "$ifcommand" | awk ' - /^veth[0-9]+\./ {print "removing interface " $1; system("ip link del " $1);} - /tmp\./ {print "removing interface " $1; system("ip link del " $1);} - /gt\./ {print "removing interface " $1; system("ip link del " $1);} - /b\./ {print "removing bridge " $1; system("ip link set " $1 " down; ip link del " $1);} - /ctrl[0-9]+\./ {print "removing bridge " $1; system("ip link set " $1 " down; ip link del " $1);} -' - -nft list ruleset | awk ' - $3 ~ /^b\./ {print "removing nftables " $3; system("nft delete table bridge " $3);} -' - -rm -rf /tmp/pycore* diff --git a/package/Dockerfile.centos b/package/Dockerfile.centos new file mode 100644 index 00000000..6cbb3610 --- /dev/null +++ b/package/Dockerfile.centos @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 +FROM centos:7 +LABEL Description="CORE CentOS Image" + +# define environment +ENV DEBIAN_FRONTEND=noninteractive + +# install basic dependencies +RUN yum update -y && yum install -y python3 python3-pip python3-tkinter +RUN python3 -m pip install --upgrade pip + +# install core +WORKDIR /opt/core +COPY core_*.rpm . +RUN NO_VENV=1 yum install -y ./core_*.rpm diff --git a/package/Dockerfile.ubuntu b/package/Dockerfile.ubuntu new file mode 100644 index 00000000..5a19e326 --- /dev/null +++ b/package/Dockerfile.ubuntu @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 +FROM ubuntu:20.04 +LABEL Description="CORE Docker Ubuntu Image" + +# define environment +ENV DEBIAN_FRONTEND=noninteractive + +# install basic dependencies +RUN apt-get update && apt-get install -y python3 python3-tk python3-pip python3-venv +RUN python3 -m pip install --upgrade pip + +# install core +WORKDIR /opt/core +COPY core_*.deb . +RUN apt-get install -y ./core_*.deb diff --git a/package/after-install.sh b/package/after-install.sh new file mode 100644 index 00000000..341e909d --- /dev/null +++ b/package/after-install.sh @@ -0,0 +1,16 @@ +#!/bin/sh +if [ ! -z "${NO_PYTHON}" ]; then + exit 0 +fi + +PYTHON="${PYTHON:=python3}" +if [ ! -z "${NO_VENV}" ]; then + ${PYTHON} -m pip install /opt/core/core-*.whl + echo "DAEMON=/usr/local/bin/core-daemon" > /opt/core/service +else + ${PYTHON} -m venv /opt/core/venv + . /opt/core/venv/bin/activate + pip install --upgrade pip + pip install /opt/core/core-*.whl + echo "DAEMON=/opt/core/venv/bin/core-daemon" > /opt/core/service +fi diff --git a/package/after-remove.sh b/package/after-remove.sh new file mode 100644 index 00000000..e67a0a59 --- /dev/null +++ b/package/after-remove.sh @@ -0,0 +1,13 @@ +#!/bin/sh +if [ -v NO_PYTHON ]; then + exit 0 +fi + +PYTHON="${PYTHON:=python3}" +if [ -v NO_VENV ]; then + ${PYTHON} -m pip uninstall core +else + ${PYTHON} -m venv /opt/core/venv + . /opt/core/venv/bin/activate + pip uninstall core +fi diff --git a/package/core-daemon b/package/core-daemon new file mode 100644 index 00000000..8cb57e4c --- /dev/null +++ b/package/core-daemon @@ -0,0 +1,112 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: core-daemon +# Required-Start: $network $remote_fs +# Required-Stop: $network $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start the core-daemon CORE daemon at boot time +# Description: Starts and stops the core-daemon CORE daemon used to +# provide network emulation services for the CORE GUI +# or scripts. +### END INIT INFO +# +# chkconfig: 35 90 03 +# description: Starts and stops the CORE daemon \ +# used to provide network emulation services. +# +# config: /etc/core/ + +. /opt/core/service +NAME=`basename $0` +PIDFILE="/var/$NAME.pid" +LOG="/var/log/$NAME.log" +CMD="$DAEMON" + +get_pid() { + cat "$PIDFILE" +} + +is_alive() { + [ -f "$PIDFILE" ] && ps -p `get_pid` > /dev/null 2>&1 +} + +corestart() { + if is_alive; then + echo "$NAME already started" + else + echo "starting $NAME" + $CMD 2>&1 >> "$LOG" & + fi + + echo $! > "$PIDFILE" + if ! is_alive; then + echo "unable to start $NAME, see $LOG" + exit 1 + fi +} + +corestop() { + if is_alive; then + echo -n "stopping $NAME.." + kill `get_pid` + for i in 1 2 3 4 5; do + sleep 1 + if ! is_alive; then + break + fi + echo -n "." + done + echo + + if is_alive; then + echo "not stopped; may still be shutting down" + exit 1 + else + echo "stopped" + if [ -f "$PIDFILE" ]; then + rm -f "$PIDFILE" + fi + fi + else + echo "$NAME not running" + fi +} + +corerestart() { + corestop + corestart +} + +corestatus() { + if is_alive; then + echo "$NAME is running" + else + echo "$NAME is stopped" + exit 1 + fi +} + + +case "$1" in + start) + corestart + ;; + stop) + corestop + ;; + restart) + corerestart + ;; + force-reload) + corerestart + ;; + status) + corestatus + ;; + *) + echo "Usage: $0 {start|stop|restart|status}" + exit 1 +esac + +exit $? diff --git a/package/core-daemon.service b/package/core-daemon.service new file mode 100644 index 00000000..ede52c63 --- /dev/null +++ b/package/core-daemon.service @@ -0,0 +1,12 @@ +[Unit] +Description=Common Open Research Emulator Service +After=network.target + +[Service] +Type=simple +EnvironmentFile=/opt/core/service +ExecStart=$DAEMON +TasksMax=infinity + +[Install] +WantedBy=multi-user.target diff --git a/daemon/data/core.conf b/package/etc/core.conf similarity index 100% rename from daemon/data/core.conf rename to package/etc/core.conf diff --git a/daemon/data/logging.conf b/package/etc/logging.conf similarity index 100% rename from daemon/data/logging.conf rename to package/etc/logging.conf diff --git a/daemon/examples/configservices/switch.py b/package/examples/configservices/switch.py similarity index 100% rename from daemon/examples/configservices/switch.py rename to package/examples/configservices/switch.py diff --git a/daemon/examples/controlnet_updown b/package/examples/controlnet_updown similarity index 100% rename from daemon/examples/controlnet_updown rename to package/examples/controlnet_updown diff --git a/daemon/examples/docker/docker2core.py b/package/examples/docker/docker2core.py similarity index 100% rename from daemon/examples/docker/docker2core.py rename to package/examples/docker/docker2core.py diff --git a/daemon/examples/docker/docker2docker.py b/package/examples/docker/docker2docker.py similarity index 100% rename from daemon/examples/docker/docker2docker.py rename to package/examples/docker/docker2docker.py diff --git a/daemon/examples/docker/switch.py b/package/examples/docker/switch.py similarity index 100% rename from daemon/examples/docker/switch.py rename to package/examples/docker/switch.py diff --git a/daemon/examples/myemane/__init__.py b/package/examples/grpc/__init__.py similarity index 100% rename from daemon/examples/myemane/__init__.py rename to package/examples/grpc/__init__.py diff --git a/daemon/examples/grpc/distributed_switch.py b/package/examples/grpc/distributed_switch.py similarity index 100% rename from daemon/examples/grpc/distributed_switch.py rename to package/examples/grpc/distributed_switch.py diff --git a/daemon/examples/grpc/emane80211.py b/package/examples/grpc/emane80211.py similarity index 100% rename from daemon/examples/grpc/emane80211.py rename to package/examples/grpc/emane80211.py diff --git a/daemon/examples/grpc/peertopeer.py b/package/examples/grpc/peertopeer.py similarity index 100% rename from daemon/examples/grpc/peertopeer.py rename to package/examples/grpc/peertopeer.py diff --git a/daemon/examples/grpc/switch.py b/package/examples/grpc/switch.py similarity index 100% rename from daemon/examples/grpc/switch.py rename to package/examples/grpc/switch.py diff --git a/daemon/examples/grpc/wireless.py b/package/examples/grpc/wireless.py similarity index 100% rename from daemon/examples/grpc/wireless.py rename to package/examples/grpc/wireless.py diff --git a/daemon/examples/grpc/wlan.py b/package/examples/grpc/wlan.py similarity index 100% rename from daemon/examples/grpc/wlan.py rename to package/examples/grpc/wlan.py diff --git a/daemon/examples/lxd/lxd2core.py b/package/examples/lxd/lxd2core.py similarity index 100% rename from daemon/examples/lxd/lxd2core.py rename to package/examples/lxd/lxd2core.py diff --git a/daemon/examples/lxd/lxd2lxd.py b/package/examples/lxd/lxd2lxd.py similarity index 100% rename from daemon/examples/lxd/lxd2lxd.py rename to package/examples/lxd/lxd2lxd.py diff --git a/daemon/examples/lxd/switch.py b/package/examples/lxd/switch.py similarity index 100% rename from daemon/examples/lxd/switch.py rename to package/examples/lxd/switch.py diff --git a/package/examples/myemane/__init__.py b/package/examples/myemane/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/examples/myemane/examplemodel.py b/package/examples/myemane/examplemodel.py similarity index 100% rename from daemon/examples/myemane/examplemodel.py rename to package/examples/myemane/examplemodel.py diff --git a/daemon/examples/myservices/__init__.py b/package/examples/myservices/__init__.py similarity index 100% rename from daemon/examples/myservices/__init__.py rename to package/examples/myservices/__init__.py diff --git a/daemon/examples/myservices/exampleservice.py b/package/examples/myservices/exampleservice.py similarity index 100% rename from daemon/examples/myservices/exampleservice.py rename to package/examples/myservices/exampleservice.py diff --git a/daemon/examples/python/distributed_emane.py b/package/examples/python/distributed_emane.py similarity index 100% rename from daemon/examples/python/distributed_emane.py rename to package/examples/python/distributed_emane.py diff --git a/daemon/examples/python/distributed_lxd.py b/package/examples/python/distributed_lxd.py similarity index 100% rename from daemon/examples/python/distributed_lxd.py rename to package/examples/python/distributed_lxd.py diff --git a/daemon/examples/python/distributed_ptp.py b/package/examples/python/distributed_ptp.py similarity index 100% rename from daemon/examples/python/distributed_ptp.py rename to package/examples/python/distributed_ptp.py diff --git a/daemon/examples/python/distributed_switch.py b/package/examples/python/distributed_switch.py similarity index 100% rename from daemon/examples/python/distributed_switch.py rename to package/examples/python/distributed_switch.py diff --git a/daemon/examples/python/emane80211.py b/package/examples/python/emane80211.py similarity index 100% rename from daemon/examples/python/emane80211.py rename to package/examples/python/emane80211.py diff --git a/daemon/examples/python/peertopeer.py b/package/examples/python/peertopeer.py similarity index 100% rename from daemon/examples/python/peertopeer.py rename to package/examples/python/peertopeer.py diff --git a/daemon/examples/python/switch.py b/package/examples/python/switch.py similarity index 100% rename from daemon/examples/python/switch.py rename to package/examples/python/switch.py diff --git a/daemon/examples/python/wireless.py b/package/examples/python/wireless.py similarity index 100% rename from daemon/examples/python/wireless.py rename to package/examples/python/wireless.py diff --git a/daemon/examples/python/wlan.py b/package/examples/python/wlan.py similarity index 100% rename from daemon/examples/python/wlan.py rename to package/examples/python/wlan.py diff --git a/daemon/examples/services/sampleFirewall b/package/examples/services/sampleFirewall similarity index 100% rename from daemon/examples/services/sampleFirewall rename to package/examples/services/sampleFirewall diff --git a/daemon/examples/services/sampleIPsec b/package/examples/services/sampleIPsec similarity index 100% rename from daemon/examples/services/sampleIPsec rename to package/examples/services/sampleIPsec diff --git a/daemon/examples/services/sampleVPNClient b/package/examples/services/sampleVPNClient similarity index 100% rename from daemon/examples/services/sampleVPNClient rename to package/examples/services/sampleVPNClient diff --git a/daemon/examples/services/sampleVPNServer b/package/examples/services/sampleVPNServer similarity index 100% rename from daemon/examples/services/sampleVPNServer rename to package/examples/services/sampleVPNServer diff --git a/daemon/examples/tdma/schedule.xml b/package/examples/tdma/schedule.xml similarity index 100% rename from daemon/examples/tdma/schedule.xml rename to package/examples/tdma/schedule.xml diff --git a/tasks.py b/tasks.py index 0cf4613f..adb9dfde 100644 --- a/tasks.py +++ b/tasks.py @@ -291,25 +291,6 @@ def install_core_files(c, local=False, verbose=False, prefix=DEFAULT_PREFIX): hide = not verbose python = get_python(c) bin_dir = Path(prefix).joinpath("bin") - # install scripts - for script in Path("daemon/scripts").iterdir(): - dest = bin_dir.joinpath(script.name) - with open(script, "r") as f: - lines = f.readlines() - first = lines[0].strip() - # modify python scripts to point to virtual environment - if not local and first == "#!/usr/bin/env python3": - lines[0] = f"#!{python}\n" - temp = NamedTemporaryFile("w", delete=False) - for line in lines: - temp.write(line) - temp.close() - c.run(f"sudo cp {temp.name} {dest}", hide=hide) - c.run(f"sudo chmod 755 {dest}", hide=hide) - os.unlink(temp.name) - # copy normal links - else: - c.run(f"sudo cp {script} {dest}", hide=hide) # setup core python helper if not local: core_python = bin_dir.joinpath("core-python") @@ -325,12 +306,12 @@ def install_core_files(c, local=False, verbose=False, prefix=DEFAULT_PREFIX): # install core configuration file config_dir = "/etc/core" c.run(f"sudo mkdir -p {config_dir}", hide=hide) - c.run(f"sudo cp -n daemon/data/core.conf {config_dir}", hide=hide) - c.run(f"sudo cp -n daemon/data/logging.conf {config_dir}", hide=hide) + c.run(f"sudo cp -n package/etc/core.conf {config_dir}", hide=hide) + c.run(f"sudo cp -n package/etc/logging.conf {config_dir}", hide=hide) # install examples examples_dir = f"{prefix}/share/core" c.run(f"sudo mkdir -p {examples_dir}", hide=hide) - c.run(f"sudo cp -r daemon/examples {examples_dir}", hide=hide) + c.run(f"sudo cp -r package/examples {examples_dir}", hide=hide) @task( @@ -467,11 +448,9 @@ def uninstall( c.run("sudo -v", hide=True) with p.start("uninstalling core"): c.run("sudo make uninstall", hide=hide) - with p.start("cleaning build directory"): c.run("make clean", hide=hide) c.run("./bootstrap.sh clean", hide=hide) - if local: with p.start("uninstalling core"): python_bin = get_env_python() @@ -485,22 +464,15 @@ def uninstall( c.run("poetry run pre-commit uninstall", hide=hide) with p.start("uninstalling poetry virtual environment"): c.run(f"poetry env remove {python}", hide=hide) - # remove installed files bin_dir = Path(prefix).joinpath("bin") - with p.start("uninstalling script files"): - for script in Path("daemon/scripts").iterdir(): - dest = bin_dir.joinpath(script.name) - c.run(f"sudo rm -f {dest}", hide=hide) with p.start("uninstalling examples"): examples_dir = Path(prefix).joinpath("share/core") c.run(f"sudo rm -rf {examples_dir}") - # remove core-python symlink if not local: core_python = bin_dir.joinpath("core-python") c.run(f"sudo rm -f {core_python}", hide=hide) - # remove service systemd_dir = Path("/lib/systemd/system/") service_name = "core-daemon.service" From d8632da96b784f165634141ad25c7f182b995844 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 27 Jul 2022 16:41:23 -0700 Subject: [PATCH 013/157] install: fixed install.md error for docker build command, fixed default dockerfile example for centos package install to not use NO_VENV --- docs/install.md | 34 +++++++++++++++++++++++++++++++++- package/Dockerfile.centos | 2 +- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/install.md b/docs/install.md index 0f15133c..7a00b680 100644 --- a/docs/install.md +++ b/docs/install.md @@ -105,6 +105,38 @@ sudo yum remove core sudo apt remove core ``` +## Installing from Packages + +Starting with 9.0.0 there are pre-built rpm/deb packages. You can retrieve the +rpm/deb package from [releases](https://github.com/coreemu/core/releases) page. + +> **NOTE:** PYTHON defaults to python3 for installs below + +> **NOTE:** the python install requires python3.6+, pip, +> tk compatibility for python gui, and venv for virtualenvs + +```shell +# install core vcmd/vnoded, system dependencies, +# and core python into a venv located at /opt/core/venv + install -y ./ + +# disable the venv and install to python directly +NO_VENV=1 install -y ./ + +# change python executable used to install for venv and direct installations +PYTHON=python3.9 install -y ./ + +# disable venv and change python executable +NO_VENV=1 PYTHON=python3.9 install -y ./ + +# skip installing the python portion entirely, as you plan to carry this out yourself +# core python wheel is located at /opt/core/core--py3-none-any.whl +NO_PYTHON=1 install -y ./ + +# install python wheel into python of your choosing + -m pip install /opt/core/core--py3-none-any.whl +``` + ## Automated Install First we will need to clone and navigate to the CORE repo. ```shell @@ -218,7 +250,7 @@ You can leverage the provided Dockerfile, to run and launch CORE within a Docker git clone https://github.com/coreemu/core.git cd core # build image -sudo docker build -t core. -f Dockerfile. +sudo docker build -t core. -f Dockerfile. . # start container sudo docker run -itd --name core -e DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw --privileged core. # enable xhost access to the root user diff --git a/package/Dockerfile.centos b/package/Dockerfile.centos index 6cbb3610..8304f820 100644 --- a/package/Dockerfile.centos +++ b/package/Dockerfile.centos @@ -12,4 +12,4 @@ RUN python3 -m pip install --upgrade pip # install core WORKDIR /opt/core COPY core_*.rpm . -RUN NO_VENV=1 yum install -y ./core_*.rpm +RUN yum install -y ./core_*.rpm From d06659ff82d323ee5b40928b92b561c62fc52634 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 29 Jul 2022 12:03:20 -0700 Subject: [PATCH 014/157] install: remove debian config from centos package dockerfile --- package/Dockerfile.centos | 3 --- 1 file changed, 3 deletions(-) diff --git a/package/Dockerfile.centos b/package/Dockerfile.centos index 8304f820..babbc0c7 100644 --- a/package/Dockerfile.centos +++ b/package/Dockerfile.centos @@ -2,9 +2,6 @@ FROM centos:7 LABEL Description="CORE CentOS Image" -# define environment -ENV DEBIAN_FRONTEND=noninteractive - # install basic dependencies RUN yum update -y && yum install -y python3 python3-pip python3-tkinter RUN python3 -m pip install --upgrade pip From a201fd2903218c624fb1ef73671fbe084259fdea Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 29 Jul 2022 14:03:45 -0700 Subject: [PATCH 015/157] install: add option to avoid y/n prompt when uninstalling core during post install script --- package/after-remove.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/after-remove.sh b/package/after-remove.sh index e67a0a59..60a5cb59 100644 --- a/package/after-remove.sh +++ b/package/after-remove.sh @@ -5,9 +5,9 @@ fi PYTHON="${PYTHON:=python3}" if [ -v NO_VENV ]; then - ${PYTHON} -m pip uninstall core + ${PYTHON} -m pip uninstall -y core else ${PYTHON} -m venv /opt/core/venv . /opt/core/venv/bin/activate - pip uninstall core + pip uninstall -y core fi From c91d8df790692acfcaef1bc81fe040240e8a66ea Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 29 Jul 2022 14:12:17 -0700 Subject: [PATCH 016/157] install: added task using a subset of the previous auto install to support auto setup and building of core packages --- tasks.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tasks.py b/tasks.py index adb9dfde..0388acc6 100644 --- a/tasks.py +++ b/tasks.py @@ -314,6 +314,36 @@ def install_core_files(c, local=False, verbose=False, prefix=DEFAULT_PREFIX): c.run(f"sudo cp -r package/examples {examples_dir}", hide=hide) +@task( + help={ + "dev": "install development mode", + "verbose": "enable verbose", + "install-type": "used to force an install type, " + "can be one of the following (redhat, debian)", + "no-python": "avoid installing python system dependencies", + }, +) +def build( + c, + verbose=False, + install_type=None, + no_python=False, +): + print("setting up to build core packages") + c.run("sudo -v", hide=True) + p = Progress(verbose) + hide = not verbose + os_info = get_os(install_type) + with p.start("installing system dependencies"): + install_system(c, os_info, hide, no_python) + with p.start("installing system grpcio-tools"): + install_grpcio(c, hide) + with p.start("building core"): + build_core(c, hide) + with p.start(f"building rpm/deb packages"): + c.run("make fpm", hide=hide) + + @task( help={ "dev": "install development mode", From 018865b2a298f96eebc11b9315666512fcce0d92 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 2 Aug 2022 13:49:38 -0700 Subject: [PATCH 017/157] docs: added install note about upgrading pip to avoid building from source related issues --- docs/install.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/install.md b/docs/install.md index 7a00b680..fcc8484e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -116,6 +116,10 @@ rpm/deb package from [releases](https://github.com/coreemu/core/releases) page. > tk compatibility for python gui, and venv for virtualenvs ```shell +# recommended to upgrade to the latest version of pip before installation +# in python, can help avoid building from source issues + -m pip install --upgrade pip + # install core vcmd/vnoded, system dependencies, # and core python into a venv located at /opt/core/venv install -y ./ From e4abefe23b46c4b4bbb5488f05a8d787c203ff14 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 3 Aug 2022 17:21:31 -0700 Subject: [PATCH 018/157] grpc: added nem id and port to interface data returned from GetNode --- daemon/core/api/grpc/grpcutils.py | 10 +++++++++- daemon/core/api/grpc/server.py | 6 +++--- daemon/core/api/grpc/wrappers.py | 4 ++++ daemon/proto/core/api/grpc/core.proto | 2 ++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/daemon/core/api/grpc/grpcutils.py b/daemon/core/api/grpc/grpcutils.py index 7f63079c..d26c0bd9 100644 --- a/daemon/core/api/grpc/grpcutils.py +++ b/daemon/core/api/grpc/grpcutils.py @@ -658,10 +658,11 @@ def get_service_configuration(service: CoreService) -> NodeServiceData: ) -def iface_to_proto(iface: CoreInterface) -> core_pb2.Interface: +def iface_to_proto(session: Session, iface: CoreInterface) -> core_pb2.Interface: """ Convenience for converting a core interface to the protobuf representation. + :param session: session interface belongs to :param iface: interface to convert :return: interface proto """ @@ -672,6 +673,11 @@ def iface_to_proto(iface: CoreInterface) -> core_pb2.Interface: ip6 = str(ip6_net.ip) if ip6_net else None ip6_mask = ip6_net.prefixlen if ip6_net else None mac = str(iface.mac) if iface.mac else None + nem_id = None + nem_port = None + if isinstance(iface.net, EmaneNet): + nem_id = session.emane.get_nem_id(iface) + nem_port = session.emane.get_nem_port(iface) return core_pb2.Interface( id=iface.id, name=iface.name, @@ -682,6 +688,8 @@ def iface_to_proto(iface: CoreInterface) -> core_pb2.Interface: ip4_mask=ip4_mask, ip6=ip6, ip6_mask=ip6_mask, + nem_id=nem_id, + nem_port=nem_port, ) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 532dd91c..5967df1a 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -585,7 +585,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): ifaces = [] for iface_id in node.ifaces: iface = node.ifaces[iface_id] - iface_proto = grpcutils.iface_to_proto(iface) + iface_proto = grpcutils.iface_to_proto(session, iface) ifaces.append(iface_proto) emane_configs = grpcutils.get_emane_model_configs_dict(session) node_emane_configs = emane_configs.get(node.id, []) @@ -756,9 +756,9 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): iface1_proto = None iface2_proto = None if node1_iface: - iface1_proto = grpcutils.iface_to_proto(node1_iface) + iface1_proto = grpcutils.iface_to_proto(session, node1_iface) if node2_iface: - iface2_proto = grpcutils.iface_to_proto(node2_iface) + iface2_proto = grpcutils.iface_to_proto(session, node2_iface) return core_pb2.AddLinkResponse( result=True, iface1=iface1_proto, iface2=iface2_proto ) diff --git a/daemon/core/api/grpc/wrappers.py b/daemon/core/api/grpc/wrappers.py index a49b9932..11402cd9 100644 --- a/daemon/core/api/grpc/wrappers.py +++ b/daemon/core/api/grpc/wrappers.py @@ -481,6 +481,8 @@ class Interface: mtu: int = None node_id: int = None net2_id: int = None + nem_id: int = None + nem_port: int = None @classmethod def from_proto(cls, proto: core_pb2.Interface) -> "Interface": @@ -497,6 +499,8 @@ class Interface: mtu=proto.mtu, node_id=proto.node_id, net2_id=proto.net2_id, + nem_id=proto.nem_id, + nem_port=proto.nem_port, ) def to_proto(self) -> core_pb2.Interface: diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 4969e9c9..d2f024da 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -668,6 +668,8 @@ message Interface { int32 mtu = 10; int32 node_id = 11; int32 net2_id = 12; + int32 nem_id = 13; + int32 nem_port = 14; } message SessionLocation { From 382ff6d49bf7af68a52de240f46ce10a5330a050 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 30 Aug 2022 12:25:34 -0700 Subject: [PATCH 019/157] docs: updated install doc to provide a better example for installing emane python bindings into a virtual environment --- docs/install.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/install.md b/docs/install.md index fcc8484e..cc463b88 100644 --- a/docs/install.md +++ b/docs/install.md @@ -281,15 +281,17 @@ python3