afonso/core-extra
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #512 from coreemu/develop

Browse source 
Develop
This commit is contained in:
bharnden 2020-09-29 08:55:49 -07:00 committed by GitHub
commit 56e98f412c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 239 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
CHANGELOG.md
View file

@ -1,3 +1,13 @@
## 2020-09-29 CORE 7.2.1
* core-daemon
* fixed issue where shutting down sessions may not have removed session directories
* fixed issue with multiple emane interfaces on the same node not getting the right configuration
* Installation
* updated automated install to be a bit more robust for alternative distros
* added force install type to try and leverage a redhat/debian like install
* locked ospf mdr version installed to older commit to avoid issues with multiple interfaces on same node
## 2020-09-15 CORE 7.2.0
* Installation

2
configure.ac
View file

@ -2,7 +2,7 @@
# Process this file with autoconf to produce a configure script.
# this defines the CORE version number, must be static for AC_INIT
AC_INIT(core, 7.2.0)
AC_INIT(core, 7.2.1)
# autoconf and automake initialization
AC_CONFIG_SRCDIR([netns/version.h.in])

5
daemon/core/api/grpc/server.py
View file

@ -315,10 +315,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
"""
logging.debug("stop session: %s", request)
session = self.get_session(request.session_id, context)
session.data_collect()
session.set_state(EventTypes.DATACOLLECT_STATE, send_event=True)
session.clear()
session.set_state(EventTypes.SHUTDOWN_STATE, send_event=True)
session.shutdown()
return core_pb2.StopSessionResponse(result=True)
def CreateSession(

5
daemon/core/api/tlv/corehandlers.py
View file

@ -735,6 +735,11 @@ class CoreHandler(socketserver.BaseRequestHandler):
elif message.flags & MessageFlags.DELETE.value:
with self._shutdown_lock:
result = self.session.delete_node(node_id)
if result and self.session.get_node_count() == 0:
self.session.set_state(EventTypes.SHUTDOWN_STATE)
self.session.delete_nodes()
self.session.distributed.shutdown()
self.session.sdt.shutdown()
# if we deleted a node broadcast out its removal
if result and message.flags & MessageFlags.STRING.value:

14
daemon/core/emane/emanemanager.py
View file

@ -71,7 +71,6 @@ class EmaneState(Enum):
@dataclass
class StartData:
emane_net: EmaneNet
node: CoreNodeBase
ifaces: List[CoreInterface] = field(default_factory=list)
@ -370,9 +369,7 @@ class EmaneManager(ModelManager):
iface.name,
)
continue
start_node = node_map.setdefault(
iface.node, StartData(emane_net, iface.node)
)
start_node = node_map.setdefault(iface.node, StartData(iface.node))
start_node.ifaces.append(iface)
start_nodes = sorted(node_map.values(), key=lambda x: x.node.id)
for start_node in start_nodes:
@ -386,7 +383,7 @@ class EmaneManager(ModelManager):
emanexml.build_platform_xml(self, control_net, data)
self.start_daemon(data.node)
for iface in data.ifaces:
self.install_iface(data.emane_net, iface)
self.install_iface(iface)
def set_nem(self, nem_id: int, iface: CoreInterface) -> None:
if nem_id in self.nems_to_ifaces:
@ -606,7 +603,12 @@ class EmaneManager(ModelManager):
node.host_cmd(emanecmd, cwd=path)
logging.info("node(%s) host emane daemon running: %s", node.name, emanecmd)
def install_iface(self, emane_net: EmaneNet, iface: CoreInterface) -> None:
def install_iface(self, iface: CoreInterface) -> None:
emane_net = iface.net
if not isinstance(emane_net, EmaneNet):
raise CoreError(
f"emane interface not connected to emane net: {emane_net.name}"
)
config = self.get_iface_config(emane_net, iface)
external = config.get("external", "0")
if isinstance(iface, TunTap) and external == "0":

27
daemon/core/emulator/session.py
View file

@ -756,23 +756,18 @@ class Session:
"""
Shutdown all session nodes and remove the session directory.
"""
if self.state == EventTypes.SHUTDOWN_STATE:
return
logging.info("session(%s) state(%s) shutting down", self.id, self.state)
if self.state != EventTypes.SHUTDOWN_STATE:
self.set_state(EventTypes.DATACOLLECT_STATE, send_event=True)
self.set_state(EventTypes.SHUTDOWN_STATE, send_event=True)
# clear out current core session
self.clear()
# shutdown sdt
self.sdt.shutdown()
# remove this sessions working directory
preserve = self.options.get_config("preservedir") == "1"
if not preserve:
shutil.rmtree(self.session_dir, ignore_errors=True)
# call session shutdown handlers
for handler in self.shutdown_handlers:
handler(self)
@ -1116,7 +1111,6 @@ class Session:
if node:
node.shutdown()
self.sdt.delete_node(_id)
self.check_shutdown()
return node is not None
def delete_nodes(self) -> None:
@ -1279,25 +1273,6 @@ class Session:
self.add_remove_control_net(2, remove=True)
self.add_remove_control_net(3, remove=True)
def check_shutdown(self) -> bool:
"""
Check if we have entered the shutdown state, when no running nodes
and links remain.
:return: True if should shutdown, False otherwise
"""
node_count = self.get_node_count()
logging.debug(
"session(%s) checking shutdown: %s nodes remaining", self.id, node_count
)
shutdown = False
if node_count == 0:
shutdown = True
self.set_state(EventTypes.SHUTDOWN_STATE)
# clearing sdt saved data here for legacy gui
self.sdt.shutdown()
return shutdown
def short_session_id(self) -> str:
"""
Return a shorter version of the session ID, appropriate for

11
daemon/core/xml/emanexml.py
View file

@ -7,6 +7,7 @@ from lxml import etree
from core import utils
from core.config import Configuration
from core.emane.nodes import EmaneNet
from core.emulator.distributed import DistributedServer
from core.errors import CoreError
from core.nodes.base import CoreNode, CoreNodeBase
@ -166,8 +167,12 @@ def build_platform_xml(
add_param(platform_element, name, value)
# create nem xml entries for all interfaces
emane_net = data.emane_net
for iface in data.ifaces:
emane_net = iface.net
if not isinstance(emane_net, EmaneNet):
raise CoreError(
f"emane interface not connected to emane net: {emane_net.name}"
)
nem_id = emane_manager.next_nem_id()
emane_manager.set_nem(nem_id, iface)
emane_manager.write_nem(iface, nem_id)
@ -180,9 +185,7 @@ def build_platform_xml(
"nem", id=str(nem_id), name=iface.localname, definition=nem_definition
)
# check if this is an external transport, get default config if an interface
# specific one does not exist
config = emane_manager.get_iface_config(emane_net, iface)
# check if this is an external transport
if is_external(config):
nem_element.set("transport", "external")
platform_endpoint = "platformendpoint"

2
daemon/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "core"
version = "7.2.0"
version = "7.2.1"
description = "CORE Common Open Research Emulator"
authors = ["Boeing Research and Technology"]
license = "BSD-2-Clause"

159
docs/install.md
View file

@ -1,29 +1,72 @@
# Installation
* Table of Contents
{:toc}
## Overview
CORE provides a script to help automate the installation of dependencies,
build and install, and either generate a CORE specific python virtual environment
or build and install a python wheel.
CORE provides a script to help automate installing all required software
to build and run, including a python virtual environment to run it all in.
> **WARNING:** if Docker is installed, the default iptable rules will block CORE traffic
### Tools Used
The following tools will be leveraged during installation:
|Tool|Description|
|---|---|
|[pip](https://pip.pypa.io/en/stable/)|used to install pipx|
|[pipx](https://pipxproject.github.io/pipx/)|used to install standalone python tools (invoke, poetry)|
|[invoke](http://www.pyinvoke.org/)|used to run provided tasks (install, daemon, gui, tests, etc)|
|[poetry](https://python-poetry.org/)|used to install the managed python virtual environment for running CORE|
|[invoke](http://www.pyinvoke.org/)|used to run provided tasks (install, uninstall, reinstall, etc)|
|[poetry](https://python-poetry.org/)|used to install python virtual environment or building a python wheel|
## Required Hardware
### Files
The following is a list of files that would be installed after running the automated installation.
> **NOTE:** the default install prefix is /usr/local, but can be changed as noted below
* executable files
* <prefix>/bin/{core-daemon, core-gui, vcmd, vnoded, etc}
* tcl/tk gui files
* <prefix>/lib/core
* <prefix>/share/core/icons
* example imn files
* <prefix>/share/core/examples
* python files
* poetry virtual env
* `cd <repo>/daemon && poetry env info`
* ~/.cache/pypoetry/virtualenvs/
* local python install
* default install path for python3 installation of a wheel
* `python3 -c "import core; print(core.__file__)"`
* configuration files
* /etc/core/{core.conf, logging.conf}
* ospf mdr repository files
* <repo>/../ospf-mdr
* emane repository files
* <repo>/../emane
### Installed Executables
After the installation complete it will have installed the following scripts.
| Name | Description |
|---|---|
| core-cleanup | tool to help removed lingering core created containers, bridges, directories |
| core-cli | tool to query, open xml files, and send commands using gRPC |
| core-daemon | runs the backed core server providing TLV and gRPC APIs |
| core-gui | runs the legacy tcl/tk based GUI |
| core-imn-to-xml | tool to help automate converting a .imn file to .xml format |
| core-manage | tool to add, remove, or check for services, models, and node types |
| core-pygui | runs the new python/tk based GUI |
| core-python | provides a convenience for running the core python virtual environment |
| core-route-monitor | tool to help monitor traffic across nodes and feed that to SDT |
| core-service-update | tool to update automate modifying a legacy service to match current naming |
| coresendmsg | tool to send TLV API commands from command line |
### Required Hardware
Any computer capable of running Linux should be able to run CORE. Since the physical machine will be hosting numerous
containers, as a general rule you should select a machine having as much RAM and CPU resources as possible.
## Supported Linux Distributions
### Supported Linux Distributions
Plan is to support recent Ubuntu and CentOS LTS releases.
Verified:
@ -46,14 +89,14 @@ sudo yum install -y kernel-modules-extra
sudo modprobe sch_netem
```
## Utility Requirements
### Utility Requirements
The following are known dependencies that will result in errors when not met.
* iproute2 4.5+ is a requirement for bridge related commands
* ebtables not backed by nftables
## Upgrading
Please make sure to uninstall the previous installation of CORE cleanly
## Upgrading from Older Release
Please make sure to uninstall any previous installations of CORE cleanly
before proceeding to install.
Previous install was built from source:
@ -72,92 +115,97 @@ sudo yum remove core
sudo apt remove core
```
## Automated Installation
> **NOTE:** installing globally can have issues with dependency conflicts etc
The automated install will install do the following:
## Automated Install
The automated install will do the following:
* install base tools needed for installation
* python3, pip, pipx, invoke, poetry
* installs system dependencies for building core
* installs latest version of [OPSF MDR](https://github.com/USNavalResearchLaboratory/ospf-mdr)
* clone/build/install working version of [OPSF MDR](https://github.com/USNavalResearchLaboratory/ospf-mdr)
* installs core into poetry managed virtual environment or locally, if flag is passed
* installs scripts pointing to python interpreter being used
* installs systemd service, disabled by default
* installs scripts pointing pointing to appropriate python location based on install type
* installs systemd service pointing to appropriate python location based on install type
After installation has completed you should be able to run the various
CORE scripts for running core.
After installation has completed you should be able to run `core-daemon` and `core-gui`.
> **NOTE:** provide a prefix that will be found on path when running as sudo
> if the default prefix is not valid
> **NOTE:** installing locally comes with its own risks, it can result it potential
> dependency conflicts with system package manager installed python dependencies
> **NOTE:** provide a prefix that will be found on path when running as sudo,
> if the default prefix /usr/local will not be valid
`install.sh` will attempt to determine your OS by way of `/etc/os-release`, currently it supports
attempts to install OSs that are debian/redhat like (yum/apt).
```shell
# clone CORE repo
git clone https://github.com/coreemu/core.git
cd core
# run install script
# script usage: install.sh [-v] [-d] [-l] [-p <prefix>]
#
# -v enable verbose install
# -d enable developer install
# -l enable local install, not compatible with developer install
# -p install prefix, defaults to /usr/local
./install.sh
# install core to virtual environment
./install.sh -p <prefix>
# install core locally
./install.sh -p <prefix> -l
```
### Unsupported Linux Distribution
For unsupported OSs you could attempt to do the following to translate
an installation to your use case.
If you are on an unsupported distribution, you can look into the
[install.sh](https://github.com/coreemu/core/blob/master/install.sh)
and
[tasks.py](https://github.com/coreemu/core/blob/master/tasks.py)
files to see the various commands ran to install CORE and translate them to
your use case, assuming it is possible.
* make sure you have python3.6+ with venv support
* make sure you have python3 invoke available to leverage `<repo>/tasks.py`
If you get install down entirely, feel free to contribute and help others.
```shell
cd <repo>
## Installed Scripts
# Usage: inv[oke] [--core-opts] install [--options] [other tasks here ...]
#
# Docstring:
# install core, poetry, scripts, service, and ospf mdr
#
# Options:
# -d, --dev install development mode
# -i STRING, --install-type=STRING
# -l, --local determines if core will install to local system, default is False
# -p STRING, --prefix=STRING prefix where scripts are installed, default is /usr/local
# -v, --verbose enable verbose
After the installation complete it will have installed the following scripts.
# install virtual environment
inv install -p <prefix>
| Name | Description |
|---|---|
| core-cleanup | tool to help removed lingering core created containers, bridges, directories |
| core-cli | tool to query, open xml files, and send commands using gRPC |
| core-daemon | runs the backed core server providing TLV and gRPC APIs |
| core-gui | runs the legacy tcl/tk based GUI |
| core-imn-to-xml | tool to help automate converting a .imn file to .xml format |
| core-manage | tool to add, remove, or check for services, models, and node types |
| core-pygui | runs the new python/tk based GUI |
| core-python | provides a convenience for running the core python virtual environment |
| core-route-monitor | tool to help monitor traffic across nodes and feed that to SDT |
| core-service-update | tool to update automate modifying a legacy service to match current naming |
| coresendmsg | tool to send TLV API commands from command line |
# indstall locally
inv install -p <prefix> -l
# this will print the commands that would be ran for a given installation
# type without actually running them, they may help in being used as
# the basis for translating to your OS
inv install --dry -v -p <prefix> -i <install type>
```
## Running User Scripts
If you create your own python scripts to run CORE directly or using the gRPC/TLV
APIs you will need to make sure you are running them within context of the
installed virtual environment. To help support this CORE provides the `core-python`
executable. This executable will allow you to enter CORE's python virtual
environment interpreter or to run a script within it.
> **NOTE:** the following assumes CORE has been installed successfully
For installations installed to a virtual environment:
```shell
core-python <script>
```
If CORE was installed locally, then you can run scripts using the default python3
interpreter.
For local installations:
```shell
python3 <script>
```
## Installing EMANE
> **NOTE:** installng emane for the virtual environment is known to work for 1.21+
> **NOTE:** automated install currently targets 1.25
@ -205,7 +253,6 @@ poetry run pip install <EMANE_REPO>/src/python
```
## Using Invoke Tasks
The invoke tool installed by way of pipx provides conveniences for running
CORE tasks to help ensure usage of the create python virtual environment.

31
install.sh
View file

@ -3,13 +3,6 @@
# exit on error
set -e
# detect os/ver for install type
os=""
if [[ -f /etc/os-release ]]; then
. /etc/os-release
os=${ID}
fi
# parse arguments
dev=""
verbose=""
@ -42,19 +35,23 @@ while getopts "dvlp:" opt; do
done
shift $((OPTIND - 1))
echo "installing CORE for ${os}"
case ${os} in
"ubuntu")
# install pre-reqs using yum/apt
if command -v apt &> /dev/null
then
echo "setup to install CORE using apt"
sudo apt install -y python3-pip python3-venv
;;
"centos")
elif command -v yum &> /dev/null
then
echo "setup to install CORE using yum"
sudo yum install -y python3-pip
;;
*)
echo "unknown OS ID ${os} cannot install"
;;
esac
else
echo "apt/yum was not found"
echo "install python3 and invoke to run the automated install"
echo "inv -h install"
exit 1
fi
# install pip/invoke to run install with provided options
python3 -m pip install --user pipx
python3 -m pipx ensurepath
export PATH=$PATH:~/.local/bin

94
tasks.py
View file

@ -8,14 +8,22 @@ from contextlib import contextmanager
from enum import Enum
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Optional
from typing import Optional, List
from invoke import task, Context
DAEMON_DIR: str = "daemon"
DEFAULT_PREFIX: str = "/usr/local"
EMANE_CHECKOUT: str = "v1.2.5"
OSPFMDR_CHECKOUT: str = "26fe5a4401a26760c553fcadfde5311199e89450"
OSPFMDR_CHECKOUT: str = "e2b4e416b7001c5dca0224fe728249c9688e4c7a"
REDHAT_LIKE = {
"redhat",
"fedora",
}
DEBIAN_LIKE = {
"ubuntu",
"debian",
}
class Progress:
@ -55,11 +63,28 @@ class Progress:
class OsName(Enum):
UBUNTU = "ubuntu"
CENTOS = "centos"
UNKNOWN = "unknown"
@classmethod
def get(cls, name: str) -> "OsName":
try:
return OsName(name)
except ValueError:
return OsName.UNKNOWN
class OsLike(Enum):
DEBIAN = "debian"
REDHAT = "rhel fedora"
REDHAT = "rhel"
@classmethod
def get(cls, values: List[str]) -> Optional["OsLike"]:
for value in values:
if value in DEBIAN_LIKE:
return OsLike.DEBIAN
elif value in REDHAT_LIKE:
return OsLike.REDHAT
return None
class OsInfo:
@ -68,6 +93,22 @@ class OsInfo:
self.like: OsLike = like
self.version: float = version
@classmethod
def get(cls, name: str, like: List[str], version: Optional[str]) -> "OsInfo":
os_name = OsName.get(name)
os_like = OsLike.get(like)
if not os_like:
like = " ".join(like)
print(f"unsupported os install type({like})")
sys.exit(1)
if version:
try:
version = float(version)
except ValueError:
print(f"os version is not a float: {version}")
sys.exit(1)
return OsInfo(os_name, os_like, version)
def get_python(c: Context, warn: bool = False) -> str:
with c.cd(DAEMON_DIR):
@ -85,7 +126,12 @@ def get_pytest(c: Context) -> str:
return os.path.join(venv, "bin", "pytest")
def get_os() -> OsInfo:
def get_os(install_type: Optional[str]) -> OsInfo:
if install_type:
name_value = OsName.UNKNOWN.value
like_value = install_type
version_value = None
else:
d = {}
with open("/etc/os-release", "r") as f:
for line in f.readlines():
@ -97,16 +143,7 @@ def get_os() -> OsInfo:
name_value = d["ID"]
like_value = d["ID_LIKE"]
version_value = d["VERSION_ID"]
try:
name = OsName(name_value)
like = OsLike(like_value)
version = float(version_value)
except ValueError:
print(
f"unsupported os({name_value}) like({like_value}) version({version_value}"
)
sys.exit(1)
return OsInfo(name, like, version)
return OsInfo.get(name_value, like_value.split(), version_value)
def check_existing_core(c: Context, hide: bool) -> None:
@ -304,9 +341,13 @@ def install_scripts(c, local=False, verbose=False, prefix=DEFAULT_PREFIX):
"verbose": "enable verbose",
"local": "determines if core will install to local system, default is False",
"prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}",
"install-type": "used to force an install type, "
"can be one of the following (redhat, debian)",
},
)
def install(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX):
def install(
c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX, install_type=None
):
"""
install core, poetry, scripts, service, and ospf mdr
"""
@ -315,7 +356,8 @@ def install(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX):
c.run("sudo -v", hide=True)
p = Progress(verbose)
hide = not verbose
os_info = get_os()
os_info = get_os(install_type)
if not c["run"]["dry"]:
with p.start("checking for old installations"):
check_existing_core(c, hide)
with p.start("installing system dependencies"):
@ -342,16 +384,18 @@ def install(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX):
help={
"verbose": "enable verbose",
"local": "used determine if core is installed locally, default is False",
"install-type": "used to force an install type, "
"can be one of the following (redhat, debian)",
},
)
def install_emane(c, verbose=False, local=False):
def install_emane(c, verbose=False, local=False, install_type=None):
"""
install emane and the python bindings
"""
c.run("sudo -v", hide=True)
p = Progress(verbose)
hide = not verbose
os_info = get_os()
os_info = get_os(install_type)
with p.start("installing system dependencies"):
if os_info.like == OsLike.DEBIAN:
c.run(
@ -458,11 +502,19 @@ def uninstall(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX):
"verbose": "enable verbose",
"local": "determines if core will install to local system, default is False",
"prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}",
"branch": "branch to install latest code from, default is current branch"
"branch": "branch to install latest code from, default is current branch",
"install-type": "used to force an install type, "
"can be one of the following (redhat, debian)",
},
)
def reinstall(
c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX, branch=None
c,
dev=False,
verbose=False,
local=False,
prefix=DEFAULT_PREFIX,
branch=None,
install_type=None
):
"""
run the uninstall task, get latest from specified branch, and run install task
@ -479,7 +531,7 @@ def reinstall(
c.run("git pull", hide=hide)
if not Path("tasks.py").exists():
raise FileNotFoundError(f"missing tasks.py on branch: {branch}")
install(c, dev, verbose, local, prefix)
install(c, dev, verbose, local, prefix, install_type)
@task