From 2094694ca394d9cd9a9d02d152acd31090e60834 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 5 May 2022 21:54:08 -0700 Subject: [PATCH 01/15] daemon: cleaned up docker node code, updates to use nsenter in most cases --- daemon/core/nodes/docker.py | 124 ++++++++++++------------------------ 1 file changed, 40 insertions(+), 84 deletions(-) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index aa398d34..4808fe04 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -1,75 +1,28 @@ import json import logging +import shlex from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Callable, Dict, Optional +from typing import TYPE_CHECKING -from core import utils from core.emulator.distributed import DistributedServer -from core.errors import CoreCommandError +from core.errors import CoreCommandError, CoreError +from core.executables import BASH from core.nodes.base import CoreNode -from core.nodes.netclient import LinuxNetClient, get_net_client logger = logging.getLogger(__name__) if TYPE_CHECKING: from core.emulator.session import Session - -class DockerClient: - def __init__(self, name: str, image: str, run: Callable[..., str]) -> None: - self.name: str = name - self.image: str = image - self.run: Callable[..., str] = run - self.pid: Optional[str] = None - - def create_container(self) -> str: - self.run( - f"docker run -td --init --net=none --hostname {self.name} " - f"--name {self.name} --sysctl net.ipv6.conf.all.disable_ipv6=0 " - f"--privileged {self.image} /bin/bash" - ) - self.pid = self.get_pid() - return self.pid - - def get_info(self) -> Dict: - args = f"docker inspect {self.name}" - output = self.run(args) - data = json.loads(output) - if not data: - raise CoreCommandError(1, args, f"docker({self.name}) not present") - return data[0] - - def is_alive(self) -> bool: - try: - data = self.get_info() - return data["State"]["Running"] - except CoreCommandError: - return False - - def stop_container(self) -> None: - self.run(f"docker rm -f {self.name}") - - def check_cmd(self, cmd: str, wait: bool = True, shell: bool = False) -> str: - logger.info("docker cmd output: %s", cmd) - return utils.cmd(f"docker exec {self.name} {cmd}", wait=wait, shell=shell) - - def create_ns_cmd(self, cmd: str) -> str: - return f"nsenter -t {self.pid} -a {cmd}" - - def get_pid(self) -> str: - args = f"docker inspect -f '{{{{.State.Pid}}}}' {self.name}" - output = self.run(args) - self.pid = output - logger.debug("node(%s) pid: %s", self.name, self.pid) - return output - - def copy_file(self, src_path: Path, dst_path: Path) -> str: - args = f"docker cp {src_path} {self.name}:{dst_path}" - return self.run(args) +DOCKER: str = "docker" class DockerNode(CoreNode): + """ + Provides logic for creating a Docker based node. + """ + def __init__( self, session: "Session", @@ -92,17 +45,18 @@ class DockerNode(CoreNode): """ super().__init__(session, _id, name, directory, server) self.image = image if image is not None else "ubuntu" - self.client: Optional[DockerClient] = None - def create_node_net_client(self, use_ovs: bool) -> LinuxNetClient: + def _create_cmd(self, args: str, shell: bool = False) -> str: """ - Create node network client for running network commands within the nodes - container. + Create command used to run commands within the context of a node. - :param use_ovs: True for OVS bridges, False for Linux bridges - :return:node network client + :param args: command arguments + :param shell: True to run shell like, False otherwise + :return: node command """ - return get_net_client(use_ovs, self.nsenter_cmd) + if shell: + args = f"{BASH} -c {shlex.quote(args)}" + return f"nsenter -t {self.pid} -m -u -i -p -n {args}" def alive(self) -> bool: """ @@ -110,22 +64,33 @@ class DockerNode(CoreNode): :return: True if node is alive, False otherwise """ - return self.client.is_alive() + try: + running = self.host_cmd( + f"{DOCKER} inspect -f '{{{{.State.Running}}}}' {self.name}" + ) + return json.loads(running) + except CoreCommandError: + return False def startup(self) -> None: """ - Start a new namespace node by invoking the vnoded process that - allocates a new namespace. Bring up the loopback device and set - the hostname. + Create a docker container instance for the specified image. :return: nothing """ with self.lock: if self.up: - raise ValueError("starting a node that is already up") + raise CoreError(f"starting node({self.name}) that is already up") self.makenodedir() - self.client = DockerClient(self.name, self.image, self.host_cmd) - self.pid = self.client.create_container() + self.host_cmd( + f"{DOCKER} run -td --init --net=none --hostname {self.name} " + f"--name {self.name} --sysctl net.ipv6.conf.all.disable_ipv6=0 " + f"--privileged {self.image} /bin/bash" + ) + self.pid = self.host_cmd( + f"{DOCKER} inspect -f '{{{{.State.Pid}}}}' {self.name}" + ) + logger.debug("node(%s) pid: %s", self.name, self.pid) self.up = True def shutdown(self) -> None: @@ -139,17 +104,9 @@ class DockerNode(CoreNode): return with self.lock: self.ifaces.clear() - self.client.stop_container() + self.host_cmd(f"{DOCKER} rm -f {self.name}") self.up = False - def nsenter_cmd(self, args: str, wait: bool = True, shell: bool = False) -> str: - if self.server is None: - args = self.client.create_ns_cmd(args) - return utils.cmd(args, wait=wait, shell=shell) - else: - args = self.client.create_ns_cmd(args) - return self.server.remote_cmd(args, wait=wait) - def termcmdstring(self, sh: str = "/bin/sh") -> str: """ Create a terminal command string. @@ -157,7 +114,7 @@ class DockerNode(CoreNode): :param sh: shell to execute command in :return: str """ - return f"docker exec -it {self.name} bash" + return f"{DOCKER} exec -it {self.name} {sh}" def create_dir(self, dir_path: Path) -> None: """ @@ -167,8 +124,7 @@ class DockerNode(CoreNode): :return: nothing """ logger.debug("creating node dir: %s", dir_path) - args = f"mkdir -p {dir_path}" - self.cmd(args) + self.cmd(f"mkdir -p {dir_path}") def mount(self, src_path: str, target_path: str) -> None: """ @@ -201,7 +157,7 @@ class DockerNode(CoreNode): self.cmd(f"mkdir -m {0o755:o} -p {directory}") if self.server is not None: self.server.remote_put(temp_path, temp_path) - self.client.copy_file(temp_path, file_path) + self.host_cmd(f"{DOCKER} cp {temp_path} {self.name}:{file_path}") self.cmd(f"chmod {mode:o} {file_path}") if self.server is not None: self.host_cmd(f"rm -f {temp_path}") @@ -226,6 +182,6 @@ class DockerNode(CoreNode): temp_path = Path(temp.name) src_path = temp_path self.server.remote_put(src_path, temp_path) - self.client.copy_file(src_path, dst_path) + self.host_cmd(f"{DOCKER} cp {src_path} {self.name}:{dst_path}") if mode is not None: self.cmd(f"chmod {mode:o} {dst_path}") From 5f1e7223315e912f56a225107196fdefb774f2ae Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 10 May 2022 11:51:36 -0700 Subject: [PATCH 02/15] install: adjustments to better support specifying an alternative python for install using PYTHON env variable --- tasks.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index dcda33c6..8c0ffe77 100644 --- a/tasks.py +++ b/tasks.py @@ -110,6 +110,10 @@ class OsInfo: return OsInfo(os_name, os_like, version) +def get_env_python() -> str: + return os.environ.get("PYTHON", "python3") + + def get_python(c: Context, warn: bool = False) -> str: with c.cd(DAEMON_DIR): r = c.run("poetry env info -p", warn=warn, hide=True) @@ -180,8 +184,9 @@ def install_system(c: Context, os_info: OsInfo, hide: bool) -> None: def install_grpcio(c: Context, hide: bool) -> None: + python_bin = get_env_python() c.run( - "python3 -m pip install --user grpcio==1.27.2 grpcio-tools==1.27.2", + f"{python_bin} -m pip install --user grpcio==1.27.2 grpcio-tools==1.27.2", hide=hide, ) @@ -204,6 +209,8 @@ def install_poetry(c: Context, dev: bool, local: bool, hide: bool) -> None: else: args = "" if dev else "--no-dev" with c.cd(DAEMON_DIR): + python_bin = get_env_python() + c.run(f"poetry env use {python_bin}", hide=hide) c.run(f"poetry install {args}", hide=hide) if dev: c.run("poetry run pre-commit install", hide=hide) From 52b875df36559ac88a098304e9b2e13935ca510f Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 10 May 2022 13:24:19 -0700 Subject: [PATCH 03/15] install: updated locked poetry packages and specifically bumped grpc tooling to a newer version for better binary install support --- daemon/poetry.lock | 283 ++++++++++++++++++++++-------------------- daemon/pyproject.toml | 4 +- tasks.py | 2 +- 3 files changed, 150 insertions(+), 139 deletions(-) diff --git a/daemon/poetry.lock b/daemon/poetry.lock index 9055a0b5..34b95f2f 100644 --- a/daemon/poetry.lock +++ b/daemon/poetry.lock @@ -30,7 +30,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> [[package]] name = "bcrypt" -version = "3.2.0" +version = "3.2.2" description = "Modern password hashing for your software and your servers" category = "main" optional = false @@ -38,7 +38,6 @@ python-versions = ">=3.6" [package.dependencies] cffi = ">=1.1" -six = ">=1.4.1" [package.extras] tests = ["pytest (>=3.2.1,!=3.3.0)"] @@ -82,7 +81,7 @@ python-versions = ">=3.6" [[package]] name = "click" -version = "8.0.3" +version = "8.0.4" description = "Composable command line interface toolkit" category = "dev" optional = false @@ -102,7 +101,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "cryptography" -version = "36.0.1" +version = "37.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false @@ -117,7 +116,7 @@ docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] name = "dataclasses" @@ -179,26 +178,29 @@ pyflakes = ">=2.2.0,<2.3.0" [[package]] name = "grpcio" -version = "1.27.2" +version = "1.43.0" description = "HTTP/2-based RPC framework" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [package.dependencies] six = ">=1.5.2" +[package.extras] +protobuf = ["grpcio-tools (>=1.43.0)"] + [[package]] name = "grpcio-tools" -version = "1.27.2" +version = "1.43.0" description = "Protobuf code generator for gRPC" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [package.dependencies] -grpcio = ">=1.27.2" -protobuf = ">=3.5.0.post1" +grpcio = ">=1.43.0" +protobuf = ">=3.5.0.post1,<4.0dev" [[package]] name = "identify" @@ -325,7 +327,7 @@ test = ["pytest", "pytest-cov"] [[package]] name = "more-itertools" -version = "8.12.0" +version = "8.13.0" description = "More routines for operating on iterables, beyond itertools" category = "dev" optional = false @@ -360,7 +362,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "paramiko" -version = "2.9.2" +version = "2.10.4" description = "SSH2 protocol library" category = "main" optional = false @@ -370,6 +372,7 @@ python-versions = "*" bcrypt = ">=3.1.3" cryptography = ">=2.5" pynacl = ">=1.0.1" +six = "*" [package.extras] all = ["pyasn1 (>=0.1.7)", "pynacl (>=1.0.1)", "bcrypt (>=3.1.3)", "invoke (>=1.3)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] @@ -560,7 +563,7 @@ python-versions = ">=3.6" [[package]] name = "virtualenv" -version = "20.13.1" +version = "20.14.1" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -601,7 +604,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "64ea28583e46b32b3aa2be3627ee8f68c1bbf36622ec6f575062d5059745a6f9" +content-hash = "8f6ab4134c3bd5219ad80a1c93a055a87cd13b985b84a10a8ff1a7e3ca98a3c8" [metadata.files] appdirs = [ @@ -617,13 +620,17 @@ attrs = [ {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] bcrypt = [ - {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, - {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, - {file = "bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1"}, - {file = "bcrypt-3.2.0-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"}, - {file = "bcrypt-3.2.0-cp36-abi3-win32.whl", hash = "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55"}, - {file = "bcrypt-3.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34"}, - {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, + {file = "bcrypt-3.2.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:7180d98a96f00b1050e93f5b0f556e658605dd9f524d0b0e68ae7944673f525e"}, + {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:61bae49580dce88095d669226d5076d0b9d927754cedbdf76c6c9f5099ad6f26"}, + {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88273d806ab3a50d06bc6a2fc7c87d737dd669b76ad955f449c43095389bc8fb"}, + {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6d2cb9d969bfca5bc08e45864137276e4c3d3d7de2b162171def3d188bf9d34a"}, + {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521"}, + {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a2c46100e315c3a5b90fdc53e429c006c5f962529bc27e1dfd656292c20ccc40"}, + {file = "bcrypt-3.2.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7d9ba2e41e330d2af4af6b1b6ec9e6128e91343d0b4afb9282e54e5508f31baa"}, + {file = "bcrypt-3.2.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cd43303d6b8a165c29ec6756afd169faba9396a9472cdff753fe9f19b96ce2fa"}, + {file = "bcrypt-3.2.2-cp36-abi3-win32.whl", hash = "sha256:4e029cef560967fb0cf4a802bcf4d562d3d6b4b1bf81de5ec1abbe0f1adb027e"}, + {file = "bcrypt-3.2.2-cp36-abi3-win_amd64.whl", hash = "sha256:7ff2069240c6bbe49109fe84ca80508773a904f5a8cb960e02a977f7f519b129"}, + {file = "bcrypt-3.2.2.tar.gz", hash = "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb"}, ] black = [ {file = "black-19.3b0-py36-none-any.whl", hash = "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf"}, @@ -686,34 +693,36 @@ cfgv = [ {file = "cfgv-3.0.0.tar.gz", hash = "sha256:04b093b14ddf9fd4d17c53ebfd55582d27b76ed30050193c14e560770c5360eb"}, ] click = [ - {file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"}, - {file = "click-8.0.3.tar.gz", hash = "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"}, + {file = "click-8.0.4-py3-none-any.whl", hash = "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1"}, + {file = "click-8.0.4.tar.gz", hash = "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] cryptography = [ - {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b"}, - {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2"}, - {file = "cryptography-36.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f"}, - {file = "cryptography-36.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3"}, - {file = "cryptography-36.0.1-cp36-abi3-win32.whl", hash = "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca"}, - {file = "cryptography-36.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf"}, - {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9"}, - {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1"}, - {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316"}, - {file = "cryptography-36.0.1.tar.gz", hash = "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638"}, + {file = "cryptography-37.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:ef15c2df7656763b4ff20a9bc4381d8352e6640cfeb95c2972c38ef508e75181"}, + {file = "cryptography-37.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:3c81599befb4d4f3d7648ed3217e00d21a9341a9a688ecdd615ff72ffbed7336"}, + {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2bd1096476aaac820426239ab534b636c77d71af66c547b9ddcd76eb9c79e004"}, + {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:31fe38d14d2e5f787e0aecef831457da6cec68e0bb09a35835b0b44ae8b988fe"}, + {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:093cb351031656d3ee2f4fa1be579a8c69c754cf874206be1d4cf3b542042804"}, + {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59b281eab51e1b6b6afa525af2bd93c16d49358404f814fe2c2410058623928c"}, + {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:0cc20f655157d4cfc7bada909dc5cc228211b075ba8407c46467f63597c78178"}, + {file = "cryptography-37.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f8ec91983e638a9bcd75b39f1396e5c0dc2330cbd9ce4accefe68717e6779e0a"}, + {file = "cryptography-37.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:46f4c544f6557a2fefa7ac8ac7d1b17bf9b647bd20b16decc8fbcab7117fbc15"}, + {file = "cryptography-37.0.2-cp36-abi3-win32.whl", hash = "sha256:731c8abd27693323b348518ed0e0705713a36d79fdbd969ad968fbef0979a7e0"}, + {file = "cryptography-37.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:471e0d70201c069f74c837983189949aa0d24bb2d751b57e26e3761f2f782b8d"}, + {file = "cryptography-37.0.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a68254dd88021f24a68b613d8c51d5c5e74d735878b9e32cc0adf19d1f10aaf9"}, + {file = "cryptography-37.0.2-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:a7d5137e556cc0ea418dca6186deabe9129cee318618eb1ffecbd35bee55ddc1"}, + {file = "cryptography-37.0.2-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:aeaba7b5e756ea52c8861c133c596afe93dd716cbcacae23b80bc238202dc023"}, + {file = "cryptography-37.0.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95e590dd70642eb2079d280420a888190aa040ad20f19ec8c6e097e38aa29e06"}, + {file = "cryptography-37.0.2-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:1b9362d34363f2c71b7853f6251219298124aa4cc2075ae2932e64c91a3e2717"}, + {file = "cryptography-37.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e53258e69874a306fcecb88b7534d61820db8a98655662a3dd2ec7f1afd9132f"}, + {file = "cryptography-37.0.2-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:1f3bfbd611db5cb58ca82f3deb35e83af34bb8cf06043fa61500157d50a70982"}, + {file = "cryptography-37.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:419c57d7b63f5ec38b1199a9521d77d7d1754eb97827bbb773162073ccd8c8d4"}, + {file = "cryptography-37.0.2-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:dc26bb134452081859aa21d4990474ddb7e863aa39e60d1592800a8865a702de"}, + {file = "cryptography-37.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3b8398b3d0efc420e777c40c16764d6870bcef2eb383df9c6dbb9ffe12c64452"}, + {file = "cryptography-37.0.2.tar.gz", hash = "sha256:f224ad253cc9cea7568f49077007d2263efa57396a2f2f78114066fd54b5c68e"}, ] dataclasses = [ {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, @@ -736,94 +745,96 @@ flake8 = [ {file = "flake8-3.8.2.tar.gz", hash = "sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634"}, ] grpcio = [ - {file = "grpcio-1.27.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dbec0a3a154dbf2eb85b38abaddf24964fa1c059ee0a4ad55d6f39211b1a4bca"}, - {file = "grpcio-1.27.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1ef949b15a1f5f30651532a9b54edf3bd7c0b699a10931505fa2c80b2d395942"}, - {file = "grpcio-1.27.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:ed123037896a8db6709b8ad5acc0ed435453726ea0b63361d12de369624c2ab5"}, - {file = "grpcio-1.27.2-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:f9d632ce9fd485119c968ec6a7a343de698c5e014d17602ae2f110f1b05925ed"}, - {file = "grpcio-1.27.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:80c3d1ce8820dd819d1c9d6b63b6f445148480a831173b572a9174a55e7abd47"}, - {file = "grpcio-1.27.2-cp27-cp27m-win32.whl", hash = "sha256:07f82aefb4a56c7e1e52b78afb77d446847d27120a838a1a0489260182096045"}, - {file = "grpcio-1.27.2-cp27-cp27m-win_amd64.whl", hash = "sha256:28f27c64dd699b8b10f70da5f9320c1cffcaefca7dd76275b44571bd097f276c"}, - {file = "grpcio-1.27.2-cp27-cp27mu-linux_armv7l.whl", hash = "sha256:a25b84e10018875a0f294a7649d07c43e8bc3e6a821714e39e5cd607a36386d7"}, - {file = "grpcio-1.27.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:57949756a3ce1f096fa2b00f812755f5ab2effeccedb19feeb7d0deafa3d1de7"}, - {file = "grpcio-1.27.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:f3614dabd2cc8741850597b418bcf644d4f60e73615906c3acc407b78ff720b3"}, - {file = "grpcio-1.27.2-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:25c77692ea8c0929d4ad400ea9c3dcbcc4936cee84e437e0ef80da58fa73d88a"}, - {file = "grpcio-1.27.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5dab393ab96b2ce4012823b2f2ed4ee907150424d2f02b97bd6f8dd8f17cc866"}, - {file = "grpcio-1.27.2-cp35-cp35m-linux_armv7l.whl", hash = "sha256:bb2987eb3af9bcf46019be39b82c120c3d35639a95bc4ee2d08f36ecdf469345"}, - {file = "grpcio-1.27.2-cp35-cp35m-macosx_10_7_intel.whl", hash = "sha256:6f328a3faaf81a2546a3022b3dfc137cc6d50d81082dbc0c94d1678943f05df3"}, - {file = "grpcio-1.27.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:5ebc13451246de82f130e8ee7e723e8d7ae1827f14b7b0218867667b1b12c88d"}, - {file = "grpcio-1.27.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:355bd7d7ce5ff2917d217f0e8ddac568cb7403e1ce1639b35a924db7d13a39b6"}, - {file = "grpcio-1.27.2-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:d1e5563e3b7f844dbc48d709c9e4a75647e11d0387cc1fa0c861d3e9d34bc844"}, - {file = "grpcio-1.27.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:1ec8fc865d8da6d0713e2092a27eee344cd54628b2c2065a0e77fff94df4ae00"}, - {file = "grpcio-1.27.2-cp35-cp35m-win32.whl", hash = "sha256:706e2dea3de33b0d8884c4d35ecd5911b4ff04d0697c4138096666ce983671a6"}, - {file = "grpcio-1.27.2-cp35-cp35m-win_amd64.whl", hash = "sha256:d18b4c8cacbb141979bb44355ee5813dd4d307e9d79b3a36d66eca7e0a203df8"}, - {file = "grpcio-1.27.2-cp36-cp36m-linux_armv7l.whl", hash = "sha256:02aef8ef1a5ac5f0836b543e462eb421df6048a7974211a906148053b8055ea6"}, - {file = "grpcio-1.27.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b78af4d42985ab3143d9882d0006f48d12f1bc4ba88e78f23762777c3ee64571"}, - {file = "grpcio-1.27.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:9c0669ba9aebad540fb05a33beb7e659ea6e5ca35833fc5229c20f057db760e8"}, - {file = "grpcio-1.27.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:68a149a0482d0bc697aac702ec6efb9d380e0afebf9484db5b7e634146528371"}, - {file = "grpcio-1.27.2-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:a71138366d57901597bfcc52af7f076ab61c046f409c7b429011cd68de8f9fe6"}, - {file = "grpcio-1.27.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:9e9cfe55dc7ac2aa47e0fd3285ff829685f96803197042c9d2f0fb44e4b39b2c"}, - {file = "grpcio-1.27.2-cp36-cp36m-win32.whl", hash = "sha256:d22c897b65b1408509099f1c3334bd3704f5e4eb7c0486c57d0e212f71cb8f54"}, - {file = "grpcio-1.27.2-cp36-cp36m-win_amd64.whl", hash = "sha256:c59b9280284b791377b3524c8e39ca7b74ae2881ba1a6c51b36f4f1bb94cee49"}, - {file = "grpcio-1.27.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6e545908bcc2ae28e5b190ce3170f92d0438cf26a82b269611390114de0106eb"}, - {file = "grpcio-1.27.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6db7ded10b82592c472eeeba34b9f12d7b0ab1e2dcad12f081b08ebdea78d7d6"}, - {file = "grpcio-1.27.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:4d3b6e66f32528bf43ca2297caca768280a8e068820b1c3dca0fcf9f03c7d6f1"}, - {file = "grpcio-1.27.2-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:586d931736912865c9790c60ca2db29e8dc4eace160d5a79fec3e58df79a9386"}, - {file = "grpcio-1.27.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:c03ce53690fe492845e14f4ab7e67d5a429a06db99b226b5c7caa23081c1e2bb"}, - {file = "grpcio-1.27.2-cp37-cp37m-win32.whl", hash = "sha256:209927e65395feb449783943d62a3036982f871d7f4045fadb90b2d82b153ea8"}, - {file = "grpcio-1.27.2-cp37-cp37m-win_amd64.whl", hash = "sha256:9713578f187fb1c4d00ac554fe1edcc6b3ddd62f5d4eb578b81261115802df8e"}, - {file = "grpcio-1.27.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b4efde5524579a9ce0459ca35a57a48ca878a4973514b8bb88cb80d7c9d34c85"}, - {file = "grpcio-1.27.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:fb62996c61eeff56b59ab8abfcaa0859ec2223392c03d6085048b576b567459b"}, - {file = "grpcio-1.27.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a22daaf30037b8e59d6968c76fe0f7ff062c976c7a026e92fbefc4c4bf3fc5a4"}, - {file = "grpcio-1.27.2-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:4a0a33ada3f6f94f855f92460896ef08c798dcc5f17d9364d1735c5adc9d7e4a"}, - {file = "grpcio-1.27.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:8111b61eee12d7af5c58f82f2c97c2664677a05df9225ef5cbc2f25398c8c454"}, - {file = "grpcio-1.27.2-cp38-cp38-win32.whl", hash = "sha256:5121fa96c79fc0ec81825091d0be5c16865f834f41b31da40b08ee60552f9961"}, - {file = "grpcio-1.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:1cff47297ee614e7ef66243dc34a776883ab6da9ca129ea114a802c5e58af5c1"}, - {file = "grpcio-1.27.2.tar.gz", hash = "sha256:5ae532b93cf9ce5a2a549b74a2c35e3b690b171ece9358519b3039c7b84c887e"}, + {file = "grpcio-1.43.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:a4e786a8ee8b30b25d70ee52cda6d1dbba2a8ca2f1208d8e20ed8280774f15c8"}, + {file = "grpcio-1.43.0-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:af9c3742f6c13575c0d4147a8454da0ff5308c4d9469462ff18402c6416942fe"}, + {file = "grpcio-1.43.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:fdac966699707b5554b815acc272d81e619dd0999f187cd52a61aef075f870ee"}, + {file = "grpcio-1.43.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e463b4aa0a6b31cf2e57c4abc1a1b53531a18a570baeed39d8d7b65deb16b7e"}, + {file = "grpcio-1.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f11d05402e0ac3a284443d8a432d3dfc76a6bd3f7b5858cddd75617af2d7bd9b"}, + {file = "grpcio-1.43.0-cp310-cp310-win32.whl", hash = "sha256:c36f418c925a41fccada8f7ae9a3d3e227bfa837ddbfddd3d8b0ac252d12dda9"}, + {file = "grpcio-1.43.0-cp310-cp310-win_amd64.whl", hash = "sha256:772b943f34374744f70236bbbe0afe413ed80f9ae6303503f85e2b421d4bca92"}, + {file = "grpcio-1.43.0-cp36-cp36m-linux_armv7l.whl", hash = "sha256:cbc9b83211d905859dcf234ad39d7193ff0f05bfc3269c364fb0d114ee71de59"}, + {file = "grpcio-1.43.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:fb7229fa2a201a0c377ff3283174ec966da8f9fd7ffcc9a92f162d2e7fc9025b"}, + {file = "grpcio-1.43.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:17b75f220ee6923338155b4fcef4c38802b9a57bc57d112c9599a13a03e99f8d"}, + {file = "grpcio-1.43.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:6620a5b751b099b3b25553cfc03dfcd873cda06f9bb2ff7e9948ac7090e20f05"}, + {file = "grpcio-1.43.0-cp36-cp36m-manylinux_2_17_aarch64.whl", hash = "sha256:1898f999383baac5fcdbdef8ea5b1ef204f38dc211014eb6977ac6e55944d738"}, + {file = "grpcio-1.43.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47b6821238d8978014d23b1132713dac6c2d72cbb561cf257608b1673894f90a"}, + {file = "grpcio-1.43.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80398e9fb598060fa41050d1220f5a2440fe74ff082c36dda41ac3215ebb5ddd"}, + {file = "grpcio-1.43.0-cp36-cp36m-win32.whl", hash = "sha256:0110310eff07bb69782f53b7a947490268c4645de559034c43c0a635612e250f"}, + {file = "grpcio-1.43.0-cp36-cp36m-win_amd64.whl", hash = "sha256:45401d00f2ee46bde75618bf33e9df960daa7980e6e0e7328047191918c98504"}, + {file = "grpcio-1.43.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:af78ac55933811e6a25141336b1f2d5e0659c2f568d44d20539b273792563ca7"}, + {file = "grpcio-1.43.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:8b2b9dc4d7897566723b77422e11c009a0ebd397966b165b21b89a62891a9fdf"}, + {file = "grpcio-1.43.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:77ef653f966934b3bfdd00e4f2064b68880eb40cf09b0b99edfa5ee22a44f559"}, + {file = "grpcio-1.43.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:e95b5d62ec26d0cd0b90c202d73e7cb927c369c3358e027225239a4e354967dc"}, + {file = "grpcio-1.43.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:04239e8f71db832c26bbbedb4537b37550a39d77681d748ab4678e58dd6455d6"}, + {file = "grpcio-1.43.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b4a7152187a49767a47d1413edde2304c96f41f7bc92cc512e230dfd0fba095"}, + {file = "grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8cc936a29c65ab39714e1ba67a694c41218f98b6e2a64efb83f04d9abc4386b"}, + {file = "grpcio-1.43.0-cp37-cp37m-win32.whl", hash = "sha256:577e024c8dd5f27cd98ba850bc4e890f07d4b5942e5bc059a3d88843a2f48f66"}, + {file = "grpcio-1.43.0-cp37-cp37m-win_amd64.whl", hash = "sha256:138f57e3445d4a48d9a8a5af1538fdaafaa50a0a3c243f281d8df0edf221dc02"}, + {file = "grpcio-1.43.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:08cf25f2936629db062aeddbb594bd76b3383ab0ede75ef0461a3b0bc3a2c150"}, + {file = "grpcio-1.43.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:01f4b887ed703fe82ebe613e1d2dadea517891725e17e7a6134dcd00352bd28c"}, + {file = "grpcio-1.43.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:0aa8285f284338eb68962fe1a830291db06f366ea12f213399b520c062b01f65"}, + {file = "grpcio-1.43.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:0edbfeb6729aa9da33ce7e28fb7703b3754934115454ae45e8cc1db601756fd3"}, + {file = "grpcio-1.43.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:c354017819201053d65212befd1dcb65c2d91b704d8977e696bae79c47cd2f82"}, + {file = "grpcio-1.43.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50cfb7e1067ee5e00b8ab100a6b7ea322d37ec6672c0455106520b5891c4b5f5"}, + {file = "grpcio-1.43.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57f1aeb65ed17dfb2f6cd717cc109910fe395133af7257a9c729c0b9604eac10"}, + {file = "grpcio-1.43.0-cp38-cp38-win32.whl", hash = "sha256:fa26a8bbb3fe57845acb1329ff700d5c7eaf06414c3e15f4cb8923f3a466ef64"}, + {file = "grpcio-1.43.0-cp38-cp38-win_amd64.whl", hash = "sha256:ade8b79a6b6aea68adb9d4bfeba5d647667d842202c5d8f3ba37ac1dc8e5c09c"}, + {file = "grpcio-1.43.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:124e718faf96fe44c98b05f3f475076be8b5198bb4c52a13208acf88a8548ba9"}, + {file = "grpcio-1.43.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2f96142d0abc91290a63ba203f01649e498302b1b6007c67bad17f823ecde0cf"}, + {file = "grpcio-1.43.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:31e6e489ccd8f08884b9349a39610982df48535881ec34f05a11c6e6b6ebf9d0"}, + {file = "grpcio-1.43.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:0e731f660e1e68238f56f4ce11156f02fd06dc58bc7834778d42c0081d4ef5ad"}, + {file = "grpcio-1.43.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:1f16725a320460435a8a5339d8b06c4e00d307ab5ad56746af2e22b5f9c50932"}, + {file = "grpcio-1.43.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4b4543e13acb4806917d883d0f70f21ba93b29672ea81f4aaba14821aaf9bb0"}, + {file = "grpcio-1.43.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:594aaa0469f4fca7773e80d8c27bf1298e7bbce5f6da0f084b07489a708f16ab"}, + {file = "grpcio-1.43.0-cp39-cp39-win32.whl", hash = "sha256:5449ae564349e7a738b8c38583c0aad954b0d5d1dd3cea68953bfc32eaee11e3"}, + {file = "grpcio-1.43.0-cp39-cp39-win_amd64.whl", hash = "sha256:bdf41550815a831384d21a498b20597417fd31bd084deb17d31ceb39ad9acc79"}, + {file = "grpcio-1.43.0.tar.gz", hash = "sha256:735d9a437c262ab039d02defddcb9f8f545d7009ae61c0114e19dda3843febe5"}, ] grpcio-tools = [ - {file = "grpcio-tools-1.27.2.tar.gz", hash = "sha256:845a51305af9fc7f9e2078edaec9a759153195f6cf1fbb12b1fa6f077e56b260"}, - {file = "grpcio_tools-1.27.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:7a2d5fb558ac153a326e742ebfd7020eb781c43d3ffd920abd42b2e6c6fdfb37"}, - {file = "grpcio_tools-1.27.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:99961156a36aae4a402d6b14c1e7efde642794b3ddbf32c51db0cb3a199e8b11"}, - {file = "grpcio_tools-1.27.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:069826dd02ce1886444cf4519c4fe1b05ac9ef41491f26e97400640531db47f6"}, - {file = "grpcio_tools-1.27.2-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:fae91f30dc050a8d0b32d20dc700e6092f0bd2138d83e9570fff3f0372c1b27e"}, - {file = "grpcio_tools-1.27.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a14dc7a36c845991d908a7179502ca47bcba5ae1817c4426ce68cf2c97b20ad9"}, - {file = "grpcio_tools-1.27.2-cp27-cp27m-win32.whl", hash = "sha256:d1a5e5fa47ba9557a7d3b31605631805adc66cdba9d95b5d10dfc52cca1fed53"}, - {file = "grpcio_tools-1.27.2-cp27-cp27m-win_amd64.whl", hash = "sha256:7b54b283ec83190680903a9037376dc915e1f03852a2d574ba4d981b7a1fd3d0"}, - {file = "grpcio_tools-1.27.2-cp27-cp27mu-linux_armv7l.whl", hash = "sha256:4698c6b6a57f73b14d91a542c69ff33a2da8729691b7060a5d7f6383624d045e"}, - {file = "grpcio_tools-1.27.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:87e8ca2c2d2d3e09b2a2bed5d740d7b3e64028dafb7d6be543b77eec85590736"}, - {file = "grpcio_tools-1.27.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bd7f59ff1252a3db8a143b13ea1c1e93d4b8cf4b852eb48b22ef1e6942f62a84"}, - {file = "grpcio_tools-1.27.2-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:a8f892378b0b02526635b806f59141abbb429d19bec56e869e04f396502c9651"}, - {file = "grpcio_tools-1.27.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:69c4a63919b9007e845d9f8980becd2f89d808a4a431ca32b9723ee37b521cb1"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-linux_armv7l.whl", hash = "sha256:dcbc06556f3713a9348c4fce02d05d91e678fc320fb2bcf0ddf8e4bb11d17867"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:16dc3fad04fe18d50777c56af7b2d9b9984cd1cfc71184646eb431196d1645c6"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1de5a273eaffeb3d126a63345e9e848ea7db740762f700eb8b5d84c5e3e7687d"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6016c07d6566e3109a3c032cf3861902d66501ecc08a5a84c47e43027302f367"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:915a695bc112517af48126ee0ecdb6aff05ed33f3eeef28f0d076f1f6b52ef5e"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:ea4b3ad696d976d5eac74ec8df9a2c692113e455446ee38d5b3bd87f8e034fa6"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-win32.whl", hash = "sha256:a140bf853edb2b5e8692fe94869e3e34077d7599170c113d07a58286c604f4fe"}, - {file = "grpcio_tools-1.27.2-cp35-cp35m-win_amd64.whl", hash = "sha256:77e25c241e33b75612f2aa62985f746c6f6803ec4e452da508bb7f8d90a69db4"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-linux_armv7l.whl", hash = "sha256:5fd7efc2fd3370bd2c72dc58f31a407a5dff5498befa145da211b2e8c6a52c63"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9ba88c2d99bcaf7b9cb720925e3290d73b2367d238c5779363fd5598b2dc98c7"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:b56caecc16307b088a431a4038c3b3bb7d0e7f9988cbd0e9fa04ac937455ea38"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:f8514453411d72cc3cf7d481f2b6057e5b7436736d0cd39ee2b2f72088bbf497"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:c1bb8f47d58e9f7c4825abfe01e6b85eda53c8b31d2267ca4cddf3c4d0829b80"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:e17b2e0936b04ced99769e26111e1e86ba81619d1b2691b1364f795e45560953"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-win32.whl", hash = "sha256:520b7dafddd0f82cb7e4f6e9c6ba1049aa804d0e207870def9fe7f94d1e14090"}, - {file = "grpcio_tools-1.27.2-cp36-cp36m-win_amd64.whl", hash = "sha256:ee50b0cf0d28748ef9f941894eb50fc464bd61b8e96aaf80c5056bea9b80d580"}, - {file = "grpcio_tools-1.27.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:627c91923df75091d8c4d244af38d5ab7ed8d786d480751d6c2b9267fbb92fe0"}, - {file = "grpcio_tools-1.27.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ef624b6134aef737b3daa4fb7e806cb8c5749efecd0b1fa9ce4f7e060c7a0221"}, - {file = "grpcio_tools-1.27.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e6932518db389ede8bf06b4119bbd3e17f42d4626e72dec2b8955b20ec732cb6"}, - {file = "grpcio_tools-1.27.2-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:43a1573400527a23e4174d88604fde7a9d9a69bf9473c21936b7f409858f8ebb"}, - {file = "grpcio_tools-1.27.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:57f8b9e2c7f55cd45f6dd930d6de61deb42d3eb7f9788137fbc7155cf724132a"}, - {file = "grpcio_tools-1.27.2-cp37-cp37m-win32.whl", hash = "sha256:2ca280af2cae1a014a238057bd3c0a254527569a6a9169a01c07f0590081d530"}, - {file = "grpcio_tools-1.27.2-cp37-cp37m-win_amd64.whl", hash = "sha256:59fbeb5bb9a7b94eb61642ac2cee1db5233b8094ca76fc56d4e0c6c20b5dd85f"}, - {file = "grpcio_tools-1.27.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:00c5080cfb197ed20ecf0d0ff2d07f1fc9c42c724cad21c40ff2d048de5712b1"}, - {file = "grpcio_tools-1.27.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f5450aa904e720f9c6407b59e96a8951ed6a95463f49444b6d2594b067d39588"}, - {file = "grpcio_tools-1.27.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:aaa5ae26883c3d58d1a4323981f96b941fa09bb8f0f368d97c6225585280cf04"}, - {file = "grpcio_tools-1.27.2-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:1266b577abe7c720fd16a83d0a4999a192e87c4a98fc9f97e0b99b106b3e155f"}, - {file = "grpcio_tools-1.27.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:a3d2aec4b09c8e59fee8b0d1ed668d09e8c48b738f03f5d8401d7eb409111c47"}, - {file = "grpcio_tools-1.27.2-cp38-cp38-win32.whl", hash = "sha256:8e7738a4b93842bca1158cde81a3587c9b7111823e40a1ddf73292ca9d58e08b"}, - {file = "grpcio_tools-1.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:84724458c86ff9b14c29b49e321f34d80445b379f4cd4d0494c694b49b1d6f88"}, + {file = "grpcio-tools-1.43.0.tar.gz", hash = "sha256:f42f1d713096808b1b0472dd2a3749b712d13f0092dab9442d9c096446e860b2"}, + {file = "grpcio_tools-1.43.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:766771ef5b60ebcba0a3bdb302dd92fda988552eb8508451ff6d97371eac38e5"}, + {file = "grpcio_tools-1.43.0-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:178a881db5de0f89abf3aeeb260ecfd1116cc31f88fb600a45fb5b19c3323b33"}, + {file = "grpcio_tools-1.43.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:019f55929e963214471825c7a4cdab7a57069109d5621b24e4db7b428b5fe47d"}, + {file = "grpcio_tools-1.43.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6c0e1d1b47554c580882d392b739df91a55b6a8ec696b2b2e1bbc127d63df2c"}, + {file = "grpcio_tools-1.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c5c80098fa69593b828d119973744de03c3f9a6935df8a02e4329a39b7072f5"}, + {file = "grpcio_tools-1.43.0-cp310-cp310-win32.whl", hash = "sha256:53f7dcaa4218df1b64b39d0fc7236a8270e8ab2db4ab8cd1d2fda0e6d4544946"}, + {file = "grpcio_tools-1.43.0-cp310-cp310-win_amd64.whl", hash = "sha256:5be6d402b0cafef20ba3abb3baa37444961d9a9c4a6434d3d7c1f082f7697deb"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-linux_armv7l.whl", hash = "sha256:8953fdebef6905d7ff13a5a376b21b6fecd808d18bf4f0d3990ffe4a215d56eb"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:18870dcc8369ac4c37213e6796d8dc20494ea770670204f5e573f88e69eaaf0b"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:010a4be6a2fccbd6741a4809c5da7f2e39a1e9e227745e6b495be567638bbeb9"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:426f16b6b14d533ce61249a18fbcd1a23a4fa0c71a6d7ab347b1c7f862847bb8"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-manylinux_2_17_aarch64.whl", hash = "sha256:f974cb0bea88bac892c3ed16da92c6ac88cff0fea17f24bf0e1892eb4d27cd00"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55c2e604536e06248e2f81e549737fb3a180c8117832e494a0a8a81fbde44837"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f97f9ffa49348fb24692751d2d4455ef2968bd07fe536d65597caaec14222629"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-win32.whl", hash = "sha256:6eaf97414237b8670ae9fa623879a26eabcc4c635b550c79a81e17eb600d6ae3"}, + {file = "grpcio_tools-1.43.0-cp36-cp36m-win_amd64.whl", hash = "sha256:04f100c1f6a7c72c537760c33582f6970070bd6fa6676b529bccfa31cc58bc79"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:9dbb6d1f58f26d88ae689f1b49de84cfaf4786c81c01b9001d3ceea178116a07"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:63862a441a77f6326ea9fe4bb005882f0e363441a5968d9cf8621c34d3dadc2b"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:6dea0cb2e79b67593553ed8662f70e4310599fa8850fc0e056b19fcb63572b7f"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:3eb4aa5b0e578c3d9d9da8e37a2ef73654287a498b8081543acd0db7f0ec1a9c"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:09464c6b17663088144b7e6ea10e9465efdcee03d4b2ffefab39a799bd8360f8"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2458d6b0404f83d95aef00cec01f310d30e9719564a25be50e39b259f6a2da5d"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e9bb5da437364b7dcd2d3c6850747081ecbec0ba645c96c6d471f7e21fdcadb"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-win32.whl", hash = "sha256:2737f749a6ab965748629e619b35f3e1cbe5820fc79e34c88f73cb99efc71dde"}, + {file = "grpcio_tools-1.43.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c39cbe7b902bb92f9afaa035091f5e2b8be35acbac501fec8cb6a0be7d7cdbbd"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:05550ba473cff7c09e905fcfb2263fd1f7600389660194ec022b5d5a3802534b"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:ce13a922db8f5f95c5041d3a4cbf04d942b353f0cba9b251a674f69a31a2d3a6"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:f19d40690c97365c1c1bde81474e6f496d7ab76f87e6d2889c72ad01bac98f2d"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba3da574eb08fcaed541b3fc97ce217360fd86d954fa9ad6a604803d57a2e049"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:efd1eb5880001f5189cfa3a774675cc9bbc8cc51586a3e90fe796394ac8626b8"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:234c7a5af653357df5c616e013173eddda6193146c8ab38f3108c4784f66be26"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7e3662f62d410b3f81823b5fa0f79c6e0e250977a1058e4131867b85138a661"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-win32.whl", hash = "sha256:5f2e584d7644ef924e9e042fa151a3bb9f7c28ef1ae260ee6c9cb327982b5e94"}, + {file = "grpcio_tools-1.43.0-cp38-cp38-win_amd64.whl", hash = "sha256:98dcb5b756855110fb661ccd6a93a716610b7efcd5720a3aec01358a1a892c30"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:61ef6cb6ccf9b9c27bb85fffc5338194bcf444df502196c2ad0ff8df4706d41e"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:1def9b68ac9e62674929bc6590a33d89635f1cf16016657d9e16a69f41aa5c36"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:b68cc0c95a0f8c757e8d69b5fa46111d5c9d887ae62af28f827649b1d1b70fe1"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:e956b5c3b586d7b27eae49fb06f544a26288596fe12e22ffec768109717276d1"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:671e61bbc91d8d568f12c3654bb5a91fce9f3fdfd5ec2cfc60c2d3a840449aa6"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7173ed19854d1066bce9bdc09f735ca9c13e74a25d47a1cc5d1fe803b53bffb"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1adb0dbcc1c10b86dcda910b8f56e39210e401bcee923dba166ba923a5f4696a"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-win32.whl", hash = "sha256:ebfb94ddb454a6dc3a505d9531dc81c948e6364e181b8795bfad3f3f479974dc"}, + {file = "grpcio_tools-1.43.0-cp39-cp39-win_amd64.whl", hash = "sha256:d21928b680e6e29538688cffbf53f3d5a53cff0ec8f0c33139641700045bdf1a"}, ] identify = [ {file = "identify-1.6.2-py2.py3-none-any.whl", hash = "sha256:8f9879b5b7cca553878d31548a419ec2f227d3328da92fe8202bc5e546d5cbc3"}, @@ -992,8 +1003,8 @@ mock = [ {file = "mock-4.0.2.tar.gz", hash = "sha256:dd33eb70232b6118298d516bbcecd26704689c386594f0f3c4f13867b2c56f72"}, ] more-itertools = [ - {file = "more-itertools-8.12.0.tar.gz", hash = "sha256:7dc6ad46f05f545f900dd59e8dfb4e84a4827b97b3cfecb175ea0c7d247f6064"}, - {file = "more_itertools-8.12.0-py3-none-any.whl", hash = "sha256:43e6dd9942dffd72661a2c4ef383ad7da1e6a3e968a927ad7a6083ab410a688b"}, + {file = "more-itertools-8.13.0.tar.gz", hash = "sha256:a42901a0a5b169d925f6f217cd5a190e32ef54360905b9c39ee7db5313bfec0f"}, + {file = "more_itertools-8.13.0-py3-none-any.whl", hash = "sha256:c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb"}, ] netaddr = [ {file = "netaddr-0.7.19-py2.py3-none-any.whl", hash = "sha256:56b3558bd71f3f6999e4c52e349f38660e54a7a8a9943335f73dfc96883e08ca"}, @@ -1008,8 +1019,8 @@ packaging = [ {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] paramiko = [ - {file = "paramiko-2.9.2-py2.py3-none-any.whl", hash = "sha256:04097dbd96871691cdb34c13db1883066b8a13a0df2afd4cb0a92221f51c2603"}, - {file = "paramiko-2.9.2.tar.gz", hash = "sha256:944a9e5dbdd413ab6c7951ea46b0ab40713235a9c4c5ca81cfe45c6f14fa677b"}, + {file = "paramiko-2.10.4-py2.py3-none-any.whl", hash = "sha256:3c9ed6084f4b671ab66dc3c729092d32d96c3258f1426071301cb33654b09027"}, + {file = "paramiko-2.10.4.tar.gz", hash = "sha256:3d2e650b6812ce6d160abff701d6ef4434ec97934b13e95cf1ad3da70ffb5c58"}, ] pillow = [ {file = "Pillow-8.3.2-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4"}, @@ -1203,8 +1214,8 @@ typing-extensions = [ {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, ] virtualenv = [ - {file = "virtualenv-20.13.1-py2.py3-none-any.whl", hash = "sha256:45e1d053cad4cd453181ae877c4ffc053546ae99e7dd049b9ff1d9be7491abf7"}, - {file = "virtualenv-20.13.1.tar.gz", hash = "sha256:e0621bcbf4160e4e1030f05065c8834b4e93f4fcc223255db2a823440aca9c14"}, + {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, + {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, diff --git a/daemon/pyproject.toml b/daemon/pyproject.toml index 9e11d3f6..a6b0ec1b 100644 --- a/daemon/pyproject.toml +++ b/daemon/pyproject.toml @@ -19,7 +19,7 @@ exclude = ["core/constants.py.in"] python = "^3.6" dataclasses = { version = "*", python = "~3.6" } fabric = "2.5.0" -grpcio = "1.27.2" +grpcio = "1.43.0" invoke = "1.4.1" lxml = "4.6.5" mako = "1.1.3" @@ -32,7 +32,7 @@ pyyaml = "5.4" [tool.poetry.dev-dependencies] black = "==19.3b0" flake8 = "3.8.2" -grpcio-tools = "1.27.2" +grpcio-tools = "1.43.0" isort = "4.3.21" mock = "4.0.2" pre-commit = "2.1.1" diff --git a/tasks.py b/tasks.py index 8c0ffe77..2b999cfc 100644 --- a/tasks.py +++ b/tasks.py @@ -186,7 +186,7 @@ def install_system(c: Context, os_info: OsInfo, hide: bool) -> None: def install_grpcio(c: Context, hide: bool) -> None: python_bin = get_env_python() c.run( - f"{python_bin} -m pip install --user grpcio==1.27.2 grpcio-tools==1.27.2", + f"{python_bin} -m pip install --user grpcio==1.43.0 grpcio-tools==1.43.0", hide=hide, ) From 6f60fba18b0e5d22be9da40d39e2038b356e6ee7 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 10 May 2022 13:54:51 -0700 Subject: [PATCH 04/15] install: updated pyproj version to avoid the need to build from source on newer distros --- daemon/poetry.lock | 69 +++++++++++++++++++++++++++---------------- daemon/pyproject.toml | 2 +- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/daemon/poetry.lock b/daemon/poetry.lock index 34b95f2f..6278e3e9 100644 --- a/daemon/poetry.lock +++ b/daemon/poetry.lock @@ -60,6 +60,14 @@ toml = ">=0.9.4" [package.extras] d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +[[package]] +name = "certifi" +version = "2021.10.8" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "cffi" version = "1.15.0" @@ -500,11 +508,14 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyproj" -version = "2.6.1.post1" +version = "3.0.1" description = "Python interface to PROJ (cartographic projections and coordinate transformations library)" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" + +[package.dependencies] +certifi = "*" [[package]] name = "pytest" @@ -604,7 +615,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "8f6ab4134c3bd5219ad80a1c93a055a87cd13b985b84a10a8ff1a7e3ca98a3c8" +content-hash = "0a739e4b479c0c2111fa22ffb1ed55a99d26583a576a1b548204c29b726e3e33" [metadata.files] appdirs = [ @@ -636,6 +647,10 @@ black = [ {file = "black-19.3b0-py36-none-any.whl", hash = "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf"}, {file = "black-19.3b0.tar.gz", hash = "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c"}, ] +certifi = [ + {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, + {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, +] cffi = [ {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, @@ -1150,29 +1165,31 @@ pyparsing = [ {file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"}, ] pyproj = [ - {file = "pyproj-2.6.1.post1-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:457ad3856014ac26af1d86def6dc8cf69c1fa377b6e2fd6e97912d51cf66bdbe"}, - {file = "pyproj-2.6.1.post1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:6f3f36440ea61f5f6da4e6beb365dddcbe159815450001d9fb753545affa45ff"}, - {file = "pyproj-2.6.1.post1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6a212d0e5c7efa33d039f0c8b0a489e2204fcd28b56206567852ad7f5f2a653e"}, - {file = "pyproj-2.6.1.post1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:451a3d1c563b672458029ebc04acbb3266cd8b3025268eb871a9176dc3638911"}, - {file = "pyproj-2.6.1.post1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e015f900b4b84e908f8035ab16ebf02d67389c1c216c17a2196fc2e515c00762"}, - {file = "pyproj-2.6.1.post1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a13e5731b3a360ee7fbd1e9199ec9203fafcece8ebd0b1351f16d0a90cad6828"}, - {file = "pyproj-2.6.1.post1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:33c1c2968a4f4f87d517c4275a18b557e5c13907cf2609371fadea8463c3ba05"}, - {file = "pyproj-2.6.1.post1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3fef83a01c1e86dd9fa99d8214f749837cfafc34d9d6230b4b0a998fa7a68a1a"}, - {file = "pyproj-2.6.1.post1-cp36-cp36m-win32.whl", hash = "sha256:a6ac4861979cd05a0f5400fefa41d26c0269a5fb8237618aef7c998907db39e1"}, - {file = "pyproj-2.6.1.post1-cp36-cp36m-win_amd64.whl", hash = "sha256:cbf6ccf990860b06c5262ff97c4b78e1d07883981635cd53a6aa438a68d92945"}, - {file = "pyproj-2.6.1.post1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:adacb67a9f71fb54ca1b887a6ab20f32dd536fcdf2acec84a19e25ad768f7965"}, - {file = "pyproj-2.6.1.post1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e50d5d20b87758acf8f13f39a3b3eb21d5ef32339d2bc8cdeb8092416e0051df"}, - {file = "pyproj-2.6.1.post1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2518d1606e2229b82318e704b40290e02a2a52d77b40cdcb2978973d6fc27b20"}, - {file = "pyproj-2.6.1.post1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:33a5d1cfbb40a019422eb80709a0e270704390ecde7278fdc0b88f3647c56a39"}, - {file = "pyproj-2.6.1.post1-cp37-cp37m-win32.whl", hash = "sha256:daf2998e3f5bcdd579a18faf009f37f53538e9b7d0a252581a610297d31e8536"}, - {file = "pyproj-2.6.1.post1-cp37-cp37m-win_amd64.whl", hash = "sha256:a8b7c8accdc61dac8e91acab7c1f7b4590d1e102f2ee9b1f1e6399fad225958e"}, - {file = "pyproj-2.6.1.post1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9f097e8f341a162438918e908be86d105a28194ff6224633b2e9616c5031153f"}, - {file = "pyproj-2.6.1.post1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d90a5d1fdd066b0e9b22409b0f5e81933469918fa04c2cf7f9a76ce84cb29dad"}, - {file = "pyproj-2.6.1.post1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:f5a8015c74ec8f6508aebf493b58ba20ccb4da8168bf05f0c2a37faccb518da9"}, - {file = "pyproj-2.6.1.post1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d87836be6b720fb4d9c112136aa47621b6ca09a554e645c1081561eb8e2fa1f4"}, - {file = "pyproj-2.6.1.post1-cp38-cp38-win32.whl", hash = "sha256:bc2f3a15d065e206d63edd2cc4739aa0a35c05338ee276ab1dc72f56f1944bda"}, - {file = "pyproj-2.6.1.post1-cp38-cp38-win_amd64.whl", hash = "sha256:93cbad7b699e8e80def7de80c350617f35e6a0b82862f8ce3c014657c25fdb3c"}, - {file = "pyproj-2.6.1.post1.tar.gz", hash = "sha256:4f5b02b4abbd41610397c635b275a8ee4a2b5bc72a75572b98ac6ae7befa471e"}, + {file = "pyproj-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f942a976ea3de6a519cf48be30a12f465e44d0ac0c38a0d820ab3acfcc0a48a6"}, + {file = "pyproj-3.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:09db64a8088b23f001e574d92bcc3080bf7de44ddca152d0282a2b50c918a64a"}, + {file = "pyproj-3.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:cba99e171d744969e13a865ad28fa9c949c4400b0e9c431a802cdd804f52f632"}, + {file = "pyproj-3.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:81c06df20d09d621e52791c19ce3c880695fb430061e59c2472fa5467e890391"}, + {file = "pyproj-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:3e7e851e6d58c16ac2cd920a1bacb7fbb24758a6fcd7f234d594a88ebae04ec9"}, + {file = "pyproj-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:aa0a2981b25145523ca17a643c5be077fe13e514fdca9b6d1c412a95d723a5a5"}, + {file = "pyproj-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:708d6e01b9ff3d6dc62a5ad2d2ba1264a863eaa657c1a9bf713a10cc35d34553"}, + {file = "pyproj-3.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:36ba436675f9dea4ab3db7d9a32d3ff11c2fbb4d6690a83454d2f3c5c0b54041"}, + {file = "pyproj-3.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:489a96da87d8846c34c90da90e637544e4f4f50a13589b5aac54297f5ee1b01d"}, + {file = "pyproj-3.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:4a333f3e46fe8b2eb4647a3daa3a2cec52ddc6c107c653b45880526114942ee8"}, + {file = "pyproj-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:9e2ef75401f17062166d3fe53c555cd62c9577697a2f5ded916b23c54e5db497"}, + {file = "pyproj-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7bfaa34e8bb0510d4380310374deecd9e4328b9cf556925cfb45b5a94d5bbdbe"}, + {file = "pyproj-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9666d01faf4e758ac68f2c16695c90de49c3170e3760988bf76a34aae11f4e15"}, + {file = "pyproj-3.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c658afc8a6115b58b02aa53d27bf2a67c1b00b55067edb1b7711c6c7391cfaa9"}, + {file = "pyproj-3.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:fee7517bd389a1db7b8bebb18838d04dedca9eaacda01d353d98f5ee421f263e"}, + {file = "pyproj-3.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:86ef2fcd584a3222bf73e2befc24b2badd139b3371f4a1e88649978ef7649540"}, + {file = "pyproj-3.0.1-cp38-cp38-win32.whl", hash = "sha256:d27d40ec541ef69a5107bfcd85f40170e9e122ceb6315ce508cd44d199983d41"}, + {file = "pyproj-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:bc70b6adcfa713d89bc561673cb57af5fb3a1718cd7d57ec537430cd1007a864"}, + {file = "pyproj-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b845510255f9580d7e226dd3321a51c468cefb7be24e46415caf67caa4287c4"}, + {file = "pyproj-3.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:7ae8e7052f18fde1884574da449010e94fa205ad27aeeaa34a097f49a1ed6a2b"}, + {file = "pyproj-3.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:a3805e026a5547be205a5e322c08e3069f0a48c63bbd53dbc7a8e3499bc66d58"}, + {file = "pyproj-3.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1be7d54900eb7e2d1e637319080b3a047c70d1fb2f3c12d3400c0fa8a90cf440"}, + {file = "pyproj-3.0.1-cp39-cp39-win32.whl", hash = "sha256:09bead60769e69b592e8cb3ac51b5215f75e9bb9c213ce575031961deb48d6da"}, + {file = "pyproj-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:a3a8ab19232bf4f4bb2590536538881b7bd0c07df23e0c2a792402ca2476c197"}, + {file = "pyproj-3.0.1.tar.gz", hash = "sha256:bfbac35490dd17f706700673506eeb8170f8a2a63fb5878171d4e6eef242d141"}, ] pytest = [ {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, diff --git a/daemon/pyproject.toml b/daemon/pyproject.toml index a6b0ec1b..c9dd5b3e 100644 --- a/daemon/pyproject.toml +++ b/daemon/pyproject.toml @@ -26,7 +26,7 @@ mako = "1.1.3" netaddr = "0.7.19" pillow = "8.3.2" protobuf = "3.19.4" -pyproj = "2.6.1.post1" +pyproj = "3.0.1" pyyaml = "5.4" [tool.poetry.dev-dependencies] From fd3be57f57e7bbdffed004cc12e2e939768676e7 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 11 May 2022 13:51:02 -0700 Subject: [PATCH 05/15] daemon: added initial support to docker nodes for setting up volumes and bind mounts --- daemon/core/emulator/data.py | 4 +++- daemon/core/emulator/session.py | 2 ++ daemon/core/nodes/docker.py | 36 +++++++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/daemon/core/emulator/data.py b/daemon/core/emulator/data.py index 28dcb813..bd9a4e75 100644 --- a/daemon/core/emulator/data.py +++ b/daemon/core/emulator/data.py @@ -2,7 +2,7 @@ CORE data objects. """ from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple import netaddr @@ -92,6 +92,8 @@ class NodeOptions: image: str = None emane: str = None legacy: bool = False + binds: Dict[str, str] = field(default_factory=dict) + volumes: Dict[str, str] = field(default_factory=dict) def set_position(self, x: float, y: float) -> None: """ diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 6dded294..5064579f 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -521,6 +521,8 @@ class Session: kwargs = dict(_id=_id, name=name, server=server) if _class in CONTAINER_NODES: kwargs["image"] = options.image + kwargs["binds"] = options.binds + kwargs["volumes"] = options.volumes node = self.create_node(_class, start, **kwargs) # set node attributes diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 4808fe04..50fc3bb3 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -1,9 +1,10 @@ import json import logging import shlex +from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict from core.emulator.distributed import DistributedServer from core.errors import CoreCommandError, CoreError @@ -18,6 +19,13 @@ if TYPE_CHECKING: DOCKER: str = "docker" +@dataclass +class DockerVolume: + src: str + dst: str + path: str = None + + class DockerNode(CoreNode): """ Provides logic for creating a Docker based node. @@ -31,6 +39,8 @@ class DockerNode(CoreNode): directory: str = None, server: DistributedServer = None, image: str = None, + binds: Dict[str, str] = None, + volumes: Dict[str, str] = None, ) -> None: """ Create a DockerNode instance. @@ -42,9 +52,16 @@ class DockerNode(CoreNode): :param server: remote server node will run on, default is None for localhost :param image: image to start container with + :param binds: bind mounts to set for the created container + :param volumes: volume mounts to set for the created container """ super().__init__(session, _id, name, directory, server) - self.image = image if image is not None else "ubuntu" + self.image: str = image if image is not None else "ubuntu" + self.binds: Dict[str, str] = binds or {} + volumes = volumes or {} + self.volumes: Dict[str, DockerVolume] = { + k: DockerVolume(k, v) for k, v in volumes.items() + } def _create_cmd(self, args: str, shell: bool = False) -> str: """ @@ -82,14 +99,27 @@ class DockerNode(CoreNode): if self.up: raise CoreError(f"starting node({self.name}) that is already up") self.makenodedir() + binds = "" + for src, dst in self.binds.items(): + binds += f"--mount type=bind,source={src},target={dst} " + volumes = "" + for volume in self.volumes.values(): + volumes += ( + f"--mount type=volume," f"source={volume.src},target={volume.dst} " + ) self.host_cmd( f"{DOCKER} run -td --init --net=none --hostname {self.name} " f"--name {self.name} --sysctl net.ipv6.conf.all.disable_ipv6=0 " + f"{binds} {volumes} " f"--privileged {self.image} /bin/bash" ) self.pid = self.host_cmd( f"{DOCKER} inspect -f '{{{{.State.Pid}}}}' {self.name}" ) + for volume in self.volumes.values(): + volume.path = self.host_cmd( + f"{DOCKER} volume inspect -f '{{{{.Mountpoint}}}}' {volume.src}" + ) logger.debug("node(%s) pid: %s", self.name, self.pid) self.up = True @@ -105,6 +135,8 @@ class DockerNode(CoreNode): with self.lock: self.ifaces.clear() self.host_cmd(f"{DOCKER} rm -f {self.name}") + for volume in self.volumes.values(): + self.host_cmd(f"{DOCKER} volume rm {volume.src}") self.up = False def termcmdstring(self, sh: str = "/bin/sh") -> str: From 03e646031c6cfb4bc72718aac2797c2d5dd82923 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 18 May 2022 14:50:08 -0700 Subject: [PATCH 06/15] daemon: updated docker nodes to start using tail to keep alive, adjustments to support more robust volume mounting and adding symlinks to the nodes directory for bind/volume mounts --- daemon/core/emulator/data.py | 8 ++++--- daemon/core/nodes/docker.py | 42 +++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/daemon/core/emulator/data.py b/daemon/core/emulator/data.py index bd9a4e75..de5b3559 100644 --- a/daemon/core/emulator/data.py +++ b/daemon/core/emulator/data.py @@ -2,7 +2,7 @@ CORE data objects. """ from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, List, Optional, Tuple import netaddr @@ -92,8 +92,10 @@ class NodeOptions: image: str = None emane: str = None legacy: bool = False - binds: Dict[str, str] = field(default_factory=dict) - volumes: Dict[str, str] = field(default_factory=dict) + # src, dst + binds: List[Tuple[str, str]] = field(default_factory=list) + # src, dst, unique, delete + volumes: List[Tuple[str, str, bool, bool]] = field(default_factory=list) def set_position(self, x: float, y: float) -> None: """ diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 50fc3bb3..122aa9b0 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -4,7 +4,7 @@ import shlex from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Dict, List, Tuple from core.emulator.distributed import DistributedServer from core.errors import CoreCommandError, CoreError @@ -23,6 +23,8 @@ DOCKER: str = "docker" class DockerVolume: src: str dst: str + unique: bool = True + delete: bool = True path: str = None @@ -39,8 +41,8 @@ class DockerNode(CoreNode): directory: str = None, server: DistributedServer = None, image: str = None, - binds: Dict[str, str] = None, - volumes: Dict[str, str] = None, + binds: List[Tuple[str, str]] = None, + volumes: List[Tuple[str, str, bool, bool]] = None, ) -> None: """ Create a DockerNode instance. @@ -53,15 +55,16 @@ class DockerNode(CoreNode): will run on, default is None for localhost :param image: image to start container with :param binds: bind mounts to set for the created container - :param volumes: volume mounts to set for the created container + :param volumes: volume mount settings to set for the created container """ super().__init__(session, _id, name, directory, server) self.image: str = image if image is not None else "ubuntu" - self.binds: Dict[str, str] = binds or {} - volumes = volumes or {} - self.volumes: Dict[str, DockerVolume] = { - k: DockerVolume(k, v) for k, v in volumes.items() - } + self.binds: List[Tuple[str, str]] = binds or [] + self.volumes: Dict[str, DockerVolume] = {} + volumes = volumes or [] + for src, dst, unique, delete in volumes: + src_name = self._unique_name(src) if unique else src + self.volumes[src] = DockerVolume(src_name, dst, unique, delete) def _create_cmd(self, args: str, shell: bool = False) -> str: """ @@ -75,6 +78,15 @@ class DockerNode(CoreNode): args = f"{BASH} -c {shlex.quote(args)}" return f"nsenter -t {self.pid} -m -u -i -p -n {args}" + def _unique_name(self, name: str) -> str: + """ + Creates a session/node unique prefixed name for the provided input. + + :param name: name to make unique + :return: unique session/node prefixed name + """ + return f"{self.session.id}.{self.id}.{name}" + def alive(self) -> bool: """ Check if the node is alive. @@ -100,7 +112,7 @@ class DockerNode(CoreNode): raise CoreError(f"starting node({self.name}) that is already up") self.makenodedir() binds = "" - for src, dst in self.binds.items(): + for src, dst in self.binds: binds += f"--mount type=bind,source={src},target={dst} " volumes = "" for volume in self.volumes.values(): @@ -111,15 +123,20 @@ class DockerNode(CoreNode): f"{DOCKER} run -td --init --net=none --hostname {self.name} " f"--name {self.name} --sysctl net.ipv6.conf.all.disable_ipv6=0 " f"{binds} {volumes} " - f"--privileged {self.image} /bin/bash" + f"--privileged {self.image} tail -f /dev/null" ) self.pid = self.host_cmd( f"{DOCKER} inspect -f '{{{{.State.Pid}}}}' {self.name}" ) + for src, dst in self.binds: + link_path = self.host_path(Path(dst), True) + self.host_cmd(f"ln -s {src} {link_path}") for volume in self.volumes.values(): volume.path = self.host_cmd( f"{DOCKER} volume inspect -f '{{{{.Mountpoint}}}}' {volume.src}" ) + link_path = self.host_path(Path(volume.dst), True) + self.host_cmd(f"ln -s {volume.path} {link_path}") logger.debug("node(%s) pid: %s", self.name, self.pid) self.up = True @@ -136,7 +153,8 @@ class DockerNode(CoreNode): self.ifaces.clear() self.host_cmd(f"{DOCKER} rm -f {self.name}") for volume in self.volumes.values(): - self.host_cmd(f"{DOCKER} volume rm {volume.src}") + if volume.delete: + self.host_cmd(f"{DOCKER} volume rm {volume.src}") self.up = False def termcmdstring(self, sh: str = "/bin/sh") -> str: From 2e3e085522f98bea7a010cd80703970f791e9fd7 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 25 May 2022 10:51:42 -0700 Subject: [PATCH 07/15] daemon: adjustments to revamp how core nodes are created in session.add_node, nodes now provide a create_options function for node specific options that are type hinted --- daemon/core/api/grpc/grpcutils.py | 66 +++--- daemon/core/api/grpc/server.py | 19 +- daemon/core/emane/emanemanager.py | 2 +- daemon/core/emane/nodes.py | 22 +- daemon/core/emulator/session.py | 152 +++++--------- daemon/core/nodes/base.py | 210 ++++++++++++------- daemon/core/nodes/docker.py | 43 ++-- daemon/core/nodes/lxd.py | 35 +++- daemon/core/nodes/network.py | 78 ++++--- daemon/core/nodes/physical.py | 10 +- daemon/core/nodes/wireless.py | 5 +- daemon/core/xml/corexml.py | 72 ++++--- daemon/examples/configservices/testing.py | 4 +- daemon/examples/docker/docker2core.py | 5 +- daemon/examples/docker/docker2docker.py | 5 +- daemon/examples/docker/switch.py | 16 +- daemon/examples/lxd/lxd2core.py | 5 +- daemon/examples/lxd/lxd2lxd.py | 5 +- daemon/examples/lxd/switch.py | 5 +- daemon/examples/python/distributed_emane.py | 12 +- daemon/examples/python/distributed_lxd.py | 5 +- daemon/examples/python/distributed_ptp.py | 8 +- daemon/examples/python/distributed_switch.py | 4 +- daemon/examples/python/emane80211.py | 22 +- daemon/examples/python/peertopeer.py | 12 +- daemon/examples/python/switch.py | 16 +- daemon/examples/python/wireless.py | 20 +- daemon/examples/python/wlan.py | 20 +- daemon/tests/emane/test_emane.py | 99 ++++----- daemon/tests/test_core.py | 10 +- daemon/tests/test_distributed.py | 7 +- daemon/tests/test_grpc.py | 20 +- daemon/tests/test_nodes.py | 5 +- daemon/tests/test_xml.py | 9 +- docs/python.md | 96 +++++---- 35 files changed, 646 insertions(+), 478 deletions(-) diff --git a/daemon/core/api/grpc/grpcutils.py b/daemon/core/api/grpc/grpcutils.py index 2c3996ba..7f63079c 100644 --- a/daemon/core/api/grpc/grpcutils.py +++ b/daemon/core/api/grpc/grpcutils.py @@ -17,15 +17,22 @@ from core.api.grpc.services_pb2 import ( ServiceDefaults, ) from core.config import ConfigurableOptions -from core.emane.nodes import EmaneNet -from core.emulator.data import InterfaceData, LinkData, LinkOptions, NodeOptions +from core.emane.nodes import EmaneNet, EmaneOptions +from core.emulator.data import InterfaceData, LinkData, LinkOptions from core.emulator.enumerations import LinkTypes, NodeTypes from core.emulator.links import CoreLink from core.emulator.session import Session from core.errors import CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility -from core.nodes.base import CoreNode, CoreNodeBase, NodeBase -from core.nodes.docker import DockerNode +from core.nodes.base import ( + CoreNode, + CoreNodeBase, + CoreNodeOptions, + NodeBase, + NodeOptions, + Position, +) +from core.nodes.docker import DockerNode, DockerOptions from core.nodes.interface import CoreInterface from core.nodes.lxd import LxcNode from core.nodes.network import CoreNetwork, CtrlNet, PtpNet, WlanNode @@ -55,34 +62,33 @@ class CpuUsage: return (total_diff - idle_diff) / total_diff -def add_node_data(node_proto: core_pb2.Node) -> Tuple[NodeTypes, int, NodeOptions]: +def add_node_data( + _class: Type[NodeBase], node_proto: core_pb2.Node +) -> Tuple[Position, NodeOptions]: """ Convert node protobuf message to data for creating a node. + :param _class: node class to create options from :param node_proto: node proto message :return: node type, id, and options """ - _id = node_proto.id - _type = NodeTypes(node_proto.type) - options = NodeOptions( - name=node_proto.name, - model=node_proto.model, - icon=node_proto.icon, - image=node_proto.image, - services=node_proto.services, - config_services=node_proto.config_services, - canvas=node_proto.canvas, - ) - if node_proto.emane: - options.emane = node_proto.emane - if node_proto.server: - options.server = node_proto.server - position = node_proto.position - options.set_position(position.x, position.y) + options = _class.create_options() + options.icon = node_proto.icon + options.canvas = node_proto.canvas + if isinstance(options, CoreNodeOptions): + options.model = node_proto.model + options.services = node_proto.services + options.config_services = node_proto.config_services + if isinstance(options, EmaneOptions): + options.emane_model = node_proto.emane + if isinstance(options, DockerOptions): + options.image = node_proto.image + position = Position() + position.set(node_proto.position.x, node_proto.position.y) if node_proto.HasField("geo"): geo = node_proto.geo - options.set_location(geo.lat, geo.lon, geo.alt) - return _type, _id, options + position.set_geo(geo.lon, geo.lat, geo.alt) + return position, options def link_iface(iface_proto: core_pb2.Interface) -> InterfaceData: @@ -150,9 +156,17 @@ def create_nodes( """ funcs = [] for node_proto in node_protos: - _type, _id, options = add_node_data(node_proto) + _type = NodeTypes(node_proto.type) _class = session.get_node_class(_type) - args = (_class, _id, options) + position, options = add_node_data(_class, node_proto) + args = ( + _class, + node_proto.id or None, + node_proto.name or None, + node_proto.server or None, + position, + options, + ) funcs.append((session.add_node, args, {})) start = time.monotonic() results, exceptions = utils.threadpool(funcs) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 3846701b..532dd91c 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -88,7 +88,12 @@ from core.configservice.base import ConfigServiceBootError from core.emane.modelmanager import EmaneModelManager from core.emulator.coreemu import CoreEmu from core.emulator.data import InterfaceData, LinkData, LinkOptions -from core.emulator.enumerations import EventTypes, ExceptionLevels, MessageFlags +from core.emulator.enumerations import ( + EventTypes, + ExceptionLevels, + MessageFlags, + NodeTypes, +) from core.emulator.session import NT, Session from core.errors import CoreCommandError, CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility @@ -548,9 +553,17 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): """ logger.debug("add node: %s", request) session = self.get_session(request.session_id, context) - _type, _id, options = grpcutils.add_node_data(request.node) + _type = NodeTypes(request.node.type) _class = session.get_node_class(_type) - node = session.add_node(_class, _id, options) + position, options = grpcutils.add_node_data(_class, request.node) + node = session.add_node( + _class, + request.node.id or None, + request.node.name or None, + request.node.server or None, + position, + options, + ) grpcutils.configure_node(session, request.node, node, context) source = request.source if request.source else None session.broadcast_node(node, MessageFlags.ADD, source) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 92fdf93b..32949301 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -603,7 +603,7 @@ class EmaneManager: node = iface.node loglevel = str(DEFAULT_LOG_LEVEL) cfgloglevel = self.session.options.get_int("emane_log_level", 2) - realtime = self.session.options.get_bool("emane_realtime") + realtime = self.session.options.get_bool("emane_realtime", True) if cfgloglevel: logger.info("setting user-defined emane log level: %d", cfgloglevel) loglevel = str(cfgloglevel) diff --git a/daemon/core/emane/nodes.py b/daemon/core/emane/nodes.py index b8809dba..1a0b6e75 100644 --- a/daemon/core/emane/nodes.py +++ b/daemon/core/emane/nodes.py @@ -5,13 +5,14 @@ share the same MAC+PHY model. import logging import time +from dataclasses import dataclass from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type, Union from core.emulator.data import InterfaceData, LinkData, LinkOptions from core.emulator.distributed import DistributedServer from core.emulator.enumerations import EventTypes, MessageFlags, RegisterTlvs from core.errors import CoreCommandError, CoreError -from core.nodes.base import CoreNetworkBase, CoreNode +from core.nodes.base import CoreNetworkBase, CoreNode, NodeOptions from core.nodes.interface import CoreInterface logger = logging.getLogger(__name__) @@ -139,6 +140,12 @@ class TunTap(CoreInterface): self.node.node_net_client.create_address(self.name, str(ip)) +@dataclass +class EmaneOptions(NodeOptions): + emane_model: str = None + """name of emane model to associate an emane network to""" + + class EmaneNet(CoreNetworkBase): """ EMANE node contains NEM configuration and causes connected nodes @@ -152,11 +159,20 @@ class EmaneNet(CoreNetworkBase): _id: int = None, name: str = None, server: DistributedServer = None, + options: EmaneOptions = None, ) -> None: - super().__init__(session, _id, name, server) + options = options or EmaneOptions() + super().__init__(session, _id, name, server, options) self.conf: str = "" - self.wireless_model: Optional["EmaneModel"] = None self.mobility: Optional[WayPointMobility] = None + model_class = self.session.emane.get_model(options.emane_model) + self.wireless_model: Optional["EmaneModel"] = model_class(self.session, self.id) + if self.session.state == EventTypes.RUNTIME_STATE: + self.session.emane.add_node(self) + + @classmethod + def create_options(cls) -> EmaneOptions: + return EmaneOptions() def linkconfig( self, iface: CoreInterface, options: LinkOptions, iface2: CoreInterface = None diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 5064579f..4444e248 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -14,7 +14,7 @@ import tempfile import threading import time from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union from core import constants, utils from core.configservice.manager import ConfigServiceManager @@ -29,7 +29,6 @@ from core.emulator.data import ( LinkData, LinkOptions, NodeData, - NodeOptions, ) from core.emulator.distributed import DistributedController from core.emulator.enumerations import ( @@ -44,7 +43,7 @@ from core.errors import CoreError from core.location.event import EventLoop from core.location.geo import GeoLocation from core.location.mobility import BasicRangeModel, MobilityManager -from core.nodes.base import CoreNetworkBase, CoreNode, CoreNodeBase, NodeBase +from core.nodes.base import CoreNode, CoreNodeBase, NodeBase, NodeOptions, Position from core.nodes.docker import DockerNode from core.nodes.interface import DEFAULT_MTU, CoreInterface from core.nodes.lxd import LxcNode @@ -476,14 +475,23 @@ class Session: return _id def add_node( - self, _class: Type[NT], _id: int = None, options: NodeOptions = None + self, + _class: Type[NT], + _id: int = None, + name: str = None, + server: str = None, + position: Position = None, + options: NodeOptions = None, ) -> NT: """ Add a node to the session, based on the provided node data. :param _class: node class to create :param _id: id for node, defaults to None for generated id - :param options: data to create node with + :param name: name to assign to node + :param server: distributed server for node, if desired + :param position: geo or x/y/z position to set + :param options: options to create node with :return: created node :raises core.CoreError: when an invalid node type is given """ @@ -492,89 +500,31 @@ class Session: enable_rj45 = self.options.get_int("enablerj45") == 1 if _class == Rj45Node and not enable_rj45: start = False - - # determine node id - if not _id: - _id = self.next_node_id() - - # generate name if not provided - if not options: - options = NodeOptions() - options.set_position(0, 0) - name = options.name - if not name: - name = f"{_class.__name__}{_id}" - + # generate options if not provided + options = options if options else _class.create_options() # verify distributed server - server = self.distributed.servers.get(options.server) - if options.server is not None and server is None: - raise CoreError(f"invalid distributed server: {options.server}") - + dist_server = None + if server is not None: + dist_server = self.distributed.servers.get(server) + if not dist_server: + raise CoreError(f"invalid distributed server: {server}") # create node - logger.info( - "creating node(%s) id(%s) name(%s) start(%s)", - _class.__name__, - _id, - name, - start, - ) - kwargs = dict(_id=_id, name=name, server=server) - if _class in CONTAINER_NODES: - kwargs["image"] = options.image - kwargs["binds"] = options.binds - kwargs["volumes"] = options.volumes - node = self.create_node(_class, start, **kwargs) - - # set node attributes - node.icon = options.icon - node.canvas = options.canvas - - # set node position and broadcast it - has_geo = all(i is not None for i in [options.lon, options.lat, options.alt]) - if has_geo: - self.set_node_geo(node, options.lon, options.lat, options.alt) + node = self.create_node(_class, start, _id, name, dist_server, options) + # set node position + position = position or Position() + if position.has_geo(): + self.set_node_geo(node, position.lon, position.lat, position.alt) else: - self.set_node_pos(node, options.x, options.y) - - # add services to needed nodes - if isinstance(node, (CoreNode, PhysicalNode)): - node.model = options.model - if options.legacy or options.services: - logger.debug("set node type: %s", node.model) - self.services.add_services(node, node.model, options.services) - - # add config services - config_services = options.config_services - if not options.legacy and not config_services and not node.services: - config_services = self.services.default_services.get(node.model, []) - logger.info("setting node config services: %s", config_services) - for name in config_services: - service_class = self.service_manager.get_service(name) - node.add_config_service(service_class) - - # set network mtu, if configured - mtu = self.options.get_int("mtu") - if isinstance(node, CoreNetworkBase) and mtu > 0: - node.mtu = mtu - - # ensure default emane configuration - if isinstance(node, EmaneNet) and options.emane: - model_class = self.emane.get_model(options.emane) - node.wireless_model = model_class(self, node.id) - if self.state == EventTypes.RUNTIME_STATE: - self.emane.add_node(node) - - # set default wlan config if needed + self.set_node_pos(node, position.x, position.y) + # setup default wlan if isinstance(node, WlanNode): - self.mobility.set_model_config(_id, BasicRangeModel.name) - - # boot nodes after runtime CoreNodes and PhysicalNodes - is_boot_node = isinstance(node, (CoreNode, PhysicalNode)) - if self.state == EventTypes.RUNTIME_STATE and is_boot_node: + self.mobility.set_model_config(self.id, BasicRangeModel.name) + # boot core nodes after runtime + is_runtime = self.state == EventTypes.RUNTIME_STATE + if is_runtime and isinstance(node, CoreNode): self.write_nodes() self.add_remove_control_iface(node, remove=False) self.boot_node(node) - self.sdt.add_node(node) return node @@ -980,24 +930,39 @@ class Session: logger.exception("failed to set permission on %s", self.directory) def create_node( - self, _class: Type[NT], start: bool, *args: Any, **kwargs: Any + self, + _class: Type[NT], + start: bool, + _id: int = None, + name: str = None, + server: str = None, + options: NodeOptions = None, ) -> NT: """ Create an emulation node. :param _class: node class to create :param start: True to start node, False otherwise - :param args: list of arguments for the class to create - :param kwargs: dictionary of arguments for the class to create + :param _id: id for node, defaults to None for generated id + :param name: name to assign to node + :param server: distributed server for node, if desired + :param options: options to create node with :return: the created node instance :raises core.CoreError: when id of the node to create already exists """ with self.nodes_lock: - node = _class(self, *args, **kwargs) + node = _class(self, _id=_id, name=name, server=server, options=options) if node.id in self.nodes: node.shutdown() raise CoreError(f"duplicate node id {node.id} for {node.name}") self.nodes[node.id] = node + logger.info( + "created node(%s) id(%s) name(%s) start(%s)", + _class.__name__, + node.id, + node.name, + start, + ) if start: node.startup() return node @@ -1219,7 +1184,7 @@ class Session: funcs = [] start = time.monotonic() for node in self.nodes.values(): - if isinstance(node, (CoreNode, PhysicalNode)): + if isinstance(node, CoreNode): self.add_remove_control_iface(node, remove=False) funcs.append((self.boot_node, (node,), {})) results, exceptions = utils.threadpool(funcs) @@ -1354,21 +1319,18 @@ class Session: updown_script, server_iface, ) - control_net = self.create_node( - CtrlNet, - start=False, - prefix=prefix, - _id=_id, - updown_script=updown_script, - serverintf=server_iface, - ) + options = CtrlNet.create_options() + options.prefix = prefix + options.updown_script = updown_script + options.serverintf = server_iface + control_net = self.create_node(CtrlNet, False, _id, options=options) control_net.brname = f"ctrl{net_index}.{self.short_session_id()}" control_net.startup() return control_net def add_remove_control_iface( self, - node: Union[CoreNode, PhysicalNode], + node: CoreNode, net_index: int = 0, remove: bool = False, conf_required: bool = True, diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 1bac0885..404e9ab2 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -5,6 +5,7 @@ import abc import logging import shutil import threading +from dataclasses import dataclass, field from pathlib import Path from threading import RLock from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type, Union @@ -33,6 +34,94 @@ if TYPE_CHECKING: PRIVATE_DIRS: List[Path] = [Path("/var/run"), Path("/var/log")] +@dataclass +class Position: + """ + Helper class for Cartesian coordinate position + """ + + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + lon: float = None + lat: float = None + alt: float = None + + def set(self, x: float = None, y: float = None, z: float = None) -> bool: + """ + Returns True if the position has actually changed. + + :param x: x position + :param y: y position + :param z: z position + :return: True if position changed, False otherwise + """ + if self.x == x and self.y == y and self.z == z: + return False + self.x = x + self.y = y + self.z = z + return True + + def get(self) -> Tuple[float, float, float]: + """ + Retrieve x,y,z position. + + :return: x,y,z position tuple + """ + return self.x, self.y, self.z + + def has_geo(self) -> bool: + return all(x is not None for x in [self.lon, self.lat, self.alt]) + + def set_geo(self, lon: float, lat: float, alt: float) -> None: + """ + Set geo position lon, lat, alt. + + :param lon: longitude value + :param lat: latitude value + :param alt: altitude value + :return: nothing + """ + self.lon = lon + self.lat = lat + self.alt = alt + + def get_geo(self) -> Tuple[float, float, float]: + """ + Retrieve current geo position lon, lat, alt. + + :return: lon, lat, alt position tuple + """ + return self.lon, self.lat, self.alt + + +@dataclass +class NodeOptions: + """ + Base options for configuring a node. + """ + + canvas: int = None + """id of canvas for display within gui""" + icon: str = None + """custom icon for display, None for default""" + + +@dataclass +class CoreNodeOptions(NodeOptions): + model: str = "PC" + """model is used for providing a default set of services""" + services: List[str] = field(default_factory=list) + """services to start within node""" + config_services: List[str] = field(default_factory=list) + """config services to start within node""" + directory: Path = None + """directory to define node, defaults to path under the session directory""" + legacy: bool = False + """legacy nodes default to standard services""" + + class NodeBase(abc.ABC): """ Base class for CORE nodes (nodes and networks) @@ -44,6 +133,7 @@ class NodeBase(abc.ABC): _id: int = None, name: str = None, server: "DistributedServer" = None, + options: NodeOptions = None, ) -> None: """ Creates a NodeBase instance. @@ -53,26 +143,29 @@ class NodeBase(abc.ABC): :param name: object name :param server: remote server node will run on, default is None for localhost + :param options: options to create node with """ - self.session: "Session" = session - if _id is None: - _id = session.next_node_id() - self.id: int = _id - self.name: str = name or f"o{self.id}" + self.id: int = _id if _id is not None else self.session.next_node_id() + self.name: str = name or f"{self.__class__.__name__}{self.id}" self.server: "DistributedServer" = server self.model: Optional[str] = None self.services: CoreServices = [] self.ifaces: Dict[int, CoreInterface] = {} self.iface_id: int = 0 - self.canvas: Optional[int] = None - self.icon: Optional[str] = None self.position: Position = Position() self.up: bool = False self.lock: RLock = RLock() self.net_client: LinuxNetClient = get_net_client( self.session.use_ovs(), self.host_cmd ) + options = options if options else NodeOptions() + self.canvas: Optional[int] = options.canvas + self.icon: Optional[str] = options.icon + + @classmethod + def create_options(cls) -> NodeOptions: + return NodeOptions() @abc.abstractmethod def startup(self) -> None: @@ -288,6 +381,7 @@ class CoreNodeBase(NodeBase): _id: int = None, name: str = None, server: "DistributedServer" = None, + options: NodeOptions = None, ) -> None: """ Create a CoreNodeBase instance. @@ -298,7 +392,7 @@ class CoreNodeBase(NodeBase): :param server: remote server node will run on, default is None for localhost """ - super().__init__(session, _id, name, server) + super().__init__(session, _id, name, server, options) self.config_services: Dict[str, "ConfigService"] = {} self.directory: Optional[Path] = None self.tmpnodedir: bool = False @@ -460,8 +554,8 @@ class CoreNode(CoreNodeBase): session: "Session", _id: int = None, name: str = None, - directory: Path = None, server: "DistributedServer" = None, + options: CoreNodeOptions = None, ) -> None: """ Create a CoreNode instance. @@ -469,18 +563,37 @@ class CoreNode(CoreNodeBase): :param session: core session instance :param _id: object id :param name: object name - :param directory: node directory :param server: remote server node will run on, default is None for localhost + :param options: options to create node with """ - super().__init__(session, _id, name, server) - self.directory: Optional[Path] = directory + options = options or CoreNodeOptions() + super().__init__(session, _id, name, server, options) + self.directory: Optional[Path] = options.directory self.ctrlchnlname: Path = self.session.directory / self.name self.pid: Optional[int] = None self._mounts: List[Tuple[Path, Path]] = [] self.node_net_client: LinuxNetClient = self.create_node_net_client( self.session.use_ovs() ) + options = options or CoreNodeOptions() + self.model: Optional[str] = options.model + # setup services + if options.legacy or options.services: + logger.debug("set node type: %s", self.model) + self.session.services.add_services(self, self.model, options.services) + # add config services + config_services = options.config_services + if not options.legacy and not config_services and not options.services: + config_services = self.session.services.default_services.get(self.model, []) + logger.info("setting node config services: %s", config_services) + for name in config_services: + service_class = self.session.service_manager.get_service(name) + self.add_config_service(service_class) + + @classmethod + def create_options(cls) -> CoreNodeOptions: + return CoreNodeOptions() def create_node_net_client(self, use_ovs: bool) -> LinuxNetClient: """ @@ -797,6 +910,7 @@ class CoreNetworkBase(NodeBase): _id: int, name: str, server: "DistributedServer" = None, + options: NodeOptions = None, ) -> None: """ Create a CoreNetworkBase instance. @@ -806,9 +920,11 @@ class CoreNetworkBase(NodeBase): :param name: object name :param server: remote server node will run on, default is None for localhost + :param options: options to create node with """ - super().__init__(session, _id, name, server) - self.mtu: int = DEFAULT_MTU + super().__init__(session, _id, name, server, options) + mtu = self.session.options.get_int("mtu") + self.mtu: int = mtu if mtu > 0 else DEFAULT_MTU self.brname: Optional[str] = None self.linked: Dict[CoreInterface, Dict[CoreInterface, bool]] = {} self.linked_lock: threading.Lock = threading.Lock() @@ -839,69 +955,3 @@ class CoreNetworkBase(NodeBase): iface.net_id = None with self.linked_lock: del self.linked[iface] - - -class Position: - """ - Helper class for Cartesian coordinate position - """ - - def __init__(self, x: float = None, y: float = None, z: float = None) -> None: - """ - Creates a Position instance. - - :param x: x position - :param y: y position - :param z: z position - """ - self.x: float = x - self.y: float = y - self.z: float = z - self.lon: Optional[float] = None - self.lat: Optional[float] = None - self.alt: Optional[float] = None - - def set(self, x: float = None, y: float = None, z: float = None) -> bool: - """ - Returns True if the position has actually changed. - - :param x: x position - :param y: y position - :param z: z position - :return: True if position changed, False otherwise - """ - if self.x == x and self.y == y and self.z == z: - return False - self.x = x - self.y = y - self.z = z - return True - - def get(self) -> Tuple[float, float, float]: - """ - Retrieve x,y,z position. - - :return: x,y,z position tuple - """ - return self.x, self.y, self.z - - def set_geo(self, lon: float, lat: float, alt: float) -> None: - """ - Set geo position lon, lat, alt. - - :param lon: longitude value - :param lat: latitude value - :param alt: altitude value - :return: nothing - """ - self.lon = lon - self.lat = lat - self.alt = alt - - def get_geo(self) -> Tuple[float, float, float]: - """ - Retrieve current geo position lon, lat, alt. - - :return: lon, lat, alt position tuple - """ - return self.lon, self.lat, self.alt diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 122aa9b0..a5560b17 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -1,7 +1,7 @@ import json import logging import shlex -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Dict, List, Tuple @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple from core.emulator.distributed import DistributedServer from core.errors import CoreCommandError, CoreError from core.executables import BASH -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, CoreNodeOptions logger = logging.getLogger(__name__) @@ -19,6 +19,21 @@ if TYPE_CHECKING: DOCKER: str = "docker" +@dataclass +class DockerOptions(CoreNodeOptions): + image: str = "ubuntu" + """image used when creating container""" + binds: List[Tuple[str, str]] = field(default_factory=list) + """bind mount source and destinations to setup within container""" + volumes: List[Tuple[str, str, bool, bool]] = field(default_factory=list) + """ + volume mount source, destination, unique, delete to setup within container + + unique is True for node unique volume naming + delete is True for deleting volume mount during shutdown + """ + + @dataclass class DockerVolume: src: str @@ -38,11 +53,8 @@ class DockerNode(CoreNode): session: "Session", _id: int = None, name: str = None, - directory: str = None, server: DistributedServer = None, - image: str = None, - binds: List[Tuple[str, str]] = None, - volumes: List[Tuple[str, str, bool, bool]] = None, + options: DockerOptions = None, ) -> None: """ Create a DockerNode instance. @@ -50,22 +62,23 @@ class DockerNode(CoreNode): :param session: core session instance :param _id: object id :param name: object name - :param directory: node directory :param server: remote server node will run on, default is None for localhost - :param image: image to start container with - :param binds: bind mounts to set for the created container - :param volumes: volume mount settings to set for the created container + :param options: options for creating node """ - super().__init__(session, _id, name, directory, server) - self.image: str = image if image is not None else "ubuntu" - self.binds: List[Tuple[str, str]] = binds or [] + options = options or DockerOptions() + super().__init__(session, _id, name, server, options) + self.image: str = options.image + self.binds: List[Tuple[str, str]] = options.binds self.volumes: Dict[str, DockerVolume] = {} - volumes = volumes or [] - for src, dst, unique, delete in volumes: + for src, dst, unique, delete in options.volumes: src_name = self._unique_name(src) if unique else src self.volumes[src] = DockerVolume(src_name, dst, unique, delete) + @classmethod + def create_options(cls) -> DockerOptions: + return DockerOptions() + def _create_cmd(self, args: str, shell: bool = False) -> str: """ Create command used to run commands within the context of a node. diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 1c12328f..b99b4576 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -1,15 +1,16 @@ import json import logging import time +from dataclasses import dataclass, field from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Callable, Dict, Optional +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple from core import utils from core.emulator.data import InterfaceData, LinkOptions from core.emulator.distributed import DistributedServer from core.errors import CoreCommandError -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, CoreNodeOptions from core.nodes.interface import CoreInterface logger = logging.getLogger(__name__) @@ -66,15 +67,29 @@ class LxdClient: self.run(args) +@dataclass +class LxcOptions(CoreNodeOptions): + image: str = "ubuntu" + """image used when creating container""" + binds: List[Tuple[str, str]] = field(default_factory=list) + """bind mount source and destinations to setup within container""" + volumes: List[Tuple[str, str, bool, bool]] = field(default_factory=list) + """ + volume mount source, destination, unique, delete to setup within container + + unique is True for node unique volume naming + delete is True for deleting volume mount during shutdown + """ + + class LxcNode(CoreNode): def __init__( self, session: "Session", _id: int = None, name: str = None, - directory: str = None, server: DistributedServer = None, - image: str = None, + options: LxcOptions = None, ) -> None: """ Create a LxcNode instance. @@ -82,15 +97,19 @@ class LxcNode(CoreNode): :param session: core session instance :param _id: object id :param name: object name - :param directory: node directory :param server: remote server node will run on, default is None for localhost - :param image: image to start container with + :param options: option to create node with """ - super().__init__(session, _id, name, directory, server) - self.image: str = image if image is not None else "ubuntu" + options = options or LxcOptions() + super().__init__(session, _id, name, server, options) + self.image: str = options.image self.client: Optional[LxdClient] = None + @classmethod + def create_options(cls) -> LxcOptions: + return LxcOptions() + def alive(self) -> bool: """ Check if the node is alive. diff --git a/daemon/core/nodes/network.py b/daemon/core/nodes/network.py index 284a34ab..2b52cbdf 100644 --- a/daemon/core/nodes/network.py +++ b/daemon/core/nodes/network.py @@ -4,6 +4,7 @@ Defines network nodes used within core. import logging import threading +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional, Type @@ -14,7 +15,7 @@ from core.emulator.data import InterfaceData, LinkData from core.emulator.enumerations import MessageFlags, NetworkPolicy, RegisterTlvs from core.errors import CoreCommandError, CoreError from core.executables import NFTABLES -from core.nodes.base import CoreNetworkBase +from core.nodes.base import CoreNetworkBase, NodeOptions from core.nodes.interface import CoreInterface, GreTap from core.nodes.netclient import get_net_client @@ -180,6 +181,12 @@ class NftablesQueue: nft_queue: NftablesQueue = NftablesQueue() +@dataclass +class NetworkOptions(NodeOptions): + policy: NetworkPolicy = None + """allows overriding the network policy, otherwise uses class defined default""" + + class CoreNetwork(CoreNetworkBase): """ Provides linux bridge network functionality for core nodes. @@ -193,7 +200,7 @@ class CoreNetwork(CoreNetworkBase): _id: int = None, name: str = None, server: "DistributedServer" = None, - policy: NetworkPolicy = None, + options: NetworkOptions = None, ) -> None: """ Creates a CoreNetwork instance. @@ -203,18 +210,19 @@ class CoreNetwork(CoreNetworkBase): :param name: object name :param server: remote server node will run on, default is None for localhost - :param policy: network policy + :param options: options to create node with """ - super().__init__(session, _id, name, server) - if name is None: - name = str(self.id) - if policy is not None: - self.policy: NetworkPolicy = policy - self.name: Optional[str] = name + options = options or NetworkOptions() + super().__init__(session, _id, name, server, options) + self.policy: NetworkPolicy = options.policy if options.policy else self.policy sessionid = self.session.short_session_id() self.brname: str = f"b.{self.id}.{sessionid}" self.has_nftables_chain: bool = False + @classmethod + def create_options(cls) -> NetworkOptions: + return NetworkOptions() + def host_cmd( self, args: str, @@ -482,6 +490,20 @@ class GreTapBridge(CoreNetwork): self.add_ips(ips) +@dataclass +class CtrlNetOptions(NetworkOptions): + prefix: str = None + """ip4 network prefix to use for generating an address""" + updown_script: str = None + """script to execute during startup and shutdown""" + serverintf: str = None + """used to associate an interface with the control network bridge""" + assign_address: bool = True + """used to determine if a specific address should be assign using hostid""" + hostid: int = None + """used with assign address to """ + + class CtrlNet(CoreNetwork): """ Control network functionality. @@ -500,36 +522,32 @@ class CtrlNet(CoreNetwork): def __init__( self, session: "Session", - prefix: str, _id: int = None, name: str = None, - hostid: int = None, server: "DistributedServer" = None, - assign_address: bool = True, - updown_script: str = None, - serverintf: str = None, + options: CtrlNetOptions = None, ) -> None: """ Creates a CtrlNet instance. :param session: core session instance :param _id: node id - :param name: node namee - :param prefix: control network ipv4 prefix - :param hostid: host id + :param name: node name :param server: remote server node will run on, default is None for localhost - :param assign_address: assigned address - :param updown_script: updown script - :param serverintf: server interface - :return: + :param options: node options for creation """ - self.prefix: netaddr.IPNetwork = netaddr.IPNetwork(prefix).cidr - self.hostid: Optional[int] = hostid - self.assign_address: bool = assign_address - self.updown_script: Optional[str] = updown_script - self.serverintf: Optional[str] = serverintf - super().__init__(session, _id, name, server) + options = options or CtrlNetOptions() + super().__init__(session, _id, name, server, options) + self.prefix: netaddr.IPNetwork = netaddr.IPNetwork(options.prefix).cidr + self.hostid: Optional[int] = options.hostid + self.assign_address: bool = options.assign_address + self.updown_script: Optional[str] = options.updown_script + self.serverintf: Optional[str] = options.serverintf + + @classmethod + def create_options(cls) -> CtrlNetOptions: + return CtrlNetOptions() def add_addresses(self, index: int) -> None: """ @@ -669,7 +687,7 @@ class WlanNode(CoreNetwork): _id: int = None, name: str = None, server: "DistributedServer" = None, - policy: NetworkPolicy = None, + options: NetworkOptions = None, ) -> None: """ Create a WlanNode instance. @@ -679,9 +697,9 @@ class WlanNode(CoreNetwork): :param name: node name :param server: remote server node will run on, default is None for localhost - :param policy: wlan policy + :param options: options to create node with """ - super().__init__(session, _id, name, server, policy) + super().__init__(session, _id, name, server, options) # wireless and mobility models (BasicRangeModel, Ns2WaypointMobility) self.wireless_model: Optional[WirelessModel] = None self.mobility: Optional[WayPointMobility] = None diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index ccbaf956..02c200db 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -13,7 +13,7 @@ from core.emulator.distributed import DistributedServer from core.emulator.enumerations import TransportType from core.errors import CoreCommandError, CoreError from core.executables import BASH, TEST, UMOUNT -from core.nodes.base import CoreNode, CoreNodeBase +from core.nodes.base import CoreNode, CoreNodeBase, CoreNodeOptions, NodeOptions from core.nodes.interface import CoreInterface logger = logging.getLogger(__name__) @@ -34,6 +34,7 @@ class Rj45Node(CoreNodeBase): _id: int = None, name: str = None, server: DistributedServer = None, + options: NodeOptions = None, ) -> None: """ Create an RJ45Node instance. @@ -43,8 +44,9 @@ class Rj45Node(CoreNodeBase): :param name: node name :param server: remote server node will run on, default is None for localhost + :param options: option to create node with """ - super().__init__(session, _id, name, server) + super().__init__(session, _id, name, server, options) self.iface: CoreInterface = CoreInterface( self.iface_id, name, name, session.use_ovs(), node=self, server=server ) @@ -224,12 +226,12 @@ class PhysicalNode(CoreNode): session: "Session", _id: int = None, name: str = None, - directory: Path = None, server: DistributedServer = None, + options: CoreNodeOptions = None, ) -> None: if not self.server: raise CoreError("physical nodes must be assigned to a remote server") - super().__init__(session, _id, name, directory, server) + super().__init__(session, _id, name, server, options) def startup(self) -> None: with self.lock: diff --git a/daemon/core/nodes/wireless.py b/daemon/core/nodes/wireless.py index 3416b65f..ef37db35 100644 --- a/daemon/core/nodes/wireless.py +++ b/daemon/core/nodes/wireless.py @@ -14,7 +14,7 @@ from core.emulator.data import LinkData, LinkOptions from core.emulator.enumerations import LinkTypes, MessageFlags from core.errors import CoreError from core.executables import NFTABLES -from core.nodes.base import CoreNetworkBase +from core.nodes.base import CoreNetworkBase, NodeOptions from core.nodes.interface import CoreInterface if TYPE_CHECKING: @@ -108,8 +108,9 @@ class WirelessNode(CoreNetworkBase): _id: int, name: str, server: "DistributedServer" = None, + options: NodeOptions = None, ): - super().__init__(session, _id, name, server) + super().__init__(session, _id, name, server, options) self.bridges: Dict[int, Tuple[CoreInterface, str]] = {} self.links: Dict[Tuple[int, int], WirelessLink] = {} self.position_enabled: bool = CONFIG_ENABLED diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index e947dece..a483a8ee 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -8,12 +8,12 @@ import core.nodes.base import core.nodes.physical from core import utils from core.config import Configuration -from core.emane.nodes import EmaneNet -from core.emulator.data import InterfaceData, LinkOptions, NodeOptions +from core.emane.nodes import EmaneNet, EmaneOptions +from core.emulator.data import InterfaceData, LinkOptions from core.emulator.enumerations import EventTypes, NodeTypes from core.errors import CoreXmlError -from core.nodes.base import CoreNodeBase, NodeBase -from core.nodes.docker import DockerNode +from core.nodes.base import CoreNodeBase, CoreNodeOptions, NodeBase, Position +from core.nodes.docker import DockerNode, DockerOptions from core.nodes.interface import CoreInterface from core.nodes.lxd import LxcNode from core.nodes.network import CtrlNet, GreTapBridge, PtpNet, WlanNode @@ -802,68 +802,76 @@ class CoreXmlReader: clazz = device_element.get("class") image = device_element.get("image") server = device_element.get("server") - options = NodeOptions( - name=name, model=model, image=image, icon=icon, server=server - ) + canvas = get_int(device_element, "canvas") node_type = NodeTypes.DEFAULT if clazz == "docker": node_type = NodeTypes.DOCKER elif clazz == "lxc": node_type = NodeTypes.LXC _class = self.session.get_node_class(node_type) - - service_elements = device_element.find("services") - if service_elements is not None: - options.services = [x.get("name") for x in service_elements.iterchildren()] - - config_service_elements = device_element.find("configservices") - if config_service_elements is not None: - options.config_services = [ - x.get("name") for x in config_service_elements.iterchildren() - ] - + options = _class.create_options() + options.icon = icon + options.canvas = canvas + # check for special options + if isinstance(options, CoreNodeOptions): + options.model = model + service_elements = device_element.find("services") + if service_elements is not None: + options.services.extend( + x.get("name") for x in service_elements.iterchildren() + ) + config_service_elements = device_element.find("configservices") + if config_service_elements is not None: + options.config_services.extend( + x.get("name") for x in config_service_elements.iterchildren() + ) + if isinstance(options, DockerOptions): + options.image = image + # get position information position_element = device_element.find("position") + position = None if position_element is not None: + position = Position() x = get_float(position_element, "x") y = get_float(position_element, "y") if all([x, y]): - options.set_position(x, y) - + position.set(x, y) lat = get_float(position_element, "lat") lon = get_float(position_element, "lon") alt = get_float(position_element, "alt") if all([lat, lon, alt]): - options.set_location(lat, lon, alt) - + position.set_geo(lon, lat, alt) logger.info("reading node id(%s) model(%s) name(%s)", node_id, model, name) - self.session.add_node(_class, node_id, options) + self.session.add_node(_class, node_id, name, server, position, options) def read_network(self, network_element: etree.Element) -> None: node_id = get_int(network_element, "id") name = network_element.get("name") + server = network_element.get("server") node_type = NodeTypes[network_element.get("type")] _class = self.session.get_node_class(node_type) - icon = network_element.get("icon") - server = network_element.get("server") - options = NodeOptions(name=name, icon=icon, server=server) - if node_type == NodeTypes.EMANE: - model = network_element.get("model") - options.emane = model + options = _class.create_options() + options.canvas = get_int(network_element, "canvas") + options.icon = network_element.get("icon") + if isinstance(options, EmaneOptions): + options.emane_model = network_element.get("model") position_element = network_element.find("position") + position = None if position_element is not None: + position = Position() x = get_float(position_element, "x") y = get_float(position_element, "y") if all([x, y]): - options.set_position(x, y) + position.set(x, y) lat = get_float(position_element, "lat") lon = get_float(position_element, "lon") alt = get_float(position_element, "alt") if all([lat, lon, alt]): - options.set_location(lat, lon, alt) + position.set_geo(lon, lat, alt) logger.info( "reading node id(%s) node_type(%s) name(%s)", node_id, node_type, name ) - node = self.session.add_node(_class, node_id, options) + node = self.session.add_node(_class, node_id, name, server, position, options) if isinstance(node, WirelessNode): wireless_element = network_element.find("wireless") if wireless_element: diff --git a/daemon/examples/configservices/testing.py b/daemon/examples/configservices/testing.py index 9706f2c9..937c3aa8 100644 --- a/daemon/examples/configservices/testing.py +++ b/daemon/examples/configservices/testing.py @@ -1,7 +1,7 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.base import CoreNode from core.nodes.network import SwitchNode @@ -11,13 +11,13 @@ if __name__ == "__main__": # setup basic network prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(model=None) coreemu = CoreEmu() session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) switch = session.add_node(SwitchNode) # node one + options = CoreNode.create_options() options.config_services = ["DefaultRoute", "IPForward"] node1 = session.add_node(CoreNode, options=options) interface = prefixes.create_iface(node1) diff --git a/daemon/examples/docker/docker2core.py b/daemon/examples/docker/docker2core.py index ae7dae79..cd6a5a20 100644 --- a/daemon/examples/docker/docker2core.py +++ b/daemon/examples/docker/docker2core.py @@ -1,7 +1,7 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.base import CoreNode from core.nodes.docker import DockerNode @@ -14,9 +14,10 @@ if __name__ == "__main__": try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(model=None, image="ubuntu") # create node one + options = DockerNode.create_options() + options.image = "ubuntu" node1 = session.add_node(DockerNode, options=options) interface1_data = prefixes.create_iface(node1) diff --git a/daemon/examples/docker/docker2docker.py b/daemon/examples/docker/docker2docker.py index 308fd00f..5fa65778 100644 --- a/daemon/examples/docker/docker2docker.py +++ b/daemon/examples/docker/docker2docker.py @@ -1,7 +1,7 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.docker import DockerNode @@ -15,9 +15,10 @@ if __name__ == "__main__": # create nodes and interfaces try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(model=None, image="ubuntu") # create node one + options = DockerNode.create_options() + options.image = "ubuntu" node1 = session.add_node(DockerNode, options=options) interface1_data = prefixes.create_iface(node1) diff --git a/daemon/examples/docker/switch.py b/daemon/examples/docker/switch.py index fa9e4e40..a3f8e31e 100644 --- a/daemon/examples/docker/switch.py +++ b/daemon/examples/docker/switch.py @@ -1,9 +1,8 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode from core.nodes.docker import DockerNode from core.nodes.network import SwitchNode @@ -16,12 +15,15 @@ if __name__ == "__main__": try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(model=None, image="ubuntu") # create switch switch = session.add_node(SwitchNode) # node one + options = DockerNode.create_options() + options.image = "core" + options.binds.append(("/tmp/testbind", "/tmp/bind")) + options.volumes.append(("var.log", "/tmp/var.log", True, True)) node1 = session.add_node(DockerNode, options=options) interface1_data = prefixes.create_iface(node1) @@ -30,16 +32,18 @@ if __name__ == "__main__": interface2_data = prefixes.create_iface(node2) # node three - node_three = session.add_node(CoreNode) - interface_three = prefixes.create_iface(node_three) + # node_three = session.add_node(CoreNode) + # interface_three = prefixes.create_iface(node_three) # add links session.add_link(node1.id, switch.id, interface1_data) session.add_link(node2.id, switch.id, interface2_data) - session.add_link(node_three.id, switch.id, interface_three) + # session.add_link(node_three.id, switch.id, interface_three) # instantiate session.instantiate() + + print(f"{node2.name}: {node2.volumes.values()}") finally: input("continue to shutdown") coreemu.shutdown() diff --git a/daemon/examples/lxd/lxd2core.py b/daemon/examples/lxd/lxd2core.py index b41520d8..ec671b29 100644 --- a/daemon/examples/lxd/lxd2core.py +++ b/daemon/examples/lxd/lxd2core.py @@ -1,7 +1,7 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.base import CoreNode from core.nodes.lxd import LxcNode @@ -14,9 +14,10 @@ if __name__ == "__main__": try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(image="ubuntu") # create node one + options = LxcNode.create_options() + options.image = "ubuntu" node1 = session.add_node(LxcNode, options=options) interface1_data = prefixes.create_iface(node1) diff --git a/daemon/examples/lxd/lxd2lxd.py b/daemon/examples/lxd/lxd2lxd.py index 3a55e2e1..7e9e6a55 100644 --- a/daemon/examples/lxd/lxd2lxd.py +++ b/daemon/examples/lxd/lxd2lxd.py @@ -1,7 +1,7 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.lxd import LxcNode @@ -15,9 +15,10 @@ if __name__ == "__main__": # create nodes and interfaces try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(image="ubuntu:18.04") # create node one + options = LxcNode.create_options() + options.image = "ubuntu:18.04" node1 = session.add_node(LxcNode, options=options) interface1_data = prefixes.create_iface(node1) diff --git a/daemon/examples/lxd/switch.py b/daemon/examples/lxd/switch.py index 12767e71..c093fd77 100644 --- a/daemon/examples/lxd/switch.py +++ b/daemon/examples/lxd/switch.py @@ -1,7 +1,7 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.base import CoreNode from core.nodes.lxd import LxcNode @@ -16,12 +16,13 @@ if __name__ == "__main__": try: prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(image="ubuntu") # create switch switch = session.add_node(SwitchNode) # node one + options = LxcNode.create_options() + options.image = "ubuntu" node1 = session.add_node(LxcNode, options=options) interface1_data = prefixes.create_iface(node1) diff --git a/daemon/examples/python/distributed_emane.py b/daemon/examples/python/distributed_emane.py index cdc9cbc3..d19a2d87 100644 --- a/daemon/examples/python/distributed_emane.py +++ b/daemon/examples/python/distributed_emane.py @@ -9,7 +9,7 @@ import logging from core.emane.models.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.base import CoreNode @@ -50,11 +50,13 @@ def main(args): session.set_state(EventTypes.CONFIGURATION_STATE) # create local node, switch, and remote nodes - options = NodeOptions(model="mdr") - options.set_position(0, 0) + options = CoreNode.create_options() + options.model = "mdr" node1 = session.add_node(CoreNode, options=options) - emane_net = session.add_node(EmaneNet) - session.emane.set_model(emane_net, EmaneIeee80211abgModel) + options = EmaneNet.create_options() + options.emane_model = EmaneIeee80211abgModel.name + emane_net = session.add_node(EmaneNet, options=options) + options = CoreNode.create_options() options.server = server_name node2 = session.add_node(CoreNode, options=options) diff --git a/daemon/examples/python/distributed_lxd.py b/daemon/examples/python/distributed_lxd.py index 26f7caa6..70af8a29 100644 --- a/daemon/examples/python/distributed_lxd.py +++ b/daemon/examples/python/distributed_lxd.py @@ -7,7 +7,7 @@ import argparse import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.lxd import LxcNode @@ -42,7 +42,8 @@ def main(args): session.set_state(EventTypes.CONFIGURATION_STATE) # create local node, switch, and remote nodes - options = NodeOptions(image="ubuntu:18.04") + options = LxcNode.create_options() + options.image = "ubuntu:18.04" node1 = session.add_node(LxcNode, options=options) options.server = server_name node2 = session.add_node(LxcNode, options=options) diff --git a/daemon/examples/python/distributed_ptp.py b/daemon/examples/python/distributed_ptp.py index fe714e1d..30dbb6bb 100644 --- a/daemon/examples/python/distributed_ptp.py +++ b/daemon/examples/python/distributed_ptp.py @@ -7,7 +7,7 @@ import argparse import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.base import CoreNode @@ -42,10 +42,8 @@ def main(args): session.set_state(EventTypes.CONFIGURATION_STATE) # create local node, switch, and remote nodes - options = NodeOptions() - node1 = session.add_node(CoreNode, options=options) - options.server = server_name - node2 = session.add_node(CoreNode, options=options) + node1 = session.add_node(CoreNode) + node2 = session.add_node(CoreNode, server=server_name) # create node interfaces and link interface1_data = prefixes.create_iface(node1) diff --git a/daemon/examples/python/distributed_switch.py b/daemon/examples/python/distributed_switch.py index 35de1cad..59a0447f 100644 --- a/daemon/examples/python/distributed_switch.py +++ b/daemon/examples/python/distributed_switch.py @@ -7,7 +7,7 @@ import argparse import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.nodes.base import CoreNode from core.nodes.network import SwitchNode @@ -47,7 +47,7 @@ def main(args): # create local node, switch, and remote nodes node1 = session.add_node(CoreNode) switch = session.add_node(SwitchNode) - options = NodeOptions() + options = CoreNode.create_options() options.server = server_name node2 = session.add_node(CoreNode, options=options) diff --git a/daemon/examples/python/emane80211.py b/daemon/examples/python/emane80211.py index 0bcc0157..005dc436 100644 --- a/daemon/examples/python/emane80211.py +++ b/daemon/examples/python/emane80211.py @@ -2,9 +2,9 @@ from core.emane.models.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position # ip nerator for example ip_prefixes = IpPrefixes(ip4_prefix="10.0.0.0/24") @@ -21,14 +21,20 @@ session.location.refscale = 150.0 session.set_state(EventTypes.CONFIGURATION_STATE) # create emane -options = NodeOptions(x=200, y=200, emane=EmaneIeee80211abgModel.name) -emane = session.add_node(EmaneNet, options=options) +options = EmaneNet.create_options() +options.emane_model = EmaneIeee80211abgModel.name +position = Position(x=200, y=200) +emane = session.add_node(EmaneNet, position=position, options=options) # create nodes -options = NodeOptions(model="mdr", x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(model="mdr", x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position, options=options) # configure general emane settings config = session.emane.get_configs() diff --git a/daemon/examples/python/peertopeer.py b/daemon/examples/python/peertopeer.py index 56fbe258..7883cac2 100644 --- a/daemon/examples/python/peertopeer.py +++ b/daemon/examples/python/peertopeer.py @@ -1,8 +1,8 @@ # required imports from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position # ip nerator for example ip_prefixes = IpPrefixes(ip4_prefix="10.0.0.0/24") @@ -15,10 +15,10 @@ session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) # create nodes -options = NodeOptions(x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position) +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position) # link nodes together iface1 = ip_prefixes.create_iface(n1) diff --git a/daemon/examples/python/switch.py b/daemon/examples/python/switch.py index b7894bc3..a609aa03 100644 --- a/daemon/examples/python/switch.py +++ b/daemon/examples/python/switch.py @@ -1,8 +1,8 @@ # required imports from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position from core.nodes.network import SwitchNode # ip nerator for example @@ -16,14 +16,14 @@ session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) # create switch -options = NodeOptions(x=200, y=200) -switch = session.add_node(SwitchNode, options=options) +position = Position(x=200, y=200) +switch = session.add_node(SwitchNode, position=position) # create nodes -options = NodeOptions(x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position) +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position) # link nodes to switch iface1 = ip_prefixes.create_iface(n1) diff --git a/daemon/examples/python/wireless.py b/daemon/examples/python/wireless.py index 559cf2f8..574ca976 100644 --- a/daemon/examples/python/wireless.py +++ b/daemon/examples/python/wireless.py @@ -2,9 +2,9 @@ import logging from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position from core.nodes.network import WlanNode # enable info logging @@ -21,14 +21,18 @@ session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) # create wireless -options = NodeOptions(x=200, y=200) -wireless = session.add_node(WlanNode, options=options) +position = Position(x=200, y=200) +wireless = session.add_node(WlanNode, position=position) # create nodes -options = NodeOptions(model="mdr", x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(model="mdr", x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position, options=options) # link nodes to wireless iface1 = ip_prefixes.create_iface(n1) diff --git a/daemon/examples/python/wlan.py b/daemon/examples/python/wlan.py index f0dbc97a..512aea3e 100644 --- a/daemon/examples/python/wlan.py +++ b/daemon/examples/python/wlan.py @@ -1,9 +1,9 @@ # required imports from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.location.mobility import BasicRangeModel -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position from core.nodes.network import WlanNode # ip nerator for example @@ -17,14 +17,18 @@ session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) # create wlan -options = NodeOptions(x=200, y=200) -wlan = session.add_node(WlanNode, options=options) +position = Position(x=200, y=200) +wlan = session.add_node(WlanNode, position=position) # create nodes -options = NodeOptions(model="mdr", x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(model="mdr", x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position, options=options) # configuring wlan session.mobility.set_model_config( diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 5cb14bdc..2ddb1a5d 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -16,10 +16,10 @@ from core.emane.models.ieee80211abg import EmaneIeee80211abgModel from core.emane.models.rfpipe import EmaneRfPipeModel from core.emane.models.tdma import EmaneTdmaModel from core.emane.nodes import EmaneNet -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.session import Session from core.errors import CoreCommandError, CoreError -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position _EMANE_MODELS = [ EmaneIeee80211abgModel, @@ -53,19 +53,22 @@ class TestEmane: """ # create emane node for networking the core nodes session.set_location(47.57917, -122.13232, 2.00000, 1.0) - options = NodeOptions() - options.set_position(80, 50) - options.emane = EmaneIeee80211abgModel.name - emane_net1 = session.add_node(EmaneNet, options=options) - options.emane = EmaneRfPipeModel.name - emane_net2 = session.add_node(EmaneNet, options=options) + options = EmaneNet.create_options() + options.emane_model = EmaneIeee80211abgModel.name + position = Position(x=80, y=50) + emane_net1 = session.add_node(EmaneNet, position=position, options=options) + options = EmaneNet.create_options() + options.emane_model = EmaneRfPipeModel.name + position = Position(x=80, y=50) + emane_net2 = session.add_node(EmaneNet, position=position, options=options) # create nodes - options = NodeOptions(model="mdr") - options.set_position(150, 150) - node1 = session.add_node(CoreNode, options=options) - options.set_position(300, 150) - node2 = session.add_node(CoreNode, options=options) + options = CoreNode.create_options() + options.model = "mdr" + position = Position(x=150, y=150) + node1 = session.add_node(CoreNode, position=position, options=options) + position = Position(x=300, y=150) + node2 = session.add_node(CoreNode, position=position, options=options) # create interfaces ip_prefix1 = IpPrefixes("10.0.0.0/24") @@ -100,9 +103,10 @@ class TestEmane: # create emane node for networking the core nodes session.set_location(47.57917, -122.13232, 2.00000, 1.0) - options = NodeOptions(emane=model.name) - options.set_position(80, 50) - emane_network = session.add_node(EmaneNet, options=options) + options = EmaneNet.create_options() + options.emane_model = model.name + position = Position(x=80, y=50) + emane_network = session.add_node(EmaneNet, position=position, options=options) # configure tdma if model == EmaneTdmaModel: @@ -111,11 +115,12 @@ class TestEmane: ) # create nodes - options = NodeOptions(model="mdr") - options.set_position(150, 150) - node1 = session.add_node(CoreNode, options=options) - options.set_position(300, 150) - node2 = session.add_node(CoreNode, options=options) + options = CoreNode.create_options() + options.model = "mdr" + position = Position(x=150, y=150) + node1 = session.add_node(CoreNode, position=position, options=options) + position = Position(x=300, y=150) + node2 = session.add_node(CoreNode, position=position, options=options) for i, node in enumerate([node1, node2]): node.setposition(x=150 * (i + 1), y=150) @@ -141,9 +146,10 @@ class TestEmane: """ # create emane node for networking the core nodes session.set_location(47.57917, -122.13232, 2.00000, 1.0) - options = NodeOptions(emane=EmaneIeee80211abgModel.name) - options.set_position(80, 50) - emane_network = session.add_node(EmaneNet, options=options) + options = EmaneNet.create_options() + options.emane_model = EmaneIeee80211abgModel.name + position = Position(x=80, y=50) + emane_network = session.add_node(EmaneNet, position=position, options=options) config_key = "txpower" config_value = "10" session.emane.set_config( @@ -151,11 +157,12 @@ class TestEmane: ) # create nodes - options = NodeOptions(model="mdr") - options.set_position(150, 150) - node1 = session.add_node(CoreNode, options=options) - options.set_position(300, 150) - node2 = session.add_node(CoreNode, options=options) + options = CoreNode.create_options() + options.model = "mdr" + position = Position(x=150, y=150) + node1 = session.add_node(CoreNode, position=position, options=options) + position = Position(x=300, y=150) + node2 = session.add_node(CoreNode, position=position, options=options) for i, node in enumerate([node1, node2]): node.setposition(x=150 * (i + 1), y=150) @@ -205,14 +212,17 @@ class TestEmane: self, session: Session, tmpdir: TemporaryFile, ip_prefixes: IpPrefixes ): # create nodes - options = NodeOptions(model="mdr", x=50, y=50) - node1 = session.add_node(CoreNode, options=options) + options = CoreNode.create_options() + options.model = "mdr" + position = Position(x=50, y=50) + node1 = session.add_node(CoreNode, position=position, options=options) iface1_data = ip_prefixes.create_iface(node1) - node2 = session.add_node(CoreNode, options=options) + node2 = session.add_node(CoreNode, position=position, options=options) iface2_data = ip_prefixes.create_iface(node2) # create emane node - options = NodeOptions(model=None, emane=EmaneRfPipeModel.name) + options = EmaneNet.create_options() + options.emane_model = EmaneRfPipeModel.name emane_node = session.add_node(EmaneNet, options=options) # create links @@ -255,11 +265,7 @@ class TestEmane: assert session.get_node(node1.id, CoreNode) assert session.get_node(node2.id, CoreNode) assert session.get_node(emane_node.id, EmaneNet) - links = [] - for node_id in session.nodes: - node = session.nodes[node_id] - links += node.links() - assert len(links) == 2 + assert len(session.link_manager.links()) == 2 config = session.emane.get_config(node1.id, EmaneRfPipeModel.name) assert config["datarate"] == datarate @@ -267,14 +273,17 @@ class TestEmane: self, session: Session, tmpdir: TemporaryFile, ip_prefixes: IpPrefixes ): # create nodes - options = NodeOptions(model="mdr", x=50, y=50) - node1 = session.add_node(CoreNode, options=options) + options = CoreNode.create_options() + options.model = "mdr" + position = Position(x=50, y=50) + node1 = session.add_node(CoreNode, position=position, options=options) iface1_data = ip_prefixes.create_iface(node1) - node2 = session.add_node(CoreNode, options=options) + node2 = session.add_node(CoreNode, position=position, options=options) iface2_data = ip_prefixes.create_iface(node2) # create emane node - options = NodeOptions(model=None, emane=EmaneRfPipeModel.name) + options = EmaneNet.create_options() + options.emane_model = EmaneRfPipeModel.name emane_node = session.add_node(EmaneNet, options=options) # create links @@ -318,10 +327,6 @@ class TestEmane: assert session.get_node(node1.id, CoreNode) assert session.get_node(node2.id, CoreNode) assert session.get_node(emane_node.id, EmaneNet) - links = [] - for node_id in session.nodes: - node = session.nodes[node_id] - links += node.links() - assert len(links) == 2 + assert len(session.link_manager.links()) == 2 config = session.emane.get_config(config_id, EmaneRfPipeModel.name) assert config["datarate"] == datarate diff --git a/daemon/tests/test_core.py b/daemon/tests/test_core.py index 71a3d972..919e4478 100644 --- a/daemon/tests/test_core.py +++ b/daemon/tests/test_core.py @@ -8,7 +8,7 @@ from typing import List, Type import pytest -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.session import Session from core.errors import CoreCommandError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility @@ -75,8 +75,8 @@ class TestCore: session.mobility.set_model(wlan_node, BasicRangeModel) # create nodes - options = NodeOptions(model="mdr") - options.set_position(0, 0) + options = CoreNode.create_options() + options.model = "mdr" node1 = session.add_node(CoreNode, options=options) node2 = session.add_node(CoreNode, options=options) @@ -105,8 +105,8 @@ class TestCore: session.mobility.set_model(wlan_node, BasicRangeModel) # create nodes - options = NodeOptions(model="mdr") - options.set_position(0, 0) + options = CoreNode.create_options() + options.model = "mdr" node1 = session.add_node(CoreNode, options=options) node2 = session.add_node(CoreNode, options=options) diff --git a/daemon/tests/test_distributed.py b/daemon/tests/test_distributed.py index 35b7af4e..3a9d43fb 100644 --- a/daemon/tests/test_distributed.py +++ b/daemon/tests/test_distributed.py @@ -1,4 +1,3 @@ -from core.emulator.data import NodeOptions from core.emulator.session import Session from core.nodes.base import CoreNode from core.nodes.network import HubNode @@ -12,8 +11,7 @@ class TestDistributed: # when session.distributed.add_server(server_name, host) - options = NodeOptions(server=server_name) - node = session.add_node(CoreNode, options=options) + node = session.add_node(CoreNode, server=server_name) session.instantiate() # then @@ -30,8 +28,7 @@ class TestDistributed: # when session.distributed.add_server(server_name, host) node1 = session.add_node(HubNode) - options = NodeOptions(server=server_name) - node2 = session.add_node(HubNode, options=options) + node2 = session.add_node(HubNode, server=server_name) session.add_link(node1.id, node2.id) session.instantiate() diff --git a/daemon/tests/test_grpc.py b/daemon/tests/test_grpc.py index a33fd5b8..9aed3395 100644 --- a/daemon/tests/test_grpc.py +++ b/daemon/tests/test_grpc.py @@ -34,7 +34,7 @@ from core.api.grpc.wrappers import ( ) from core.emane.models.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet -from core.emulator.data import EventData, IpPrefixes, NodeData, NodeOptions +from core.emulator.data import EventData, IpPrefixes, NodeData from core.emulator.enumerations import EventTypes, ExceptionLevels, MessageFlags from core.errors import CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility @@ -350,8 +350,7 @@ class TestGrpc: client = CoreGrpcClient() session = grpc_server.coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) - options = NodeOptions(model="Host") - node = session.add_node(CoreNode, options=options) + node = session.add_node(CoreNode) session.instantiate() expected_output = "hello world" expected_status = 0 @@ -369,8 +368,7 @@ class TestGrpc: client = CoreGrpcClient() session = grpc_server.coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) - options = NodeOptions(model="Host") - node = session.add_node(CoreNode, options=options) + node = session.add_node(CoreNode) session.instantiate() # then @@ -444,10 +442,12 @@ class TestGrpc: # given client = CoreGrpcClient() session = grpc_server.coreemu.create_session() + session.set_state(EventTypes.CONFIGURATION_STATE) switch = session.add_node(SwitchNode) node = session.add_node(CoreNode) iface_data = ip_prefixes.create_iface(node) iface, _ = session.add_link(node.id, switch.id, iface_data) + session.instantiate() options = LinkOptions(bandwidth=30000) assert iface.options.bandwidth != options.bandwidth link = Link(node.id, switch.id, iface1=Interface(id=iface.id), options=options) @@ -535,7 +535,8 @@ class TestGrpc: client = CoreGrpcClient() session = grpc_server.coreemu.create_session() session.set_location(47.57917, -122.13232, 2.00000, 1.0) - options = NodeOptions(emane=EmaneIeee80211abgModel.name) + options = EmaneNet.create_options() + options.emane_model = EmaneIeee80211abgModel.name emane_network = session.add_node(EmaneNet, options=options) session.emane.node_models[emane_network.id] = EmaneIeee80211abgModel.name config_key = "bandwidth" @@ -565,7 +566,8 @@ class TestGrpc: client = CoreGrpcClient() session = grpc_server.coreemu.create_session() session.set_location(47.57917, -122.13232, 2.00000, 1.0) - options = NodeOptions(emane=EmaneIeee80211abgModel.name) + options = EmaneNet.create_options() + options.emane_model = EmaneIeee80211abgModel.name emane_network = session.add_node(EmaneNet, options=options) session.emane.node_models[emane_network.id] = EmaneIeee80211abgModel.name @@ -685,7 +687,8 @@ class TestGrpc: # given client = CoreGrpcClient() session = grpc_server.coreemu.create_session() - options = NodeOptions(legacy=True) + options = CoreNode.create_options() + options.legacy = True node = session.add_node(CoreNode, options=options) service_name = "DefaultRoute" @@ -932,6 +935,7 @@ class TestGrpc: # given client = CoreGrpcClient() session = grpc_server.coreemu.create_session() + session.set_state(EventTypes.CONFIGURATION_STATE) wlan = session.add_node(WlanNode) node1 = session.add_node(CoreNode) node2 = session.add_node(CoreNode) diff --git a/daemon/tests/test_nodes.py b/daemon/tests/test_nodes.py index 112045fa..bb76bb4e 100644 --- a/daemon/tests/test_nodes.py +++ b/daemon/tests/test_nodes.py @@ -1,6 +1,6 @@ import pytest -from core.emulator.data import InterfaceData, NodeOptions +from core.emulator.data import InterfaceData from core.emulator.session import Session from core.errors import CoreError from core.nodes.base import CoreNode @@ -14,7 +14,8 @@ class TestNodes: @pytest.mark.parametrize("model", MODELS) def test_node_add(self, session: Session, model: str): # given - options = NodeOptions(model=model) + options = CoreNode.create_options() + options.model = model # when node = session.add_node(CoreNode, options=options) diff --git a/daemon/tests/test_xml.py b/daemon/tests/test_xml.py index b3e57bef..6841da8e 100644 --- a/daemon/tests/test_xml.py +++ b/daemon/tests/test_xml.py @@ -4,7 +4,7 @@ from xml.etree import ElementTree import pytest -from core.emulator.data import IpPrefixes, LinkOptions, NodeOptions +from core.emulator.data import IpPrefixes, LinkOptions from core.emulator.enumerations import EventTypes from core.emulator.session import Session from core.errors import CoreError @@ -116,8 +116,7 @@ class TestXml: :param ip_prefixes: generates ip addresses for nodes """ # create nodes - options = NodeOptions(model="host") - node1 = session.add_node(CoreNode, options=options) + node1 = session.add_node(CoreNode) node2 = session.add_node(CoreNode) # link nodes to ptp net @@ -180,8 +179,8 @@ class TestXml: session.mobility.set_model(wlan, BasicRangeModel, {"test": "1"}) # create nodes - options = NodeOptions(model="mdr") - options.set_position(0, 0) + options = CoreNode.create_options() + options.model = "mdr" node1 = session.add_node(CoreNode, options=options) node2 = session.add_node(CoreNode, options=options) diff --git a/docs/python.md b/docs/python.md index ba4b4f33..bf9a74e1 100644 --- a/docs/python.md +++ b/docs/python.md @@ -1,7 +1,7 @@ # Python API * Table of Contents -{:toc} + {:toc} ## Overview @@ -21,13 +21,13 @@ When creating nodes of type `core.nodes.base.CoreNode` these are the default mod and the services they map to. * mdr - * zebra, OSPFv3MDR, IPForward + * zebra, OSPFv3MDR, IPForward * PC - * DefaultRoute + * DefaultRoute * router - * zebra, OSPFv2, OSPFv3, IPForward + * zebra, OSPFv2, OSPFv3, IPForward * host - * DefaultRoute, SSH + * DefaultRoute, SSH ### Interface Helper @@ -36,8 +36,10 @@ when creating interface data for nodes. Alternatively one can manually create a `core.emulator.data.InterfaceData` class instead with appropriate information. Manually creating interface data: + ```python from core.emulator.data import InterfaceData + # id is optional and will set to the next available id # name is optional and will default to eth # mac is optional and will result in a randomly generated mac @@ -52,6 +54,7 @@ iface_data = InterfaceData( ``` Leveraging the interface prefixes helper class: + ```python from core.emulator.data import IpPrefixes @@ -69,6 +72,7 @@ iface_data = ip_prefixes.create_iface( Various events that can occur within a session can be listened to. Event types: + * session - events for changes in session state and mobility start/stop/pause * node - events for node movements and icon changes * link - events for link configuration changes and wireless link add/delete @@ -80,6 +84,7 @@ Event types: def event_listener(event): print(event) + # add an event listener to event type you want to listen to # each handler will receive an object unique to that type session.event_handlers.append(event_listener) @@ -95,6 +100,7 @@ session.config_handlers.append(event_listener) Links can be configured at the time of creation or during runtime. Currently supported configuration options: + * bandwidth (bps) * delay (us) * dup (%) @@ -119,12 +125,13 @@ session.update_link(n1_id, n2_id, iface1_id, iface2_id, options) ``` ### Peer to Peer Example + ```python # required imports from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position # ip nerator for example ip_prefixes = IpPrefixes(ip4_prefix="10.0.0.0/24") @@ -137,10 +144,10 @@ session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) # create nodes -options = NodeOptions(x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position) +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position) # link nodes together iface1 = ip_prefixes.create_iface(n1) @@ -158,12 +165,13 @@ session.shutdown() ``` ### Switch/Hub Example + ```python # required imports from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position from core.nodes.network import SwitchNode # ip nerator for example @@ -177,14 +185,14 @@ session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) # create switch -options = NodeOptions(x=200, y=200) -switch = session.add_node(SwitchNode, options=options) +position = Position(x=200, y=200) +switch = session.add_node(SwitchNode, position=position) # create nodes -options = NodeOptions(x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position) +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position) # link nodes to switch iface1 = ip_prefixes.create_iface(n1) @@ -203,13 +211,14 @@ session.shutdown() ``` ### WLAN Example + ```python # required imports from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes from core.location.mobility import BasicRangeModel -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position from core.nodes.network import WlanNode # ip nerator for example @@ -223,14 +232,16 @@ session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) # create wlan -options = NodeOptions(x=200, y=200) -wlan = session.add_node(WlanNode, options=options) +position = Position(x=200, y=200) +wlan = session.add_node(WlanNode, position=position) # create nodes -options = NodeOptions(model="mdr", x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(model="mdr", x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position, options=options) +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position, options=options) # configuring wlan session.mobility.set_model_config(wlan.id, BasicRangeModel.name, { @@ -263,6 +274,7 @@ For EMANE you can import and use one of the existing models and use its name for configuration. Current models: + * core.emane.ieee80211abg.EmaneIeee80211abgModel * core.emane.rfpipe.EmaneRfPipeModel * core.emane.tdma.EmaneTdmaModel @@ -281,9 +293,9 @@ will use the defaults. When no configuration is used, the defaults are used. from core.emane.models.ieee80211abg import EmaneIeee80211abgModel from core.emane.nodes import EmaneNet from core.emulator.coreemu import CoreEmu -from core.emulator.data import IpPrefixes, NodeOptions +from core.emulator.data import IpPrefixes from core.emulator.enumerations import EventTypes -from core.nodes.base import CoreNode +from core.nodes.base import CoreNode, Position # ip nerator for example ip_prefixes = IpPrefixes(ip4_prefix="10.0.0.0/24") @@ -300,25 +312,29 @@ session.location.refscale = 150.0 session.set_state(EventTypes.CONFIGURATION_STATE) # create emane -options = NodeOptions(x=200, y=200, emane=EmaneIeee80211abgModel.name) -emane = session.add_node(EmaneNet, options=options) +options = EmaneNet.create_options() +options.emane_model = EmaneIeee80211abgModel.name +position = Position(x=200, y=200) +emane = session.add_node(EmaneNet, position=position, options=options) # create nodes -options = NodeOptions(model="mdr", x=100, y=100) -n1 = session.add_node(CoreNode, options=options) -options = NodeOptions(model="mdr", x=300, y=100) -n2 = session.add_node(CoreNode, options=options) +options = CoreNode.create_options() +options.model = "mdr" +position = Position(x=100, y=100) +n1 = session.add_node(CoreNode, position=position, options=options) +position = Position(x=300, y=100) +n2 = session.add_node(CoreNode, position=position, options=options) # configure general emane settings config = session.emane.get_configs() config.update({ - "eventservicettl": "2" + "eventservicettl": "2" }) # configure emane model settings # using a dict mapping currently support values as strings session.emane.set_model_config(emane.id, EmaneIeee80211abgModel.name, { - "unicastrate": "3", + "unicastrate": "3", }) # link nodes to emane @@ -338,6 +354,7 @@ session.shutdown() ``` EMANE Model Configuration: + ```python from core import utils @@ -358,6 +375,7 @@ Configuring the files of a service results in a specific hard coded script being generated, instead of the default scripts, that may leverage dynamic generation. The following features can be configured for a service: + * configs - files that will be generated * dirs - directories that will be mounted unique to the node * startup - commands to run start a service @@ -365,6 +383,7 @@ The following features can be configured for a service: * shutdown - commands to run to stop a service Editing service properties: + ```python # configure a service, for a node, for a given session session.services.set_service(node_id, service_name) @@ -380,6 +399,7 @@ When editing a service file, it must be the name of `config` file that the service will generate. Editing a service file: + ```python # to edit the contents of a generated file you can specify # the service, the file name, and its contents @@ -397,10 +417,12 @@ File versions of the network examples can be found [here](https://github.com/coreemu/core/tree/master/daemon/examples/python). ## Executing Scripts from GUI + To execute a python script from a GUI you need have the following. The builtin name check here to know it is being executed from the GUI, this can be avoided if your script does not use a name check. + ```python if __name__ in ["__main__", "__builtin__"]: main() From 28d3deeee8bd553d6b70b5d08ffa8a90904e6c68 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 25 May 2022 21:43:52 -0700 Subject: [PATCH 08/15] install: updated and added dockerfiles for ubuntu/centos, updated notes on install page to correlate --- Dockerfile | 100 ---------------------------------------------- Dockerfile.centos | 39 ++++++++++++++++++ Dockerfile.ubuntu | 34 ++++++++++++++++ docs/install.md | 4 +- 4 files changed, 75 insertions(+), 102 deletions(-) delete mode 100644 Dockerfile create mode 100644 Dockerfile.centos create mode 100644 Dockerfile.ubuntu diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a116becb..00000000 --- a/Dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM ubuntu:20.04 -LABEL Description="CORE Docker Image" - -# define variables -ARG DEBIAN_FRONTEND=noninteractive -ARG PREFIX=/usr/local -ARG BRANCH=master -ARG CORE_TARBALL=core.tar.gz -ARG OSPF_TARBALL=ospf.tar.gz - -# install system dependencies -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - automake \ - bash \ - ca-certificates \ - ethtool \ - gawk \ - gcc \ - g++ \ - iproute2 \ - iputils-ping \ - libc-dev \ - libev-dev \ - libreadline-dev \ - libtool \ - libtk-img \ - make \ - nftables \ - python3 \ - python3-pip \ - python3-tk \ - pkg-config \ - systemctl \ - tk \ - wget \ - xauth \ - xterm \ - && apt-get clean -# install python dependencies -RUN python3 -m pip install \ - grpcio==1.27.2 \ - grpcio-tools==1.27.2 \ - poetry==1.1.7 -# retrieve, build, and install core -RUN wget -q -O ${CORE_TARBALL} https://api.github.com/repos/coreemu/core/tarball/${BRANCH} && \ - tar xf ${CORE_TARBALL} && \ - cd coreemu-core* && \ - ./bootstrap.sh && \ - ./configure && \ - make -j $(nproc) && \ - make install && \ - cd daemon && \ - python3 -m poetry build -f wheel && \ - python3 -m pip install dist/* && \ - cp scripts/* ${PREFIX}/bin && \ - mkdir /etc/core && \ - cp -n data/core.conf /etc/core && \ - cp -n data/logging.conf /etc/core && \ - mkdir -p ${PREFIX}/share/core && \ - cp -r examples ${PREFIX}/share/core && \ - echo '\ -[Unit]\n\ -Description=Common Open Research Emulator Service\n\ -After=network.target\n\ -\n\ -[Service]\n\ -Type=simple\n\ -ExecStart=/usr/local/bin/core-daemon\n\ -TasksMax=infinity\n\ -\n\ -[Install]\n\ -WantedBy=multi-user.target\ -' > /lib/systemd/system/core-daemon.service && \ - cd ../.. && \ - rm ${CORE_TARBALL} && \ - rm -rf coreemu-core* -# retrieve, build, and install ospf mdr -RUN wget -q -O ${OSPF_TARBALL} https://github.com/USNavalResearchLaboratory/ospf-mdr/tarball/master && \ - tar xf ${OSPF_TARBALL} && \ - cd USNavalResearchLaboratory-ospf-mdr* && \ - ./bootstrap.sh && \ - ./configure --disable-doc --enable-user=root --enable-group=root \ - --with-cflags=-ggdb --sysconfdir=/usr/local/etc/quagga --enable-vtysh \ - --localstatedir=/var/run/quagga && \ - make -j $(nproc) && \ - make install && \ - cd .. && \ - rm ${OSPF_TARBALL} && \ - rm -rf USNavalResearchLaboratory-ospf-mdr* -# retrieve and install emane packages -RUN wget -q https://adjacentlink.com/downloads/emane/emane-1.2.7-release-1.ubuntu-20_04.amd64.tar.gz && \ - tar xf emane*.tar.gz && \ - cd emane-1.2.7-release-1/debs/ubuntu-20_04/amd64 && \ - apt-get install -y ./emane*.deb ./python3-emane_*.deb && \ - cd ../../../.. && \ - rm emane-1.2.7-release-1.ubuntu-20_04.amd64.tar.gz && \ - rm -rf emane-1.2.7-release-1 -CMD ["systemctl", "start", "core-daemon"] diff --git a/Dockerfile.centos b/Dockerfile.centos new file mode 100644 index 00000000..509e6dfe --- /dev/null +++ b/Dockerfile.centos @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1 +FROM centos:7 +LABEL Description="CORE Docker CentOS Image" + +# define variables +ARG PREFIX=/usr +ARG BRANCH=master + +# define environment +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG en_US.UTF-8 + +# install core +RUN yum -y update && \ + yum install -y git sudo wget tzdata unzip python3 +WORKDIR /root +RUN git clone https://github.com/coreemu/core +WORKDIR /root/core +RUN git checkout ${BRANCH} +RUN ./setup.sh +RUN . /root/.bashrc && inv install -v -p ${PREFIX} +# install emane packages and python bindings +WORKDIR /root +RUN wget -q https://adjacentlink.com/downloads/emane/emane-1.3.3-release-1.el7.x86_64.tar.gz && \ + tar xf emane-1.3.3-release-1.el7.x86_64.tar.gz && \ + cd emane-1.3.3-release-1/rpms/el7/x86_64 && \ + yum install -y epel-release && \ + yum install -y ./openstatistic*.rpm ./emane*.rpm ./python3-emane_*.rpm && \ + cd ../../../.. && \ + rm emane-1.3.3-release-1.el7.x86_64.tar.gz && \ + rm -rf emane-1.3.3-release-1 +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip && \ + mkdir protoc && \ + unzip protoc-3.7.1-linux-x86_64.zip -d protoc +WORKDIR /root/core +RUN . /root/.bashrc && PATH=/root/protoc/bin:$PATH inv install-emane -v -e v1.3.3 + +# run daemon +CMD ["core-daemon"] diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu new file mode 100644 index 00000000..32dcecf6 --- /dev/null +++ b/Dockerfile.ubuntu @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1 +FROM ubuntu:20.04 +LABEL Description="CORE Docker Ubuntu Image" + +# define variables +ARG PREFIX=/usr/local +ARG BRANCH=master + +# define environment +ENV DEBIAN_FRONTEND=noninteractive + +# install core +RUN apt-get update && \ + apt-get install -y git sudo wget tzdata +WORKDIR /root +RUN git clone https://github.com/coreemu/core +WORKDIR /root/core +RUN git checkout ${BRANCH} +RUN ./setup.sh +RUN . /root/.bashrc && inv install -v -p ${PREFIX} +# install emane packages and python bindings +WORKDIR /root +RUN wget -q https://adjacentlink.com/downloads/emane/emane-1.3.3-release-1.ubuntu-20_04.amd64.tar.gz && \ + tar xf emane*.tar.gz && \ + cd emane-1.3.3-release-1/debs/ubuntu-20_04/amd64 && \ + apt-get install -y ./emane*.deb ./python3-emane_*.deb && \ + cd ../../../.. && \ + rm emane-1.3.3-release-1.ubuntu-20_04.amd64.tar.gz && \ + rm -rf emane-1.3.3-release-1 +WORKDIR /root/core +RUN . /root/.bashrc && inv install-emane -v -e v1.3.3 + +# run daemon +CMD ["core-daemon"] diff --git a/docs/install.md b/docs/install.md index 7af62ee3..0f15133c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -218,9 +218,9 @@ You can leverage the provided Dockerfile, to run and launch CORE within a Docker git clone https://github.com/coreemu/core.git cd core # build image -sudo docker build -t core . +sudo docker build -t core. -f Dockerfile. # start container -sudo docker run -itd --name core -e DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw --privileged core +sudo docker run -itd --name core -e DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw --privileged core. # enable xhost access to the root user xhost +local:root # launch core-gui From 31bc0c64973e2a9e1af1ac7b2a6f813012799f24 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 May 2022 10:15:34 -0700 Subject: [PATCH 09/15] scripts: fixed bug in setting type when creating nodes using core-cli --- daemon/scripts/core-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/scripts/core-cli b/daemon/scripts/core-cli index 6f2c1f5b..b9de2615 100755 --- a/daemon/scripts/core-cli +++ b/daemon/scripts/core-cli @@ -28,7 +28,7 @@ from core.api.grpc.wrappers import ( Position, ) -NODE_TYPES = [x for x in NodeType if x != NodeType.PEER_TO_PEER] +NODE_TYPES = [x.name for x in NodeType if x != NodeType.PEER_TO_PEER] def protobuf_to_json(message: Any) -> Dict[str, Any]: From bbcd4664ff0a1c575f9d48daf4631575f26dde1a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 May 2022 10:43:59 -0700 Subject: [PATCH 10/15] daemon: removed session.add_node_file as it is not needed and was only used by the old tlv api --- daemon/core/emulator/session.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 4444e248..7f8144d0 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -599,28 +599,6 @@ class Session: logger.info("immediately running new state hook") self.run_hook(hook) - def add_node_file( - self, - node_id: int, - src_path: Optional[Path], - file_path: Path, - data: Optional[str], - ) -> None: - """ - Add a file to a node. - - :param node_id: node to add file to - :param src_path: source file path - :param file_path: file path to add - :param data: file data - :return: nothing - """ - node = self.get_node(node_id, CoreNode) - if src_path is not None: - node.copy_file(src_path, file_path) - elif data is not None: - node.create_file(file_path, data) - def clear(self) -> None: """ Clear all CORE session data. (nodes, hooks, etc) From 7173e488cb02c44ad00b0e9dc982d95a3755670f Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 May 2022 10:57:26 -0700 Subject: [PATCH 11/15] daemon: fixed issue with not properly starting emane event monitoring when enabled --- daemon/core/emane/emanemanager.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 32949301..294ae528 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -382,6 +382,8 @@ class EmaneManager: service = EmaneEventService( self, event_net.brname, eventgroup, int(eventport) ) + if self.doeventmonitor(): + service.start() self.services[event_net.brname] = service self.nem_service[nem_id] = service except EventServiceException: @@ -636,20 +638,13 @@ class EmaneManager: """ Returns boolean whether or not EMANE events will be monitored. """ - # this support must be explicitly turned on; by default, CORE will - # generate the EMANE events when nodes are moved return self.session.options.get_bool("emane_event_monitor", False) def genlocationevents(self) -> bool: """ Returns boolean whether or not EMANE events will be generated. """ - # By default, CORE generates EMANE location events when nodes - # are moved; this can be explicitly disabled in core.conf - tmp = self.session.options.get_bool("emane_event_generate", True) - if tmp is None: - tmp = not self.doeventmonitor() - return tmp + return self.session.options.get_bool("emane_event_generate", True) def handlelocationevent(self, rxnemid: int, eid: int, data: str) -> None: """ From d77ed9c47363aaeec8d218030b767435e2e65725 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 May 2022 20:10:48 -0700 Subject: [PATCH 12/15] install: adjustments to install scripts to better support alternative versions of python, PYTHON_DEP added for system package adjustments --- setup.sh | 10 ++++++---- tasks.py | 31 +++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/setup.sh b/setup.sh index eb81a451..80f51b8e 100755 --- a/setup.sh +++ b/setup.sh @@ -1,14 +1,16 @@ #!/bin/bash # install pre-reqs using yum/apt +PYTHON="${PYTHON:=python3}" +PYTHON_DEP="${PYTHON_DEP:=python3}" if command -v apt &> /dev/null then echo "setup to install CORE using apt" - sudo apt install -y python3-pip python3-venv + sudo apt install -y ${PYTHON_DEP}-pip ${PYTHON_DEP}-venv elif command -v yum &> /dev/null then echo "setup to install CORE using yum" - sudo yum install -y python3-pip + sudo yum install -y ${PYTHON_DEP}-pip else echo "apt/yum was not found" echo "install python3, pip, venv, pipx, and invoke to run the automated install" @@ -16,8 +18,8 @@ else fi # install tooling for invoke based installation -python3 -m pip install --user pipx==0.16.4 -python3 -m pipx ensurepath +${PYTHON} -m pip install --user pipx==0.16.4 +${PYTHON} -m pipx ensurepath export PATH=$PATH:~/.local/bin pipx install invoke==1.4.1 pipx install poetry==1.1.12 diff --git a/tasks.py b/tasks.py index 2b999cfc..63231de5 100644 --- a/tasks.py +++ b/tasks.py @@ -114,6 +114,10 @@ def get_env_python() -> str: return os.environ.get("PYTHON", "python3") +def get_env_python_dep() -> str: + return os.environ.get("PYTHON_DEP", "python3") + + def get_python(c: Context, warn: bool = False) -> str: with c.cd(DAEMON_DIR): r = c.run("poetry env info -p", warn=warn, hide=True) @@ -153,23 +157,27 @@ def get_os(install_type: Optional[str]) -> OsInfo: def check_existing_core(c: Context, hide: bool) -> None: if c.run("python -c \"import core\"", warn=True, hide=hide): raise SystemError("existing python2 core installation detected, please remove") - if c.run("python3 -c \"import core\"", warn=True, hide=hide): - raise SystemError("existing python3 core installation detected, please remove") + python_bin = get_env_python() + if c.run(f"{python_bin} -c \"import core\"", warn=True, hide=hide): + raise SystemError( + f"existing {python_bin} core installation detected, please remove" + ) if c.run("which core-daemon", warn=True, hide=hide): raise SystemError("core scripts found, please remove old installation") def install_system(c: Context, os_info: OsInfo, hide: bool) -> None: + python_dep = get_env_python_dep() if os_info.like == OsLike.DEBIAN: c.run( "sudo apt install -y automake pkg-config gcc libev-dev nftables " - "iproute2 ethtool tk python3-tk bash", + f"iproute2 ethtool tk {python_dep}-tk bash", hide=hide ) elif os_info.like == OsLike.REDHAT: c.run( "sudo yum install -y automake pkgconf-pkg-config gcc gcc-c++ " - "libev-devel nftables iproute python3-devel python3-tkinter " + f"libev-devel nftables iproute {python_dep}-devel {python_dep}-tkinter " "tk ethtool make bash", hide=hide ) @@ -202,14 +210,14 @@ def install_core(c: Context, hide: bool) -> None: def install_poetry(c: Context, dev: bool, local: bool, hide: bool) -> None: + python_bin = get_env_python() if local: with c.cd(DAEMON_DIR): c.run("poetry build -f wheel", hide=hide) - c.run("sudo python3 -m pip install dist/*") + c.run(f"sudo {python_bin} -m pip install dist/*") else: args = "" if dev else "--no-dev" with c.cd(DAEMON_DIR): - python_bin = get_env_python() c.run(f"poetry env use {python_bin}", hide=hide) c.run(f"poetry install {args}", hide=hide) if dev: @@ -388,12 +396,13 @@ def install_emane(c, emane_version, verbose=False, install_type=None): p = Progress(verbose) hide = not verbose os_info = get_os(install_type) + python_dep = get_env_python_dep() with p.start("installing system dependencies"): if os_info.like == OsLike.DEBIAN: c.run( "sudo apt install -y gcc g++ automake libtool libxml2-dev " "libprotobuf-dev libpcap-dev libpcre3-dev uuid-dev pkg-config " - "protobuf-compiler git python3-protobuf python3-setuptools", + f"protobuf-compiler git {python_dep}-protobuf {python_dep}-setuptools", hide=hide, ) elif os_info.like == OsLike.REDHAT: @@ -402,7 +411,7 @@ def install_emane(c, emane_version, verbose=False, install_type=None): c.run( "sudo yum install -y autoconf automake git libtool libxml2-devel " "libpcap-devel pcre-devel libuuid-devel make gcc-c++ protobuf-compiler " - "protobuf-devel python3-setuptools", + f"protobuf-devel {python_dep}-setuptools", hide=hide, ) emane_dir = "../emane" @@ -411,10 +420,11 @@ def install_emane(c, emane_version, verbose=False, install_type=None): with p.start("cloning emane"): c.run(f"git clone {emane_url} {emane_dir}", hide=hide) with p.start("setup emane"): + python_bin = get_env_python() with c.cd(emane_dir): c.run(f"git checkout {emane_version}", hide=hide) c.run("./autogen.sh", hide=hide) - c.run("PYTHON=python3 ./configure --prefix=/usr", hide=hide) + c.run(f"PYTHON={python_bin} ./configure --prefix=/usr", hide=hide) with p.start("build emane python bindings"): with c.cd(str(emane_python_dir)): c.run("make -j$(nproc)", hide=hide) @@ -456,7 +466,8 @@ def uninstall( if local: with p.start("uninstalling core"): - c.run("sudo python3 -m pip uninstall -y core", hide=hide) + python_bin = get_env_python() + c.run(f"sudo {python_bin} -m pip uninstall -y core", hide=hide) else: python = get_python(c, warn=True) if python: From f40de9f838833b988f9e9721c96d669fe6f6f43b Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 27 May 2022 20:54:47 -0700 Subject: [PATCH 13/15] install: added dockerfile for oracle linux support --- Dockerfile.oracle | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 Dockerfile.oracle diff --git a/Dockerfile.oracle b/Dockerfile.oracle new file mode 100644 index 00000000..28a488b0 --- /dev/null +++ b/Dockerfile.oracle @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1 +FROM oraclelinux:8 +LABEL Description="CORE Docker Oracle Linux Image" + +# define variables +ARG PREFIX=/usr +ARG BRANCH=docker-updates + +# define environment +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG en_US.UTF-8 + +# install core +RUN yum -y update && \ + yum install -y git sudo wget tzdata unzip python39 which iproute-tc hostname xterm +WORKDIR /root +RUN git clone https://github.com/coreemu/core +WORKDIR /root/core +RUN git checkout ${BRANCH} +RUN PYTHON=python3.9 PYTHON_DEP=python39 ./setup.sh +RUN . /root/.bashrc && PYTHON=python3.9 PYTHON_DEP=python39 inv install -v -p ${PREFIX} +# install emane packages and python bindings +WORKDIR /root +RUN yum config-manager --set-enabled ol8_codeready_builder && yum install -y protobuf-devel +RUN wget -q https://adjacentlink.com/downloads/emane/emane-1.3.3-release-1.el8.x86_64.tar.gz && \ + tar xf emane-1.3.3-release-1.el8.x86_64.tar.gz && \ + cd emane-1.3.3-release-1/rpms/el8/x86_64 && \ + yum install -y epel-release && \ + yum install -y ./openstatistic*.rpm ./emane*.rpm ./python3-emane-1.3.3*.rpm && \ + cd ../../../.. && \ + rm emane-1.3.3-release-1.el8.x86_64.tar.gz && \ + rm -rf emane-1.3.3-release-1 +WORKDIR /root/core +RUN . /root/.bashrc && PYTHON=python3.9 PATH=/root/protoc/bin:$PATH inv install-emane -v -e v1.3.3 + +# run daemon +CMD ["core-daemon"] From 9fa3e77d12d31104e574835c0ef7065c581d52dc Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 31 May 2022 14:25:49 -0700 Subject: [PATCH 14/15] daemon: added further python docs to docker.py --- daemon/core/nodes/docker.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index a5560b17..a97ba08c 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -37,10 +37,15 @@ class DockerOptions(CoreNodeOptions): @dataclass class DockerVolume: src: str + """volume mount name""" dst: str + """volume mount destination directory""" unique: bool = True + """True to create a node unique prefixed name for this volume""" delete: bool = True + """True to delete the volume during shutdown""" path: str = None + """path to the volume on the host""" class DockerNode(CoreNode): @@ -60,8 +65,8 @@ class DockerNode(CoreNode): Create a DockerNode instance. :param session: core session instance - :param _id: object id - :param name: object name + :param _id: node id + :param name: node name :param server: remote server node will run on, default is None for localhost :param options: options for creating node @@ -77,6 +82,11 @@ class DockerNode(CoreNode): @classmethod def create_options(cls) -> DockerOptions: + """ + Return default creation options, which can be used during node creation. + + :return: docker options + """ return DockerOptions() def _create_cmd(self, args: str, shell: bool = False) -> str: From e5d6299f0a6523022900249ab7a7be9ef7d2575c Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 1 Jun 2022 11:12:20 -0700 Subject: [PATCH 15/15] daemon: fixed error in requirements check and sorted executable definitions --- daemon/core/executables.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/daemon/core/executables.py b/daemon/core/executables.py index 3d0e80f6..95c97378 100644 --- a/daemon/core/executables.py +++ b/daemon/core/executables.py @@ -1,30 +1,31 @@ from typing import List BASH: str = "bash" -VNODED: str = "vnoded" -VCMD: str = "vcmd" -SYSCTL: str = "sysctl" -IP: str = "ip" ETHTOOL: str = "ethtool" -TC: str = "tc" +IP: str = "ip" MOUNT: str = "mount" -UMOUNT: str = "umount" -OVS_VSCTL: str = "ovs-vsctl" -TEST: str = "test" NFTABLES: str = "nft" +OVS_VSCTL: str = "ovs-vsctl" +SYSCTL: str = "sysctl" +TC: str = "tc" +TEST: str = "test" +UMOUNT: str = "umount" +VCMD: str = "vcmd" +VNODED: str = "vnoded" COMMON_REQUIREMENTS: List[str] = [ BASH, - NFTABLES, ETHTOOL, IP, MOUNT, + NFTABLES, SYSCTL, TC, - UMOUNT, TEST, + UMOUNT, + VCMD, + VNODED, ] -VCMD_REQUIREMENTS: List[str] = [VNODED, VCMD] OVS_REQUIREMENTS: List[str] = [OVS_VSCTL] @@ -38,6 +39,4 @@ def get_requirements(use_ovs: bool) -> List[str]: requirements = COMMON_REQUIREMENTS if use_ovs: requirements += OVS_REQUIREMENTS - else: - requirements += VCMD_REQUIREMENTS return requirements