afonso/core-extra
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

daemon: refactoring to remove usage of os.path where possible and pathlib.Path instead

Browse source
This commit is contained in:
Blake Harnden 2021-03-19 16:54:24 -07:00
parent d0a55dd471
commit 1c970bbe00
38 changed files with 520 additions and 606 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
daemon/core/api/grpc/grpcutils.py
View file

@ -271,11 +271,11 @@ def get_node_proto(session: Session, node: NodeBase) -> core_pb2.Node:
node_dir = None
config_services = []
if isinstance(node, CoreNodeBase):
node_dir = node.nodedir
node_dir = str(node.nodedir)
config_services = [x for x in node.config_services]
channel = None
if isinstance(node, CoreNode):
channel = node.ctrlchnlname
channel = str(node.ctrlchnlname)
emane_model = None
if isinstance(node, EmaneNet):
emane_model = node.model.name

37
daemon/core/api/grpc/server.py
View file

@ -6,6 +6,7 @@ import tempfile
import threading
import time
from concurrent import futures
from pathlib import Path
from typing import Iterable, Optional, Pattern, Type
import grpc
@ -221,8 +222,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
# clear previous state and setup for creation
session.clear()
if not os.path.exists(session.session_dir):
os.mkdir(session.session_dir)
session.session_dir.mkdir(exist_ok=True)
session.set_state(EventTypes.CONFIGURATION_STATE)
# location
@ -366,12 +366,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
sessions = []
for session_id in self.coreemu.sessions:
session = self.coreemu.sessions[session_id]
session_file = str(session.file_path) if session.file_path else None
session_summary = core_pb2.SessionSummary(
id=session_id,
state=session.state.value,
nodes=session.get_node_count(),
file=session.file_name,
dir=session.session_dir,
file=session_file,
dir=str(session.session_dir),
)
sessions.append(session_summary)
return core_pb2.GetSessionsResponse(sessions=sessions)
@ -423,14 +424,11 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
"""
logging.debug("set session state: %s", request)
session = self.get_session(request.session_id, context)
try:
state = EventTypes(request.state)
session.set_state(state)
if state == EventTypes.INSTANTIATION_STATE:
if not os.path.exists(session.session_dir):
os.mkdir(session.session_dir)
session.session_dir.mkdir(exist_ok=True)
session.instantiate()
elif state == EventTypes.SHUTDOWN_STATE:
session.shutdown()
@ -438,11 +436,9 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
session.data_collect()
elif state == EventTypes.DEFINITION_STATE:
session.clear()
result = True
except KeyError:
result = False
return core_pb2.SetSessionStateResponse(result=result)
def SetSessionUser(
@ -573,12 +569,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
mobility_configs = grpcutils.get_mobility_configs(session)
service_configs = grpcutils.get_node_service_configs(session)
config_service_configs = grpcutils.get_node_config_service_configs(session)
session_file = str(session.file_path) if session.file_path else None
session_proto = core_pb2.Session(
id=session.id,
state=session.state.value,
nodes=nodes,
links=links,
dir=session.session_dir,
dir=str(session.session_dir),
user=session.user,
default_services=default_services,
location=location,
@ -591,7 +588,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
config_service_configs=config_service_configs,
mobility_configs=mobility_configs,
metadata=session.metadata,
file=session.file_name,
file=session_file,
)
return core_pb2.GetSessionResponse(session=session_proto)
@ -1508,15 +1505,15 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
"""
logging.debug("open xml: %s", request)
session = self.coreemu.create_session()
temp = tempfile.NamedTemporaryFile(delete=False)
temp.write(request.data.encode("utf-8"))
temp.close()
temp_path = Path(temp.name)
file_path = Path(request.file)
try:
session.open_xml(temp.name, request.start)
session.name = os.path.basename(request.file)
session.file_name = request.file
session.open_xml(temp_path, request.start)
session.name = file_path.name
session.file_path = file_path
return core_pb2.OpenXmlResponse(session_id=session.id, result=True)
except IOError:
logging.exception("error opening session file")
@ -1733,12 +1730,10 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer):
def ExecuteScript(self, request, context):
existing_sessions = set(self.coreemu.sessions.keys())
file_path = Path(request.script)
thread = threading.Thread(
target=utils.execute_file,
args=(
request.script,
{"__file__": request.script, "coreemu": self.coreemu},
),
args=(file_path, {"coreemu": self.coreemu}),
daemon=True,
)
thread.start()

84
daemon/core/api/tlv/corehandlers.py
View file

@ -3,7 +3,6 @@ socket server request handlers leveraged by core servers.
"""
import logging
import os
import shlex
import shutil
import socketserver
@ -11,6 +10,7 @@ import sys
import threading
import time
from itertools import repeat
from pathlib import Path
from queue import Empty, Queue
from typing import Optional
@ -167,39 +167,27 @@ class CoreHandler(socketserver.BaseRequestHandler):
date_list = []
thumb_list = []
num_sessions = 0
with self._sessions_lock:
for _id in self.coreemu.sessions:
session = self.coreemu.sessions[_id]
num_sessions += 1
id_list.append(str(_id))
name = session.name
if not name:
name = ""
name_list.append(name)
file_name = session.file_name
if not file_name:
file_name = ""
file_list.append(file_name)
file_name = str(session.file_path) if session.file_path else ""
file_list.append(str(file_name))
node_count_list.append(str(session.get_node_count()))
date_list.append(time.ctime(session.state_time))
thumb = session.thumbnail
if not thumb:
thumb = ""
thumb = str(session.thumbnail) if session.thumbnail else ""
thumb_list.append(thumb)
session_ids = "|".join(id_list)
names = "|".join(name_list)
files = "|".join(file_list)
node_counts = "|".join(node_count_list)
dates = "|".join(date_list)
thumbs = "|".join(thumb_list)
if num_sessions > 0:
tlv_data = b""
if len(session_ids) > 0:
@ -221,7 +209,6 @@ class CoreHandler(socketserver.BaseRequestHandler):
message = coreapi.CoreSessionMessage.pack(flags, tlv_data)
else:
message = None
return message
def handle_broadcast_event(self, event_data):
@ -931,22 +918,18 @@ class CoreHandler(socketserver.BaseRequestHandler):
if message.flags & MessageFlags.STRING.value:
old_session_ids = set(self.coreemu.sessions.keys())
sys.argv = shlex.split(execute_server)
file_name = sys.argv[0]
if os.path.splitext(file_name)[1].lower() == ".xml":
file_path = Path(sys.argv[0])
if file_path.suffix == ".xml":
session = self.coreemu.create_session()
try:
session.open_xml(file_name)
session.open_xml(file_path)
except Exception:
self.coreemu.delete_session(session.id)
raise
else:
thread = threading.Thread(
target=utils.execute_file,
args=(
file_name,
{"__file__": file_name, "coreemu": self.coreemu},
),
args=(file_path, {"coreemu": self.coreemu}),
daemon=True,
)
thread.start()
@ -1465,10 +1448,12 @@ class CoreHandler(socketserver.BaseRequestHandler):
:return: reply messages
"""
if message.flags & MessageFlags.ADD.value:
node_num = message.get_tlv(FileTlvs.NODE.value)
node_id = message.get_tlv(FileTlvs.NODE.value)
file_name = message.get_tlv(FileTlvs.NAME.value)
file_type = message.get_tlv(FileTlvs.TYPE.value)
source_name = message.get_tlv(FileTlvs.SOURCE_NAME.value)
src_path = message.get_tlv(FileTlvs.SOURCE_NAME.value)
if src_path:
src_path = Path(src_path)
data = message.get_tlv(FileTlvs.DATA.value)
compressed_data = message.get_tlv(FileTlvs.COMPRESSED_DATA.value)
@ -1478,7 +1463,7 @@ class CoreHandler(socketserver.BaseRequestHandler):
)
return ()
if source_name and data:
if src_path and data:
logging.warning(
"ignoring invalid File message: source and data TLVs are both present"
)
@ -1490,7 +1475,7 @@ class CoreHandler(socketserver.BaseRequestHandler):
if file_type.startswith("service:"):
_, service_name = file_type.split(":")[:2]
self.session.services.set_service_file(
node_num, service_name, file_name, data
node_id, service_name, file_name, data
)
return ()
elif file_type.startswith("hook:"):
@ -1500,19 +1485,20 @@ class CoreHandler(socketserver.BaseRequestHandler):
return ()
state = int(state)
state = EventTypes(state)
self.session.add_hook(state, file_name, data, source_name)
self.session.add_hook(state, file_name, data, src_path)
return ()
# writing a file to the host
if node_num is None:
if source_name is not None:
shutil.copy2(source_name, file_name)
if node_id is None:
if src_path is not None:
shutil.copy2(src_path, file_name)
else:
with open(file_name, "w") as open_file:
open_file.write(data)
with file_name.open("w") as f:
f.write(data)
return ()
self.session.add_node_file(node_num, source_name, file_name, data)
file_path = Path(file_name)
self.session.add_node_file(node_id, src_path, file_path, data)
else:
raise NotImplementedError
@ -1567,26 +1553,32 @@ class CoreHandler(socketserver.BaseRequestHandler):
"dropping unhandled event message for node: %s", node.name
)
return ()
self.session.set_state(event_type)
if event_type == EventTypes.DEFINITION_STATE:
self.session.set_state(event_type)
# clear all session objects in order to receive new definitions
self.session.clear()
elif event_type == EventTypes.CONFIGURATION_STATE:
self.session.set_state(event_type)
elif event_type == EventTypes.INSTANTIATION_STATE:
self.session.set_state(event_type)
if len(self.handler_threads) > 1:
# TODO: sync handler threads here before continuing
time.sleep(2.0) # XXX
# done receiving node/link configuration, ready to instantiate
self.session.instantiate()
# after booting nodes attempt to send emulation id for nodes waiting on status
# after booting nodes attempt to send emulation id for nodes
# waiting on status
for _id in self.session.nodes:
self.send_node_emulation_id(_id)
elif event_type == EventTypes.RUNTIME_STATE:
self.session.set_state(event_type)
logging.warning("Unexpected event message: RUNTIME state received")
elif event_type == EventTypes.DATACOLLECT_STATE:
self.session.data_collect()
elif event_type == EventTypes.SHUTDOWN_STATE:
self.session.set_state(event_type)
logging.warning("Unexpected event message: SHUTDOWN state received")
elif event_type in {
EventTypes.START,
@ -1613,13 +1605,13 @@ class CoreHandler(socketserver.BaseRequestHandler):
name,
)
elif event_type == EventTypes.FILE_OPEN:
filename = event_data.name
self.session.open_xml(filename, start=False)
file_path = Path(event_data.name)
self.session.open_xml(file_path, start=False)
self.send_objects()
return ()
elif event_type == EventTypes.FILE_SAVE:
filename = event_data.name
self.session.save_xml(filename)
file_path = Path(event_data.name)
self.session.save_xml(file_path)
elif event_type == EventTypes.SCHEDULED:
etime = event_data.time
node_id = event_data.node
@ -1733,20 +1725,16 @@ class CoreHandler(socketserver.BaseRequestHandler):
session = self.session
else:
session = self.coreemu.sessions.get(session_id)
if session is None:
logging.warning("session %s not found", session_id)
continue
if names is not None:
session.name = names[index]
if files is not None:
session.file_name = files[index]
session.file_path = Path(files[index])
if thumb:
thumb = Path(thumb)
session.set_thumbnail(thumb)
if user:
session.set_user(user)
elif (

19
daemon/core/configservice/base.py
View file

@ -2,8 +2,8 @@ import abc
import enum
import inspect
import logging
import pathlib
import time
from pathlib import Path
from typing import Any, Dict, List
from mako import exceptions
@ -46,7 +46,7 @@ class ConfigService(abc.ABC):
"""
self.node: CoreNode = node
class_file = inspect.getfile(self.__class__)
templates_path = pathlib.Path(class_file).parent.joinpath(TEMPLATES_DIR)
templates_path = Path(class_file).parent.joinpath(TEMPLATES_DIR)
self.templates: TemplateLookup = TemplateLookup(directories=templates_path)
self.config: Dict[str, Configuration] = {}
self.custom_templates: Dict[str, str] = {}
@ -176,9 +176,10 @@ class ConfigService(abc.ABC):
:raises CoreError: when there is a failure creating a directory
"""
for directory in self.directories:
dir_path = Path(directory)
try:
self.node.privatedir(directory)
except (CoreCommandError, ValueError):
self.node.privatedir(dir_path)
except (CoreCommandError, CoreError):
raise CoreError(
f"node({self.node.name}) service({self.name}) "
f"failure to create service directory: {directory}"
@ -220,7 +221,7 @@ class ConfigService(abc.ABC):
"""
templates = {}
for name in self.files:
basename = pathlib.Path(name).name
basename = Path(name).name
if name in self.custom_templates:
template = self.custom_templates[name]
template = self.clean_text(template)
@ -240,12 +241,12 @@ class ConfigService(abc.ABC):
"""
data = self.data()
for name in self.files:
basename = pathlib.Path(name).name
file_path = Path(name)
if name in self.custom_templates:
text = self.custom_templates[name]
rendered = self.render_text(text, data)
elif self.templates.has_template(basename):
rendered = self.render_template(basename, data)
elif self.templates.has_template(file_path.name):
rendered = self.render_template(file_path.name, data)
else:
text = self.get_text_template(name)
rendered = self.render_text(text, data)
@ -256,7 +257,7 @@ class ConfigService(abc.ABC):
name,
rendered,
)
self.node.nodefile(name, rendered)
self.node.nodefile(file_path, rendered)
def run_startup(self, wait: bool) -> None:
"""

5
daemon/core/configservice/manager.py
View file

@ -1,5 +1,6 @@
import logging
import pathlib
from pathlib import Path
from typing import Dict, List, Type
from core import utils
@ -58,7 +59,7 @@ class ConfigServiceManager:
# make service available
self.services[name] = service
def load(self, path: str) -> List[str]:
def load(self, path: Path) -> List[str]:
"""
Search path provided for configurable services and add them for being managed.
@ -71,7 +72,7 @@ class ConfigServiceManager:
service_errors = []
for subdir in subdirs:
logging.debug("loading config services from: %s", subdir)
services = utils.load_classes(str(subdir), ConfigService)
services = utils.load_classes(subdir, ConfigService)
for service in services:
try:
self.add(service)

8
daemon/core/constants.py.in
View file

@ -1,3 +1,5 @@
COREDPY_VERSION = "@PACKAGE_VERSION@"
CORE_CONF_DIR = "@CORE_CONF_DIR@"
CORE_DATA_DIR = "@CORE_DATA_DIR@"
from pathlib import Path
COREDPY_VERSION: str = "@PACKAGE_VERSION@"
CORE_CONF_DIR: Path = Path("@CORE_CONF_DIR@")
CORE_DATA_DIR: Path = Path("@CORE_DATA_DIR@")

6
daemon/core/emane/commeffect.py
View file

@ -3,7 +3,7 @@ commeffect.py: EMANE CommEffect model for CORE
"""
import logging
import os
from pathlib import Path
from typing import Dict, List
from lxml import etree
@ -48,8 +48,8 @@ class EmaneCommEffectModel(emanemodel.EmaneModel):
external_config: List[Configuration] = []
@classmethod
def load(cls, emane_prefix: str) -> None:
shim_xml_path = os.path.join(emane_prefix, "share/emane/manifest", cls.shim_xml)
def load(cls, emane_prefix: Path) -> None:
shim_xml_path = emane_prefix / "share/emane/manifest" / cls.shim_xml
cls.config_shim = emanemanifest.parse(shim_xml_path, cls.shim_defaults)
@classmethod

30
daemon/core/emane/emanemanager.py
View file

@ -8,6 +8,7 @@ import threading
from collections import OrderedDict
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type
from core import utils
@ -175,17 +176,15 @@ class EmaneManager(ModelManager):
if not path:
logging.info("emane is not installed")
return
# get version
emane_version = utils.cmd("emane --version")
logging.info("using emane: %s", emane_version)
# load default emane models
self.load_models(EMANE_MODELS)
# load custom models
custom_models_path = self.session.options.get_config("emane_models_dir")
if custom_models_path:
if custom_models_path is not None:
custom_models_path = Path(custom_models_path)
emane_models = utils.load_classes(custom_models_path, EmaneModel)
self.load_models(emane_models)
@ -246,6 +245,7 @@ class EmaneManager(ModelManager):
emane_prefix = self.session.options.get_config(
"emane_prefix", default=DEFAULT_EMANE_PREFIX
)
emane_prefix = Path(emane_prefix)
emane_model.load(emane_prefix)
self.models[emane_model.name] = emane_model
@ -398,9 +398,9 @@ class EmaneManager(ModelManager):
return self.ifaces_to_nems.get(iface)
def write_nem(self, iface: CoreInterface, nem_id: int) -> None:
path = os.path.join(self.session.session_dir, "emane_nems")
path = self.session.session_dir / "emane_nems"
try:
with open(path, "a") as f:
with path.open("a") as f:
f.write(f"{iface.node.name} {iface.name} {nem_id}\n")
except IOError:
logging.exception("error writing to emane nem file")
@ -590,18 +590,17 @@ class EmaneManager(ModelManager):
if eventservicenetidx >= 0 and eventgroup != otagroup:
node.node_net_client.create_route(eventgroup, eventdev)
# start emane
log_file = os.path.join(node.nodedir, f"{node.name}-emane.log")
platform_xml = os.path.join(node.nodedir, f"{node.name}-platform.xml")
log_file = node.nodedir / f"{node.name}-emane.log"
platform_xml = node.nodedir / f"{node.name}-platform.xml"
args = f"{emanecmd} -f {log_file} {platform_xml}"
node.cmd(args)
logging.info("node(%s) emane daemon running: %s", node.name, args)
else:
path = self.session.session_dir
log_file = os.path.join(path, f"{node.name}-emane.log")
platform_xml = os.path.join(path, f"{node.name}-platform.xml")
emanecmd += f" -f {log_file} {platform_xml}"
node.host_cmd(emanecmd, cwd=path)
logging.info("node(%s) host emane daemon running: %s", node.name, emanecmd)
log_file = self.session.session_dir / f"{node.name}-emane.log"
platform_xml = self.session.session_dir / f"{node.name}-platform.xml"
args = f"{emanecmd} -f {log_file} {platform_xml}"
node.host_cmd(args, cwd=self.session.session_dir)
logging.info("node(%s) host emane daemon running: %s", node.name, args)
def install_iface(self, iface: CoreInterface) -> None:
emane_net = iface.net
@ -869,7 +868,8 @@ class EmaneGlobalModel:
emane_prefix = self.session.options.get_config(
"emane_prefix", default=DEFAULT_EMANE_PREFIX
)
emulator_xml = os.path.join(emane_prefix, "share/emane/manifest/nemmanager.xml")
emane_prefix = Path(emane_prefix)
emulator_xml = emane_prefix / "share/emane/manifest/nemmanager.xml"
emulator_defaults = {
"eventservicedevice": DEFAULT_DEV,
"eventservicegroup": "224.1.2.8:45703",

8
daemon/core/emane/emanemanifest.py
View file

@ -1,4 +1,5 @@
import logging
from pathlib import Path
from typing import Dict, List
from core.config import Configuration
@ -71,9 +72,10 @@ def _get_default(config_type_name: str, config_value: List[str]) -> str:
return config_default
def parse(manifest_path: str, defaults: Dict[str, str]) -> List[Configuration]:
def parse(manifest_path: Path, defaults: Dict[str, str]) -> List[Configuration]:
"""
Parses a valid emane manifest file and converts the provided configuration values into ones used by core.
Parses a valid emane manifest file and converts the provided configuration values
into ones used by core.
:param manifest_path: absolute manifest file path
:param defaults: used to override default values for configurations
@ -85,7 +87,7 @@ def parse(manifest_path: str, defaults: Dict[str, str]) -> List[Configuration]:
return []
# load configuration file
manifest_file = manifest.Manifest(manifest_path)
manifest_file = manifest.Manifest(str(manifest_path))
manifest_configurations = manifest_file.getAllConfiguration()
configurations = []

9
daemon/core/emane/emanemodel.py
View file

@ -2,7 +2,7 @@
Defines Emane Models used within CORE.
"""
import logging
import os
from pathlib import Path
from typing import Dict, List, Optional, Set
from core.config import ConfigGroup, Configuration
@ -53,7 +53,7 @@ class EmaneModel(WirelessModel):
config_ignore: Set[str] = set()
@classmethod
def load(cls, emane_prefix: str) -> None:
def load(cls, emane_prefix: Path) -> None:
"""
Called after being loaded within the EmaneManager. Provides configured emane_prefix for
parsing xml files.
@ -63,11 +63,10 @@ class EmaneModel(WirelessModel):
"""
manifest_path = "share/emane/manifest"
# load mac configuration
mac_xml_path = os.path.join(emane_prefix, manifest_path, cls.mac_xml)
mac_xml_path = emane_prefix / manifest_path / cls.mac_xml
cls.mac_config = emanemanifest.parse(mac_xml_path, cls.mac_defaults)
# load phy configuration
phy_xml_path = os.path.join(emane_prefix, manifest_path, cls.phy_xml)
phy_xml_path = emane_prefix / manifest_path / cls.phy_xml
cls.phy_config = emanemanifest.parse(phy_xml_path, cls.phy_defaults)
@classmethod

8
daemon/core/emane/ieee80211abg.py
View file

@ -1,7 +1,7 @@
"""
ieee80211abg.py: EMANE IEEE 802.11abg model for CORE
"""
import os
from pathlib import Path
from core.emane import emanemodel
@ -15,8 +15,8 @@ class EmaneIeee80211abgModel(emanemodel.EmaneModel):
mac_xml: str = "ieee80211abgmaclayer.xml"
@classmethod
def load(cls, emane_prefix: str) -> None:
cls.mac_defaults["pcrcurveuri"] = os.path.join(
emane_prefix, "share/emane/xml/models/mac/ieee80211abg/ieee80211pcr.xml"
def load(cls, emane_prefix: Path) -> None:
cls.mac_defaults["pcrcurveuri"] = str(
emane_prefix / "share/emane/xml/models/mac/ieee80211abg/ieee80211pcr.xml"
)
super().load(emane_prefix)

8
daemon/core/emane/rfpipe.py
View file

@ -1,7 +1,7 @@
"""
rfpipe.py: EMANE RF-PIPE model for CORE
"""
import os
from pathlib import Path
from core.emane import emanemodel
@ -15,8 +15,8 @@ class EmaneRfPipeModel(emanemodel.EmaneModel):
mac_xml: str = "rfpipemaclayer.xml"
@classmethod
def load(cls, emane_prefix: str) -> None:
cls.mac_defaults["pcrcurveuri"] = os.path.join(
emane_prefix, "share/emane/xml/models/mac/rfpipe/rfpipepcr.xml"
def load(cls, emane_prefix: Path) -> None:
cls.mac_defaults["pcrcurveuri"] = str(
emane_prefix / "share/emane/xml/models/mac/rfpipe/rfpipepcr.xml"
)
super().load(emane_prefix)

22
daemon/core/emane/tdma.py
View file

@ -3,7 +3,7 @@ tdma.py: EMANE TDMA model bindings for CORE
"""
import logging
import os
from pathlib import Path
from typing import Set
from core import constants, utils
@ -22,27 +22,25 @@ class EmaneTdmaModel(emanemodel.EmaneModel):
# add custom schedule options and ignore it when writing emane xml
schedule_name: str = "schedule"
default_schedule: str = os.path.join(
constants.CORE_DATA_DIR, "examples", "tdma", "schedule.xml"
default_schedule: Path = (
constants.CORE_DATA_DIR / "examples" / "tdma" / "schedule.xml"
)
config_ignore: Set[str] = {schedule_name}
@classmethod
def load(cls, emane_prefix: str) -> None:
cls.mac_defaults["pcrcurveuri"] = os.path.join(
emane_prefix,
"share/emane/xml/models/mac/tdmaeventscheduler/tdmabasemodelpcr.xml",
def load(cls, emane_prefix: Path) -> None:
cls.mac_defaults["pcrcurveuri"] = str(
emane_prefix
/ "share/emane/xml/models/mac/tdmaeventscheduler/tdmabasemodelpcr.xml"
)
super().load(emane_prefix)
cls.mac_config.insert(
0,
Configuration(
config_item = Configuration(
_id=cls.schedule_name,
_type=ConfigDataTypes.STRING,
default=cls.default_schedule,
default=str(cls.default_schedule),
label="TDMA schedule file (core)",
),
)
cls.mac_config.insert(0, config_item)
def post_startup(self) -> None:
"""

11
daemon/core/emulator/coreemu.py
View file

@ -3,6 +3,7 @@ import logging
import os
import signal
import sys
from pathlib import Path
from typing import Dict, List, Type
import core.services
@ -60,10 +61,11 @@ class CoreEmu:
# config services
self.service_manager: ConfigServiceManager = ConfigServiceManager()
config_services_path = os.path.abspath(os.path.dirname(configservices.__file__))
config_services_path = Path(configservices.__file__).resolve().parent
self.service_manager.load(config_services_path)
custom_dir = self.config.get("custom_config_services_dir")
if custom_dir:
if custom_dir is not None:
custom_dir = Path(custom_dir)
self.service_manager.load(custom_dir)
# check executables exist on path
@ -91,13 +93,12 @@ class CoreEmu:
"""
# load default services
self.service_errors = core.services.load()
# load custom services
service_paths = self.config.get("custom_services_dir")
logging.debug("custom service paths: %s", service_paths)
if service_paths:
if service_paths is not None:
for service_path in service_paths.split(","):
service_path = service_path.strip()
service_path = Path(service_path.strip())
custom_service_errors = ServiceManager.add_services(service_path)
self.service_errors.extend(custom_service_errors)

20
daemon/core/emulator/distributed.py
View file

@ -6,6 +6,7 @@ import logging
import os
import threading
from collections import OrderedDict
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Callable, Dict, Tuple
@ -79,23 +80,23 @@ class DistributedServer:
stdout, stderr = e.streams_for_display()
raise CoreCommandError(e.result.exited, cmd, stdout, stderr)
def remote_put(self, source: str, destination: str) -> None:
def remote_put(self, src_path: Path, dst_path: Path) -> None:
"""
Push file to remote server.
:param source: source file to push
:param destination: destination file location
:param src_path: source file to push
:param dst_path: destination file location
:return: nothing
"""
with self.lock:
self.conn.put(source, destination)
self.conn.put(str(src_path), str(dst_path))
def remote_put_temp(self, destination: str, data: str) -> None:
def remote_put_temp(self, dst_path: Path, data: str) -> None:
"""
Remote push file contents to a remote server, using a temp file as an
intermediate step.
:param destination: file destination for data
:param dst_path: file destination for data
:param data: data to store in remote file
:return: nothing
"""
@ -103,7 +104,7 @@ class DistributedServer:
temp = NamedTemporaryFile(delete=False)
temp.write(data.encode("utf-8"))
temp.close()
self.conn.put(temp.name, destination)
self.conn.put(temp.name, str(dst_path))
os.unlink(temp.name)
@ -170,13 +171,11 @@ class DistributedController:
tunnels = self.tunnels[key]
for tunnel in tunnels:
tunnel.shutdown()
# remove all remote session directories
for name in self.servers:
server = self.servers[name]
cmd = f"rm -rf {self.session.session_dir}"
server.remote_cmd(cmd)
# clear tunnels
self.tunnels.clear()
@ -212,14 +211,12 @@ class DistributedController:
tunnel = self.tunnels.get(key)
if tunnel is not None:
return tunnel
# local to server
logging.info(
"local tunnel node(%s) to remote(%s) key(%s)", node.name, host, key
)
local_tap = GreTap(session=self.session, remoteip=host, key=key)
local_tap.net_client.set_iface_master(node.brname, local_tap.localname)
# server to local
logging.info(
"remote tunnel node(%s) to local(%s) key(%s)", node.name, self.address, key
@ -228,7 +225,6 @@ class DistributedController:
session=self.session, remoteip=self.address, key=key, server=server
)
remote_tap.net_client.set_iface_master(node.brname, remote_tap.localname)
# save tunnels for shutdown
tunnel = (local_tap, remote_tap)
self.tunnels[key] = tunnel

95
daemon/core/emulator/session.py
View file

@ -103,13 +103,13 @@ class Session:
self.id: int = _id
# define and create session directory when desired
self.session_dir: str = os.path.join(tempfile.gettempdir(), f"pycore.{self.id}")
self.session_dir: Path = Path(tempfile.gettempdir()) / f"pycore.{self.id}"
if mkdir:
os.mkdir(self.session_dir)
self.session_dir.mkdir()
self.name: Optional[str] = None
self.file_name: Optional[str] = None
self.thumbnail: Optional[str] = None
self.file_path: Optional[Path] = None
self.thumbnail: Optional[Path] = None
self.user: Optional[str] = None
self.event_loop: EventLoop = EventLoop()
self.link_colors: Dict[int, str] = {}
@ -641,42 +641,39 @@ class Session:
logging.info("session(%s) checking if active: %s", self.id, result)
return result
def open_xml(self, file_name: str, start: bool = False) -> None:
def open_xml(self, file_path: Path, start: bool = False) -> None:
"""
Import a session from the EmulationScript XML format.
:param file_name: xml file to load session from
:param file_path: xml file to load session from
:param start: instantiate session if true, false otherwise
:return: nothing
"""
logging.info("opening xml: %s", file_name)
logging.info("opening xml: %s", file_path)
# clear out existing session
self.clear()
# set state and read xml
state = EventTypes.CONFIGURATION_STATE if start else EventTypes.DEFINITION_STATE
self.set_state(state)
self.name = os.path.basename(file_name)
self.file_name = file_name
CoreXmlReader(self).read(file_name)
self.name = file_path.name
self.file_path = file_path
CoreXmlReader(self).read(file_path)
# start session if needed
if start:
self.set_state(EventTypes.INSTANTIATION_STATE)
self.instantiate()
def save_xml(self, file_name: str) -> None:
def save_xml(self, file_path: Path) -> None:
"""
Export a session to the EmulationScript XML format.
:param file_name: file name to write session xml to
:param file_path: file name to write session xml to
:return: nothing
"""
CoreXmlWriter(self).write(file_name)
CoreXmlWriter(self).write(file_path)
def add_hook(
self, state: EventTypes, file_name: str, data: str, source_name: str = None
self, state: EventTypes, file_name: str, data: str, src_name: str = None
) -> None:
"""
Store a hook from a received file message.
@ -684,11 +681,11 @@ class Session:
:param state: when to run hook
:param file_name: file name for hook
:param data: hook data
:param source_name: source name
:param src_name: source name
:return: nothing
"""
logging.info(
"setting state hook: %s - %s source(%s)", state, file_name, source_name
"setting state hook: %s - %s source(%s)", state, file_name, src_name
)
hook = file_name, data
state_hooks = self.hooks.setdefault(state, [])
@ -700,22 +697,22 @@ class Session:
self.run_hook(hook)
def add_node_file(
self, node_id: int, source_name: str, file_name: str, data: str
self, node_id: int, src_path: Path, file_path: Path, data: str
) -> None:
"""
Add a file to a node.
:param node_id: node to add file to
:param source_name: source file name
:param file_name: file name to add
:param src_path: source file path
:param file_path: file path to add
:param data: file data
:return: nothing
"""
node = self.get_node(node_id, CoreNodeBase)
if source_name is not None:
node.addfile(source_name, file_name)
if src_path is not None:
node.addfile(src_path, file_path)
elif data is not None:
node.nodefile(file_name, data)
node.nodefile(file_path, data)
def clear(self) -> None:
"""
@ -879,9 +876,9 @@ class Session:
:param state: state to write to file
:return: nothing
"""
state_file = os.path.join(self.session_dir, "state")
state_file = self.session_dir / "state"
try:
with open(state_file, "w") as f:
with state_file.open("w") as f:
f.write(f"{state.value} {state.name}\n")
except IOError:
logging.exception("error writing state file: %s", state.name)
@ -907,12 +904,12 @@ class Session:
"""
file_name, data = hook
logging.info("running hook %s", file_name)
file_path = os.path.join(self.session_dir, file_name)
log_path = os.path.join(self.session_dir, f"{file_name}.log")
file_path = self.session_dir / file_name
log_path = self.session_dir / f"{file_name}.log"
try:
with open(file_path, "w") as f:
with file_path.open("w") as f:
f.write(data)
with open(log_path, "w") as f:
with log_path.open("w") as f:
args = ["/bin/sh", file_name]
subprocess.check_call(
args,
@ -983,10 +980,10 @@ class Session:
"""
self.emane.poststartup()
# create session deployed xml
xml_file_name = os.path.join(self.session_dir, "session-deployed.xml")
xml_writer = corexml.CoreXmlWriter(self)
corexmldeployment.CoreXmlDeployment(self, xml_writer.scenario)
xml_writer.write(xml_file_name)
xml_file_path = self.session_dir / "session-deployed.xml"
xml_writer.write(xml_file_path)
def get_environment(self, state: bool = True) -> Dict[str, str]:
"""
@ -1001,9 +998,9 @@ class Session:
env["CORE_PYTHON"] = sys.executable
env["SESSION"] = str(self.id)
env["SESSION_SHORT"] = self.short_session_id()
env["SESSION_DIR"] = self.session_dir
env["SESSION_DIR"] = str(self.session_dir)
env["SESSION_NAME"] = str(self.name)
env["SESSION_FILENAME"] = str(self.file_name)
env["SESSION_FILENAME"] = str(self.file_path)
env["SESSION_USER"] = str(self.user)
if state:
env["SESSION_STATE"] = str(self.state)
@ -1011,8 +1008,8 @@ class Session:
# /etc/core/environment
# /home/user/.core/environment
# /tmp/pycore.<session id>/environment
core_env_path = Path(constants.CORE_CONF_DIR) / "environment"
session_env_path = Path(self.session_dir) / "environment"
core_env_path = constants.CORE_CONF_DIR / "environment"
session_env_path = self.session_dir / "environment"
if self.user:
user_home_path = Path(f"~{self.user}").expanduser()
user_env1 = user_home_path / ".core" / "environment"
@ -1028,20 +1025,20 @@ class Session:
logging.exception("error reading environment file: %s", path)
return env
def set_thumbnail(self, thumb_file: str) -> None:
def set_thumbnail(self, thumb_file: Path) -> None:
"""
Set the thumbnail filename. Move files from /tmp to session dir.
:param thumb_file: tumbnail file to set for session
:return: nothing
"""
if not os.path.exists(thumb_file):
if not thumb_file.is_file():
logging.error("thumbnail file to set does not exist: %s", thumb_file)
self.thumbnail = None
return
destination_file = os.path.join(self.session_dir, os.path.basename(thumb_file))
shutil.copy(thumb_file, destination_file)
self.thumbnail = destination_file
dst_path = self.session_dir / thumb_file.name
shutil.copy(thumb_file, dst_path)
self.thumbnail = dst_path
def set_user(self, user: str) -> None:
"""
@ -1054,7 +1051,7 @@ class Session:
if user:
try:
uid = pwd.getpwnam(user).pw_uid
gid = os.stat(self.session_dir).st_gid
gid = self.session_dir.stat().st_gid
os.chown(self.session_dir, uid, gid)
except IOError:
logging.exception("failed to set permission on %s", self.session_dir)
@ -1140,10 +1137,10 @@ class Session:
Write nodes to a 'nodes' file in the session dir.
The 'nodes' file lists: number, name, api-type, class-type
"""
file_path = os.path.join(self.session_dir, "nodes")
file_path = self.session_dir / "nodes"
try:
with self.nodes_lock:
with open(file_path, "w") as f:
with file_path.open("w") as f:
for _id, node in self.nodes.items():
f.write(f"{_id} {node.name} {node.apitype} {type(node)}\n")
except IOError:
@ -1268,13 +1265,11 @@ class Session:
# stop event loop
self.event_loop.stop()
# stop node services
# stop mobility and node services
with self.nodes_lock:
funcs = []
for node_id in self.nodes:
node = self.nodes[node_id]
if not isinstance(node, CoreNodeBase) or not node.up:
continue
for node in self.nodes.values():
if isinstance(node, CoreNodeBase) and node.up:
args = (node,)
funcs.append((self.services.stop_services, args, {}))
utils.threadpool(funcs)

13
daemon/core/gui/dialogs/serviceconfig.py
View file

@ -1,6 +1,6 @@
import logging
import os
import tkinter as tk
from pathlib import Path
from tkinter import filedialog, ttk
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
@ -579,11 +579,12 @@ class ServiceConfigDialog(Dialog):
self.directory_entry.insert("end", d)
def add_directory(self) -> None:
d = self.directory_entry.get()
if os.path.isdir(d):
if d not in self.temp_directories:
self.dir_list.listbox.insert("end", d)
self.temp_directories.append(d)
directory = self.directory_entry.get()
directory = Path(directory)
if directory.is_dir():
if str(directory) not in self.temp_directories:
self.dir_list.listbox.insert("end", directory)
self.temp_directories.append(directory)
def remove_directory(self) -> None:
d = self.directory_entry.get()

29
daemon/core/gui/menubar.py
View file

@ -1,8 +1,8 @@
import logging
import os
import tkinter as tk
import webbrowser
from functools import partial
from pathlib import Path
from tkinter import filedialog, messagebox
from typing import TYPE_CHECKING, Optional
@ -272,12 +272,12 @@ class Menubar(tk.Menu):
menu.add_command(label="About", command=self.click_about)
self.add_cascade(label="Help", menu=menu)
def open_recent_files(self, filename: str) -> None:
if os.path.isfile(filename):
logging.debug("Open recent file %s", filename)
self.open_xml_task(filename)
def open_recent_files(self, file_path: Path) -> None:
if file_path.is_file():
logging.debug("Open recent file %s", file_path)
self.open_xml_task(file_path)
else:
logging.warning("File does not exist %s", filename)
logging.warning("File does not exist %s", file_path)
def update_recent_files(self) -> None:
self.recent_menu.delete(0, tk.END)
@ -286,7 +286,7 @@ class Menubar(tk.Menu):
label=i, command=partial(self.open_recent_files, i)
)
def click_save(self, _event=None) -> None:
def click_save(self, _event: tk.Event = None) -> None:
if self.core.session.file:
self.core.save_xml()
else:
@ -314,7 +314,7 @@ class Menubar(tk.Menu):
if file_path:
self.open_xml_task(file_path)
def open_xml_task(self, file_path: str) -> None:
def open_xml_task(self, file_path: Path) -> None:
self.add_recent_file_to_gui_config(file_path)
self.prompt_save_running_session()
task = ProgressTask(self.app, "Open XML", self.core.open_xml, args=(file_path,))
@ -324,21 +324,14 @@ class Menubar(tk.Menu):
dialog = ExecutePythonDialog(self.app)
dialog.show()
def add_recent_file_to_gui_config(self, file_path) -> None:
def add_recent_file_to_gui_config(self, file_path: Path) -> None:
recent_files = self.app.guiconfig.recentfiles
num_files = len(recent_files)
if num_files == 0:
recent_files.insert(0, file_path)
elif 0 < num_files <= MAX_FILES:
file_path = str(file_path)
if file_path in recent_files:
recent_files.remove(file_path)
recent_files.insert(0, file_path)
else:
if num_files == MAX_FILES:
if len(recent_files) > MAX_FILES:
recent_files.pop()
recent_files.insert(0, file_path)
else:
logging.error("unexpected number of recent files")
self.app.save_config()
self.app.menubar.update_recent_files()

48
daemon/core/location/mobility.py
View file

@ -419,7 +419,7 @@ class BasicRangeModel(WirelessModel):
self.wlan.link(a, b)
self.sendlinkmsg(a, b)
except KeyError:
logging.exception("error getting interfaces during calclinkS")
logging.exception("error getting interfaces during calclink")
@staticmethod
def calcdistance(
@ -920,7 +920,7 @@ class Ns2ScriptedMobility(WayPointMobility):
:param _id: object id
"""
super().__init__(session, _id)
self.file: Optional[str] = None
self.file: Optional[Path] = None
self.autostart: Optional[str] = None
self.nodemap: Dict[int, int] = {}
self.script_start: Optional[str] = None
@ -928,7 +928,7 @@ class Ns2ScriptedMobility(WayPointMobility):
self.script_stop: Optional[str] = None
def update_config(self, config: Dict[str, str]) -> None:
self.file = config["file"]
self.file = Path(config["file"])
logging.info(
"ns-2 scripted mobility configured for WLAN %d using file: %s",
self.id,
@ -953,15 +953,15 @@ class Ns2ScriptedMobility(WayPointMobility):
:return: nothing
"""
filename = self.findfile(self.file)
file_path = self.findfile(self.file)
try:
f = open(filename, "r")
f = file_path.open("r")
except IOError:
logging.exception(
"ns-2 scripted mobility failed to load file: %s", self.file
)
return
logging.info("reading ns-2 script file: %s", filename)
logging.info("reading ns-2 script file: %s", file_path)
ln = 0
ix = iy = iz = None
inodenum = None
@ -977,13 +977,13 @@ class Ns2ScriptedMobility(WayPointMobility):
# waypoints:
# $ns_ at 1.00 "$node_(6) setdest 500.0 178.0 25.0"
parts = line.split()
time = float(parts[2])
line_time = float(parts[2])
nodenum = parts[3][1 + parts[3].index("(") : parts[3].index(")")]
x = float(parts[5])
y = float(parts[6])
z = None
speed = float(parts[7].strip('"'))
self.addwaypoint(time, self.map(nodenum), x, y, z, speed)
self.addwaypoint(line_time, self.map(nodenum), x, y, z, speed)
elif line[:7] == "$node_(":
# initial position (time=0, speed=0):
# $node_(6) set X_ 780.0
@ -1011,31 +1011,31 @@ class Ns2ScriptedMobility(WayPointMobility):
if ix is not None and iy is not None:
self.addinitial(self.map(inodenum), ix, iy, iz)
def findfile(self, file_name: str) -> str:
def findfile(self, file_path: Path) -> Path:
"""
Locate a script file. If the specified file doesn't exist, look in the
same directory as the scenario file, or in gui directories.
:param file_name: file name to find
:param file_path: file name to find
:return: absolute path to the file
:raises CoreError: when file is not found
"""
file_path = Path(file_name).expanduser()
file_path = file_path.expanduser()
if file_path.exists():
return str(file_path)
if self.session.file_name:
file_path = Path(self.session.file_name).parent / file_name
if file_path.exists():
return str(file_path)
return file_path
if self.session.file_path:
session_file_path = self.session.file_path.parent / file_path
if session_file_path.exists():
return session_file_path
if self.session.user:
user_path = Path(f"~{self.session.user}").expanduser()
file_path = user_path / ".core" / "configs" / file_name
if file_path.exists():
return str(file_path)
file_path = user_path / ".coregui" / "mobility" / file_name
if file_path.exists():
return str(file_path)
raise CoreError(f"invalid file: {file_name}")
configs_path = user_path / ".core" / "configs" / file_path
if configs_path.exists():
return configs_path
mobility_path = user_path / ".coregui" / "mobility" / file_path
if mobility_path.exists():
return mobility_path
raise CoreError(f"invalid file: {file_path}")
def parsemap(self, mapstr: str) -> None:
"""
@ -1047,7 +1047,6 @@ class Ns2ScriptedMobility(WayPointMobility):
self.nodemap = {}
if mapstr.strip() == "":
return
for pair in mapstr.split(","):
parts = pair.split(":")
try:
@ -1152,6 +1151,7 @@ class Ns2ScriptedMobility(WayPointMobility):
filename = self.script_stop
if filename is None or filename == "":
return
filename = Path(filename)
filename = self.findfile(filename)
args = f"{BASH} {filename} {typestr}"
utils.cmd(

161
daemon/core/nodes/base.py
View file

@ -3,9 +3,9 @@ Defines the base logic for nodes used within core.
"""
import abc
import logging
import os
import shutil
import threading
from pathlib import Path
from threading import RLock
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type, Union
@ -30,6 +30,8 @@ if TYPE_CHECKING:
CoreServices = List[Union[CoreService, Type[CoreService]]]
ConfigServiceType = Type[ConfigService]
PRIVATE_DIRS: List[Path] = [Path("/var/run"), Path("/var/log")]
class NodeBase(abc.ABC):
"""
@ -97,7 +99,7 @@ class NodeBase(abc.ABC):
self,
args: str,
env: Dict[str, str] = None,
cwd: str = None,
cwd: Path = None,
wait: bool = True,
shell: bool = False,
) -> str:
@ -221,7 +223,7 @@ class CoreNodeBase(NodeBase):
"""
super().__init__(session, _id, name, server)
self.config_services: Dict[str, "ConfigService"] = {}
self.nodedir: Optional[str] = None
self.nodedir: Optional[Path] = None
self.tmpnodedir: bool = False
@abc.abstractmethod
@ -233,11 +235,11 @@ class CoreNodeBase(NodeBase):
raise NotImplementedError
@abc.abstractmethod
def nodefile(self, filename: str, contents: str, mode: int = 0o644) -> None:
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create a node file with a given mode.
:param filename: name of file to create
:param file_path: name of file to create
:param contents: contents of file
:param mode: mode for file
:return: nothing
@ -245,12 +247,12 @@ class CoreNodeBase(NodeBase):
raise NotImplementedError
@abc.abstractmethod
def addfile(self, srcname: str, filename: str) -> None:
def addfile(self, src_path: Path, file_path: Path) -> None:
"""
Add a file.
:param srcname: source file name
:param filename: file name to add
:param src_path: source file path
:param file_path: file name to add
:return: nothing
:raises CoreCommandError: when a non-zero exit status occurs
"""
@ -302,6 +304,21 @@ class CoreNodeBase(NodeBase):
"""
raise NotImplementedError
def host_path(self, path: Path, is_dir: bool = False) -> Path:
"""
Return the name of a node"s file on the host filesystem.
:param path: path to translate to host path
:param is_dir: True if path is a directory path, False otherwise
:return: path to file
"""
if is_dir:
directory = str(path).strip("/").replace("/", ".")
return self.nodedir / directory
else:
directory = str(path.parent).strip("/").replace("/", ".")
return self.nodedir / directory / path.name
def add_config_service(self, service_class: "ConfigServiceType") -> None:
"""
Adds a configuration service to the node.
@ -346,7 +363,7 @@ class CoreNodeBase(NodeBase):
:return: nothing
"""
if self.nodedir is None:
self.nodedir = os.path.join(self.session.session_dir, self.name + ".conf")
self.nodedir = self.session.session_dir / f"{self.name}.conf"
self.host_cmd(f"mkdir -p {self.nodedir}")
self.tmpnodedir = True
else:
@ -458,7 +475,7 @@ class CoreNode(CoreNodeBase):
session: "Session",
_id: int = None,
name: str = None,
nodedir: str = None,
nodedir: Path = None,
server: "DistributedServer" = None,
) -> None:
"""
@ -472,14 +489,12 @@ class CoreNode(CoreNodeBase):
will run on, default is None for localhost
"""
super().__init__(session, _id, name, server)
self.nodedir: Optional[str] = nodedir
self.ctrlchnlname: str = os.path.abspath(
os.path.join(self.session.session_dir, self.name)
)
self.nodedir: Optional[Path] = nodedir
self.ctrlchnlname: Path = self.session.session_dir / self.name
self.client: Optional[VnodeClient] = None
self.pid: Optional[int] = None
self.lock: RLock = RLock()
self._mounts: List[Tuple[str, str]] = []
self._mounts: List[Tuple[Path, Path]] = []
self.node_net_client: LinuxNetClient = self.create_node_net_client(
self.session.use_ovs()
)
@ -549,8 +564,8 @@ class CoreNode(CoreNodeBase):
self.up = True
# create private directories
self.privatedir("/var/run")
self.privatedir("/var/log")
for dir_path in PRIVATE_DIRS:
self.privatedir(dir_path)
def shutdown(self) -> None:
"""
@ -561,29 +576,24 @@ class CoreNode(CoreNodeBase):
# nothing to do if node is not up
if not self.up:
return
with self.lock:
try:
# unmount all targets (NOTE: non-persistent mount namespaces are
# removed by the kernel when last referencing process is killed)
self._mounts = []
# shutdown all interfaces
for iface in self.get_ifaces():
iface.shutdown()
# kill node process if present
try:
self.host_cmd(f"kill -9 {self.pid}")
except CoreCommandError:
logging.exception("error killing process")
# remove node directory if present
try:
self.host_cmd(f"rm -rf {self.ctrlchnlname}")
except CoreCommandError:
logging.exception("error removing node directory")
# clear interface data, close client, and mark self and not up
self.ifaces.clear()
self.client.close()
@ -636,35 +646,32 @@ class CoreNode(CoreNodeBase):
else:
return f"ssh -X -f {self.server.host} xterm -e {terminal}"
def privatedir(self, path: str) -> None:
def privatedir(self, dir_path: Path) -> None:
"""
Create a private directory.
:param path: path to create
:param dir_path: path to create
:return: nothing
"""
if path[0] != "/":
raise ValueError(f"path not fully qualified: {path}")
hostpath = os.path.join(
self.nodedir, os.path.normpath(path).strip("/").replace("/", ".")
)
self.host_cmd(f"mkdir -p {hostpath}")
self.mount(hostpath, path)
if not str(dir_path).startswith("/"):
raise CoreError(f"private directory path not fully qualified: {dir_path}")
host_path = self.host_path(dir_path, is_dir=True)
self.host_cmd(f"mkdir -p {host_path}")
self.mount(host_path, dir_path)
def mount(self, source: str, target: str) -> None:
def mount(self, src_path: Path, target_path: Path) -> None:
"""
Create and mount a directory.
:param source: source directory to mount
:param target: target directory to create
:param src_path: source directory to mount
:param target_path: target directory to create
:return: nothing
:raises CoreCommandError: when a non-zero exit status occurs
"""
source = os.path.abspath(source)
logging.debug("node(%s) mounting: %s at %s", self.name, source, target)
self.cmd(f"mkdir -p {target}")
self.cmd(f"{MOUNT} -n --bind {source} {target}")
self._mounts.append((source, target))
logging.debug("node(%s) mounting: %s at %s", self.name, src_path, target_path)
self.cmd(f"mkdir -p {target_path}")
self.cmd(f"{MOUNT} -n --bind {src_path} {target_path}")
self._mounts.append((src_path, target_path))
def next_iface_id(self) -> int:
"""
@ -851,86 +858,66 @@ class CoreNode(CoreNodeBase):
self.ifup(iface_id)
return self.get_iface(iface_id)
def addfile(self, srcname: str, filename: str) -> None:
def addfile(self, src_path: Path, file_path: Path) -> None:
"""
Add a file.
:param srcname: source file name
:param filename: file name to add
:param src_path: source file path
:param file_path: file name to add
:return: nothing
:raises CoreCommandError: when a non-zero exit status occurs
"""
logging.info("adding file from %s to %s", srcname, filename)
directory = os.path.dirname(filename)
logging.info("adding file from %s to %s", src_path, file_path)
directory = file_path.parent
if self.server is None:
self.client.check_cmd(f"mkdir -p {directory}")
self.client.check_cmd(f"mv {srcname} {filename}")
self.client.check_cmd(f"mv {src_path} {file_path}")
self.client.check_cmd("sync")
else:
self.host_cmd(f"mkdir -p {directory}")
self.server.remote_put(srcname, filename)
self.server.remote_put(src_path, file_path)
def hostfilename(self, filename: str) -> str:
"""
Return the name of a node"s file on the host filesystem.
:param filename: host file name
:return: path to file
"""
dirname, basename = os.path.split(filename)
if not basename:
raise ValueError(f"no basename for filename: {filename}")
if dirname and dirname[0] == "/":
dirname = dirname[1:]
dirname = dirname.replace("/", ".")
dirname = os.path.join(self.nodedir, dirname)
return os.path.join(dirname, basename)
def nodefile(self, filename: str, contents: str, mode: int = 0o644) -> None:
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create a node file with a given mode.
:param filename: name of file to create
:param file_path: name of file to create
:param contents: contents of file
:param mode: mode for file
:return: nothing
"""
hostfilename = self.hostfilename(filename)
dirname, _basename = os.path.split(hostfilename)
host_path = self.host_path(file_path)
directory = host_path.parent
if self.server is None:
if not os.path.isdir(dirname):
os.makedirs(dirname, mode=0o755)
with open(hostfilename, "w") as open_file:
open_file.write(contents)
os.chmod(open_file.name, mode)
if not directory.exists():
directory.mkdir(parents=True, mode=0o755)
with host_path.open("w") as f:
f.write(contents)
host_path.chmod(mode)
else:
self.host_cmd(f"mkdir -m {0o755:o} -p {dirname}")
self.server.remote_put_temp(hostfilename, contents)
self.host_cmd(f"chmod {mode:o} {hostfilename}")
logging.debug(
"node(%s) added file: %s; mode: 0%o", self.name, hostfilename, mode
)
self.host_cmd(f"mkdir -m {0o755:o} -p {directory}")
self.server.remote_put_temp(host_path, contents)
self.host_cmd(f"chmod {mode:o} {host_path}")
logging.debug("node(%s) added file: %s; mode: 0%o", self.name, host_path, mode)
def nodefilecopy(self, filename: str, srcfilename: str, mode: int = None) -> None:
def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
"""
Copy a file to a node, following symlinks and preserving metadata.
Change file mode if specified.
:param filename: file name to copy file to
:param srcfilename: file to copy
:param file_path: file name to copy file to
:param src_path: file to copy
:param mode: mode to copy to
:return: nothing
"""
hostfilename = self.hostfilename(filename)
host_path = self.host_path(file_path)
if self.server is None:
shutil.copy2(srcfilename, hostfilename)
shutil.copy2(src_path, host_path)
else:
self.server.remote_put(srcfilename, hostfilename)
self.server.remote_put(src_path, host_path)
if mode is not None:
self.host_cmd(f"chmod {mode:o} {hostfilename}")
logging.info(
"node(%s) copied file: %s; mode: %s", self.name, hostfilename, mode
)
self.host_cmd(f"chmod {mode:o} {host_path}")
logging.info("node(%s) copied file: %s; mode: %s", self.name, host_path, mode)
class CoreNetworkBase(NodeBase):

5
daemon/core/nodes/client.py
View file

@ -3,6 +3,7 @@ client.py: implementation of the VnodeClient class for issuing commands
over a control channel to the vnoded process running in a network namespace.
The control channel can be accessed via calls using the vcmd shell.
"""
from pathlib import Path
from core import utils
from core.executables import BASH, VCMD
@ -13,7 +14,7 @@ class VnodeClient:
Provides client functionality for interacting with a virtual node.
"""
def __init__(self, name: str, ctrlchnlname: str) -> None:
def __init__(self, name: str, ctrlchnlname: Path) -> None:
"""
Create a VnodeClient instance.
@ -21,7 +22,7 @@ class VnodeClient:
:param ctrlchnlname: control channel name
"""
self.name: str = name
self.ctrlchnlname: str = ctrlchnlname
self.ctrlchnlname: Path = ctrlchnlname
def _verify_connection(self) -> None:
"""

72
daemon/core/nodes/docker.py
View file

@ -1,6 +1,6 @@
import json
import logging
import os
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Callable, Dict, Optional
@ -63,8 +63,8 @@ class DockerClient:
logging.debug("node(%s) pid: %s", self.name, self.pid)
return output
def copy_file(self, source: str, destination: str) -> str:
args = f"docker cp {source} {self.name}:{destination}"
def copy_file(self, src_path: Path, dst_path: Path) -> str:
args = f"docker cp {src_path} {self.name}:{dst_path}"
return self.run(args)
@ -162,77 +162,73 @@ class DockerNode(CoreNode):
"""
return f"docker exec -it {self.name} bash"
def privatedir(self, path: str) -> None:
def privatedir(self, dir_path: str) -> None:
"""
Create a private directory.
:param path: path to create
:param dir_path: path to create
:return: nothing
"""
logging.debug("creating node dir: %s", path)
args = f"mkdir -p {path}"
logging.debug("creating node dir: %s", dir_path)
args = f"mkdir -p {dir_path}"
self.cmd(args)
def mount(self, source: str, target: str) -> None:
def mount(self, src_path: str, target_path: str) -> None:
"""
Create and mount a directory.
:param source: source directory to mount
:param target: target directory to create
:param src_path: source directory to mount
:param target_path: target directory to create
:return: nothing
:raises CoreCommandError: when a non-zero exit status occurs
"""
logging.debug("mounting source(%s) target(%s)", source, target)
logging.debug("mounting source(%s) target(%s)", src_path, target_path)
raise Exception("not supported")
def nodefile(self, filename: str, contents: str, mode: int = 0o644) -> None:
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create a node file with a given mode.
:param filename: name of file to create
:param file_path: name of file to create
:param contents: contents of file
:param mode: mode for file
:return: nothing
"""
logging.debug("nodefile filename(%s) mode(%s)", filename, mode)
directory = os.path.dirname(filename)
logging.debug("nodefile filename(%s) mode(%s)", file_path, mode)
temp = NamedTemporaryFile(delete=False)
temp.write(contents.encode("utf-8"))
temp.close()
if directory:
temp_path = Path(temp.name)
directory = file_path.name
if str(directory) != ".":
self.cmd(f"mkdir -m {0o755:o} -p {directory}")
if self.server is not None:
self.server.remote_put(temp.name, temp.name)
self.client.copy_file(temp.name, filename)
self.cmd(f"chmod {mode:o} {filename}")
self.server.remote_put(temp_path, temp_path)
self.client.copy_file(temp_path, file_path)
self.cmd(f"chmod {mode:o} {file_path}")
if self.server is not None:
self.host_cmd(f"rm -f {temp.name}")
os.unlink(temp.name)
logging.debug("node(%s) added file: %s; mode: 0%o", self.name, filename, mode)
self.host_cmd(f"rm -f {temp_path}")
temp_path.unlink()
logging.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode)
def nodefilecopy(self, filename: str, srcfilename: str, mode: int = None) -> None:
def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
"""
Copy a file to a node, following symlinks and preserving metadata.
Change file mode if specified.
:param filename: file name to copy file to
:param srcfilename: file to copy
:param file_path: file name to copy file to
:param src_path: file to copy
:param mode: mode to copy to
:return: nothing
"""
logging.info(
"node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode
"node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode
)
directory = os.path.dirname(filename)
self.cmd(f"mkdir -p {directory}")
if self.server is None:
source = srcfilename
else:
self.cmd(f"mkdir -p {file_path.parent}")
if self.server:
temp = NamedTemporaryFile(delete=False)
source = temp.name
self.server.remote_put(source, temp.name)
self.client.copy_file(source, filename)
self.cmd(f"chmod {mode:o} {filename}")
temp_path = Path(temp.name)
src_path = temp_path
self.server.remote_put(src_path, temp_path)
self.client.copy_file(src_path, file_path)
self.cmd(f"chmod {mode:o} {file_path}")

3
daemon/core/nodes/interface.py
View file

@ -4,6 +4,7 @@ virtual ethernet classes that implement the interfaces available under Linux.
import logging
import time
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
import netaddr
@ -79,7 +80,7 @@ class CoreInterface:
self,
args: str,
env: Dict[str, str] = None,
cwd: str = None,
cwd: Path = None,
wait: bool = True,
shell: bool = False,
) -> str:

78
daemon/core/nodes/lxd.py
View file

@ -1,7 +1,7 @@
import json
import logging
import os
import time
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Callable, Dict, Optional
@ -57,11 +57,10 @@ class LxdClient:
args = self.create_cmd(cmd)
return utils.cmd(args, wait=wait, shell=shell)
def copy_file(self, source: str, destination: str) -> None:
if destination[0] != "/":
destination = os.path.join("/root/", destination)
args = f"lxc file push {source} {self.name}/{destination}"
def copy_file(self, src_path: Path, dst_path: Path) -> None:
if not str(dst_path).startswith("/"):
dst_path = Path("/root/") / dst_path
args = f"lxc file push {src_path} {self.name}/{dst_path}"
self.run(args)
@ -139,81 +138,76 @@ class LxcNode(CoreNode):
"""
return f"lxc exec {self.name} -- {sh}"
def privatedir(self, path: str) -> None:
def privatedir(self, dir_path: Path) -> None:
"""
Create a private directory.
:param path: path to create
:param dir_path: path to create
:return: nothing
"""
logging.info("creating node dir: %s", path)
args = f"mkdir -p {path}"
logging.info("creating node dir: %s", dir_path)
args = f"mkdir -p {dir_path}"
self.cmd(args)
def mount(self, source: str, target: str) -> None:
def mount(self, src_path: Path, target_path: Path) -> None:
"""
Create and mount a directory.
:param source: source directory to mount
:param target: target directory to create
:param src_path: source directory to mount
:param target_path: target directory to create
:return: nothing
:raises CoreCommandError: when a non-zero exit status occurs
"""
logging.debug("mounting source(%s) target(%s)", source, target)
logging.debug("mounting source(%s) target(%s)", src_path, target_path)
raise Exception("not supported")
def nodefile(self, filename: str, contents: str, mode: int = 0o644) -> None:
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create a node file with a given mode.
:param filename: name of file to create
:param file_path: name of file to create
:param contents: contents of file
:param mode: mode for file
:return: nothing
"""
logging.debug("nodefile filename(%s) mode(%s)", filename, mode)
directory = os.path.dirname(filename)
logging.debug("nodefile filename(%s) mode(%s)", file_path, mode)
temp = NamedTemporaryFile(delete=False)
temp.write(contents.encode("utf-8"))
temp.close()
if directory:
temp_path = Path(temp.name)
directory = file_path.parent
if str(directory) != ".":
self.cmd(f"mkdir -m {0o755:o} -p {directory}")
if self.server is not None:
self.server.remote_put(temp.name, temp.name)
self.client.copy_file(temp.name, filename)
self.cmd(f"chmod {mode:o} {filename}")
self.server.remote_put(temp_path, temp_path)
self.client.copy_file(temp_path, file_path)
self.cmd(f"chmod {mode:o} {file_path}")
if self.server is not None:
self.host_cmd(f"rm -f {temp.name}")
os.unlink(temp.name)
logging.debug("node(%s) added file: %s; mode: 0%o", self.name, filename, mode)
self.host_cmd(f"rm -f {temp_path}")
temp_path.unlink()
logging.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode)
def nodefilecopy(self, filename: str, srcfilename: str, mode: int = None) -> None:
def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
"""
Copy a file to a node, following symlinks and preserving metadata.
Change file mode if specified.
:param filename: file name to copy file to
:param srcfilename: file to copy
:param file_path: file name to copy file to
:param src_path: file to copy
:param mode: mode to copy to
:return: nothing
"""
logging.info(
"node file copy file(%s) source(%s) mode(%s)", filename, srcfilename, mode
"node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode
)
directory = os.path.dirname(filename)
self.cmd(f"mkdir -p {directory}")
if self.server is None:
source = srcfilename
else:
self.cmd(f"mkdir -p {file_path.parent}")
if self.server:
temp = NamedTemporaryFile(delete=False)
source = temp.name
self.server.remote_put(source, temp.name)
self.client.copy_file(source, filename)
self.cmd(f"chmod {mode:o} {filename}")
temp_path = Path(temp.name)
src_path = temp_path
self.server.remote_put(src_path, temp_path)
self.client.copy_file(src_path, file_path)
self.cmd(f"chmod {mode:o} {file_path}")
def add_iface(self, iface: CoreInterface, iface_id: int) -> None:
super().add_iface(iface, iface_id)

10
daemon/core/nodes/network.py
View file

@ -6,6 +6,7 @@ import logging
import math
import threading
import time
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type
import netaddr
@ -292,7 +293,7 @@ class CoreNetwork(CoreNetworkBase):
self,
args: str,
env: Dict[str, str] = None,
cwd: str = None,
cwd: Path = None,
wait: bool = True,
shell: bool = False,
) -> str:
@ -333,9 +334,7 @@ class CoreNetwork(CoreNetworkBase):
"""
if not self.up:
return
ebq.stopupdateloop(self)
try:
self.net_client.delete_bridge(self.brname)
if self.has_ebtables_chain:
@ -346,11 +345,9 @@ class CoreNetwork(CoreNetworkBase):
ebtablescmds(self.host_cmd, cmds)
except CoreCommandError:
logging.exception("error during shutdown")
# removes veth pairs used for bridge-to-bridge connections
for iface in self.get_ifaces():
iface.shutdown()
self.ifaces.clear()
self._linked.clear()
del self.session
@ -389,10 +386,8 @@ class CoreNetwork(CoreNetworkBase):
# check if the network interfaces are attached to this network
if self.ifaces[iface1.net_id] != iface1:
raise ValueError(f"inconsistency for interface {iface1.name}")
if self.ifaces[iface2.net_id] != iface2:
raise ValueError(f"inconsistency for interface {iface2.name}")
try:
linked = self._linked[iface1][iface2]
except KeyError:
@ -403,7 +398,6 @@ class CoreNetwork(CoreNetworkBase):
else:
raise Exception(f"unknown policy: {self.policy.value}")
self._linked[iface1][iface2] = linked
return linked
def unlink(self, iface1: CoreInterface, iface2: CoreInterface) -> None:

84
daemon/core/nodes/physical.py
View file

@ -3,9 +3,9 @@ PhysicalNode class for including real systems in the emulated network.
"""
import logging
import os
import threading
from typing import IO, TYPE_CHECKING, List, Optional, Tuple
from pathlib import Path
from typing import TYPE_CHECKING, List, Optional, Tuple
from core.emulator.data import InterfaceData, LinkOptions
from core.emulator.distributed import DistributedServer
@ -26,15 +26,15 @@ class PhysicalNode(CoreNodeBase):
session: "Session",
_id: int = None,
name: str = None,
nodedir: str = None,
nodedir: Path = None,
server: DistributedServer = None,
) -> None:
super().__init__(session, _id, name, server)
if not self.server:
raise CoreError("physical nodes must be assigned to a remote server")
self.nodedir: Optional[str] = nodedir
self.nodedir: Optional[Path] = nodedir
self.lock: threading.RLock = threading.RLock()
self._mounts: List[Tuple[str, str]] = []
self._mounts: List[Tuple[Path, Path]] = []
def startup(self) -> None:
with self.lock:
@ -44,15 +44,12 @@ class PhysicalNode(CoreNodeBase):
def shutdown(self) -> None:
if not self.up:
return
with self.lock:
while self._mounts:
_source, target = self._mounts.pop(-1)
self.umount(target)
_, target_path = self._mounts.pop(-1)
self.umount(target_path)
for iface in self.get_ifaces():
iface.shutdown()
self.rmnodedir()
def path_exists(self, path: str) -> bool:
@ -186,55 +183,40 @@ class PhysicalNode(CoreNodeBase):
self.adopt_iface(iface, iface_id, iface_data.mac, ips)
return iface
def privatedir(self, path: str) -> None:
if path[0] != "/":
raise ValueError(f"path not fully qualified: {path}")
hostpath = os.path.join(
self.nodedir, os.path.normpath(path).strip("/").replace("/", ".")
)
os.mkdir(hostpath)
self.mount(hostpath, path)
def privatedir(self, dir_path: Path) -> None:
if not str(dir_path).startswith("/"):
raise CoreError(f"private directory path not fully qualified: {dir_path}")
host_path = self.host_path(dir_path, is_dir=True)
self.host_cmd(f"mkdir -p {host_path}")
self.mount(host_path, dir_path)
def mount(self, source: str, target: str) -> None:
source = os.path.abspath(source)
logging.info("mounting %s at %s", source, target)
os.makedirs(target)
self.host_cmd(f"{MOUNT} --bind {source} {target}", cwd=self.nodedir)
self._mounts.append((source, target))
def mount(self, src_path: Path, target_path: Path) -> None:
logging.debug("node(%s) mounting: %s at %s", self.name, src_path, target_path)
self.cmd(f"mkdir -p {target_path}")
self.host_cmd(f"{MOUNT} --bind {src_path} {target_path}", cwd=self.nodedir)
self._mounts.append((src_path, target_path))
def umount(self, target: str) -> None:
logging.info("unmounting '%s'", target)
def umount(self, target_path: Path) -> None:
logging.info("unmounting '%s'", target_path)
try:
self.host_cmd(f"{UMOUNT} -l {target}", cwd=self.nodedir)
self.host_cmd(f"{UMOUNT} -l {target_path}", cwd=self.nodedir)
except CoreCommandError:
logging.exception("unmounting failed for %s", target)
logging.exception("unmounting failed for %s", target_path)
def opennodefile(self, filename: str, mode: str = "w") -> IO:
dirname, basename = os.path.split(filename)
if not basename:
raise ValueError("no basename for filename: " + filename)
if dirname and dirname[0] == "/":
dirname = dirname[1:]
dirname = dirname.replace("/", ".")
dirname = os.path.join(self.nodedir, dirname)
if not os.path.isdir(dirname):
os.makedirs(dirname, mode=0o755)
hostfilename = os.path.join(dirname, basename)
return open(hostfilename, mode)
def nodefile(self, filename: str, contents: str, mode: int = 0o644) -> None:
with self.opennodefile(filename, "w") as f:
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
host_path = self.host_path(file_path)
directory = host_path.parent
if not directory.is_dir():
directory.mkdir(parents=True, mode=0o755)
with host_path.open("w") as f:
f.write(contents)
os.chmod(f.name, mode)
logging.info("created nodefile: '%s'; mode: 0%o", f.name, mode)
host_path.chmod(mode)
logging.info("created nodefile: '%s'; mode: 0%o", host_path, mode)
def cmd(self, args: str, wait: bool = True, shell: bool = False) -> str:
return self.host_cmd(args, wait=wait)
def addfile(self, srcname: str, filename: str) -> None:
def addfile(self, src_path: str, file_path: str) -> None:
raise CoreError("physical node does not support addfile")
@ -464,10 +446,10 @@ class Rj45Node(CoreNodeBase):
def termcmdstring(self, sh: str) -> str:
raise CoreError("rj45 does not support terminal commands")
def addfile(self, srcname: str, filename: str) -> None:
def addfile(self, src_path: str, file_path: str) -> None:
raise CoreError("rj45 does not support addfile")
def nodefile(self, filename: str, contents: str, mode: int = 0o644) -> None:
def nodefile(self, file_path: str, contents: str, mode: int = 0o644) -> None:
raise CoreError("rj45 does not support nodefile")
def cmd(self, args: str, wait: bool = True, shell: bool = False) -> str:

4
daemon/core/services/__init__.py
View file

@ -4,11 +4,11 @@ Services
Services available to nodes can be put in this directory. Everything listed in
__all__ is automatically loaded by the main core module.
"""
import os
from pathlib import Path
from core.services.coreservices import ServiceManager
_PATH = os.path.abspath(os.path.dirname(__file__))
_PATH: Path = Path(__file__).resolve().parent
def load():

29
daemon/core/services/coreservices.py
View file

@ -10,6 +10,7 @@ services.
import enum
import logging
import time
from pathlib import Path
from typing import (
TYPE_CHECKING,
Dict,
@ -264,7 +265,7 @@ class ServiceManager:
return service
@classmethod
def add_services(cls, path: str) -> List[str]:
def add_services(cls, path: Path) -> List[str]:
"""
Method for retrieving all CoreServices from a given path.
@ -276,7 +277,6 @@ class ServiceManager:
for service in services:
if not service.name:
continue
try:
cls.add(service)
except (CoreError, ValueError) as e:
@ -488,9 +488,10 @@ class CoreServices:
# create service directories
for directory in service.dirs:
dir_path = Path(directory)
try:
node.privatedir(directory)
except (CoreCommandError, ValueError) as e:
node.privatedir(dir_path)
except (CoreCommandError, CoreError) as e:
logging.warning(
"error mounting private dir '%s' for service '%s': %s",
directory,
@ -534,14 +535,14 @@ class CoreServices:
"node(%s) service(%s) failed validation" % (node.name, service.name)
)
def copy_service_file(self, node: CoreNode, filename: str, cfg: str) -> bool:
def copy_service_file(self, node: CoreNode, file_path: Path, cfg: str) -> bool:
"""
Given a configured service filename and config, determine if the
config references an existing file that should be copied.
Returns True for local files, False for generated.
:param node: node to copy service for
:param filename: file name for a configured service
:param file_path: file name for a configured service
:param cfg: configuration string
:return: True if successful, False otherwise
"""
@ -550,7 +551,7 @@ class CoreServices:
src = src.split("\n")[0]
src = utils.expand_corepath(src, node.session, node)
# TODO: glob here
node.nodefilecopy(filename, src, mode=0o644)
node.nodefilecopy(file_path, src, mode=0o644)
return True
return False
@ -729,8 +730,8 @@ class CoreServices:
config_files = service.configs
if not service.custom:
config_files = service.get_configs(node)
for file_name in config_files:
file_path = Path(file_name)
logging.debug(
"generating service config custom(%s): %s", service.custom, file_name
)
@ -738,18 +739,16 @@ class CoreServices:
cfg = service.config_data.get(file_name)
if cfg is None:
cfg = service.generate_config(node, file_name)
# cfg may have a file:/// url for copying from a file
try:
if self.copy_service_file(node, file_name, cfg):
if self.copy_service_file(node, file_path, cfg):
continue
except IOError:
logging.exception("error copying service file: %s", file_name)
continue
else:
cfg = service.generate_config(node, file_name)
node.nodefile(file_name, cfg)
node.nodefile(file_path, cfg)
def service_reconfigure(self, node: CoreNode, service: "CoreService") -> None:
"""
@ -762,17 +761,15 @@ class CoreServices:
config_files = service.configs
if not service.custom:
config_files = service.get_configs(node)
for file_name in config_files:
file_path = Path(file_name)
if file_name[:7] == "file:///":
# TODO: implement this
raise NotImplementedError
cfg = service.config_data.get(file_name)
if cfg is None:
cfg = service.generate_config(node, file_name)
node.nodefile(file_name, cfg)
node.nodefile(file_path, cfg)
class CoreService:

67
daemon/core/utils.py
View file

@ -46,11 +46,10 @@ IFACE_CONFIG_FACTOR: int = 1000
def execute_file(
path: str, exec_globals: Dict[str, str] = None, exec_locals: Dict[str, str] = None
path: Path, exec_globals: Dict[str, str] = None, exec_locals: Dict[str, str] = None
) -> None:
"""
Provides an alternative way to run execfile to be compatible for
both python2/3.
Provides a way to execute a file.
:param path: path of file to execute
:param exec_globals: globals values to pass to execution
@ -59,8 +58,8 @@ def execute_file(
"""
if exec_globals is None:
exec_globals = {}
exec_globals.update({"__file__": path, "__name__": "__main__"})
with open(path, "rb") as f:
exec_globals.update({"__file__": str(path), "__name__": "__main__"})
with path.open("rb") as f:
data = compile(f.read(), path, "exec")
exec(data, exec_globals, exec_locals)
@ -92,24 +91,19 @@ def _detach_init() -> None:
os.setsid()
def _valid_module(path: str, file_name: str) -> bool:
def _valid_module(path: Path) -> bool:
"""
Check if file is a valid python module.
:param path: path to file
:param file_name: file name to check
:return: True if a valid python module file, False otherwise
"""
file_path = os.path.join(path, file_name)
if not os.path.isfile(file_path):
if not path.is_file():
return False
if file_name.startswith("_"):
if path.name.startswith("_"):
return False
if not file_name.endswith(".py"):
if not path.suffix == ".py":
return False
return True
@ -124,13 +118,10 @@ def _is_class(module: Any, member: Type, clazz: Type) -> bool:
"""
if not inspect.isclass(member):
return False
if not issubclass(member, clazz):
return False
if member.__module__ != module.__name__:
return False
return True
@ -196,7 +187,7 @@ def mute_detach(args: str, **kwargs: Dict[str, Any]) -> int:
def cmd(
args: str,
env: Dict[str, str] = None,
cwd: str = None,
cwd: Path = None,
wait: bool = True,
shell: bool = False,
) -> str:
@ -282,7 +273,7 @@ def file_demunge(pathname: str, header: str) -> None:
def expand_corepath(
pathname: str, session: "Session" = None, node: "CoreNode" = None
) -> str:
) -> Path:
"""
Expand a file path given session information.
@ -294,14 +285,12 @@ def expand_corepath(
if session is not None:
pathname = pathname.replace("~", f"/home/{session.user}")
pathname = pathname.replace("%SESSION%", str(session.id))
pathname = pathname.replace("%SESSION_DIR%", session.session_dir)
pathname = pathname.replace("%SESSION_DIR%", str(session.session_dir))
pathname = pathname.replace("%SESSION_USER%", session.user)
if node is not None:
pathname = pathname.replace("%NODE%", str(node.id))
pathname = pathname.replace("%NODENAME%", node.name)
return pathname
return Path(pathname)
def sysctl_devname(devname: str) -> Optional[str]:
@ -337,7 +326,7 @@ def load_config(file_path: Path, d: Dict[str, str]) -> None:
logging.exception("error reading file to dict: %s", file_path)
def load_classes(path: str, clazz: Generic[T]) -> T:
def load_classes(path: Path, clazz: Generic[T]) -> T:
"""
Dynamically load classes for use within CORE.
@ -347,24 +336,19 @@ def load_classes(path: str, clazz: Generic[T]) -> T:
"""
# validate path exists
logging.debug("attempting to load modules from path: %s", path)
if not os.path.isdir(path):
if not path.is_dir():
logging.warning("invalid custom module directory specified" ": %s", path)
# check if path is in sys.path
parent_path = os.path.dirname(path)
if parent_path not in sys.path:
logging.debug("adding parent path to allow imports: %s", parent_path)
sys.path.append(parent_path)
# retrieve potential service modules, and filter out invalid modules
base_module = os.path.basename(path)
module_names = os.listdir(path)
module_names = filter(lambda x: _valid_module(path, x), module_names)
module_names = map(lambda x: x[:-3], module_names)
parent = str(path.parent)
if parent not in sys.path:
logging.debug("adding parent path to allow imports: %s", parent)
sys.path.append(parent)
# import and add all service modules in the path
classes = []
for module_name in module_names:
import_statement = f"{base_module}.{module_name}"
for p in path.iterdir():
if not _valid_module(p):
continue
import_statement = f"{path.name}.{p.stem}"
logging.debug("importing custom module: %s", import_statement)
try:
module = importlib.import_module(import_statement)
@ -376,19 +360,18 @@ def load_classes(path: str, clazz: Generic[T]) -> T:
logging.exception(
"unexpected error during import, skipping: %s", import_statement
)
return classes
def load_logging_config(config_path: str) -> None:
def load_logging_config(config_path: Path) -> None:
"""
Load CORE logging configuration file.
:param config_path: path to logging config file
:return: nothing
"""
with open(config_path, "r") as log_config_file:
log_config = json.load(log_config_file)
with config_path.open("r") as f:
log_config = json.load(f)
logging.config.dictConfig(log_config)

18
daemon/core/xml/corexml.py
View file

@ -1,4 +1,5 @@
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type, TypeVar
from lxml import etree
@ -25,7 +26,7 @@ T = TypeVar("T")
def write_xml_file(
xml_element: etree.Element, file_path: str, doctype: str = None
xml_element: etree.Element, file_path: Path, doctype: str = None
) -> None:
xml_data = etree.tostring(
xml_element,
@ -34,8 +35,8 @@ def write_xml_file(
encoding="UTF-8",
doctype=doctype,
)
with open(file_path, "wb") as xml_file:
xml_file.write(xml_data)
with file_path.open("wb") as f:
f.write(xml_data)
def get_type(element: etree.Element, name: str, _type: Generic[T]) -> Optional[T]:
@ -293,13 +294,12 @@ class CoreXmlWriter:
self.write_session_metadata()
self.write_default_services()
def write(self, file_name: str) -> None:
self.scenario.set("name", file_name)
def write(self, path: Path) -> None:
self.scenario.set("name", str(path))
# write out generated xml
xml_tree = etree.ElementTree(self.scenario)
xml_tree.write(
file_name, xml_declaration=True, pretty_print=True, encoding="UTF-8"
str(path), xml_declaration=True, pretty_print=True, encoding="UTF-8"
)
def write_session_origin(self) -> None:
@ -580,8 +580,8 @@ class CoreXmlReader:
self.session: "Session" = session
self.scenario: Optional[etree.ElementTree] = None
def read(self, file_name: str) -> None:
xml_tree = etree.parse(file_name)
def read(self, file_path: Path) -> None:
xml_tree = etree.parse(str(file_path))
self.scenario = xml_tree.getroot()
# read xml session content

20
daemon/core/xml/emanexml.py
View file

@ -1,5 +1,5 @@
import logging
import os
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
@ -54,7 +54,7 @@ def _value_to_params(value: str) -> Optional[Tuple[str]]:
def create_file(
xml_element: etree.Element,
doc_name: str,
file_path: str,
file_path: Path,
server: DistributedServer = None,
) -> None:
"""
@ -71,10 +71,11 @@ def create_file(
)
if server:
temp = NamedTemporaryFile(delete=False)
corexml.write_xml_file(xml_element, temp.name, doctype=doctype)
temp_path = Path(temp.name)
corexml.write_xml_file(xml_element, temp_path, doctype=doctype)
temp.close()
server.remote_put(temp.name, file_path)
os.unlink(temp.name)
server.remote_put(temp_path, file_path)
temp_path.unlink()
else:
corexml.write_xml_file(xml_element, file_path, doctype=doctype)
@ -92,9 +93,9 @@ def create_node_file(
:return:
"""
if isinstance(node, CoreNode):
file_path = os.path.join(node.nodedir, file_name)
file_path = node.nodedir / file_name
else:
file_path = os.path.join(node.session.session_dir, file_name)
file_path = node.session.session_dir / file_name
create_file(xml_element, doc_name, file_path, node.server)
@ -316,7 +317,7 @@ def create_event_service_xml(
group: str,
port: str,
device: str,
file_directory: str,
file_directory: Path,
server: DistributedServer = None,
) -> None:
"""
@ -340,8 +341,7 @@ def create_event_service_xml(
):
sub_element = etree.SubElement(event_element, name)
sub_element.text = value
file_name = "libemaneeventservice.xml"
file_path = os.path.join(file_directory, file_name)
file_path = file_directory / "libemaneeventservice.xml"
create_file(event_element, "emaneeventmsgsvc", file_path, server)

4
daemon/scripts/core-daemon
View file

@ -12,6 +12,7 @@ import sys
import threading
import time
from configparser import ConfigParser
from pathlib import Path
from core import constants
from core.api.grpc.server import CoreGrpcServer
@ -148,7 +149,8 @@ def main():
:return: nothing
"""
cfg = get_merged_config(f"{CORE_CONF_DIR}/core.conf")
load_logging_config(cfg["logfile"])
log_config_path = Path(cfg["logfile"])
load_logging_config(log_config_path)
banner()
try:
cored(cfg)

9
daemon/tests/emane/test_emane.py
View file

@ -1,7 +1,7 @@
"""
Unit tests for testing CORE EMANE networks.
"""
import os
from pathlib import Path
from tempfile import TemporaryFile
from typing import Type
from xml.etree import ElementTree
@ -28,7 +28,8 @@ _EMANE_MODELS = [
EmaneCommEffectModel,
EmaneTdmaModel,
]
_DIR = os.path.dirname(os.path.abspath(__file__))
_DIR: Path = Path(__file__).resolve().parent
_SCHEDULE: Path = _DIR / "../../examples/tdma/schedule.xml"
def ping(
@ -107,9 +108,7 @@ class TestEmane:
# configure tdma
if model == EmaneTdmaModel:
session.emane.set_model_config(
emane_network.id,
EmaneTdmaModel.name,
{"schedule": os.path.join(_DIR, "../../examples/tdma/schedule.xml")},
emane_network.id, EmaneTdmaModel.name, {"schedule": str(_SCHEDULE)}
)
# create nodes

10
daemon/tests/test_config_services.py
View file

@ -1,3 +1,4 @@
from pathlib import Path
from unittest import mock
import pytest
@ -68,7 +69,8 @@ class TestConfigServices:
service.create_dirs()
# then
node.privatedir.assert_called_with(MyService.directories[0])
directory = Path(MyService.directories[0])
node.privatedir.assert_called_with(directory)
def test_create_files_custom(self):
# given
@ -81,7 +83,8 @@ class TestConfigServices:
service.create_files()
# then
node.nodefile.assert_called_with(MyService.files[0], text)
file_path = Path(MyService.files[0])
node.nodefile.assert_called_with(file_path, text)
def test_create_files_text(self):
# given
@ -92,7 +95,8 @@ class TestConfigServices:
service.create_files()
# then
node.nodefile.assert_called_with(MyService.files[0], TEMPLATE_TEXT)
file_path = Path(MyService.files[0])
node.nodefile.assert_called_with(file_path, TEMPLATE_TEXT)
def test_run_startup(self):
# given

12
daemon/tests/test_core.py
View file

@ -2,9 +2,9 @@
Unit tests for testing basic CORE networks.
"""
import os
import threading
from typing import Type
from pathlib import Path
from typing import List, Type
import pytest
@ -16,9 +16,9 @@ from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility
from core.nodes.base import CoreNode, NodeBase
from core.nodes.network import HubNode, PtpNet, SwitchNode, WlanNode
_PATH = os.path.abspath(os.path.dirname(__file__))
_MOBILITY_FILE = os.path.join(_PATH, "mobility.scen")
_WIRED = [PtpNet, HubNode, SwitchNode]
_PATH: Path = Path(__file__).resolve().parent
_MOBILITY_FILE: Path = _PATH / "mobility.scen"
_WIRED: List = [PtpNet, HubNode, SwitchNode]
def ping(from_node: CoreNode, to_node: CoreNode, ip_prefixes: IpPrefixes):
@ -195,7 +195,7 @@ class TestCore:
# configure mobility script for session
config = {
"file": _MOBILITY_FILE,
"file": str(_MOBILITY_FILE),
"refresh_ms": "50",
"loop": "1",
"autostart": "0.0",

23
daemon/tests/test_gui.py
View file

@ -1,8 +1,8 @@
"""
Tests for testing tlv message handling.
"""
import os
import time
from pathlib import Path
from typing import Optional
import mock
@ -425,7 +425,7 @@ class TestGui:
assert file_data == service_file.data
def test_file_node_file_copy(self, request, coretlv: CoreHandler):
file_name = "/var/log/test/node.log"
file_path = Path("/var/log/test/node.log")
node = coretlv.session.add_node(CoreNode)
node.makenodedir()
file_data = "echo hello"
@ -433,7 +433,7 @@ class TestGui:
MessageFlags.ADD.value,
[
(FileTlvs.NODE, node.id),
(FileTlvs.NAME, file_name),
(FileTlvs.NAME, str(file_path)),
(FileTlvs.DATA, file_data),
],
)
@ -441,10 +441,10 @@ class TestGui:
coretlv.handle_message(message)
if not request.config.getoption("mock"):
directory, basename = os.path.split(file_name)
directory = str(file_path.parent)
created_directory = directory[1:].replace("/", ".")
create_path = os.path.join(node.nodedir, created_directory, basename)
assert os.path.exists(create_path)
create_path = node.nodedir / created_directory / file_path.name
assert create_path.exists()
def test_exec_node_tty(self, coretlv: CoreHandler):
coretlv.dispatch_replies = mock.MagicMock()
@ -547,20 +547,21 @@ class TestGui:
0,
[(EventTlvs.TYPE, EventTypes.FILE_SAVE.value), (EventTlvs.NAME, file_path)],
)
coretlv.handle_message(message)
assert os.path.exists(file_path)
assert Path(file_path).exists()
def test_event_open_xml(self, coretlv: CoreHandler, tmpdir):
xml_file = tmpdir.join("coretlv.session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
node = coretlv.session.add_node(CoreNode)
coretlv.session.save_xml(file_path)
coretlv.session.delete_node(node.id)
message = coreapi.CoreEventMessage.create(
0,
[(EventTlvs.TYPE, EventTypes.FILE_OPEN.value), (EventTlvs.NAME, file_path)],
[
(EventTlvs.TYPE, EventTypes.FILE_OPEN.value),
(EventTlvs.NAME, str(file_path)),
],
)
coretlv.handle_message(message)

12
daemon/tests/test_services.py
View file

@ -1,5 +1,5 @@
import itertools
import os
from pathlib import Path
import pytest
from mock import MagicMock
@ -9,8 +9,8 @@ from core.errors import CoreCommandError
from core.nodes.base import CoreNode
from core.services.coreservices import CoreService, ServiceDependencies, ServiceManager
_PATH = os.path.abspath(os.path.dirname(__file__))
_SERVICES_PATH = os.path.join(_PATH, "myservices")
_PATH: Path = Path(__file__).resolve().parent
_SERVICES_PATH = _PATH / "myservices"
SERVICE_ONE = "MyService"
SERVICE_TWO = "MyService2"
@ -64,15 +64,15 @@ class TestServices:
ServiceManager.add_services(_SERVICES_PATH)
my_service = ServiceManager.get(SERVICE_ONE)
node = session.add_node(CoreNode)
file_name = my_service.configs[0]
file_path = node.hostfilename(file_name)
file_path = Path(my_service.configs[0])
file_path = node.host_path(file_path)
# when
session.services.create_service_files(node, my_service)
# then
if not request.config.getoption("mock"):
assert os.path.exists(file_path)
assert file_path.exists()
def test_service_validate(self, session: Session):
# given

17
daemon/tests/test_xml.py
View file

@ -1,3 +1,4 @@
from pathlib import Path
from tempfile import TemporaryFile
from xml.etree import ElementTree
@ -34,7 +35,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed
@ -85,7 +86,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed
@ -148,7 +149,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed
@ -210,7 +211,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed
@ -261,7 +262,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed
@ -321,7 +322,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed
@ -390,7 +391,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed
@ -471,7 +472,7 @@ class TestXml:
# save xml
xml_file = tmpdir.join("session.xml")
file_path = xml_file.strpath
file_path = Path(xml_file.strpath)
session.save_xml(file_path)
# verify xml file was created and can be parsed