updates to fail early for required executables that are not in PATH

daemon/core/__init__.py

@@ -1,41 +1,4 @@
-import json
 import logging.config
-import os
-import subprocess
-
-from core import constants
 
 # setup default null handler
 logging.getLogger(__name__).addHandler(logging.NullHandler())
-
-
-def load_logging_config(config_path=None):
-    """
-    Load CORE logging configuration file.
-
-    :param str config_path: path to logging config file,
-        when None defaults to /etc/core/logging.conf
-    :return: nothing
-    """
-    if not config_path:
-        config_path = os.path.join(constants.CORE_CONF_DIR, "logging.conf")
-    with open(config_path, "r") as log_config_file:
-        log_config = json.load(log_config_file)
-        logging.config.dictConfig(log_config)
-
-
-class CoreCommandError(subprocess.CalledProcessError):
-    """
-    Used when encountering internal CORE command errors.
-    """
-
-    def __str__(self):
-        return "Command(%s), Status(%s):\n%s" % (self.cmd, self.returncode, self.output)
-
-
-class CoreError(Exception):
-    """
-    Used for errors when dealing with CoreEmu and Sessions.
-    """
-
-    pass

daemon/core/api/grpc/server.py

@@ -10,7 +10,6 @@ from queue import Empty, Queue
 
 import grpc
 
-from core import CoreError
 from core.api.grpc import core_pb2, core_pb2_grpc
 from core.emane.nodes import EmaneNet
 from core.emulator.data import (
@@ -23,6 +22,7 @@ from core.emulator.data import (
 )
 from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions
 from core.emulator.enumerations import EventTypes, LinkTypes, NodeTypes
+from core.errors import CoreError
 from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility
 from core.nodes.base import CoreNetworkBase
 from core.nodes.docker import DockerNode

daemon/core/api/tlv/corehandlers.py

@@ -14,7 +14,7 @@ from builtins import range
 from itertools import repeat
 from queue import Empty, Queue
 
-from core import CoreError, utils
+from core import utils
 from core.api.tlv import coreapi, dataconversion, structutils
 from core.config import ConfigShim
 from core.emulator.data import ConfigData, EventData, ExceptionData, FileData
@@ -37,6 +37,7 @@ from core.emulator.enumerations import (
     RegisterTlvs,
     SessionTlvs,
 )
+from core.errors import CoreError
 from core.location.mobility import BasicRangeModel
 from core.nodes.network import WlanNode
 from core.services.coreservices import ServiceManager, ServiceShim

daemon/core/constants.py.in

@@ -1,29 +1,20 @@
-import os
+from core.utils import which
 
 COREDPY_VERSION = "@PACKAGE_VERSION@"
-CORE_STATE_DIR = "@CORE_STATE_DIR@"
 CORE_CONF_DIR = "@CORE_CONF_DIR@"
 CORE_DATA_DIR = "@CORE_DATA_DIR@"
 QUAGGA_STATE_DIR = "@CORE_STATE_DIR@/run/quagga"
 FRR_STATE_DIR = "@CORE_STATE_DIR@/run/frr"
 
-
+VNODED_BIN = which("vnoded", required=True)
-def which(command):
+VCMD_BIN = which("vcmd", required=True)
-    for path in os.environ["PATH"].split(os.pathsep):
+BRCTL_BIN = which("brctl", required=True)
-        command_path = os.path.join(path, command)
+SYSCTL_BIN = which("sysctl", required=True)
-        if os.path.isfile(command_path) and os.access(command_path, os.X_OK):
+IP_BIN = which("ip", required=True)
-            return command_path
+ETHTOOL_BIN = which("ethtool", required=True)
-
+TC_BIN = which("tc", required=True)
-
+EBTABLES_BIN = which("ebtables", required=True)
-VNODED_BIN = which("vnoded")
+MOUNT_BIN = which("mount", required=True)
-VCMD_BIN = which("vcmd")
+UMOUNT_BIN = which("umount", required=True)
-BRCTL_BIN = which("brctl")
+OVS_BIN = which("ovs-vsctl", required=False)
-SYSCTL_BIN = which("sysctl")
+OVS_FLOW_BIN = which("ovs-ofctl", required=False)
-IP_BIN = which("ip")
-ETHTOOL_BIN = which("ethtool")
-TC_BIN = which("tc")
-EBTABLES_BIN = which("ebtables")
-MOUNT_BIN = which("mount")
-UMOUNT_BIN = which("umount")
-OVS_BIN = which("ovs-vsctl")
-OVS_FLOW_BIN = which("ovs-ofctl")

daemon/core/emane/emanemanager.py

@@ -7,7 +7,7 @@ import logging
 import os
 import threading
 
-from core import CoreCommandError, CoreError, constants, utils
+from core import constants, utils
 from core.api.tlv import coreapi, dataconversion
 from core.config import ConfigGroup, ConfigShim, Configuration, ModelManager
 from core.emane import emanemanifest
@@ -26,6 +26,7 @@ from core.emulator.enumerations import (
     MessageTypes,
     RegisterTlvs,
 )
+from core.errors import CoreCommandError, CoreError
 from core.xml import emanexml
 
 try:

daemon/core/emane/emanemodel.py

@@ -4,10 +4,10 @@ Defines Emane Models used within CORE.
 import logging
 import os
 
-from core import CoreError
 from core.config import ConfigGroup, Configuration
 from core.emane import emanemanifest
 from core.emulator.enumerations import ConfigDataTypes
+from core.errors import CoreError
 from core.location.mobility import WirelessModel
 from core.xml import emanexml
 

daemon/core/emulator/session.py

@@ -14,7 +14,7 @@ import threading
 import time
 from multiprocessing.pool import ThreadPool
 
-from core import CoreError, constants, utils
+from core import constants, utils
 from core.api.tlv import coreapi
 from core.api.tlv.broker import CoreBroker
 from core.emane.emanemanager import EmaneManager
@@ -29,6 +29,7 @@ from core.emulator.emudata import (
 )
 from core.emulator.enumerations import EventTypes, ExceptionLevels, LinkTypes, NodeTypes
 from core.emulator.sessionconfig import SessionConfig, SessionMetaData
+from core.errors import CoreError
 from core.location.corelocation import CoreLocation
 from core.location.event import EventLoop
 from core.location.mobility import MobilityManager

daemon/core/errors.py

@@ -0,0 +1,21 @@
+"""
+Provides CORE specific errors.
+"""
+import subprocess
+
+
+class CoreCommandError(subprocess.CalledProcessError):
+    """
+    Used when encountering internal CORE command errors.
+    """
+
+    def __str__(self):
+        return "Command(%s), Status(%s):\n%s" % (self.cmd, self.returncode, self.output)
+
+
+class CoreError(Exception):
+    """
+    Used for errors when dealing with CoreEmu and Sessions.
+    """
+
+    pass

daemon/core/location/mobility.py

@@ -11,7 +11,7 @@ import time
 from builtins import int
 from functools import total_ordering
 
-from core import CoreError, utils
+from core import utils
 from core.config import ConfigGroup, ConfigurableOptions, Configuration, ModelManager
 from core.emulator.data import EventData, LinkData
 from core.emulator.enumerations import (
@@ -23,6 +23,7 @@ from core.emulator.enumerations import (
     NodeTlvs,
     RegisterTlvs,
 )
+from core.errors import CoreError
 from core.nodes.base import CoreNodeBase
 from core.nodes.ipaddress import IpAddress
 

daemon/core/nodes/base.py

@@ -14,17 +14,16 @@ import threading
 from builtins import range
 from socket import AF_INET, AF_INET6
 
-from core import CoreCommandError, constants, utils
+from core import constants, utils
 from core.emulator.data import LinkData, NodeData
 from core.emulator.enumerations import LinkTypes, NodeTypes
+from core.errors import CoreCommandError
 from core.nodes import client, ipaddress
 from core.nodes.interface import CoreInterface, TunTap, Veth
 from core.nodes.netclient import LinuxNetClient, OvsNetClient
 
 _DEFAULT_MTU = 1500
 
-utils.check_executables([constants.IP_BIN])
-
 
 class NodeBase(object):
     """

daemon/core/nodes/client.py

@@ -8,7 +8,8 @@ import logging
 import os
 from subprocess import PIPE, Popen
 
-from core import CoreCommandError, constants, utils
+from core import constants, utils
+from core.errors import CoreCommandError
 
 
 class VnodeClient(object):

daemon/core/nodes/docker.py

@@ -2,8 +2,9 @@ import json
 import logging
 import os
 
-from core import CoreCommandError, utils
+from core import utils
 from core.emulator.enumerations import NodeTypes
+from core.errors import CoreCommandError
 from core.nodes.base import CoreNode
 
 

daemon/core/nodes/interface.py

@@ -6,9 +6,8 @@ import logging
 import time
 from builtins import int, range
 
-from core import CoreCommandError, constants, utils
+from core import constants, utils
-
+from core.errors import CoreCommandError
-utils.check_executables([constants.IP_BIN])
 
 
 class CoreInterface(object):

daemon/core/nodes/lxd.py

@@ -3,8 +3,9 @@ import logging
 import os
 import time
 
-from core import CoreCommandError, utils
+from core import utils
 from core.emulator.enumerations import NodeTypes
+from core.errors import CoreCommandError
 from core.nodes.base import CoreNode
 
 

daemon/core/nodes/network.py

@@ -9,17 +9,14 @@ import threading
 import time
 from socket import AF_INET, AF_INET6
 
-from core import CoreCommandError, CoreError, constants, utils
+from core import constants, utils
 from core.emulator.data import LinkData
 from core.emulator.enumerations import LinkTypes, NodeTypes, RegisterTlvs
+from core.errors import CoreCommandError, CoreError
 from core.nodes import ipaddress
 from core.nodes.base import CoreNetworkBase
 from core.nodes.interface import GreTap, Veth
 
-utils.check_executables(
-    [constants.BRCTL_BIN, constants.IP_BIN, constants.EBTABLES_BIN, constants.TC_BIN]
-)
-
 ebtables_lock = threading.Lock()
 
 

daemon/core/nodes/physical.py

@@ -7,8 +7,9 @@ import os
 import subprocess
 import threading
 
-from core import CoreCommandError, constants, utils
+from core import constants, utils
 from core.emulator.enumerations import NodeTypes
+from core.errors import CoreCommandError
 from core.nodes.base import CoreNodeBase
 from core.nodes.interface import CoreInterface
 from core.nodes.network import CoreNetwork, GreTap

daemon/core/plugins/sdt.py

@@ -7,7 +7,7 @@ import socket
 
 from future.moves.urllib.parse import urlparse
 
-from core import CoreError, constants
+from core import constants
 from core.emane.nodes import EmaneNet
 from core.emulator.enumerations import (
     EventTypes,
@@ -18,6 +18,7 @@ from core.emulator.enumerations import (
     NodeTlvs,
     NodeTypes,
 )
+from core.errors import CoreError
 from core.nodes.base import CoreNetworkBase, NodeBase
 from core.nodes.network import WlanNode
 

daemon/core/services/coreservices.py

@@ -12,10 +12,11 @@ import logging
 import time
 from multiprocessing.pool import ThreadPool
 
-from core import CoreCommandError, utils
+from core import utils
 from core.constants import which
 from core.emulator.data import FileData
 from core.emulator.enumerations import MessageFlags, RegisterTlvs
+from core.errors import CoreCommandError
 
 
 class ServiceBootError(Exception):
@@ -258,13 +259,7 @@ class ServiceManager(object):
 
         # validate dependent executables are present
         for executable in service.executables:
-            if not which(executable):
+            which(executable, required=True)
-                logging.debug(
-                    "service(%s) missing executable: %s", service.name, executable
-                )
-                raise ValueError(
-                    "service(%s) missing executable: %s" % (service.name, executable)
-                )
 
         # make service available
         cls.services[name] = service
@@ -300,7 +295,7 @@ class ServiceManager(object):
                 cls.add(service)
             except ValueError as e:
                 service_errors.append(service.name)
-                logging.debug("not loading service: %s", e)
+                logging.debug("not loading service(%s): %s", service.name, e)
         return service_errors
 
 

daemon/core/services/utility.py

@@ -4,7 +4,8 @@ utility.py: defines miscellaneous utility services.
 
 import os
 
-from core import CoreCommandError, constants, utils
+from core import constants, utils
+from core.errors import CoreCommandError
 from core.nodes.ipaddress import Ipv4Prefix, Ipv6Prefix
 from core.services.coreservices import CoreService
 

daemon/core/utils.py

@@ -6,7 +6,9 @@ import fcntl
 import hashlib
 import importlib
 import inspect
+import json
 import logging
+import logging.config
 import os
 import shlex
 import subprocess
@@ -14,7 +16,8 @@ import sys
 
 from past.builtins import basestring
 
-from core import CoreCommandError
+from core import constants
+from core.errors import CoreCommandError
 
 DEVNULL = open(os.devnull, "wb")
 
@@ -109,17 +112,6 @@ def _is_class(module, member, clazz):
     return True
 
 
-def _is_exe(file_path):
-    """
-    Check if a given file path exists and is an executable file.
-
-    :param str file_path: file path to check
-    :return: True if the file is considered and executable file, False otherwise
-    :rtype: bool
-    """
-    return os.path.isfile(file_path) and os.access(file_path, os.X_OK)
-
-
 def close_onexec(fd):
     """
     Close on execution of a shell process.
@@ -131,17 +123,26 @@ def close_onexec(fd):
     fcntl.fcntl(fd, fcntl.F_SETFD, fdflags | fcntl.FD_CLOEXEC)
 
 
-def check_executables(executables):
+def which(command, required):
     """
-    Check executables, verify they exist and are executable.
+    Find location of desired executable within current PATH.
 
-    :param list[str] executables: executable to check
+    :param str command: command to find location for
-    :return: nothing
+    :param bool required: command is required to be found, false otherwise
-    :raises EnvironmentError: when an executable doesn't exist or is not executable
+    :return: command location or None
+    :raises ValueError: when not found and required
     """
-    for executable in executables:
+    found_path = None
-        if not _is_exe(executable):
+    for path in os.environ["PATH"].split(os.pathsep):
-            raise EnvironmentError("executable not found: %s" % executable)
+        command_path = os.path.join(path, command)
+        if os.path.isfile(command_path) and os.access(command_path, os.X_OK):
+            found_path = command_path
+            break
+
+    if found_path is None and required:
+        raise ValueError("failed to find required executable(%s) in path" % command)
+
+    return found_path
 
 
 def make_tuple(obj):
@@ -167,7 +168,8 @@ def make_tuple_fromstr(s, value_type):
     :return: tuple from string
     :rtype: tuple
     """
-    # remove tuple braces and strip commands and space from all values in the tuple string
+    # remove tuple braces and strip commands and space from all values in the tuple
+    # string
     values = []
     for x in s.strip("(), ").split(","):
         x = x.strip("' ")
@@ -178,7 +180,8 @@ def make_tuple_fromstr(s, value_type):
 
 def split_args(args):
     """
-    Convenience method for splitting potential string commands into a shell-like syntax list.
+    Convenience method for splitting potential string commands into a shell-like
+    syntax list.
 
     :param list/str args: command list or string
     :return: shell-like syntax list
@@ -227,8 +230,8 @@ def cmd(args, wait=True):
 
 def cmd_output(args):
     """
-    Execute a command on the host and return a tuple containing the exit status and result string. stderr output
+    Execute a command on the host and return a tuple containing the exit status and
-    is folded into the stdout result string.
+    result string. stderr output is folded into the stdout result string.
 
     :param list[str]|str args: command arguments
     :return: command status and stdout
@@ -248,14 +251,15 @@ def cmd_output(args):
 
 def check_cmd(args, **kwargs):
     """
-    Execute a command on the host and return a tuple containing the exit status and result string. stderr output
+    Execute a command on the host and return a tuple containing the exit status and
-    is folded into the stdout result string.
+    result string. stderr output is folded into the stdout result string.
 
     :param list[str]|str args: command arguments
     :param dict kwargs: keyword arguments to pass to subprocess.Popen
     :return: combined stdout and stderr
     :rtype: str
-    :raises CoreCommandError: when there is a non-zero exit status or the file to execute is not found
+    :raises CoreCommandError: when there is a non-zero exit status or the file to
+        execute is not found
     """
     kwargs["stdout"] = subprocess.PIPE
     kwargs["stderr"] = subprocess.STDOUT
@@ -350,7 +354,7 @@ def expand_corepath(pathname, session=None, node=None):
     Expand a file path given session information.
 
     :param str pathname: file path to expand
-    :param core.emulator.session.Session session: core session object to expand path with
+    :param core.emulator.session.Session session: core session object to expand path
     :param core.nodes.base.CoreNode node: node to expand path with
     :return: expanded path
     :rtype: str
@@ -383,7 +387,8 @@ def sysctl_devname(devname):
 
 def load_config(filename, d):
     """
-    Read key=value pairs from a file, into a dict. Skip comments; strip newline characters and spacing.
+    Read key=value pairs from a file, into a dict. Skip comments; strip newline
+    characters and spacing.
 
     :param str filename: file to read into a dictionary
     :param dict d: dictionary to read file into
@@ -444,3 +449,18 @@ def load_classes(path, clazz):
             )
 
     return classes
+
+
+def load_logging_config(config_path=None):
+    """
+    Load CORE logging configuration file.
+
+    :param str config_path: path to logging config file,
+        when None defaults to /etc/core/logging.conf
+    :return: nothing
+    """
+    if not config_path:
+        config_path = os.path.join(constants.CORE_CONF_DIR, "logging.conf")
+    with open(config_path, "r") as log_config_file:
+        log_config = json.load(log_config_file)
+        logging.config.dictConfig(log_config)

daemon/examples/api/emane80211.py

@@ -6,11 +6,11 @@ import datetime
 import parser
 from builtins import range
 
-from core import load_logging_config
 from core.emane.ieee80211abg import EmaneIeee80211abgModel
 from core.emulator.coreemu import CoreEmu
 from core.emulator.emudata import IpPrefixes
 from core.emulator.enumerations import EventTypes
+from core.utils import load_logging_config
 
 load_logging_config()
 

daemon/examples/api/switch.py

@@ -9,10 +9,10 @@ import datetime
 import parser
 from builtins import range
 
-from core import load_logging_config
 from core.emulator.coreemu import CoreEmu
 from core.emulator.emudata import IpPrefixes
 from core.emulator.enumerations import EventTypes, NodeTypes
+from core.utils import load_logging_config
 
 load_logging_config()
 

daemon/examples/api/switch_inject.py

@@ -6,9 +6,9 @@
 # nodestep
 from builtins import range
 
-from core import load_logging_config
 from core.emulator.emudata import IpPrefixes
 from core.emulator.enumerations import EventTypes, NodeTypes
+from core.utils import load_logging_config
 
 load_logging_config()
 

daemon/examples/api/wlan.py

@@ -9,11 +9,11 @@ import datetime
 import parser
 from builtins import range
 
-from core import load_logging_config
 from core.emulator.coreemu import CoreEmu
 from core.emulator.emudata import IpPrefixes, NodeOptions
 from core.emulator.enumerations import EventTypes, NodeTypes
 from core.location.mobility import BasicRangeModel
+from core.utils import load_logging_config
 
 load_logging_config()
 

daemon/scripts/core-daemon

@@ -12,12 +12,12 @@ import threading
 import time
 from configparser import ConfigParser
 
-from core import constants, load_logging_config
+from core import constants
 from core.api.grpc.server import CoreGrpcServer
 from core.api.tlv.corehandlers import CoreHandler, CoreUdpHandler
 from core.api.tlv.coreserver import CoreServer, CoreUdpServer
 from core.emulator import enumerations
-from core.utils import close_onexec
+from core.utils import close_onexec, load_logging_config
 
 
 def banner():

daemon/tests/emane/test_emane.py

@@ -6,13 +6,13 @@ from xml.etree import ElementTree
 
 import pytest
 
-from core import CoreError
 from core.emane.bypass import EmaneBypassModel
 from core.emane.commeffect import EmaneCommEffectModel
 from core.emane.ieee80211abg import EmaneIeee80211abgModel
 from core.emane.rfpipe import EmaneRfPipeModel
 from core.emane.tdma import EmaneTdmaModel
 from core.emulator.emudata import NodeOptions
+from core.errors import CoreError
 
 _EMANE_MODELS = [
     EmaneIeee80211abgModel,

daemon/tests/test_grpc.py

@@ -5,7 +5,6 @@ from queue import Queue
 import grpc
 import pytest
 
-from core import CoreError
 from core.api.grpc import core_pb2
 from core.api.grpc.client import CoreGrpcClient
 from core.config import ConfigShim
@@ -18,6 +17,7 @@ from core.emulator.enumerations import (
     ExceptionLevels,
     NodeTypes,
 )
+from core.errors import CoreError
 from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility
 
 

daemon/tests/test_gui.py

@@ -7,7 +7,6 @@ import time
 import mock
 import pytest
 
-from core import CoreError
 from core.api.tlv import coreapi
 from core.emane.ieee80211abg import EmaneIeee80211abgModel
 from core.emulator.enumerations import (
@@ -24,6 +23,7 @@ from core.emulator.enumerations import (
     RegisterTlvs,
     SessionTlvs,
 )
+from core.errors import CoreError
 from core.location.mobility import BasicRangeModel
 from core.nodes.ipaddress import Ipv4Prefix
 

daemon/tests/test_nodes.py

@@ -3,9 +3,10 @@ import time
 
 import pytest
 
-from core import CoreError, utils
+from core import utils
 from core.emulator.emudata import NodeOptions
 from core.emulator.enumerations import NodeTypes
+from core.errors import CoreError
 
 MODELS = ["router", "host", "PC", "mdr"]
 

daemon/tests/test_xml.py

@@ -2,9 +2,9 @@ from xml.etree import ElementTree
 
 import pytest
 
-from core import CoreError
 from core.emulator.emudata import LinkOptions, NodeOptions
 from core.emulator.enumerations import NodeTypes
+from core.errors import CoreError
 from core.location.mobility import BasicRangeModel
 from core.services.utility import SshService