core-extra/daemon/core/misc/utils.py

"""
Miscellaneous utility functions, wrappers around some subprocess procedures.
"""

import importlib
import inspect
import os
import shlex
import subprocess
import sys

import fcntl
import resource

from core import logger

DEVNULL = open(os.devnull, "wb")


def _detach_init():
    """
    Fork a child process and exit.

    :return: nothing
    """
    if os.fork():
        # parent exits
        os._exit(0)
    os.setsid()


def _valid_module(path, file_name):
    """
    Check if file is a valid python module.

    :param str path: path to file
    :param str file_name: file name to check
    :return: True if a valid python module file, False otherwise
    :rtype: bool
    """
    file_path = os.path.join(path, file_name)
    if not os.path.isfile(file_path):
        return False

    if file_name.startswith("_"):
        return False

    if not file_name.endswith(".py"):
        return False

    return True


def _is_class(module, member, clazz):
    """
    Validates if a module member is a class and an instance of a CoreService.

    :param module: module to validate for service
    :param member: member to validate for service
    :param clazz: clazz type to check for validation
    :return: True if a valid service, False otherwise
    :rtype: bool
    """
    if not inspect.isclass(member):
        return False

    if not issubclass(member, clazz):
        return False

    if member.__module__ != module.__name__:
        return False

    return True


def _is_exe(file_path):
    """
    Check if a given file path exists and is an executable file.

    :param str file_path: file path to check
    :return: True if the file is considered and executable file, False otherwise
    :rtype: bool
    """
    return os.path.isfile(file_path) and os.access(file_path, os.X_OK)


def close_onexec(fd):
    """
    Close on execution of a shell process.

    :param fd: file descriptor to close
    :return: nothing
    """
    fdflags = fcntl.fcntl(fd, fcntl.F_GETFD)
    fcntl.fcntl(fd, fcntl.F_SETFD, fdflags | fcntl.FD_CLOEXEC)


def check_executables(executables):
    """
    Check executables, verify they exist and are executable.

    :param list[str] executables: executable to check
    :return: nothing
    :raises EnvironmentError: when an executable doesn't exist or is not executable
    """
    for executable in executables:
        if not _is_exe(executable):
            raise EnvironmentError("executable not found: %s" % executable)


def make_tuple(obj):
    """
    Create a tuple from an object, or return the object itself.

    :param obj: object to convert to a tuple
    :return: converted tuple or the object itself
    :rtype: tuple
    """
    if hasattr(obj, "__iter__"):
        return tuple(obj)
    else:
        return obj,


def make_tuple_fromstr(s, value_type):
    """
    Create a tuple from a string.

    :param str s: string to convert to a tuple
    :param value_type: type of values to be contained within tuple
    :return: tuple from string
    :rtype: tuple
    """
    # remove tuple braces and strip commands and space from all values in the tuple string
    values = [x.strip("' ") for x in s.strip("(), ").split(",")]
    return tuple(value_type(i) for i in values)


def split_args(args):
    """
    Convenience method for splitting potential string commands into a shell-like syntax list.

    :param list/str args: command list or string
    :return: shell-like syntax list
    :rtype: list
    """
    if type(args) == str:
        args = shlex.split(args)
    return args


def mute_detach(args, **kwargs):
    """
    Run a muted detached process by forking it.

    :param list[str]|str args: arguments for the command
    :param dict kwargs: keyword arguments for the command
    :return: process id of the command
    :rtype: int
    """
    args = split_args(args)
    kwargs["preexec_fn"] = _detach_init
    kwargs["stdout"] = DEVNULL
    kwargs["stderr"] = subprocess.STDOUT
    return subprocess.Popen(args, **kwargs).pid


def cmd(args, wait=True):
    """
    Runs a command on and returns the exit status.

    :param list[str]|str args: command arguments
    :param bool wait: wait for command to end or not
    :return: command status
    :rtype: int
    """
    args = split_args(args)
    try:
        p = subprocess.Popen(args)
        if not wait:
            return 0
        return p.wait()
    except OSError:
        raise subprocess.CalledProcessError(-1, args)


def cmd_output(args):
    """
    Execute a command on the host and return a tuple containing the exit status and result string. stderr output
    is folded into the stdout result string.

    :param list[str]|str args: command arguments
    :return: command status and stdout
    :rtype: tuple[int, str]
    :raises subprocess.CalledProcessError: when the file to execute is not found
    """
    args = split_args(args)
    try:
        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        stdout, _ = p.communicate()
        status = p.wait()
        return status, stdout.strip()
    except OSError:
        raise subprocess.CalledProcessError(-1, args)


def check_cmd(args, **kwargs):
    """
    Execute a command on the host and return a tuple containing the exit status and result string. stderr output
    is folded into the stdout result string.

    :param list[str]|str args: command arguments
    :param dict kwargs: keyword arguments to pass to subprocess.Popen
    :return: combined stdout and stderr
    :rtype: str
    :raises subprocess.CalledProcessError: when there is a non-zero exit status or the file to execute is not found
    """
    kwargs["stdout"] = subprocess.PIPE
    kwargs["stderr"] = subprocess.STDOUT
    args = split_args(args)
    try:
        p = subprocess.Popen(args, **kwargs)
        stdout, _ = p.communicate()
        status = p.wait()
        if status != 0:
            raise subprocess.CalledProcessError(status, args, stdout)
        return stdout.strip()
    except OSError:
        raise subprocess.CalledProcessError(-1, args)


def hex_dump(s, bytes_per_word=2, words_per_line=8):
    """
    Hex dump of a string.

    :param str s: string to hex dump
    :param bytes_per_word: number of bytes per word
    :param words_per_line: number of words per line
    :return: hex dump of string
    """
    dump = ""
    count = 0
    total_bytes = bytes_per_word * words_per_line

    while s:
        line = s[:total_bytes]
        s = s[total_bytes:]
        tmp = map(lambda x: ("%02x" * bytes_per_word) % x,
                  zip(*[iter(map(ord, line))] * bytes_per_word))
        if len(line) % 2:
            tmp.append("%x" % ord(line[-1]))
        dump += "0x%08x: %s\n" % (count, " ".join(tmp))
        count += len(line)
    return dump[:-1]


def file_munge(pathname, header, text):
    """
    Insert text at the end of a file, surrounded by header comments.

    :param str pathname: file path to add text to
    :param str header: header text comments
    :param str text: text to append to file
    :return: nothing
    """
    # prevent duplicates
    file_demunge(pathname, header)

    with open(pathname, "a") as append_file:
        append_file.write("# BEGIN %s\n" % header)
        append_file.write(text)
        append_file.write("# END %s\n" % header)


def file_demunge(pathname, header):
    """
    Remove text that was inserted in a file surrounded by header comments.

    :param str pathname: file path to open for removing a header
    :param str header: header text to target for removal
    :return: nothing
    """
    with open(pathname, "r") as read_file:
        lines = read_file.readlines()

    start = None
    end = None

    for i in range(len(lines)):
        if lines[i] == "# BEGIN %s\n" % header:
            start = i
        elif lines[i] == "# END %s\n" % header:
            end = i + 1

    if start is None or end is None:
        return

    with open(pathname, "w") as write_file:
        lines = lines[:start] + lines[end:]
        write_file.write("".join(lines))


def expand_corepath(pathname, session=None, node=None):
    """
    Expand a file path given session information.

    :param str pathname: file path to expand
    :param core.session.Session session: core session object to expand path with
    :param core.netns.LxcNode node: node to expand path with
    :return: expanded path
    :rtype: str
    """
    if session is not None:
        pathname = pathname.replace("~", "/home/%s" % session.user)
        pathname = pathname.replace("%SESSION%", str(session.session_id))
        pathname = pathname.replace("%SESSION_DIR%", session.session_dir)
        pathname = pathname.replace("%SESSION_USER%", session.user)

    if node is not None:
        pathname = pathname.replace("%NODE%", str(node.objid))
        pathname = pathname.replace("%NODENAME%", node.name)

    return pathname


def sysctl_devname(devname):
    """
    Translate a device name to the name used with sysctl.

    :param str devname: device name to translate
    :return: translated device name
    :rtype: str
    """
    if devname is None:
        return None
    return devname.replace(".", "/")


def daemonize(rootdir="/", umask=0, close_fds=False, dontclose=(),
              stdin=os.devnull, stdout=os.devnull, stderr=os.devnull,
              stdoutmode=0644, stderrmode=0644, pidfilename=None,
              defaultmaxfd=1024):
    """
    Run the background process as a daemon.

    :param str rootdir: root directory for daemon
    :param int umask: umask for daemon
    :param bool close_fds: flag to close file descriptors
    :param dontclose: dont close options
    :param stdin: stdin for daemon
    :param stdout: stdout for daemon
    :param stderr: stderr for daemon
    :param int stdoutmode: stdout mode
    :param int stderrmode: stderr mode
    :param str pidfilename: pid file name
    :param int defaultmaxfd: default max file descriptors
    :return: nothing
    """
    if not hasattr(dontclose, "__contains__"):
        if not isinstance(dontclose, int):
            raise TypeError("dontclose must be an integer")
        dontclose = (int(dontclose),)
    else:
        for fd in dontclose:
            if not isinstance(fd, int):
                raise TypeError("dontclose must contain only integers")

    # redirect stdin
    if stdin:
        fd = os.open(stdin, os.O_RDONLY)
        os.dup2(fd, 0)
        os.close(fd)

    # redirect stdout
    if stdout:
        fd = os.open(stdout, os.O_WRONLY | os.O_CREAT | os.O_APPEND,
                     stdoutmode)
        os.dup2(fd, 1)
        if stdout == stderr:
            os.dup2(1, 2)
        os.close(fd)

    # redirect stderr
    if stderr and (stderr != stdout):
        fd = os.open(stderr, os.O_WRONLY | os.O_CREAT | os.O_APPEND,
                     stderrmode)
        os.dup2(fd, 2)
        os.close(fd)

    if os.fork():
        # parent exits
        os._exit(0)

    os.setsid()
    pid = os.fork()
    if pid:
        if pidfilename:
            try:
                f = open(pidfilename, "w")
                f.write("%s\n" % pid)
                f.close()
            except IOError:
                logger.exception("error writing to file: %s", pidfilename)
        # parent exits
        os._exit(0)

    if rootdir:
        os.chdir(rootdir)

    os.umask(umask)
    if close_fds:
        try:
            maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
            if maxfd == resource.RLIM_INFINITY:
                raise ValueError
        except:
            maxfd = defaultmaxfd

        for fd in xrange(3, maxfd):
            if fd in dontclose:
                continue
            try:
                os.close(fd)
            except IOError:
                logger.exception("error closing file descriptor")


def load_config(filename, d):
    """
    Read key=value pairs from a file, into a dict. Skip comments; strip newline characters and spacing.

    :param str filename: file to read into a dictionary
    :param dict d: dictionary to read file into
    :return: nothing
    """
    with open(filename, "r") as f:
        lines = f.readlines()

    for line in lines:
        if line[:1] == "#":
            continue

        try:
            key, value = line.split("=", 1)
            d[key] = value.strip()
        except ValueError:
            logger.exception("error reading file to dict: %s", filename)


def load_classes(path, clazz):
    """
    Dynamically load classes for use within CORE.

    :param path: path to load classes from
    :param clazz: class type expected to be inherited from for loading
    :return: list of classes loaded
    """
    # validate path exists
    logger.info("attempting to load modules from path: %s", path)
    if not os.path.isdir(path):
        logger.warn("invalid custom module directory specified" ": %s" % path)
    # check if path is in sys.path
    parent_path = os.path.dirname(path)
    if parent_path not in sys.path:
        logger.info("adding parent path to allow imports: %s", parent_path)
        sys.path.append(parent_path)

    # retrieve potential service modules, and filter out invalid modules
    base_module = os.path.basename(path)
    module_names = os.listdir(path)
    module_names = filter(lambda x: _valid_module(path, x), module_names)
    module_names = map(lambda x: x[:-3], module_names)

    # import and add all service modules in the path
    classes = []
    for module_name in module_names:
        import_statement = "%s.%s" % (base_module, module_name)
        logger.info("importing custom module: %s", import_statement)
        try:
            module = importlib.import_module(import_statement)
            members = inspect.getmembers(module, lambda x: _is_class(module, x, clazz))
            for member in members:
                valid_class = member[1]
                classes.append(valid_class)
        except:
            logger.exception("unexpected error during import, skipping: %s", import_statement)

    return classes