Merge pull request #509 from coreemu/enhancement/local-install-option

Enhancement/local install option

daemon/pyproject.toml

@@ -6,7 +6,14 @@ authors = ["Boeing Research and Technology"]
 license = "BSD-2-Clause"
 repository = "https://github.com/coreemu/core"
 documentation = "https://coreemu.github.io/core/"
-include = ["core/gui/data/**/*", "core/configservices/*/templates"]
+include = [
+    "core/api/grpc/*",
+    "core/configservices/*/templates",
+    "core/constants.py",
+    "core/gui/data/**/*",
+]
+exclude = ["core/constants.py.in"]
+
 
 [tool.poetry.dependencies]
 python = "^3.6"

docs/install.md

@@ -74,11 +74,16 @@ sudo apt remove core
 
 ## Automated Installation
 
-The automated install will install the various tools needed to help automate
+> **NOTE:** installing globally can have issues with dependency conflicts etc
-the CORE installation (python3, pip, pipx, invoke, poetry). The script will
+
-also automatically clone, build, and install the latest version of OSPF MDR.
+The automated install will install do the following:
-Finally it will install CORE scripts and a systemd service, which have
+* install base tools needed for installation
-been modified to use the installed poetry created virtual environment.
+  * python3, pip, pipx, invoke, poetry
+* installs system dependencies for building core
+* installs latest version of [OPSF MDR](https://github.com/USNavalResearchLaboratory/ospf-mdr)
+* installs core into poetry managed virtual environment or locally, if flag is passed
+* installs scripts pointing to python interpreter being used
+* installs systemd service, disabled by default
 
 After installation has completed you should be able to run the various
 CORE scripts for running core.
@@ -92,10 +97,11 @@ git clone https://github.com/coreemu/core.git
 cd core
 
 # run install script
-# script usage: install.sh [-d] [-v]
+# script usage: install.sh [-v] [-d] [-l] [-p <prefix>]
 #
 # -v enable verbose install
 # -d enable developer install
+# -l enable local install, not compatible with developer install
 # -p install prefix, defaults to /usr/local
 ./install.sh
 ```
@@ -117,16 +123,17 @@ After the installation complete it will have installed the following scripts.
 
 | Name | Description |
 |---|---|
+| core-cleanup | tool to help removed lingering core created containers, bridges, directories |
+| core-cli | tool to query, open xml files, and send commands using gRPC |
 | core-daemon | runs the backed core server providing TLV and gRPC APIs |
 | core-gui | runs the legacy tcl/tk based GUI |
-| core-pygui | runs the new python/tk based GUI |
-| core-cleanup | tool to help removed lingering core created containers, bridges, directories |
 | core-imn-to-xml | tool to help automate converting a .imn file to .xml format |
+| core-manage | tool to add, remove, or check for services, models, and node types |
+| core-pygui | runs the new python/tk based GUI |
+| core-python | provides a convenience for running the core python virtual environment |
 | core-route-monitor | tool to help monitor traffic across nodes and feed that to SDT |
 | core-service-update | tool to update automate modifying a legacy service to match current naming |
 | coresendmsg | tool to send TLV API commands from command line |
-| core-cli | tool to query, open xml files, and send commands using gRPC |
-| core-manage | tool to add, remove, or check for services, models, and node types |
 
 ## Running User Scripts
 
@@ -142,28 +149,57 @@ environment interpreter or to run a script within it.
 core-python <script>
 ```
 
-## Manually Install EMANE
+If CORE was installed locally, then you can run scripts using the default python3
+interpreter.
 
-EMANE can be installed from deb or RPM packages or from source. See the
+```shell
-[EMANE GitHub](https://github.com/adjacentlink/emane) for full details.
+python3 <script>
+```
 
-There is an invoke task to help with installing EMANE, but has issues,
+## Installing EMANE
-which attempts to build EMANE from source, but has issue on systems with
+
- older protobuf-compilers.
+> **NOTE:** installng emane for the virtual environment is known to work for 1.21+
+> **NOTE:** automated install currently targets 1.25
+
+There is an invoke task to help with installing EMANE, which attempts to
+build EMANE from source, but has issue on systems with older protobuf-compilers.
 
 ```shell
 cd <CORE_REPO>
+
+# install to virtual environment
 inv install-emane
+
+# install locally to system python3
+inv install-emane -l
 ```
 
-Alternatively, you can
+Alternatively EMANE can be installed from deb or RPM packages or from source. See the
+[EMANE GitHub](https://github.com/adjacentlink/emane) for full details.
+With the caveat that the python bindings need to be installed into CORE's
+virtualenv, unless installed locally.
+
+### Installing EMANE Python Bindings for Virtual Environment
+
+If you need to just install the EMANE python bindings to the CORE virtual
+environment, since you are installing EMANE itself from pre-built packages.
+You can run the following
+
+Leveraging the following wiki:
 [build EMANE](https://github.com/adjacentlink/emane/wiki/Build)
-from source and install the python
-bindings into the core virtual environment.
 
 The following would install the EMANE python bindings after being
 successfully built.
 ```shell
+# clone and build emane python bindings
+git clone https://github.com/adjacentlink/emane.git
+cd emane
+./autogen.sh
+PYTHON=python3 ./configure --prefix=/usr
+cd src/python
+make
+
+# install to core virtual environment
 cd <CORE_REPO>/daemon
 poetry run pip install <EMANE_REPO>/src/python
 ```

install.sh

@@ -14,7 +14,8 @@ fi
 dev=""
 verbose=""
 prefix=""
-while getopts "dvp:" opt; do
+local=""
+while getopts "dvlp:" opt; do
   case ${opt} in
   d)
     dev="-d"
@@ -22,14 +23,18 @@ while getopts "dvp:" opt; do
   v)
     verbose="-v"
     ;;
+  l)
+    local="-l"
+    ;;
   p)
     prefix="-p ${OPTARG}"
     ;;
   \?)
-    echo "script usage: $(basename $0) [-d] [-v]" >&2
+    echo "script usage: $(basename $0) [-v] [-d] [-l] [-p <prefix>]" >&2
     echo "" >&2
     echo "-v enable verbose install" >&2
     echo "-d enable developer install" >&2
+    echo "-l enable local install, not compatible with developer install" >&2
     echo "-p install prefix, defaults to /usr/local" >&2
     exit 1
     ;;
@@ -54,4 +59,4 @@ python3 -m pip install --user pipx
 python3 -m pipx ensurepath
 export PATH=$PATH:~/.local/bin
 pipx install invoke
-inv install ${dev} ${verbose} ${prefix}
+inv install ${dev} ${verbose} ${local} ${prefix}

tasks.py

@@ -171,13 +171,18 @@ def install_core(c: Context, hide: bool) -> None:
     c.run("sudo make install", hide=hide)
 
 
-def install_poetry(c: Context, dev: bool, hide: bool) -> None:
+def install_poetry(c: Context, dev: bool, local: bool, hide: bool) -> None:
     c.run("pipx install poetry", hide=hide)
-    args = "" if dev else "--no-dev"
+    if local:
-    with c.cd(DAEMON_DIR):
+        with c.cd(DAEMON_DIR):
-        c.run(f"poetry install {args}", hide=hide)
+            c.run("poetry build -f wheel", hide=hide)
-        if dev:
+            c.run("sudo python3 -m pip install dist/*")
-            c.run("poetry run pre-commit install", hide=hide)
+    else:
+        args = "" if dev else "--no-dev"
+        with c.cd(DAEMON_DIR):
+            c.run(f"poetry install {args}", hide=hide)
+            if dev:
+                c.run("poetry run pre-commit install", hide=hide)
 
 
 def install_ospf_mdr(c: Context, os_info: OsInfo, hide: bool) -> None:
@@ -243,10 +248,11 @@ def install_service(c, verbose=False, prefix=DEFAULT_PREFIX):
 @task(
     help={
         "verbose": "enable verbose",
-        "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}"
+        "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}",
+        "local": "determines if core will install to local system, default is False",
     },
 )
-def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX):
+def install_scripts(c, local=False, verbose=False, prefix=DEFAULT_PREFIX):
     """
     install core script files, modified to leverage virtual environment
     """
@@ -259,7 +265,7 @@ def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX):
             lines = f.readlines()
         first = lines[0].strip()
         # modify python scripts to point to virtual environment
-        if first == "#!/usr/bin/env python3":
+        if not local and first == "#!/usr/bin/env python3":
             lines[0] = f"#!{python}\n"
             temp = NamedTemporaryFile("w", delete=False)
             for line in lines:
@@ -273,16 +279,17 @@ def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX):
             c.run(f"sudo cp {script} {dest}", hide=hide)
 
     # setup core python helper
-    core_python = bin_dir.joinpath("core-python")
+    if not local:
-    temp = NamedTemporaryFile("w", delete=False)
+        core_python = bin_dir.joinpath("core-python")
-    temp.writelines([
+        temp = NamedTemporaryFile("w", delete=False)
-        "#!/bin/bash\n",
+        temp.writelines([
-        f'exec "{python}" "$@"\n',
+            "#!/bin/bash\n",
-    ])
+            f'exec "{python}" "$@"\n',
-    temp.close()
+        ])
-    c.run(f"sudo cp {temp.name} {core_python}", hide=hide)
+        temp.close()
-    c.run(f"sudo chmod 755 {core_python}", hide=hide)
+        c.run(f"sudo cp {temp.name} {core_python}", hide=hide)
-    os.unlink(temp.name)
+        c.run(f"sudo chmod 755 {core_python}", hide=hide)
+        os.unlink(temp.name)
 
     # install core configuration file
     config_dir = "/etc/core"
@@ -295,13 +302,15 @@ def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX):
     help={
         "dev": "install development mode",
         "verbose": "enable verbose",
-        "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}"
+        "local": "determines if core will install to local system, default is False",
+        "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}",
     },
 )
-def install(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX):
+def install(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX):
     """
     install core, poetry, scripts, service, and ospf mdr
     """
+    print(f"installing core locally: {local}")
     print(f"installing core with prefix: {prefix}")
     c.run("sudo -v", hide=True)
     p = Progress(verbose)
@@ -317,10 +326,11 @@ def install(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX):
         build_core(c, hide, prefix)
     with p.start("installing vcmd/gui"):
         install_core(c, hide)
-    with p.start("installing poetry virtual environment"):
+    install_type = "core" if local else "core virtual environment"
-        install_poetry(c, dev, hide)
+    with p.start(f"installing {install_type}"):
+        install_poetry(c, dev, local, hide)
     with p.start("installing scripts and /etc/core"):
-        install_scripts(c, hide, prefix)
+        install_scripts(c, local, hide, prefix)
     with p.start("installing systemd service"):
         install_service(c, hide, prefix)
     with p.start("installing ospf mdr"):
@@ -331,9 +341,10 @@ def install(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX):
 @task(
     help={
         "verbose": "enable verbose",
+        "local": "used determine if core is installed locally, default is False",
     },
 )
-def install_emane(c, verbose=False):
+def install_emane(c, verbose=False, local=False):
     """
     install emane and the python bindings
     """
@@ -373,18 +384,25 @@ def install_emane(c, verbose=False):
         with c.cd(emane_dir):
             c.run("sudo make install", hide=hide)
     with p.start("installing python binding for core"):
-        with c.cd(DAEMON_DIR):
+        if local:
-            c.run(f"poetry run pip install {emane_python_dir.absolute()}", hide=hide)
+            with c.cd(str(emane_python_dir)):
+                c.run("sudo python3 -m pip install .", hide=hide)
+        else:
+            with c.cd(DAEMON_DIR):
+                c.run(
+                    f"poetry run pip install {emane_python_dir.absolute()}", hide=hide
+                )
 
 
 @task(
     help={
         "dev": "uninstall development mode",
         "verbose": "enable verbose",
+        "local": "determines if core was installed local to system, default is False",
         "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}"
     },
 )
-def uninstall(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX):
+def uninstall(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX):
     """
     uninstall core, scripts, service, virtual environment, and clean build directory
     """
@@ -399,14 +417,18 @@ def uninstall(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX):
         c.run("make clean", hide=hide)
         c.run("./bootstrap.sh clean", hide=hide)
 
-    python = get_python(c, warn=True)
+    if local:
-    if python:
+        with p.start("uninstalling core"):
-        with c.cd(DAEMON_DIR):
+            c.run("sudo python3 -m pip uninstall -y core", hide=hide)
-            if dev:
+    else:
-                with p.start("uninstalling pre-commit"):
+        python = get_python(c, warn=True)
-                    c.run("poetry run pre-commit uninstall", hide=hide)
+        if python:
-            with p.start("uninstalling poetry virtual environment"):
+            with c.cd(DAEMON_DIR):
-                c.run(f"poetry env remove {python}", hide=hide)
+                if dev:
+                    with p.start("uninstalling pre-commit"):
+                        c.run("poetry run pre-commit uninstall", hide=hide)
+                with p.start("uninstalling poetry virtual environment"):
+                    c.run(f"poetry env remove {python}", hide=hide)
 
     # remove installed files
     bin_dir = Path(prefix).joinpath("bin")
@@ -416,10 +438,11 @@ def uninstall(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX):
             c.run(f"sudo rm -f {dest}", hide=hide)
 
     # remove core-python symlink
-    core_python = bin_dir.joinpath("core-python")
+    if not local:
-    c.run(f"sudo rm -f {core_python}", hide=hide)
+        core_python = bin_dir.joinpath("core-python")
+        c.run(f"sudo rm -f {core_python}", hide=hide)
 
-    # install service
+    # remove service
     systemd_dir = Path("/lib/systemd/system/")
     service_name = "core-daemon.service"
     service_file = systemd_dir.joinpath(service_name)
@@ -433,15 +456,18 @@ def uninstall(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX):
     help={
         "dev": "reinstall development mode",
         "verbose": "enable verbose",
+        "local": "determines if core will install to local system, default is False",
         "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}",
         "branch": "branch to install latest code from, default is current branch"
     },
 )
-def reinstall(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX, branch=None):
+def reinstall(
+    c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX, branch=None
+):
     """
     run the uninstall task, get latest from specified branch, and run install task
     """
-    uninstall(c, dev, verbose, prefix)
+    uninstall(c, dev, verbose, local, prefix)
     hide = not verbose
     p = Progress(verbose)
     with p.start("pulling latest code"):
@@ -453,7 +479,7 @@ def reinstall(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX, branch=None):
         c.run("git pull", hide=hide)
         if not Path("tasks.py").exists():
             raise FileNotFoundError(f"missing tasks.py on branch: {branch}")
-    install(c, dev, verbose, prefix)
+    install(c, dev, verbose, local, prefix)
 
 
 @task