From 988ca76cec78836f8aa09c385e7ddb2d6a9646b5 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 16 Jan 2020 15:48:21 -0800 Subject: [PATCH 01/37] initial code towards services that will generate files from templates and provide configuration --- daemon/Pipfile | 1 + daemon/Pipfile.lock | 294 ++++++++++-------- daemon/core/configservice/__init__.py | 0 daemon/core/configservice/base.py | 128 ++++++++ daemon/core/configservices/__init__.py | 0 .../configservices/defaultroute/__init__.py | 0 .../configservices/defaultroute/service.py | 47 +++ .../defaultroute/templates/defaultroute.sh | 5 + 8 files changed, 346 insertions(+), 129 deletions(-) create mode 100644 daemon/core/configservice/__init__.py create mode 100644 daemon/core/configservice/base.py create mode 100644 daemon/core/configservices/__init__.py create mode 100644 daemon/core/configservices/defaultroute/__init__.py create mode 100644 daemon/core/configservices/defaultroute/service.py create mode 100644 daemon/core/configservices/defaultroute/templates/defaultroute.sh diff --git a/daemon/Pipfile b/daemon/Pipfile index d55b248f..63a1d240 100644 --- a/daemon/Pipfile +++ b/daemon/Pipfile @@ -21,3 +21,4 @@ mock = "*" [packages] core = {editable = true,path = "."} +mako = "*" diff --git a/daemon/Pipfile.lock b/daemon/Pipfile.lock index 71c11fd2..c2ffa006 100644 --- a/daemon/Pipfile.lock +++ b/daemon/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d702e6eed5a1362bf261543572bbffd2e8a87140b8d8cb07b99fb0d25220a2b5" + "sha256": "3d8bcb3cbfddc5803b6f2309330c77e815b44ed81f42644f4d42b625e65b5d24" }, "pipfile-spec": 6, "requires": {}, @@ -162,11 +162,11 @@ }, "invoke": { "hashes": [ - "sha256:c52274d2e8a6d64ef0d61093e1983268ea1fc0cd13facb9448c4ef0c9a7ac7da", - "sha256:f4ec8a134c0122ea042c8912529f87652445d9f4de590b353d23f95bfa1f0efd", - "sha256:fc803a5c9052f15e63310aa81a43498d7c55542beb18564db88a9d75a176fa44" + "sha256:4668a4a594a47f2da2f0672ec2f7b1566f809cebf10bcd95ce2de9ecf39b95d1", + "sha256:ae7b4513638bde9afcda0825e9535599637a3f65bd819a27098356027bb17c8a", + "sha256:e04faba8ea7cdf6f5c912be42dcafd5c1074b7f2f306998992c4bfb40a9a690b" ], - "version": "==1.3.0" + "version": "==1.4.0" }, "lxml": { "hashes": [ @@ -199,6 +199,46 @@ ], "version": "==4.4.2" }, + "mako": { + "hashes": [ + "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "markupsafe": { + "hashes": [ + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" + ], + "version": "==1.1.1" + }, "netaddr": { "hashes": [ "sha256:38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd", @@ -215,59 +255,53 @@ }, "pillow": { "hashes": [ - "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", - "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", - "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", - "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", - "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", - "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", - "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", - "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", - "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", - "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", - "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", - "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", - "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", - "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", - "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", - "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", - "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", - "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", - "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", - "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", - "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", - "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", - "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", - "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", - "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", - "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", - "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", - "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", - "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", - "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" + "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be", + "sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946", + "sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837", + "sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f", + "sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00", + "sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d", + "sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533", + "sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a", + "sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358", + "sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda", + "sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435", + "sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2", + "sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313", + "sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff", + "sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317", + "sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2", + "sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614", + "sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0", + "sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386", + "sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9", + "sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636", + "sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865" ], - "version": "==6.2.1" + "version": "==7.0.0" }, "protobuf": { "hashes": [ - "sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd", - "sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed", - "sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057", - "sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce", - "sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03", - "sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46", - "sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33", - "sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c", - "sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9", - "sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef", - "sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b", - "sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d", - "sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8", - "sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6", - "sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941", - "sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13" + "sha256:0329e86a397db2a83f9dcbe21d9be55a47f963cdabc893c3a24f4d3a8f117c37", + "sha256:0a7219254afec0d488211f3d482d8ed57e80ae735394e584a98d8f30a8c88a36", + "sha256:14d6ac53df9cb5bb87c4f91b677c1bc5cec9c0fd44327f367a3c9562de2877c4", + "sha256:180fc364b42907a1d2afa183ccbeffafe659378c236b1ec3daca524950bb918d", + "sha256:3d7a7d8d20b4e7a8f63f62de2d192cfd8b7a53c56caba7ece95367ca2b80c574", + "sha256:3f509f7e50d806a434fe4a5fbf602516002a0f092889209fff7db82060efffc0", + "sha256:4571da974019849201fc1ec6626b9cea54bd11b6bed140f8f737c0a33ea37de5", + "sha256:56bd1d84fbf4505c7b73f04de987eef5682e5752c811141b0186a3809bfb396f", + "sha256:680c668d00b5eff08b86aef9e5ba9a705e621ea05d39071cfea8e28cb2400946", + "sha256:6b5b947dc8b3f2aec0eaad65b0b5113fcd642c358c31357c647da6281ee31104", + "sha256:6e96dffaf4d0a9a329e528b353ba62fd9ef13599688723d96bc9c165d0b6871e", + "sha256:919f0d6f6addc836d08658eba3b52be2e92fd3e76da3ce00c325d8e9826d17c7", + "sha256:9c7b19c30cf0644afd0e4218b13f637ce54382fdcb1c8f75bf3e84e49a5f6d0a", + "sha256:a2e6f57114933882ec701807f217df2fb4588d47f71f227c0a163446b930d507", + "sha256:a6b970a2eccfcbabe1acf230fbf112face1c4700036c95e195f3554d7bcb04c1", + "sha256:bc45641cbcdea068b67438244c926f9fd3e5cbdd824448a4a64370610df7c593", + "sha256:d61b14a9090da77fe87e38ba4c6c43d3533dcbeb5d84f5474e7ac63c532dcc9c", + "sha256:d6faf5dbefb593e127463f58076b62fcfe0784187be8fe1aa9167388f24a22a1" ], - "version": "==3.11.1" + "version": "==3.11.2" }, "pycparser": { "hashes": [ @@ -303,26 +337,26 @@ }, "pyyaml": { "hashes": [ - "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", - "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", - "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", - "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", - "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", - "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", - "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", - "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", - "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", - "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", - "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], - "version": "==5.2" + "version": "==5.3" }, "six": { "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.13.0" + "version": "==1.14.0" } }, "develop": { @@ -483,18 +517,18 @@ }, "identify": { "hashes": [ - "sha256:7782115794ec28b011702815d9f5e532244560cd2bf0789c4f09381d43befd90", - "sha256:9e7521e9abeaede4d2d1092a106e418c65ddf6b3182b43930bcb3c8cfb974488" + "sha256:418f3b2313ac0b531139311a6b426854e9cbdfcfb6175447a5039aa6291d8b30", + "sha256:8ad99ed1f3a965612dcb881435bf58abcfbeb05e230bb8c352b51e8eac103360" ], - "version": "==1.4.8" + "version": "==1.4.10" }, "importlib-metadata": { "hashes": [ - "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", - "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" + "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", + "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" ], "markers": "python_version < '3.8'", - "version": "==1.3.0" + "version": "==1.4.0" }, "importlib-resources": { "hashes": [ @@ -529,23 +563,23 @@ }, "more-itertools": { "hashes": [ - "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d", - "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564" + "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39", + "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288" ], - "version": "==8.0.2" + "version": "==8.1.0" }, "nodeenv": { "hashes": [ - "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a" + "sha256:561057acd4ae3809e665a9aaaf214afff110bbb6a6d5c8a96121aea6878408b3" ], - "version": "==1.3.3" + "version": "==1.3.4" }, "packaging": { "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", + "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" ], - "version": "==19.2" + "version": "==20.0" }, "pluggy": { "hashes": [ @@ -556,39 +590,41 @@ }, "pre-commit": { "hashes": [ - "sha256:9f152687127ec90642a2cc3e4d9e1e6240c4eb153615cb02aa1ad41d331cbb6e", - "sha256:c2e4810d2d3102d354947907514a78c5d30424d299dc0fe48f5aa049826e9b50" + "sha256:8f48d8637bdae6fa70cc97db9c1dd5aa7c5c8bf71968932a380628c25978b850", + "sha256:f92a359477f3252452ae2e8d3029de77aec59415c16ae4189bcfba40b757e029" ], "index": "pypi", - "version": "==1.20.0" + "version": "==1.21.0" }, "protobuf": { "hashes": [ - "sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd", - "sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed", - "sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057", - "sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce", - "sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03", - "sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46", - "sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33", - "sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c", - "sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9", - "sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef", - "sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b", - "sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d", - "sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8", - "sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6", - "sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941", - "sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13" + "sha256:0329e86a397db2a83f9dcbe21d9be55a47f963cdabc893c3a24f4d3a8f117c37", + "sha256:0a7219254afec0d488211f3d482d8ed57e80ae735394e584a98d8f30a8c88a36", + "sha256:14d6ac53df9cb5bb87c4f91b677c1bc5cec9c0fd44327f367a3c9562de2877c4", + "sha256:180fc364b42907a1d2afa183ccbeffafe659378c236b1ec3daca524950bb918d", + "sha256:3d7a7d8d20b4e7a8f63f62de2d192cfd8b7a53c56caba7ece95367ca2b80c574", + "sha256:3f509f7e50d806a434fe4a5fbf602516002a0f092889209fff7db82060efffc0", + "sha256:4571da974019849201fc1ec6626b9cea54bd11b6bed140f8f737c0a33ea37de5", + "sha256:56bd1d84fbf4505c7b73f04de987eef5682e5752c811141b0186a3809bfb396f", + "sha256:680c668d00b5eff08b86aef9e5ba9a705e621ea05d39071cfea8e28cb2400946", + "sha256:6b5b947dc8b3f2aec0eaad65b0b5113fcd642c358c31357c647da6281ee31104", + "sha256:6e96dffaf4d0a9a329e528b353ba62fd9ef13599688723d96bc9c165d0b6871e", + "sha256:919f0d6f6addc836d08658eba3b52be2e92fd3e76da3ce00c325d8e9826d17c7", + "sha256:9c7b19c30cf0644afd0e4218b13f637ce54382fdcb1c8f75bf3e84e49a5f6d0a", + "sha256:a2e6f57114933882ec701807f217df2fb4588d47f71f227c0a163446b930d507", + "sha256:a6b970a2eccfcbabe1acf230fbf112face1c4700036c95e195f3554d7bcb04c1", + "sha256:bc45641cbcdea068b67438244c926f9fd3e5cbdd824448a4a64370610df7c593", + "sha256:d61b14a9090da77fe87e38ba4c6c43d3533dcbeb5d84f5474e7ac63c532dcc9c", + "sha256:d6faf5dbefb593e127463f58076b62fcfe0784187be8fe1aa9167388f24a22a1" ], - "version": "==3.11.1" + "version": "==3.11.2" }, "py": { "hashes": [ - "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", - "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", + "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" ], - "version": "==1.8.0" + "version": "==1.8.1" }, "pycodestyle": { "hashes": [ @@ -606,10 +642,10 @@ }, "pyparsing": { "hashes": [ - "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", - "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.5" + "version": "==2.4.6" }, "pytest": { "hashes": [ @@ -621,26 +657,26 @@ }, "pyyaml": { "hashes": [ - "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", - "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", - "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", - "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", - "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", - "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", - "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", - "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", - "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", - "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", - "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], - "version": "==5.2" + "version": "==5.3" }, "six": { "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.13.0" + "version": "==1.14.0" }, "toml": { "hashes": [ @@ -658,17 +694,17 @@ }, "wcwidth": { "hashes": [ - "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", - "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", + "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" ], - "version": "==0.1.7" + "version": "==0.1.8" }, "zipp": { "hashes": [ - "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", - "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" + "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", + "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" ], - "version": "==0.6.0" + "version": "==1.0.0" } } } diff --git a/daemon/core/configservice/__init__.py b/daemon/core/configservice/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py new file mode 100644 index 00000000..46990333 --- /dev/null +++ b/daemon/core/configservice/base.py @@ -0,0 +1,128 @@ +import abc +import enum +import inspect +import logging +import pathlib +import time +from typing import Any, Dict, List + +from mako import exceptions +from mako.lookup import TemplateLookup + +from core.errors import CoreCommandError, CoreError +from core.nodes.base import CoreNode + +TEMPLATES_DIR = "templates" + + +class ConfigServiceMode(enum.Enum): + BLOCKING = 0 + NON_BLOCKING = 1 + TIMER = 2 + + +class ConfigService(abc.ABC): + # validation period in seconds, how frequent validation is attempted + validation_period = 0.5 + + # time to wait in seconds for determining if service started successfully + validation_timer = 5 + + def __init__(self, node: CoreNode) -> None: + self.node = node + class_file = inspect.getfile(self.__class__) + templates_path = pathlib.Path(class_file).parent.joinpath(TEMPLATES_DIR) + logging.info(templates_path) + self.templates = TemplateLookup(directories=templates_path) + + @property + @abc.abstractmethod + def name(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def group(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def executables(self) -> List[str]: + raise NotImplementedError + + @property + @abc.abstractmethod + def dependencies(self) -> List[str]: + raise NotImplementedError + + @property + @abc.abstractmethod + def startup(self) -> List[str]: + raise NotImplementedError + + @property + @abc.abstractmethod + def validate(self) -> List[str]: + raise NotImplementedError + + @property + @abc.abstractmethod + def shutdown(self) -> List[str]: + raise NotImplementedError + + @property + @abc.abstractmethod + def validation_mode(self) -> ConfigServiceMode: + raise NotImplementedError + + def start(self) -> None: + if not self.startup: + return + wait = self.validation_mode == ConfigServiceMode.BLOCKING + start = time.monotonic() + index = 0 + cmds = self.startup[:] + while cmds: + cmd = cmds[index] + try: + self.node.cmd(cmd, wait=wait) + del cmds[index] + index += 1 + except CoreCommandError: + logging.exception("error starting command") + time.sleep(self.validation_period) + + if time.monotonic() - start > 0: + raise CoreError( + f"node({self.node.name}) service({self.name()}) failed to start" + ) + + def stop(self) -> None: + if not self.shutdown: + return + for cmd in self.shutdown: + self.node.cmd(cmd, wait=False) + + def restart(self): + self.stop() + self.start() + + def run(self, cmd: str, wait: bool = True): + self.node.cmd(cmd, wait) + + def create_files(self) -> None: + raise NotImplementedError + + def render(self, name: str, data: Dict[str, Any] = None) -> None: + if data is None: + data = {} + try: + template = self.templates.get_template(name) + rendered = template.render_unicode(node=self.node, **data) + print(rendered) + # self.node.nodefile(name, rendered) + except Exception: + raise CoreError( + f"error rendering template: {name}" + f"{exceptions.text_error_template().render()}" + ) diff --git a/daemon/core/configservices/__init__.py b/daemon/core/configservices/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/core/configservices/defaultroute/__init__.py b/daemon/core/configservices/defaultroute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/core/configservices/defaultroute/service.py b/daemon/core/configservices/defaultroute/service.py new file mode 100644 index 00000000..dab779a0 --- /dev/null +++ b/daemon/core/configservices/defaultroute/service.py @@ -0,0 +1,47 @@ +import logging + +import netaddr + +from core.configservice.base import ConfigService, ConfigServiceMode +from core.emulator.session import Session +from core.nodes.base import CoreNode +from core.nodes.interface import Veth + + +class DefaultRoute(ConfigService): + name = "DefaultRoute" + group = "Utility" + executables = [] + dependencies = [] + startup = [] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + + def create_files(self): + self.create_default_route() + + def create_default_route(self): + addresses = [] + for netif in self.node.netifs(): + if getattr(netif, "control", False): + continue + for addr in netif.addrlist: + net = netaddr.IPNetwork(addr) + if net[1] != net[-2]: + addresses.append(net[1]) + data = dict(addresses=addresses) + self.render("defaultroute.sh", data) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + session = Session(1, mkdir=False) + node = CoreNode(session, _id=1, start=False) + netif = Veth(session, node, "eth0", "eth0", start=False) + netif.addaddr("10.0.0.1/24") + node.addnetif(netif, 0) + service = DefaultRoute(node) + service.create_files() + # data = service.render(node, "defaultroute.sh", dict(addresses=[])) + # print(data) diff --git a/daemon/core/configservices/defaultroute/templates/defaultroute.sh b/daemon/core/configservices/defaultroute/templates/defaultroute.sh new file mode 100644 index 00000000..879a8861 --- /dev/null +++ b/daemon/core/configservices/defaultroute/templates/defaultroute.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# auto-generated by DefaultRoute service +% for address in addresses: +ip route add default via ${address} +% endfor From cf7dda816cdff4aa7ce21a884b091b0ab2ce0beb Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 16 Jan 2020 17:14:42 -0800 Subject: [PATCH 02/37] improvements to config services, start/stop/validate basics, abc levergae to enforce definitions for custom services --- daemon/core/configservice/base.py | 89 ++++++++++++++----- .../configservices/defaultroute/service.py | 5 +- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 46990333..dd0ceb60 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -37,12 +37,17 @@ class ConfigService(abc.ABC): @property @abc.abstractmethod - def name(self): + def name(self) -> str: raise NotImplementedError @property @abc.abstractmethod - def group(self): + def group(self) -> str: + raise NotImplementedError + + @property + @abc.abstractmethod + def directories(self) -> List[str]: raise NotImplementedError @property @@ -76,8 +81,49 @@ class ConfigService(abc.ABC): raise NotImplementedError def start(self) -> None: - if not self.startup: - return + self.create_dirs() + self.create_files() + self.run_startup() + self.run_validation() + + def stop(self) -> None: + for cmd in self.shutdown: + try: + self.node.cmd(cmd) + except CoreCommandError: + logging.exception( + f"node({self.node.name}) service({self.name}) " + f"failed shutdown: {cmd}" + ) + + def restart(self) -> None: + self.stop() + self.start() + + def create_dirs(self) -> None: + for directory in self.directories: + try: + self.node.privatedir(directory) + except (CoreCommandError, ValueError): + raise CoreError( + f"node({self.node.name}) service({self.name}) " + f"failure to create service directory: {directory}" + ) + + def create_files(self) -> None: + raise NotImplementedError + + def run_startup(self) -> None: + for cmd in self.startup: + try: + self.node.cmd(cmd) + except CoreCommandError: + raise CoreError( + f"node({self.node.name}) service({self.name}) " + f"failed startup: {cmd}" + ) + + def run_validation(self) -> None: wait = self.validation_mode == ConfigServiceMode.BLOCKING start = time.monotonic() index = 0 @@ -89,40 +135,35 @@ class ConfigService(abc.ABC): del cmds[index] index += 1 except CoreCommandError: - logging.exception("error starting command") + logging.debug( + f"node({self.node.name}) service({self.name}) " + f"validate command failed: {cmd}" + ) time.sleep(self.validation_period) if time.monotonic() - start > 0: raise CoreError( - f"node({self.node.name}) service({self.name()}) failed to start" + f"node({self.node.name}) service({self.name}) " + f"failed to validate" ) - def stop(self) -> None: - if not self.shutdown: - return - for cmd in self.shutdown: - self.node.cmd(cmd, wait=False) - - def restart(self): - self.stop() - self.start() - - def run(self, cmd: str, wait: bool = True): - self.node.cmd(cmd, wait) - - def create_files(self) -> None: - raise NotImplementedError - def render(self, name: str, data: Dict[str, Any] = None) -> None: if data is None: data = {} try: template = self.templates.get_template(name) rendered = template.render_unicode(node=self.node, **data) - print(rendered) + logging.info( + "node(%s) service(%s) template(%s): \n%s", + self.node.name, + self.name, + name, + rendered, + ) # self.node.nodefile(name, rendered) except Exception: raise CoreError( - f"error rendering template: {name}" + f"node({self.node.name}) service({self.name}) " + f"error rendering template({name}): " f"{exceptions.text_error_template().render()}" ) diff --git a/daemon/core/configservices/defaultroute/service.py b/daemon/core/configservices/defaultroute/service.py index dab779a0..445899c9 100644 --- a/daemon/core/configservices/defaultroute/service.py +++ b/daemon/core/configservices/defaultroute/service.py @@ -11,6 +11,7 @@ from core.nodes.interface import Veth class DefaultRoute(ConfigService): name = "DefaultRoute" group = "Utility" + directories = [] executables = [] dependencies = [] startup = [] @@ -42,6 +43,4 @@ if __name__ == "__main__": netif.addaddr("10.0.0.1/24") node.addnetif(netif, 0) service = DefaultRoute(node) - service.create_files() - # data = service.render(node, "defaultroute.sh", dict(addresses=[])) - # print(data) + service.start() From 433327c0ae76cdceaa9094286c11fef65a99d504 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 17 Jan 2020 11:40:29 -0800 Subject: [PATCH 03/37] added config service manager, added custom loading of subdirs for config based services, added configurations for config services --- daemon/Pipfile.lock | 6 +- daemon/core/configservice/base.py | 68 ++++++++++- .../configservices/defaultroute/service.py | 46 -------- .../__init__.py | 0 .../templates/defaultroute.sh | 1 + .../serviceutils/templates/ipforward.sh | 16 +++ .../core/configservices/serviceutils/utils.py | 109 ++++++++++++++++++ 7 files changed, 194 insertions(+), 52 deletions(-) delete mode 100644 daemon/core/configservices/defaultroute/service.py rename daemon/core/configservices/{defaultroute => serviceutils}/__init__.py (100%) rename daemon/core/configservices/{defaultroute => serviceutils}/templates/defaultroute.sh (86%) create mode 100644 daemon/core/configservices/serviceutils/templates/ipforward.sh create mode 100644 daemon/core/configservices/serviceutils/utils.py diff --git a/daemon/Pipfile.lock b/daemon/Pipfile.lock index c2ffa006..aa6bcd51 100644 --- a/daemon/Pipfile.lock +++ b/daemon/Pipfile.lock @@ -649,11 +649,11 @@ }, "pytest": { "hashes": [ - "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa", - "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4" + "sha256:9f8d44f4722b3d06b41afaeb8d177cfbe0700f8351b1fc755dd27eedaa3eb9e0", + "sha256:f5d3d0e07333119fe7d4af4ce122362dc4053cdd34a71d2766290cf5369c64ad" ], "index": "pypi", - "version": "==5.3.2" + "version": "==5.3.3" }, "pyyaml": { "hashes": [ diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index dd0ceb60..06f48be0 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -9,6 +9,8 @@ from typing import Any, Dict, List from mako import exceptions from mako.lookup import TemplateLookup +from core import utils +from core.config import Configuration from core.errors import CoreCommandError, CoreError from core.nodes.base import CoreNode @@ -21,6 +23,42 @@ class ConfigServiceMode(enum.Enum): TIMER = 2 +class ConfigServiceManager: + def __init__(self): + self.services = {} + + def add(self, service: "ConfigService") -> None: + name = service.name + logging.debug("loading service: class(%s) name(%s)", service.__class__, name) + + # avoid duplicate services + if name in self.services: + raise CoreError(f"duplicate service being added: {name}") + + # validate dependent executables are present + for executable in service.executables: + utils.which(executable, required=True) + + # make service available + self.services[name] = service + + def load(self, path: str) -> List[str]: + path = pathlib.Path(path) + subdirs = [x for x in path.iterdir() if x.is_dir()] + service_errors = [] + for subdir in subdirs: + logging.info("loading config services from: %s", subdir) + services = utils.load_classes(str(subdir), ConfigService) + for service in services: + logging.info("found service: %s", service) + try: + self.add(service) + except CoreError as e: + service_errors.append(service.name) + logging.debug("not loading service(%s): %s", service.name, e) + return service_errors + + class ConfigService(abc.ABC): # validation period in seconds, how frequent validation is attempted validation_period = 0.5 @@ -34,6 +72,9 @@ class ConfigService(abc.ABC): templates_path = pathlib.Path(class_file).parent.joinpath(TEMPLATES_DIR) logging.info(templates_path) self.templates = TemplateLookup(directories=templates_path) + self.config = {} + configs = self.default_configs[:] + self._define_config(configs) @property @abc.abstractmethod @@ -50,6 +91,11 @@ class ConfigService(abc.ABC): def directories(self) -> List[str]: raise NotImplementedError + @property + @abc.abstractmethod + def default_configs(self) -> List[Configuration]: + raise NotImplementedError + @property @abc.abstractmethod def executables(self) -> List[str]: @@ -127,7 +173,7 @@ class ConfigService(abc.ABC): wait = self.validation_mode == ConfigServiceMode.BLOCKING start = time.monotonic() index = 0 - cmds = self.startup[:] + cmds = self.validate[:] while cmds: cmd = cmds[index] try: @@ -152,7 +198,9 @@ class ConfigService(abc.ABC): data = {} try: template = self.templates.get_template(name) - rendered = template.render_unicode(node=self.node, **data) + rendered = template.render_unicode( + node=self.node, config=self.render_config(), **data + ) logging.info( "node(%s) service(%s) template(%s): \n%s", self.node.name, @@ -160,10 +208,24 @@ class ConfigService(abc.ABC): name, rendered, ) - # self.node.nodefile(name, rendered) + self.node.nodefile(name, rendered) except Exception: raise CoreError( f"node({self.node.name}) service({self.name}) " f"error rendering template({name}): " f"{exceptions.text_error_template().render()}" ) + + def _define_config(self, configs: List[Configuration]) -> None: + for config in configs: + self.config[config.id] = config + + def render_config(self) -> Dict[str, str]: + return {k: v.default for k, v in self.config.items()} + + def set_config(self, data: Dict[str, str]) -> None: + for key, value in data.items(): + config = self.config.get(key) + if config is None: + raise CoreError(f"unknown config: {key}") + config.default = value diff --git a/daemon/core/configservices/defaultroute/service.py b/daemon/core/configservices/defaultroute/service.py deleted file mode 100644 index 445899c9..00000000 --- a/daemon/core/configservices/defaultroute/service.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging - -import netaddr - -from core.configservice.base import ConfigService, ConfigServiceMode -from core.emulator.session import Session -from core.nodes.base import CoreNode -from core.nodes.interface import Veth - - -class DefaultRoute(ConfigService): - name = "DefaultRoute" - group = "Utility" - directories = [] - executables = [] - dependencies = [] - startup = [] - validate = [] - shutdown = [] - validation_mode = ConfigServiceMode.BLOCKING - - def create_files(self): - self.create_default_route() - - def create_default_route(self): - addresses = [] - for netif in self.node.netifs(): - if getattr(netif, "control", False): - continue - for addr in netif.addrlist: - net = netaddr.IPNetwork(addr) - if net[1] != net[-2]: - addresses.append(net[1]) - data = dict(addresses=addresses) - self.render("defaultroute.sh", data) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - session = Session(1, mkdir=False) - node = CoreNode(session, _id=1, start=False) - netif = Veth(session, node, "eth0", "eth0", start=False) - netif.addaddr("10.0.0.1/24") - node.addnetif(netif, 0) - service = DefaultRoute(node) - service.start() diff --git a/daemon/core/configservices/defaultroute/__init__.py b/daemon/core/configservices/serviceutils/__init__.py similarity index 100% rename from daemon/core/configservices/defaultroute/__init__.py rename to daemon/core/configservices/serviceutils/__init__.py diff --git a/daemon/core/configservices/defaultroute/templates/defaultroute.sh b/daemon/core/configservices/serviceutils/templates/defaultroute.sh similarity index 86% rename from daemon/core/configservices/defaultroute/templates/defaultroute.sh rename to daemon/core/configservices/serviceutils/templates/defaultroute.sh index 879a8861..952a7ef1 100644 --- a/daemon/core/configservices/defaultroute/templates/defaultroute.sh +++ b/daemon/core/configservices/serviceutils/templates/defaultroute.sh @@ -1,5 +1,6 @@ #!/bin/sh # auto-generated by DefaultRoute service +# config: ${config} % for address in addresses: ip route add default via ${address} % endfor diff --git a/daemon/core/configservices/serviceutils/templates/ipforward.sh b/daemon/core/configservices/serviceutils/templates/ipforward.sh new file mode 100644 index 00000000..a8d3abed --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/ipforward.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# auto-generated by IPForward service (utility.py) +sysctl -w net.ipv4.conf.all.forwarding=1 +sysctl -w net.ipv4.conf.default.forwarding=1 +sysctl -w net.ipv6.conf.all.forwarding=1 +sysctl -w net.ipv6.conf.default.forwarding=1 +sysctl -w net.ipv4.conf.all.send_redirects=0 +sysctl -w net.ipv4.conf.default.send_redirects=0 +sysctl -w net.ipv4.conf.all.rp_filter=0 +sysctl -w net.ipv4.conf.default.rp_filter=0 +# setup forwarding for node interfaces +% for devname in devnames: +sysctl -w net.ipv4.conf.${devname}.forwarding=1 +sysctl -w net.ipv4.conf.${devname}.send_redirects=0 +sysctl -w net.ipv4.conf.${devname}.rp_filter=0 +% endfor diff --git a/daemon/core/configservices/serviceutils/utils.py b/daemon/core/configservices/serviceutils/utils.py new file mode 100644 index 00000000..ef975a0e --- /dev/null +++ b/daemon/core/configservices/serviceutils/utils.py @@ -0,0 +1,109 @@ +import logging +import os + +import netaddr + +from core import utils +from core.config import Configuration +from core.configservice.base import ( + ConfigService, + ConfigServiceManager, + ConfigServiceMode, +) +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import ConfigDataTypes, EventTypes, NodeTypes + +GROUP_NAME = "Utility" + + +class DefaultRoute(ConfigService): + name = "DefaultRoute" + group = GROUP_NAME + directories = [] + executables = [] + dependencies = [] + startup = ["sh defaultroute.sh"] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [ + Configuration(_id="value1", _type=ConfigDataTypes.STRING, label="Value 1"), + Configuration(_id="value2", _type=ConfigDataTypes.STRING, label="Value 2"), + Configuration(_id="value3", _type=ConfigDataTypes.STRING, label="Value 3"), + ] + + def create_files(self): + addresses = [] + for netif in self.node.netifs(): + if getattr(netif, "control", False): + continue + for addr in netif.addrlist: + net = netaddr.IPNetwork(addr) + if net[1] != net[-2]: + addresses.append(net[1]) + data = dict(addresses=addresses) + self.render("defaultroute.sh", data) + + +class IpForwardService(ConfigService): + name = "IPForward" + group = GROUP_NAME + directories = [] + executables = ["sysctl"] + dependencies = [] + startup = ["sh ipforward.sh"] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + + def create_files(self) -> None: + devnames = [] + for ifc in self.node.netifs(): + devname = utils.sysctl_devname(ifc.name) + devnames.append(devname) + data = dict(devnames=devnames) + self.render("ipforward.sh", data) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + # setup basic network + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + options = NodeOptions(model="nothing") + # options.services = [] + coreemu = CoreEmu() + session = coreemu.create_session() + session.set_state(EventTypes.CONFIGURATION_STATE) + switch = session.add_node(_type=NodeTypes.SWITCH) + + # node one + node_one = session.add_node(options=options) + interface = prefixes.create_interface(node_one) + session.add_link(node_one.id, switch.id, interface_one=interface) + + # node two + node_two = session.add_node(options=options) + interface = prefixes.create_interface(node_two) + session.add_link(node_two.id, switch.id, interface_one=interface) + + session.instantiate() + + # manager load config services + manager = ConfigServiceManager() + path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) + manager.load(path) + + clazz = manager.services["DefaultRoute"] + dr_service = clazz(node_one) + dr_service.set_config({"value1": "custom"}) + dr_service.start() + + clazz = manager.services["IPForward"] + dr_service = clazz(node_one) + dr_service.start() + + input("press enter to exit") + session.shutdown() From dbc77d81f67fcc7fa9a8efd007fc23f9ab6bf6bf Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 17 Jan 2020 13:47:55 -0800 Subject: [PATCH 04/37] updates naming for built in config services, broke out current example script to examples dir, broke out config service manager to separate file --- daemon/core/configservice/base.py | 37 ----------- daemon/core/configservice/manager.py | 48 +++++++++++++++ .../sercurityservices/__init__.py | 0 .../sercurityservices/services.py | 19 ++++++ .../sercurityservices/templates/vpnclient.sh | 61 +++++++++++++++++++ .../serviceutils/{utils.py => services.py} | 57 +---------------- daemon/examples/configservices/testing.py | 49 +++++++++++++++ 7 files changed, 180 insertions(+), 91 deletions(-) create mode 100644 daemon/core/configservice/manager.py create mode 100644 daemon/core/configservices/sercurityservices/__init__.py create mode 100644 daemon/core/configservices/sercurityservices/services.py create mode 100644 daemon/core/configservices/sercurityservices/templates/vpnclient.sh rename daemon/core/configservices/serviceutils/{utils.py => services.py} (50%) create mode 100644 daemon/examples/configservices/testing.py diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 06f48be0..93de9ab4 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -9,7 +9,6 @@ from typing import Any, Dict, List from mako import exceptions from mako.lookup import TemplateLookup -from core import utils from core.config import Configuration from core.errors import CoreCommandError, CoreError from core.nodes.base import CoreNode @@ -23,42 +22,6 @@ class ConfigServiceMode(enum.Enum): TIMER = 2 -class ConfigServiceManager: - def __init__(self): - self.services = {} - - def add(self, service: "ConfigService") -> None: - name = service.name - logging.debug("loading service: class(%s) name(%s)", service.__class__, name) - - # avoid duplicate services - if name in self.services: - raise CoreError(f"duplicate service being added: {name}") - - # validate dependent executables are present - for executable in service.executables: - utils.which(executable, required=True) - - # make service available - self.services[name] = service - - def load(self, path: str) -> List[str]: - path = pathlib.Path(path) - subdirs = [x for x in path.iterdir() if x.is_dir()] - service_errors = [] - for subdir in subdirs: - logging.info("loading config services from: %s", subdir) - services = utils.load_classes(str(subdir), ConfigService) - for service in services: - logging.info("found service: %s", service) - try: - self.add(service) - except CoreError as e: - service_errors.append(service.name) - logging.debug("not loading service(%s): %s", service.name, e) - return service_errors - - class ConfigService(abc.ABC): # validation period in seconds, how frequent validation is attempted validation_period = 0.5 diff --git a/daemon/core/configservice/manager.py b/daemon/core/configservice/manager.py new file mode 100644 index 00000000..b0fe6d2e --- /dev/null +++ b/daemon/core/configservice/manager.py @@ -0,0 +1,48 @@ +import logging +import pathlib +from typing import List + +from core import utils +from core.configservice.base import ConfigService +from core.errors import CoreError + + +class ConfigServiceManager: + def __init__(self): + self.services = {} + + def add(self, service: ConfigService) -> None: + name = service.name + logging.debug("loading service: class(%s) name(%s)", service.__class__, name) + + # avoid duplicate services + if name in self.services: + raise CoreError(f"duplicate service being added: {name}") + + # validate dependent executables are present + for executable in service.executables: + try: + utils.which(executable, required=True) + except ValueError: + raise CoreError( + f"service({service.name}) missing executable {executable}" + ) + + # make service available + self.services[name] = service + + def load(self, path: str) -> List[str]: + path = pathlib.Path(path) + subdirs = [x for x in path.iterdir() if x.is_dir()] + service_errors = [] + for subdir in subdirs: + logging.info("loading config services from: %s", subdir) + services = utils.load_classes(str(subdir), ConfigService) + for service in services: + logging.info("found service: %s", service) + try: + self.add(service) + except CoreError as e: + service_errors.append(service.name) + logging.debug("not loading service(%s): %s", service.name, e) + return service_errors diff --git a/daemon/core/configservices/sercurityservices/__init__.py b/daemon/core/configservices/sercurityservices/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/core/configservices/sercurityservices/services.py b/daemon/core/configservices/sercurityservices/services.py new file mode 100644 index 00000000..b063a8e2 --- /dev/null +++ b/daemon/core/configservices/sercurityservices/services.py @@ -0,0 +1,19 @@ +from core.configservice.base import ConfigService, ConfigServiceMode + +GROUP_NAME = "Security" + + +class VpnClient(ConfigService): + name = "VPNClient" + group = GROUP_NAME + directories = [] + executables = ["openvpn", "ip", "killall"] + dependencies = [] + startup = ["sh vpnclient.sh"] + validate = ["pidof openvpn"] + shutdown = ["killall openvpn"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + + def create_files(self): + self.render("vpnclient.sh") diff --git a/daemon/core/configservices/sercurityservices/templates/vpnclient.sh b/daemon/core/configservices/sercurityservices/templates/vpnclient.sh new file mode 100644 index 00000000..7d620b46 --- /dev/null +++ b/daemon/core/configservices/sercurityservices/templates/vpnclient.sh @@ -0,0 +1,61 @@ +# -------- CUSTOMIZATION REQUIRED -------- +# +# The VPNClient service builds a VPN tunnel to the specified VPN server using +# OpenVPN software and a virtual TUN/TAP device. + +# directory containing the certificate and key described below +keydir=/etc/core/keys + +# the name used for a "$keyname.crt" certificate and "$keyname.key" private key. +keyname=client1 + +# the public IP address of the VPN server this client should connect with +vpnserver="10.0.2.10" + +# optional next hop for adding a static route to reach the VPN server +nexthop="10.0.1.1" + +# --------- END CUSTOMIZATION -------- + +# validate addresses +if [ "$(dpkg -l | grep " sipcalc ")" = "" ]; then + echo "WARNING: ip validation disabled because package sipcalc not installed + " > $PWD/vpnclient.log +else + if [ "$(sipcalc "$vpnserver" "$nexthop" | grep ERR)" != "" ]; then + echo "ERROR: invalide address $vpnserver or $nexthop " > $PWD/vpnclient.log + fi +fi + +# validate key and certification files +if [ ! -e $keydir\/$keyname.key ] || [ ! -e $keydir\/$keyname.crt ] \ + || [ ! -e $keydir\/ca.crt ] || [ ! -e $keydir\/dh1024.pem ]; then + echo "ERROR: missing certification or key files under $keydir $keyname.key or $keyname.crt or ca.crt or dh1024.pem" >> $PWD/vpnclient.log +fi + +# if necessary, add a static route for reaching the VPN server IP via the IF +vpnservernet=${vpnserver%.*}.0/24 +if [ "$nexthop" != "" ]; then + ip route add $vpnservernet via $nexthop +fi + +# create openvpn client.conf +( +cat << EOF +client +dev tun +proto udp +remote $vpnserver 1194 +nobind +ca $keydir/ca.crt +cert $keydir/$keyname.crt +key $keydir/$keyname.key +dh $keydir/dh1024.pem +cipher AES-256-CBC +log $PWD/openvpn-client.log +verb 4 +daemon +EOF +) > client.conf + +openvpn --config client.conf diff --git a/daemon/core/configservices/serviceutils/utils.py b/daemon/core/configservices/serviceutils/services.py similarity index 50% rename from daemon/core/configservices/serviceutils/utils.py rename to daemon/core/configservices/serviceutils/services.py index ef975a0e..24afc17f 100644 --- a/daemon/core/configservices/serviceutils/utils.py +++ b/daemon/core/configservices/serviceutils/services.py @@ -1,18 +1,9 @@ -import logging -import os - import netaddr from core import utils from core.config import Configuration -from core.configservice.base import ( - ConfigService, - ConfigServiceManager, - ConfigServiceMode, -) -from core.emulator.coreemu import CoreEmu -from core.emulator.emudata import IpPrefixes, NodeOptions -from core.emulator.enumerations import ConfigDataTypes, EventTypes, NodeTypes +from core.configservice.base import ConfigService, ConfigServiceMode +from core.emulator.enumerations import ConfigDataTypes GROUP_NAME = "Utility" @@ -21,7 +12,7 @@ class DefaultRoute(ConfigService): name = "DefaultRoute" group = GROUP_NAME directories = [] - executables = [] + executables = ["ip"] dependencies = [] startup = ["sh defaultroute.sh"] validate = [] @@ -65,45 +56,3 @@ class IpForwardService(ConfigService): devnames.append(devname) data = dict(devnames=devnames) self.render("ipforward.sh", data) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - # setup basic network - prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") - options = NodeOptions(model="nothing") - # options.services = [] - coreemu = CoreEmu() - session = coreemu.create_session() - session.set_state(EventTypes.CONFIGURATION_STATE) - switch = session.add_node(_type=NodeTypes.SWITCH) - - # node one - node_one = session.add_node(options=options) - interface = prefixes.create_interface(node_one) - session.add_link(node_one.id, switch.id, interface_one=interface) - - # node two - node_two = session.add_node(options=options) - interface = prefixes.create_interface(node_two) - session.add_link(node_two.id, switch.id, interface_one=interface) - - session.instantiate() - - # manager load config services - manager = ConfigServiceManager() - path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) - manager.load(path) - - clazz = manager.services["DefaultRoute"] - dr_service = clazz(node_one) - dr_service.set_config({"value1": "custom"}) - dr_service.start() - - clazz = manager.services["IPForward"] - dr_service = clazz(node_one) - dr_service.start() - - input("press enter to exit") - session.shutdown() diff --git a/daemon/examples/configservices/testing.py b/daemon/examples/configservices/testing.py new file mode 100644 index 00000000..ad9def1e --- /dev/null +++ b/daemon/examples/configservices/testing.py @@ -0,0 +1,49 @@ +import logging +import os + +from core import configservices +from core.configservice.manager import ConfigServiceManager +from core.emulator.coreemu import CoreEmu +from core.emulator.emudata import IpPrefixes, NodeOptions +from core.emulator.enumerations import EventTypes, NodeTypes + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + + # setup basic network + prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") + options = NodeOptions(model="nothing") + # options.services = [] + coreemu = CoreEmu() + session = coreemu.create_session() + session.set_state(EventTypes.CONFIGURATION_STATE) + switch = session.add_node(_type=NodeTypes.SWITCH) + + # node one + node_one = session.add_node(options=options) + interface = prefixes.create_interface(node_one) + session.add_link(node_one.id, switch.id, interface_one=interface) + + # node two + node_two = session.add_node(options=options) + interface = prefixes.create_interface(node_two) + session.add_link(node_two.id, switch.id, interface_one=interface) + + session.instantiate() + + # manager load config services + manager = ConfigServiceManager() + path = os.path.dirname(os.path.abspath(configservices.__file__)) + manager.load(path) + + clazz = manager.services["DefaultRoute"] + dr_service = clazz(node_one) + dr_service.set_config({"value1": "custom"}) + dr_service.start() + + clazz = manager.services["IPForward"] + dr_service = clazz(node_one) + dr_service.start() + + input("press enter to exit") + session.shutdown() From 191a9e9909a22cfc234f491eb229b8b098df5f88 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 17 Jan 2020 16:57:49 -0800 Subject: [PATCH 05/37] added some code to keep track of config services separately within core nodes, added function for starting config services during session instantiation --- daemon/core/configservice/base.py | 52 ++++++++++++++----- daemon/core/configservice/manager.py | 9 ++++ .../sercurityservices/services.py | 2 +- .../configservices/serviceutils/services.py | 4 +- daemon/core/configservices/simpleservice.py | 22 ++++++++ daemon/core/emulator/session.py | 1 + daemon/core/nodes/base.py | 10 ++++ daemon/examples/configservices/testing.py | 13 ++--- 8 files changed, 87 insertions(+), 26 deletions(-) create mode 100644 daemon/core/configservices/simpleservice.py diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 93de9ab4..ecbac826 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List from mako import exceptions from mako.lookup import TemplateLookup +from mako.template import Template from core.config import Configuration from core.errors import CoreCommandError, CoreError @@ -39,6 +40,12 @@ class ConfigService(abc.ABC): configs = self.default_configs[:] self._define_config(configs) + def __hash__(self) -> int: + return hash(self.name) + + def __eq__(self, other: "ConfigService") -> bool: + return self.name == other.name + @property @abc.abstractmethod def name(self) -> str: @@ -156,27 +163,44 @@ class ConfigService(abc.ABC): f"failed to validate" ) - def render(self, name: str, data: Dict[str, Any] = None) -> None: + def _render( + self, name: str, template: Template, data: Dict[str, Any] = None + ) -> None: if data is None: data = {} + rendered = template.render_unicode( + node=self.node, config=self.render_config(), **data + ) + logging.info( + "node(%s) service(%s) template(%s): \n%s", + self.node.name, + self.name, + name, + rendered, + ) + self.node.nodefile(name, rendered) + + def render_text(self, name: str, text: str, data: Dict[str, Any] = None) -> None: try: - template = self.templates.get_template(name) - rendered = template.render_unicode( - node=self.node, config=self.render_config(), **data - ) - logging.info( - "node(%s) service(%s) template(%s): \n%s", - self.node.name, - self.name, - name, - rendered, - ) - self.node.nodefile(name, rendered) + text = inspect.cleandoc(text) + template = Template(text) + self._render(name, template, data) except Exception: raise CoreError( f"node({self.node.name}) service({self.name}) " f"error rendering template({name}): " - f"{exceptions.text_error_template().render()}" + f"{exceptions.text_error_template().render_unicode()}" + ) + + def render_template(self, name: str, data: Dict[str, Any] = None) -> None: + try: + template = self.templates.get_template(name) + self._render(name, template, data) + except Exception: + raise CoreError( + f"node({self.node.name}) service({self.name}) " + f"error rendering template({name}): " + f"{exceptions.text_error_template().render_template()}" ) def _define_config(self, configs: List[Configuration]) -> None: diff --git a/daemon/core/configservice/manager.py b/daemon/core/configservice/manager.py index b0fe6d2e..747369c4 100644 --- a/daemon/core/configservice/manager.py +++ b/daemon/core/configservice/manager.py @@ -5,12 +5,20 @@ from typing import List from core import utils from core.configservice.base import ConfigService from core.errors import CoreError +from core.nodes.base import CoreNode class ConfigServiceManager: def __init__(self): self.services = {} + def set_service(self, node: CoreNode, name: str) -> None: + service_class = self.services.get(name) + if service_class in node.config_services: + raise CoreError(f"node already has service {name}") + service = service_class(node) + node.config_services.add(service) + def add(self, service: ConfigService) -> None: name = service.name logging.debug("loading service: class(%s) name(%s)", service.__class__, name) @@ -34,6 +42,7 @@ class ConfigServiceManager: def load(self, path: str) -> List[str]: path = pathlib.Path(path) subdirs = [x for x in path.iterdir() if x.is_dir()] + subdirs.append(path) service_errors = [] for subdir in subdirs: logging.info("loading config services from: %s", subdir) diff --git a/daemon/core/configservices/sercurityservices/services.py b/daemon/core/configservices/sercurityservices/services.py index b063a8e2..6fd1fba0 100644 --- a/daemon/core/configservices/sercurityservices/services.py +++ b/daemon/core/configservices/sercurityservices/services.py @@ -16,4 +16,4 @@ class VpnClient(ConfigService): default_configs = [] def create_files(self): - self.render("vpnclient.sh") + self.render_template("vpnclient.sh") diff --git a/daemon/core/configservices/serviceutils/services.py b/daemon/core/configservices/serviceutils/services.py index 24afc17f..d15c08c2 100644 --- a/daemon/core/configservices/serviceutils/services.py +++ b/daemon/core/configservices/serviceutils/services.py @@ -34,7 +34,7 @@ class DefaultRoute(ConfigService): if net[1] != net[-2]: addresses.append(net[1]) data = dict(addresses=addresses) - self.render("defaultroute.sh", data) + self.render_template("defaultroute.sh", data) class IpForwardService(ConfigService): @@ -55,4 +55,4 @@ class IpForwardService(ConfigService): devname = utils.sysctl_devname(ifc.name) devnames.append(devname) data = dict(devnames=devnames) - self.render("ipforward.sh", data) + self.render_template("ipforward.sh", data) diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py new file mode 100644 index 00000000..8fb40938 --- /dev/null +++ b/daemon/core/configservices/simpleservice.py @@ -0,0 +1,22 @@ +from core.configservice.base import ConfigService, ConfigServiceMode + + +class SimpleService(ConfigService): + name = "Simple" + group = "SimpleGroup" + directories = [] + executables = [] + dependencies = [] + startup = [] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + + def create_files(self): + text = """ + # sample script + # node id(${node.id}) name(${node.name}) + echo hello + """ + self.render_text("test1.sh", text) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index c3083e70..780d2bcd 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -1602,6 +1602,7 @@ class Session: logging.info("booting node(%s): %s", node.name, [x.name for x in node.services]) self.add_remove_control_interface(node=node, remove=False) self.services.boot_services(node) + node.start_config_services() def boot_nodes(self) -> List[Exception]: """ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index d31701b8..b6b818c7 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -277,9 +277,19 @@ class CoreNodeBase(NodeBase): """ super().__init__(session, _id, name, start, server) self.services = [] + self.config_services = set() self.nodedir = None self.tmpnodedir = False + def start_config_services(self) -> None: + """ + Start configuration services for this node. + + :return: nothing + """ + for service in self.config_services: + service.start() + def makenodedir(self) -> None: """ Create the node directory. diff --git a/daemon/examples/configservices/testing.py b/daemon/examples/configservices/testing.py index ad9def1e..bb88be13 100644 --- a/daemon/examples/configservices/testing.py +++ b/daemon/examples/configservices/testing.py @@ -29,21 +29,16 @@ if __name__ == "__main__": interface = prefixes.create_interface(node_two) session.add_link(node_two.id, switch.id, interface_one=interface) - session.instantiate() - # manager load config services manager = ConfigServiceManager() path = os.path.dirname(os.path.abspath(configservices.__file__)) manager.load(path) - clazz = manager.services["DefaultRoute"] - dr_service = clazz(node_one) - dr_service.set_config({"value1": "custom"}) - dr_service.start() + manager.set_service(node_one, "DefaultRoute") + manager.set_service(node_one, "IPForward") - clazz = manager.services["IPForward"] - dr_service = clazz(node_one) - dr_service.start() + # start session and run services + session.instantiate() input("press enter to exit") session.shutdown() From dcc683dd38268e25c0992cfa45552b692bac5c32 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 17 Jan 2020 21:09:51 -0800 Subject: [PATCH 06/37] added config service manager to CoreEmu and made it possible to create a session and nodes with config services from a script --- daemon/core/configservice/base.py | 6 ----- daemon/core/configservice/manager.py | 12 ++++----- .../serviceutils/templates/defaultroute.sh | 1 - daemon/core/emulator/coreemu.py | 8 ++++++ daemon/core/emulator/emudata.py | 1 + daemon/core/emulator/session.py | 11 +++++++- daemon/core/nodes/base.py | 26 ++++++++++++------- daemon/examples/configservices/testing.py | 13 +--------- 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index ecbac826..b35307a0 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -40,12 +40,6 @@ class ConfigService(abc.ABC): configs = self.default_configs[:] self._define_config(configs) - def __hash__(self) -> int: - return hash(self.name) - - def __eq__(self, other: "ConfigService") -> bool: - return self.name == other.name - @property @abc.abstractmethod def name(self) -> str: diff --git a/daemon/core/configservice/manager.py b/daemon/core/configservice/manager.py index 747369c4..2cdbd817 100644 --- a/daemon/core/configservice/manager.py +++ b/daemon/core/configservice/manager.py @@ -1,23 +1,21 @@ import logging import pathlib -from typing import List +from typing import List, Type from core import utils from core.configservice.base import ConfigService from core.errors import CoreError -from core.nodes.base import CoreNode class ConfigServiceManager: def __init__(self): self.services = {} - def set_service(self, node: CoreNode, name: str) -> None: + def get_service(self, name: str) -> Type[ConfigService]: service_class = self.services.get(name) - if service_class in node.config_services: - raise CoreError(f"node already has service {name}") - service = service_class(node) - node.config_services.add(service) + if service_class is None: + raise CoreError(f"service does not exit {name}") + return service_class def add(self, service: ConfigService) -> None: name = service.name diff --git a/daemon/core/configservices/serviceutils/templates/defaultroute.sh b/daemon/core/configservices/serviceutils/templates/defaultroute.sh index 952a7ef1..879a8861 100644 --- a/daemon/core/configservices/serviceutils/templates/defaultroute.sh +++ b/daemon/core/configservices/serviceutils/templates/defaultroute.sh @@ -1,6 +1,5 @@ #!/bin/sh # auto-generated by DefaultRoute service -# config: ${config} % for address in addresses: ip route add default via ${address} % endfor diff --git a/daemon/core/emulator/coreemu.py b/daemon/core/emulator/coreemu.py index 3fe4a40f..4041cf40 100644 --- a/daemon/core/emulator/coreemu.py +++ b/daemon/core/emulator/coreemu.py @@ -6,6 +6,8 @@ import sys from typing import Mapping, Type import core.services +from core import configservices +from core.configservice.manager import ConfigServiceManager from core.emulator.session import Session from core.services.coreservices import ServiceManager @@ -55,6 +57,11 @@ class CoreEmu: self.service_errors = [] self.load_services() + # config services + self.service_manager = ConfigServiceManager() + config_services_path = os.path.abspath(os.path.dirname(configservices.__file__)) + self.service_manager.load(config_services_path) + # catch exit event atexit.register(self.shutdown) @@ -97,6 +104,7 @@ class CoreEmu: while _id in self.sessions: _id += 1 session = _cls(_id, config=self.config) + session.service_manager = self.service_manager logging.info("created session: %s", _id) self.sessions[_id] = session return session diff --git a/daemon/core/emulator/emudata.py b/daemon/core/emulator/emudata.py index 96eab522..6a0ec8a6 100644 --- a/daemon/core/emulator/emudata.py +++ b/daemon/core/emulator/emudata.py @@ -75,6 +75,7 @@ class NodeOptions: self.icon = None self.opaque = None self.services = [] + self.config_services = [] self.x = None self.y = None self.lat = None diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 780d2bcd..0b1e00bf 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -161,6 +161,9 @@ class Session: "host": ("DefaultRoute", "SSH"), } + # config services + self.service_manager = None + @classmethod def get_node_class(cls, _type: NodeTypes) -> Type[NodeBase]: """ @@ -726,6 +729,11 @@ class Session: logging.debug("set node type: %s", node.type) self.services.add_services(node, node.type, options.services) + # add config services + for name in options.config_services: + service_class = self.service_manager.get_service(name) + node.add_config_service(service_class) + # ensure default emane configuration if isinstance(node, EmaneNet) and options.emane: self.emane.set_model_config(_id, options.emane) @@ -1602,7 +1610,8 @@ class Session: logging.info("booting node(%s): %s", node.name, [x.name for x in node.services]) self.add_remove_control_interface(node=node, remove=False) self.services.boot_services(node) - node.start_config_services() + for service in node.config_services.values(): + service.start() def boot_nodes(self) -> List[Exception]: """ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index b6b818c7..144c9f30 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -6,7 +6,7 @@ import logging import os import shutil import threading -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type import netaddr @@ -14,7 +14,7 @@ from core import utils from core.constants import MOUNT_BIN, VNODED_BIN from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes -from core.errors import CoreCommandError +from core.errors import CoreCommandError, CoreError from core.nodes import client from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, get_net_client @@ -22,6 +22,9 @@ from core.nodes.netclient import LinuxNetClient, get_net_client if TYPE_CHECKING: from core.emulator.distributed import DistributedServer from core.emulator.session import Session + from core.configservice.base import ConfigService + + ConfigServiceType = Type[ConfigService] _DEFAULT_MTU = 1500 @@ -277,18 +280,21 @@ class CoreNodeBase(NodeBase): """ super().__init__(session, _id, name, start, server) self.services = [] - self.config_services = set() + self.config_services = {} self.nodedir = None self.tmpnodedir = False - def start_config_services(self) -> None: - """ - Start configuration services for this node. + def add_config_service(self, service_class: "ConfigServiceType"): + name = service_class.name + if name in self.config_services: + raise CoreError(f"node({self.name}) already has service({name})") + self.config_services[name] = service_class(self) - :return: nothing - """ - for service in self.config_services: - service.start() + def set_service_config(self, name: str, data: Dict[str, str]) -> None: + service = self.config_services.get(name) + if service is None: + raise CoreError(f"node({self.name}) does not have service({name})") + service.set_config(data) def makenodedir(self) -> None: """ diff --git a/daemon/examples/configservices/testing.py b/daemon/examples/configservices/testing.py index bb88be13..5b193aee 100644 --- a/daemon/examples/configservices/testing.py +++ b/daemon/examples/configservices/testing.py @@ -1,8 +1,5 @@ import logging -import os -from core import configservices -from core.configservice.manager import ConfigServiceManager from core.emulator.coreemu import CoreEmu from core.emulator.emudata import IpPrefixes, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes @@ -13,13 +10,13 @@ if __name__ == "__main__": # setup basic network prefixes = IpPrefixes(ip4_prefix="10.83.0.0/16") options = NodeOptions(model="nothing") - # options.services = [] coreemu = CoreEmu() session = coreemu.create_session() session.set_state(EventTypes.CONFIGURATION_STATE) switch = session.add_node(_type=NodeTypes.SWITCH) # node one + options.config_services = ["DefaultRoute", "IPForward"] node_one = session.add_node(options=options) interface = prefixes.create_interface(node_one) session.add_link(node_one.id, switch.id, interface_one=interface) @@ -29,14 +26,6 @@ if __name__ == "__main__": interface = prefixes.create_interface(node_two) session.add_link(node_two.id, switch.id, interface_one=interface) - # manager load config services - manager = ConfigServiceManager() - path = os.path.dirname(os.path.abspath(configservices.__file__)) - manager.load(path) - - manager.set_service(node_one, "DefaultRoute") - manager.set_service(node_one, "IPForward") - # start session and run services session.instantiate() From 0af6a7a6204b3f8451a82cd3fe3c35dc6ac82370 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 18 Jan 2020 13:52:15 -0800 Subject: [PATCH 07/37] improved gitignore for generated proto files, added new initial separate config service proto, incorporated into grpc api --- .gitignore | 4 +- Pipfile | 0 daemon/core/api/grpc/client.py | 28 ++++++++ daemon/core/api/grpc/server.py | 65 +++++++++++++++++++ daemon/proto/Makefile.am | 4 +- .../proto/core/api/grpc/configservices.proto | 54 +++++++++++++++ daemon/proto/core/api/grpc/core.proto | 10 +++ 7 files changed, 161 insertions(+), 4 deletions(-) delete mode 100644 Pipfile create mode 100644 daemon/proto/core/api/grpc/configservices.proto diff --git a/.gitignore b/.gitignore index 12e9577d..bcfbadeb 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,8 @@ debian stamp-h1 # generated protobuf files -daemon/core/api/grpc/core_pb2.py -daemon/core/api/grpc/core_pb2_grpc.py +*_pb2.py +*_pb2_grpc.py # python build directory dist diff --git a/Pipfile b/Pipfile deleted file mode 100644 index e69de29b..00000000 diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index 07e9d8ef..cedf812a 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -12,6 +12,14 @@ import netaddr from core import utils from core.api.grpc import core_pb2, core_pb2_grpc +from core.api.grpc.configservices_pb2 import ( + GetConfigServiceRequest, + GetConfigServiceResponse, + GetConfigServicesRequest, + GetConfigServicesResponse, + SetConfigServiceRequest, + SetConfigServiceResponse, +) class InterfaceHelper: @@ -1078,6 +1086,26 @@ class CoreGrpcClient: request = core_pb2.GetInterfacesRequest() return self.stub.GetInterfaces(request) + def get_config_services(self) -> GetConfigServicesResponse: + request = GetConfigServicesRequest() + return self.stub.GetConfigServices(request) + + def get_config_service( + self, session_id: int, node_id: int, name: str + ) -> GetConfigServiceResponse: + request = GetConfigServiceRequest( + session_id=session_id, node_id=node_id, name=name + ) + return self.stub.GetConfigService(request) + + def set_config_service( + self, session_id: int, node_id: int, name: str, config: Dict[str, str] + ) -> SetConfigServiceResponse: + request = SetConfigServiceRequest( + session_id=session_id, node_id=node_id, name=name, config=config + ) + return self.stub.SetConfigService(request) + def connect(self) -> None: """ Open connection to server, must be closed manually. diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 3d24b981..61fcbade 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,6 +10,15 @@ import grpc from grpc import ServicerContext from core.api.grpc import core_pb2, core_pb2_grpc, grpcutils +from core.api.grpc.configservices_pb2 import ( + ConfigService, + GetConfigServiceRequest, + GetConfigServiceResponse, + GetConfigServicesRequest, + GetConfigServicesResponse, + SetConfigServiceRequest, + SetConfigServiceResponse, +) from core.api.grpc.events import EventStreamer from core.api.grpc.grpcutils import ( get_config_options, @@ -101,6 +110,10 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): except CoreError: context.abort(grpc.StatusCode.NOT_FOUND, f"node {node_id} not found") + def validate_service(self, name: str, context: ServicerContext) -> None: + if name not in self.coreemu.service_manager.services: + context.abort(grpc.StatusCode.NOT_FOUND, f"unknown service {name}") + def StartSession( self, request: core_pb2.StartSessionRequest, context: ServicerContext ) -> core_pb2.StartSessionResponse: @@ -1429,3 +1442,55 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): return core_pb2.EmaneLinkResponse(result=True) else: return core_pb2.EmaneLinkResponse(result=False) + + def GetConfigServices( + self, request: GetConfigServicesRequest, context: ServicerContext + ) -> GetConfigServicesResponse: + services = [] + for service in self.coreemu.service_manager.services.values(): + service_proto = ConfigService( + name=service.name, + group=service.group, + executables=service.executables, + dependencies=service.dependencies, + directories=service.directories, + startup=service.startup, + validate=service.validate, + shutdown=service.shutdown, + validation_mode=service.validation_mode.value, + validation_timer=service.validation_timer, + validation_period=service.validation_period, + ) + services.append(service_proto) + return GetConfigServicesResponse(services=services) + + def GetConfigService( + self, request: GetConfigServiceRequest, context: ServicerContext + ) -> GetConfigServiceResponse: + session = self.get_session(request.session_id, context) + node = self.get_node(session, request.node_id, context) + self.validate_service(request.name, context) + service = node.config_services.get(request.name) + if service: + config = service.get_config() + + else: + service = self.coreemu.service_manager.get_service(request.name) + config = {x.id: x.default for x in service.default_configs} + return GetConfigServiceResponse(config=config) + + def SetConfigService( + self, request: SetConfigServiceRequest, context: ServicerContext + ) -> SetConfigServiceResponse: + session = self.get_session(request.session_id, context) + node = self.get_node(session, request.node_id, context) + self.validate_service(request.name, context) + service = node.config_services.get(request.name) + if service: + service.set_config(request.config) + return SetConfigServiceResponse(result=True) + else: + context.abort( + grpc.StatusCode.NOT_FOUND, + f"node {node.name} missing service {request.name}", + ) diff --git a/daemon/proto/Makefile.am b/daemon/proto/Makefile.am index 05f2d394..e63179ca 100644 --- a/daemon/proto/Makefile.am +++ b/daemon/proto/Makefile.am @@ -1,5 +1,5 @@ all: - $(PYTHON) -m grpc_tools.protoc -I . --python_out=.. --grpc_python_out=.. core/api/grpc/core.proto + $(PYTHON) -m grpc_tools.protoc -I . --python_out=.. --grpc_python_out=.. core/api/grpc/*.proto clean: - -rm -f ../core/api/grpc/core_pb2* + -rm -f ../core/api/grpc/*_pb2* diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto new file mode 100644 index 00000000..926e30f7 --- /dev/null +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -0,0 +1,54 @@ +syntax = "proto3"; + +package configservices; + +message ConfigServiceValidationMode { + enum Enum { + BLOCKING = 0; + NON_BLOCKING = 1; + TIMER = 2; + } +} + +message ConfigService { + string group = 1; + string name = 2; + repeated string executables = 3; + repeated string dependencies = 4; + repeated string directories = 5; + repeated string startup = 6; + repeated string validate = 7; + repeated string shutdown = 8; + ConfigServiceValidationMode.Enum validation_mode = 9; + int32 validation_timer = 10; + float validation_period = 11; +} + +message GetConfigServicesRequest { + int32 session_id = 1; +} + +message GetConfigServicesResponse { + repeated ConfigService services = 1; +} + +message GetConfigServiceRequest { + int32 session_id = 1; + int32 node_id = 2; + string name = 3; +} + +message GetConfigServiceResponse { + map config = 1; +} + +message SetConfigServiceRequest { + int32 session_id = 1; + int32 node_id = 2; + string name = 3; + map config = 4; +} + +message SetConfigServiceResponse { + bool result = 1; +} diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 856ed7aa..5930639e 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -5,6 +5,8 @@ package core; option java_package = "com.core.client.grpc"; option java_outer_classname = "CoreProto"; +import "core/api/grpc/configservices.proto"; + service CoreApi { // session rpc rpc StartSession (StartSessionRequest) returns (StartSessionResponse) { @@ -102,6 +104,14 @@ service CoreApi { rpc ServiceAction (ServiceActionRequest) returns (ServiceActionResponse) { } + // config services + rpc GetConfigServices (configservices.GetConfigServicesRequest) returns (configservices.GetConfigServicesResponse) { + } + rpc GetConfigService (configservices.GetConfigServiceRequest) returns (configservices.GetConfigServiceResponse) { + } + rpc SetConfigService (configservices.SetConfigServiceRequest) returns (configservices.SetConfigServiceResponse) { + } + // wlan rpc rpc GetWlanConfigs (GetWlanConfigsRequest) returns (GetWlanConfigsResponse) { } From a7764214d207c5153d7c7b0870f5b787330f143a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 18 Jan 2020 21:23:46 -0800 Subject: [PATCH 08/37] updating github actions to build all protos --- .github/workflows/daemon-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daemon-checks.yml b/.github/workflows/daemon-checks.yml index 21dd95dc..00537c8e 100644 --- a/.github/workflows/daemon-checks.yml +++ b/.github/workflows/daemon-checks.yml @@ -35,7 +35,7 @@ jobs: - name: grpc run: | cd daemon/proto - pipenv run python -m grpc_tools.protoc -I . --python_out=.. --grpc_python_out=.. core/api/grpc/core.proto + pipenv run python -m grpc_tools.protoc -I . --python_out=.. --grpc_python_out=.. core/api/grpc/*.proto - name: test run: | cd daemon From 0e6d1535db97b7d3c907223b45d0a5a486b573a9 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Sun, 19 Jan 2020 19:27:57 -0800 Subject: [PATCH 09/37] added grpc to get current config services for a node --- daemon/core/api/grpc/client.py | 32 ++++++++++------- daemon/core/api/grpc/server.py | 34 ++++++++++++------- .../proto/core/api/grpc/configservices.proto | 17 +++++++--- daemon/proto/core/api/grpc/core.proto | 6 ++-- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index cedf812a..8a52c5c6 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -13,12 +13,14 @@ import netaddr from core import utils from core.api.grpc import core_pb2, core_pb2_grpc from core.api.grpc.configservices_pb2 import ( - GetConfigServiceRequest, - GetConfigServiceResponse, GetConfigServicesRequest, GetConfigServicesResponse, - SetConfigServiceRequest, - SetConfigServiceResponse, + GetNodeConfigServiceRequest, + GetNodeConfigServiceResponse, + GetNodeConfigServicesRequest, + GetNodeConfigServicesResponse, + SetNodeConfigServiceRequest, + SetNodeConfigServiceResponse, ) @@ -1090,21 +1092,27 @@ class CoreGrpcClient: request = GetConfigServicesRequest() return self.stub.GetConfigServices(request) - def get_config_service( + def get_node_config_service( self, session_id: int, node_id: int, name: str - ) -> GetConfigServiceResponse: - request = GetConfigServiceRequest( + ) -> GetNodeConfigServiceResponse: + request = GetNodeConfigServiceRequest( session_id=session_id, node_id=node_id, name=name ) - return self.stub.GetConfigService(request) + return self.stub.GetNodeConfigService(request) - def set_config_service( + def get_node_config_services( + self, session_id: int, node_id: int + ) -> GetNodeConfigServicesResponse: + request = GetNodeConfigServicesRequest(session_id=session_id, node_id=node_id) + return self.stub.GetNodeConfigServices(request) + + def set_node_config_service( self, session_id: int, node_id: int, name: str, config: Dict[str, str] - ) -> SetConfigServiceResponse: - request = SetConfigServiceRequest( + ) -> SetNodeConfigServiceResponse: + request = SetNodeConfigServiceRequest( session_id=session_id, node_id=node_id, name=name, config=config ) - return self.stub.SetConfigService(request) + return self.stub.SetNodeConfigService(request) def connect(self) -> None: """ diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 61fcbade..ed97eb34 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -12,12 +12,14 @@ from grpc import ServicerContext from core.api.grpc import core_pb2, core_pb2_grpc, grpcutils from core.api.grpc.configservices_pb2 import ( ConfigService, - GetConfigServiceRequest, - GetConfigServiceResponse, GetConfigServicesRequest, GetConfigServicesResponse, - SetConfigServiceRequest, - SetConfigServiceResponse, + GetNodeConfigServiceRequest, + GetNodeConfigServiceResponse, + GetNodeConfigServicesRequest, + GetNodeConfigServicesResponse, + SetNodeConfigServiceRequest, + SetNodeConfigServiceResponse, ) from core.api.grpc.events import EventStreamer from core.api.grpc.grpcutils import ( @@ -1464,9 +1466,9 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): services.append(service_proto) return GetConfigServicesResponse(services=services) - def GetConfigService( - self, request: GetConfigServiceRequest, context: ServicerContext - ) -> GetConfigServiceResponse: + def GetNodeConfigService( + self, request: GetNodeConfigServiceRequest, context: ServicerContext + ) -> GetNodeConfigServiceResponse: session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) self.validate_service(request.name, context) @@ -1477,18 +1479,26 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): else: service = self.coreemu.service_manager.get_service(request.name) config = {x.id: x.default for x in service.default_configs} - return GetConfigServiceResponse(config=config) + return GetNodeConfigServiceResponse(config=config) - def SetConfigService( - self, request: SetConfigServiceRequest, context: ServicerContext - ) -> SetConfigServiceResponse: + def GetNodeConfigServices( + self, request: GetNodeConfigServicesRequest, context: ServicerContext + ) -> GetNodeConfigServicesResponse: + session = self.get_session(request.session_id, context) + node = self.get_node(session, request.node_id, context) + services = node.config_services.keys() + return GetNodeConfigServicesResponse(services=services) + + def SetNodeConfigService( + self, request: SetNodeConfigServiceRequest, context: ServicerContext + ) -> SetNodeConfigServiceResponse: session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) self.validate_service(request.name, context) service = node.config_services.get(request.name) if service: service.set_config(request.config) - return SetConfigServiceResponse(result=True) + return SetNodeConfigServiceResponse(result=True) else: context.abort( grpc.StatusCode.NOT_FOUND, diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto index 926e30f7..a6d3e95a 100644 --- a/daemon/proto/core/api/grpc/configservices.proto +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -32,23 +32,32 @@ message GetConfigServicesResponse { repeated ConfigService services = 1; } -message GetConfigServiceRequest { +message GetNodeConfigServiceRequest { int32 session_id = 1; int32 node_id = 2; string name = 3; } -message GetConfigServiceResponse { +message GetNodeConfigServiceResponse { map config = 1; } -message SetConfigServiceRequest { +message GetNodeConfigServicesRequest { + int32 session_id = 1; + int32 node_id = 2; +} + +message GetNodeConfigServicesResponse { + repeated string services = 1; +} + +message SetNodeConfigServiceRequest { int32 session_id = 1; int32 node_id = 2; string name = 3; map config = 4; } -message SetConfigServiceResponse { +message SetNodeConfigServiceResponse { bool result = 1; } diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 5930639e..215a23d5 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -107,9 +107,11 @@ service CoreApi { // config services rpc GetConfigServices (configservices.GetConfigServicesRequest) returns (configservices.GetConfigServicesResponse) { } - rpc GetConfigService (configservices.GetConfigServiceRequest) returns (configservices.GetConfigServiceResponse) { + rpc GetNodeConfigService (configservices.GetNodeConfigServiceRequest) returns (configservices.GetNodeConfigServiceResponse) { } - rpc SetConfigService (configservices.SetConfigServiceRequest) returns (configservices.SetConfigServiceResponse) { + rpc GetNodeConfigServices (configservices.GetNodeConfigServicesRequest) returns (configservices.GetNodeConfigServicesResponse) { + } + rpc SetNodeConfigService (configservices.SetNodeConfigServiceRequest) returns (configservices.SetNodeConfigServiceResponse) { } // wlan rpc From 9447ddb94ffcd0204872277af8911ca167fc8bdc Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 20 Jan 2020 12:17:11 -0800 Subject: [PATCH 10/37] initial changes to add config services to coretk gui --- daemon/core/gui/coreclient.py | 7 + .../core/gui/dialogs/configserviceconfig.py | 494 ++++++++++++++++++ daemon/core/gui/dialogs/nodeconfigservice.py | 169 ++++++ daemon/core/gui/graph/node.py | 10 + daemon/proto/core/api/grpc/core.proto | 1 + 5 files changed, 681 insertions(+) create mode 100644 daemon/core/gui/dialogs/configserviceconfig.py create mode 100644 daemon/core/gui/dialogs/nodeconfigservice.py diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index f4371715..1e44715a 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -63,6 +63,7 @@ class CoreClient: self.app = app self.master = app.master self.services = {} + self.config_services = {} self.default_services = {} self.emane_models = [] self.observer = None @@ -413,6 +414,12 @@ class CoreClient: group_services = self.services.setdefault(service.group, set()) group_services.add(service.name) + # get config service informations + response = self.client.get_config_services() + for service in response.services: + group_services = self.config_services.setdefault(service.group, set()) + group_services.add(service.name) + # if there are no sessions, create a new session, else join a session response = self.client.get_sessions() logging.info("current sessions: %s", response) diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py new file mode 100644 index 00000000..b017212b --- /dev/null +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -0,0 +1,494 @@ +""" +Service configuration dialog +""" +import tkinter as tk +from tkinter import ttk +from typing import TYPE_CHECKING, Any, List + +import grpc + +from core.api.grpc import core_pb2 +from core.gui.dialogs.copyserviceconfig import CopyServiceConfigDialog +from core.gui.dialogs.dialog import Dialog +from core.gui.errors import show_grpc_error +from core.gui.images import ImageEnum, Images +from core.gui.themes import FRAME_PAD, PADX, PADY +from core.gui.widgets import CodeText, ListboxScroll + +if TYPE_CHECKING: + from core.gui.app import Application + + +class ConfigServiceConfigDialog(Dialog): + def __init__( + self, master: Any, app: "Application", service_name: str, node_id: int + ): + title = f"{service_name} Config Service" + super().__init__(master, app, title, modal=True) + self.master = master + self.app = app + self.core = app.core + self.node_id = node_id + self.service_name = service_name + self.service_configs = app.core.service_configs + self.file_configs = app.core.file_configs + + self.radiovar = tk.IntVar() + self.radiovar.set(2) + self.filenames = [] + self.dependencies = [] + self.executables = [] + self.startup_commands = [] + self.validation_commands = [] + self.shutdown_commands = [] + self.default_startup = [] + self.default_validate = [] + self.default_shutdown = [] + self.validation_mode = None + self.validation_time = None + self.validation_period = None + self.documentnew_img = Images.get(ImageEnum.DOCUMENTNEW, 16) + self.editdelete_img = Images.get(ImageEnum.EDITDELETE, 16) + + self.notebook = None + self.filename_combobox = None + self.startup_commands_listbox = None + self.shutdown_commands_listbox = None + self.validate_commands_listbox = None + self.validation_time_entry = None + self.validation_mode_entry = None + self.service_file_data = None + self.validation_period_entry = None + self.original_service_files = {} + self.temp_service_files = {} + self.modified_files = set() + self.load() + self.draw() + + def load(self): + try: + self.app.core.create_nodes_and_links() + default_config = self.app.core.get_node_service( + self.node_id, self.service_name + ) + self.default_startup = default_config.startup[:] + self.default_validate = default_config.validate[:] + self.default_shutdown = default_config.shutdown[:] + custom_configs = self.service_configs + if ( + self.node_id in custom_configs + and self.service_name in custom_configs[self.node_id] + ): + service_config = custom_configs[self.node_id][self.service_name] + else: + service_config = default_config + + self.dependencies = service_config.dependencies[:] + self.executables = service_config.executables[:] + self.filenames = service_config.configs[:] + self.startup_commands = service_config.startup[:] + self.validation_commands = service_config.validate[:] + self.shutdown_commands = service_config.shutdown[:] + self.validation_mode = service_config.validation_mode + self.validation_time = service_config.validation_timer + self.original_service_files = { + x: self.app.core.get_node_service_file( + self.node_id, self.service_name, x + ) + for x in self.filenames + } + self.temp_service_files = dict(self.original_service_files) + file_configs = self.file_configs + if ( + self.node_id in file_configs + and self.service_name in file_configs[self.node_id] + ): + for file, data in file_configs[self.node_id][self.service_name].items(): + self.temp_service_files[file] = data + except grpc.RpcError as e: + show_grpc_error(e) + + def draw(self): + self.top.columnconfigure(0, weight=1) + self.top.rowconfigure(1, weight=1) + + # draw notebook + self.notebook = ttk.Notebook(self.top) + self.notebook.grid(sticky="nsew", pady=PADY) + self.draw_tab_files() + self.draw_tab_directories() + self.draw_tab_startstop() + self.draw_tab_configuration() + + self.draw_buttons() + + def draw_tab_files(self): + tab = ttk.Frame(self.notebook, padding=FRAME_PAD) + tab.grid(sticky="nsew") + tab.columnconfigure(0, weight=1) + self.notebook.add(tab, text="Files") + + label = ttk.Label( + tab, text="Config files and scripts that are generated for this service." + ) + label.grid() + + frame = ttk.Frame(tab) + frame.grid(sticky="ew", pady=PADY) + frame.columnconfigure(1, weight=1) + label = ttk.Label(frame, text="File Name") + label.grid(row=0, column=0, padx=PADX, sticky="w") + self.filename_combobox = ttk.Combobox( + frame, values=self.filenames, state="readonly" + ) + self.filename_combobox.bind( + "<>", self.display_service_file_data + ) + self.filename_combobox.grid(row=0, column=1, sticky="ew", padx=PADX) + button = ttk.Button(frame, image=self.documentnew_img, state="disabled") + button.bind("", self.add_filename) + button.grid(row=0, column=2, padx=PADX) + button = ttk.Button(frame, image=self.editdelete_img, state="disabled") + button.bind("", self.delete_filename) + button.grid(row=0, column=3) + + frame = ttk.Frame(tab) + frame.grid(sticky="ew", pady=PADY) + frame.columnconfigure(1, weight=1) + button = ttk.Radiobutton( + frame, + variable=self.radiovar, + text="Copy Source File", + value=1, + state=tk.DISABLED, + ) + button.grid(row=0, column=0, sticky="w", padx=PADX) + entry = ttk.Entry(frame, state=tk.DISABLED) + entry.grid(row=0, column=1, sticky="ew", padx=PADX) + image = Images.get(ImageEnum.FILEOPEN, 16) + button = ttk.Button(frame, image=image) + button.image = image + button.grid(row=0, column=2) + + frame = ttk.Frame(tab) + frame.grid(sticky="ew", pady=PADY) + frame.columnconfigure(0, weight=1) + button = ttk.Radiobutton( + frame, + variable=self.radiovar, + text="Use text below for file contents", + value=2, + ) + button.grid(row=0, column=0, sticky="ew") + image = Images.get(ImageEnum.FILEOPEN, 16) + button = ttk.Button(frame, image=image) + button.image = image + button.grid(row=0, column=1) + image = Images.get(ImageEnum.DOCUMENTSAVE, 16) + button = ttk.Button(frame, image=image) + button.image = image + button.grid(row=0, column=2) + + self.service_file_data = CodeText(tab) + self.service_file_data.grid(sticky="nsew") + tab.rowconfigure(self.service_file_data.grid_info()["row"], weight=1) + if len(self.filenames) > 0: + self.filename_combobox.current(0) + self.service_file_data.text.delete(1.0, "end") + self.service_file_data.text.insert( + "end", self.temp_service_files[self.filenames[0]] + ) + self.service_file_data.text.bind( + "", self.update_temp_service_file_data + ) + + def draw_tab_directories(self): + tab = ttk.Frame(self.notebook, padding=FRAME_PAD) + tab.grid(sticky="nsew") + tab.columnconfigure(0, weight=1) + self.notebook.add(tab, text="Directories") + + label = ttk.Label( + tab, + text="Directories required by this service that are unique for each node.", + ) + label.grid() + + def draw_tab_startstop(self): + tab = ttk.Frame(self.notebook, padding=FRAME_PAD) + tab.grid(sticky="nsew") + tab.columnconfigure(0, weight=1) + for i in range(3): + tab.rowconfigure(i, weight=1) + self.notebook.add(tab, text="Startup/Shutdown") + commands = [] + # tab 3 + for i in range(3): + label_frame = None + if i == 0: + label_frame = ttk.LabelFrame( + tab, text="Startup Commands", padding=FRAME_PAD + ) + commands = self.startup_commands + elif i == 1: + label_frame = ttk.LabelFrame( + tab, text="Shutdown Commands", padding=FRAME_PAD + ) + commands = self.shutdown_commands + elif i == 2: + label_frame = ttk.LabelFrame( + tab, text="Validation Commands", padding=FRAME_PAD + ) + commands = self.validation_commands + label_frame.columnconfigure(0, weight=1) + label_frame.rowconfigure(1, weight=1) + label_frame.grid(row=i, column=0, sticky="nsew", pady=PADY) + + frame = ttk.Frame(label_frame) + frame.grid(row=0, column=0, sticky="nsew", pady=PADY) + frame.columnconfigure(0, weight=1) + entry = ttk.Entry(frame, textvariable=tk.StringVar()) + entry.grid(row=0, column=0, stick="ew", padx=PADX) + button = ttk.Button(frame, image=self.documentnew_img) + button.bind("", self.add_command) + button.grid(row=0, column=1, sticky="ew", padx=PADX) + button = ttk.Button(frame, image=self.editdelete_img) + button.grid(row=0, column=2, sticky="ew") + button.bind("", self.delete_command) + listbox_scroll = ListboxScroll(label_frame) + listbox_scroll.listbox.bind("<>", self.update_entry) + for command in commands: + listbox_scroll.listbox.insert("end", command) + listbox_scroll.listbox.config(height=4) + listbox_scroll.grid(row=1, column=0, sticky="nsew") + if i == 0: + self.startup_commands_listbox = listbox_scroll.listbox + elif i == 1: + self.shutdown_commands_listbox = listbox_scroll.listbox + elif i == 2: + self.validate_commands_listbox = listbox_scroll.listbox + + def draw_tab_configuration(self): + tab = ttk.Frame(self.notebook, padding=FRAME_PAD) + tab.grid(sticky="nsew") + tab.columnconfigure(0, weight=1) + self.notebook.add(tab, text="Configuration", sticky="nsew") + + frame = ttk.Frame(tab) + frame.grid(sticky="ew", pady=PADY) + frame.columnconfigure(1, weight=1) + + label = ttk.Label(frame, text="Validation Time") + label.grid(row=0, column=0, sticky="w", padx=PADX) + self.validation_time_entry = ttk.Entry(frame) + self.validation_time_entry.insert("end", self.validation_time) + self.validation_time_entry.config(state=tk.DISABLED) + self.validation_time_entry.grid(row=0, column=1, sticky="ew", pady=PADY) + + label = ttk.Label(frame, text="Validation Mode") + label.grid(row=1, column=0, sticky="w", padx=PADX) + if self.validation_mode == core_pb2.ServiceValidationMode.BLOCKING: + mode = "BLOCKING" + elif self.validation_mode == core_pb2.ServiceValidationMode.NON_BLOCKING: + mode = "NON_BLOCKING" + else: + mode = "TIMER" + self.validation_mode_entry = ttk.Entry( + frame, textvariable=tk.StringVar(value=mode) + ) + self.validation_mode_entry.insert("end", mode) + self.validation_mode_entry.config(state=tk.DISABLED) + self.validation_mode_entry.grid(row=1, column=1, sticky="ew", pady=PADY) + + label = ttk.Label(frame, text="Validation Period") + label.grid(row=2, column=0, sticky="w", padx=PADX) + self.validation_period_entry = ttk.Entry( + frame, state=tk.DISABLED, textvariable=tk.StringVar() + ) + self.validation_period_entry.grid(row=2, column=1, sticky="ew", pady=PADY) + + label_frame = ttk.LabelFrame(tab, text="Executables", padding=FRAME_PAD) + label_frame.grid(sticky="nsew", pady=PADY) + label_frame.columnconfigure(0, weight=1) + label_frame.rowconfigure(0, weight=1) + listbox_scroll = ListboxScroll(label_frame) + listbox_scroll.grid(sticky="nsew") + tab.rowconfigure(listbox_scroll.grid_info()["row"], weight=1) + for executable in self.executables: + listbox_scroll.listbox.insert("end", executable) + + label_frame = ttk.LabelFrame(tab, text="Dependencies", padding=FRAME_PAD) + label_frame.grid(sticky="nsew", pady=PADY) + label_frame.columnconfigure(0, weight=1) + label_frame.rowconfigure(0, weight=1) + listbox_scroll = ListboxScroll(label_frame) + listbox_scroll.grid(sticky="nsew") + tab.rowconfigure(listbox_scroll.grid_info()["row"], weight=1) + for dependency in self.dependencies: + listbox_scroll.listbox.insert("end", dependency) + + def draw_buttons(self): + frame = ttk.Frame(self.top) + frame.grid(sticky="ew") + for i in range(4): + frame.columnconfigure(i, weight=1) + button = ttk.Button(frame, text="Apply", command=self.click_apply) + button.grid(row=0, column=0, sticky="ew", padx=PADX) + button = ttk.Button(frame, text="Defaults", command=self.click_defaults) + button.grid(row=0, column=1, sticky="ew", padx=PADX) + button = ttk.Button(frame, text="Copy...", command=self.click_copy) + button.grid(row=0, column=2, sticky="ew", padx=PADX) + button = ttk.Button(frame, text="Cancel", command=self.destroy) + button.grid(row=0, column=3, sticky="ew") + + def add_filename(self, event: tk.Event): + # not worry about it for now + return + frame_contains_button = event.widget.master + combobox = frame_contains_button.grid_slaves(row=0, column=1)[0] + filename = combobox.get() + if filename not in combobox["values"]: + combobox["values"] += (filename,) + + def delete_filename(self, event: tk.Event): + # not worry about it for now + return + frame_comntains_button = event.widget.master + combobox = frame_comntains_button.grid_slaves(row=0, column=1)[0] + filename = combobox.get() + if filename in combobox["values"]: + combobox["values"] = tuple([x for x in combobox["values"] if x != filename]) + combobox.set("") + + def add_command(self, event: tk.Event): + frame_contains_button = event.widget.master + listbox = frame_contains_button.master.grid_slaves(row=1, column=0)[0].listbox + command_to_add = frame_contains_button.grid_slaves(row=0, column=0)[0].get() + if command_to_add == "": + return + for cmd in listbox.get(0, tk.END): + if cmd == command_to_add: + return + listbox.insert(tk.END, command_to_add) + + def update_entry(self, event: tk.Event): + listbox = event.widget + current_selection = listbox.curselection() + if len(current_selection) > 0: + cmd = listbox.get(current_selection[0]) + entry = listbox.master.master.grid_slaves(row=0, column=0)[0].grid_slaves( + row=0, column=0 + )[0] + entry.delete(0, "end") + entry.insert(0, cmd) + + def delete_command(self, event: tk.Event): + button = event.widget + frame_contains_button = button.master + listbox = frame_contains_button.master.grid_slaves(row=1, column=0)[0].listbox + current_selection = listbox.curselection() + if len(current_selection) > 0: + listbox.delete(current_selection[0]) + entry = frame_contains_button.grid_slaves(row=0, column=0)[0] + entry.delete(0, tk.END) + + def click_apply(self): + current_listbox = self.master.current.listbox + if not self.is_custom_service_config() and not self.is_custom_service_file(): + if self.node_id in self.service_configs: + self.service_configs[self.node_id].pop(self.service_name, None) + current_listbox.itemconfig(current_listbox.curselection()[0], bg="") + self.destroy() + return + + try: + if self.is_custom_service_config(): + startup_commands = self.startup_commands_listbox.get(0, "end") + shutdown_commands = self.shutdown_commands_listbox.get(0, "end") + validate_commands = self.validate_commands_listbox.get(0, "end") + config = self.core.set_node_service( + self.node_id, + self.service_name, + startup_commands, + validate_commands, + shutdown_commands, + ) + if self.node_id not in self.service_configs: + self.service_configs[self.node_id] = {} + self.service_configs[self.node_id][self.service_name] = config + + for file in self.modified_files: + if self.node_id not in self.file_configs: + self.file_configs[self.node_id] = {} + if self.service_name not in self.file_configs[self.node_id]: + self.file_configs[self.node_id][self.service_name] = {} + self.file_configs[self.node_id][self.service_name][ + file + ] = self.temp_service_files[file] + + self.app.core.set_node_service_file( + self.node_id, self.service_name, file, self.temp_service_files[file] + ) + all_current = current_listbox.get(0, tk.END) + current_listbox.itemconfig(all_current.index(self.service_name), bg="green") + except grpc.RpcError as e: + show_grpc_error(e) + self.destroy() + + def display_service_file_data(self, event: tk.Event): + combobox = event.widget + filename = combobox.get() + self.service_file_data.text.delete(1.0, "end") + self.service_file_data.text.insert("end", self.temp_service_files[filename]) + + def update_temp_service_file_data(self, event: tk.Event): + scrolledtext = event.widget + filename = self.filename_combobox.get() + self.temp_service_files[filename] = scrolledtext.get(1.0, "end") + if self.temp_service_files[filename] != self.original_service_files[filename]: + self.modified_files.add(filename) + else: + self.modified_files.discard(filename) + + def is_custom_service_config(self): + startup_commands = self.startup_commands_listbox.get(0, "end") + shutdown_commands = self.shutdown_commands_listbox.get(0, "end") + validate_commands = self.validate_commands_listbox.get(0, "end") + return ( + set(self.default_startup) != set(startup_commands) + or set(self.default_validate) != set(validate_commands) + or set(self.default_shutdown) != set(shutdown_commands) + ) + + def is_custom_service_file(self): + return len(self.modified_files) > 0 + + def click_defaults(self): + if self.node_id in self.service_configs: + self.service_configs[self.node_id].pop(self.service_name, None) + if self.node_id in self.file_configs: + self.file_configs[self.node_id].pop(self.service_name, None) + self.temp_service_files = dict(self.original_service_files) + filename = self.filename_combobox.get() + self.service_file_data.text.delete(1.0, "end") + self.service_file_data.text.insert("end", self.temp_service_files[filename]) + self.startup_commands_listbox.delete(0, tk.END) + self.validate_commands_listbox.delete(0, tk.END) + self.shutdown_commands_listbox.delete(0, tk.END) + for cmd in self.default_startup: + self.startup_commands_listbox.insert(tk.END, cmd) + for cmd in self.default_validate: + self.validate_commands_listbox.insert(tk.END, cmd) + for cmd in self.default_shutdown: + self.shutdown_commands_listbox.insert(tk.END, cmd) + + def click_copy(self): + dialog = CopyServiceConfigDialog(self, self.app, self.node_id) + dialog.show() + + def append_commands( + self, commands: List[str], listbox: tk.Listbox, to_add: List[str] + ): + for cmd in to_add: + commands.append(cmd) + listbox.insert(tk.END, cmd) diff --git a/daemon/core/gui/dialogs/nodeconfigservice.py b/daemon/core/gui/dialogs/nodeconfigservice.py new file mode 100644 index 00000000..46897d60 --- /dev/null +++ b/daemon/core/gui/dialogs/nodeconfigservice.py @@ -0,0 +1,169 @@ +""" +core node services +""" +import tkinter as tk +from tkinter import messagebox, ttk +from typing import TYPE_CHECKING, Any, Set + +from core.gui.dialogs.configserviceconfig import ConfigServiceConfigDialog +from core.gui.dialogs.dialog import Dialog +from core.gui.themes import FRAME_PAD, PADX, PADY +from core.gui.widgets import CheckboxList, ListboxScroll + +if TYPE_CHECKING: + from core.gui.app import Application + from core.gui.graph.node import CanvasNode + + +class NodeConfigServiceDialog(Dialog): + def __init__( + self, + master: Any, + app: "Application", + canvas_node: "CanvasNode", + services: Set[str] = None, + ): + title = f"{canvas_node.core_node.name} Config Services" + super().__init__(master, app, title, modal=True) + self.app = app + self.canvas_node = canvas_node + self.node_id = canvas_node.core_node.id + self.groups = None + self.services = None + self.current = None + if services is None: + services = set(canvas_node.core_node.config_services) + self.current_services = services + self.draw() + + def draw(self): + self.top.columnconfigure(0, weight=1) + self.top.rowconfigure(0, weight=1) + + frame = ttk.Frame(self.top) + frame.grid(stick="nsew", pady=PADY) + frame.rowconfigure(0, weight=1) + for i in range(3): + frame.columnconfigure(i, weight=1) + label_frame = ttk.LabelFrame(frame, text="Groups", padding=FRAME_PAD) + label_frame.grid(row=0, column=0, sticky="nsew") + label_frame.rowconfigure(0, weight=1) + label_frame.columnconfigure(0, weight=1) + self.groups = ListboxScroll(label_frame) + self.groups.grid(sticky="nsew") + for group in sorted(self.app.core.config_services): + self.groups.listbox.insert(tk.END, group) + self.groups.listbox.bind("<>", self.handle_group_change) + self.groups.listbox.selection_set(0) + + label_frame = ttk.LabelFrame(frame, text="Services") + label_frame.grid(row=0, column=1, sticky="nsew") + label_frame.columnconfigure(0, weight=1) + label_frame.rowconfigure(0, weight=1) + self.services = CheckboxList( + label_frame, self.app, clicked=self.service_clicked, padding=FRAME_PAD + ) + self.services.grid(sticky="nsew") + + label_frame = ttk.LabelFrame(frame, text="Selected", padding=FRAME_PAD) + label_frame.grid(row=0, column=2, sticky="nsew") + label_frame.rowconfigure(0, weight=1) + label_frame.columnconfigure(0, weight=1) + self.current = ListboxScroll(label_frame) + self.current.grid(sticky="nsew") + for service in sorted(self.current_services): + self.current.listbox.insert(tk.END, service) + if self.is_custom_service(service): + self.current.listbox.itemconfig(tk.END, bg="green") + + frame = ttk.Frame(self.top) + frame.grid(stick="ew") + for i in range(4): + frame.columnconfigure(i, weight=1) + button = ttk.Button(frame, text="Configure", command=self.click_configure) + button.grid(row=0, column=0, sticky="ew", padx=PADX) + button = ttk.Button(frame, text="Save", command=self.click_save) + button.grid(row=0, column=1, sticky="ew", padx=PADX) + button = ttk.Button(frame, text="Remove", command=self.click_remove) + button.grid(row=0, column=2, sticky="ew", padx=PADX) + button = ttk.Button(frame, text="Cancel", command=self.click_cancel) + button.grid(row=0, column=3, sticky="ew") + + # trigger group change + self.groups.listbox.event_generate("<>") + + def handle_group_change(self, event: tk.Event = None): + selection = self.groups.listbox.curselection() + if selection: + index = selection[0] + group = self.groups.listbox.get(index) + self.services.clear() + for name in sorted(self.app.core.config_services[group]): + checked = name in self.current_services + self.services.add(name, checked) + + def service_clicked(self, name: str, var: tk.IntVar): + if var.get() and name not in self.current_services: + self.current_services.add(name) + elif not var.get() and name in self.current_services: + self.current_services.remove(name) + self.current.listbox.delete(0, tk.END) + for name in sorted(self.current_services): + self.current.listbox.insert(tk.END, name) + if self.is_custom_service(name): + self.current.listbox.itemconfig(tk.END, bg="green") + self.canvas_node.core_node.config_services[:] = self.current_services + + def click_configure(self): + current_selection = self.current.listbox.curselection() + if len(current_selection): + dialog = ConfigServiceConfigDialog( + master=self, + app=self.app, + service_name=self.current.listbox.get(current_selection[0]), + node_id=self.node_id, + ) + dialog.show() + else: + messagebox.showinfo( + "Node service configuration", "Select a service to configure" + ) + + def click_save(self): + if ( + self.current_services + != self.app.core.default_services[self.canvas_node.core_node.model] + ): + self.canvas_node.core_node.config_services[:] = self.current_services + else: + if len(self.canvas_node.core_node.config_services) > 0: + self.canvas_node.core_node.config_services[:] = [] + self.destroy() + + def click_cancel(self): + self.current_services = None + self.destroy() + + def click_remove(self): + cur = self.current.listbox.curselection() + if cur: + service = self.current.listbox.get(cur[0]) + self.current.listbox.delete(cur[0]) + self.current_services.remove(service) + for checkbutton in self.services.frame.winfo_children(): + if checkbutton["text"] == service: + checkbutton.invoke() + return + + def is_custom_service(self, service: str) -> bool: + service_configs = self.app.core.service_configs + file_configs = self.app.core.file_configs + if self.node_id in service_configs and service in service_configs[self.node_id]: + return True + if ( + self.node_id in file_configs + and service in file_configs[self.node_id] + and file_configs[self.node_id][service] + ): + return True + return False diff --git a/daemon/core/gui/graph/node.py b/daemon/core/gui/graph/node.py index c1d8e075..e5974625 100644 --- a/daemon/core/gui/graph/node.py +++ b/daemon/core/gui/graph/node.py @@ -10,6 +10,7 @@ from core.gui import themes from core.gui.dialogs.emaneconfig import EmaneConfigDialog from core.gui.dialogs.mobilityconfig import MobilityConfigDialog from core.gui.dialogs.nodeconfig import NodeConfigDialog +from core.gui.dialogs.nodeconfigservice import NodeConfigServiceDialog from core.gui.dialogs.nodeservice import NodeServiceDialog from core.gui.dialogs.wlanconfig import WlanConfigDialog from core.gui.errors import show_grpc_error @@ -180,6 +181,7 @@ class CanvasNode: context.add_command(label="Configure", command=self.show_config) if NodeUtils.is_container_node(self.core_node.type): context.add_command(label="Services", state=tk.DISABLED) + context.add_command(label="Config Services", state=tk.DISABLED) if is_wlan: context.add_command(label="WLAN Config", command=self.show_wlan_config) if is_wlan and self.core_node.id in self.app.core.mobility_players: @@ -198,6 +200,9 @@ class CanvasNode: context.add_command(label="Configure", command=self.show_config) if NodeUtils.is_container_node(self.core_node.type): context.add_command(label="Services", command=self.show_services) + context.add_command( + label="Config Services", command=self.show_config_services + ) if is_emane: context.add_command( label="EMANE Config", command=self.show_emane_config @@ -253,6 +258,11 @@ class CanvasNode: dialog = NodeServiceDialog(self.app.master, self.app, self) dialog.show() + def show_config_services(self): + self.canvas.context = None + dialog = NodeConfigServiceDialog(self.app.master, self.app, self) + dialog.show() + def has_emane_link(self, interface_id: int) -> core_pb2.Node: result = None for edge in self.edges: diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 215a23d5..93435077 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -953,6 +953,7 @@ message Node { string opaque = 9; string image = 10; string server = 11; + repeated string config_services = 12; } message Link { From 7b5df11dc70346866e73f96bb8556abbec90cf41 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 20 Jan 2020 15:02:04 -0800 Subject: [PATCH 11/37] added files to config services, added default logic for creating files from templates, added new method to provide extra data to templates, updated gui to display templates raw --- daemon/core/api/grpc/client.py | 8 ++++ daemon/core/api/grpc/server.py | 20 ++++++++- daemon/core/configservice/base.py | 34 +++++++++++++- .../sercurityservices/services.py | 4 +- .../configservices/serviceutils/services.py | 14 +++--- daemon/core/configservices/simpleservice.py | 15 ++++--- daemon/core/gui/coreclient.py | 6 ++- .../core/gui/dialogs/configserviceconfig.py | 44 +++++++++---------- daemon/core/gui/dialogs/nodeconfigservice.py | 4 +- .../proto/core/api/grpc/configservices.proto | 21 ++++++--- daemon/proto/core/api/grpc/core.proto | 2 + 11 files changed, 120 insertions(+), 52 deletions(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index 8a52c5c6..297fb52d 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -15,6 +15,8 @@ from core.api.grpc import core_pb2, core_pb2_grpc from core.api.grpc.configservices_pb2 import ( GetConfigServicesRequest, GetConfigServicesResponse, + GetConfigServiceTemplatesRequest, + GetConfigServiceTemplatesResponse, GetNodeConfigServiceRequest, GetNodeConfigServiceResponse, GetNodeConfigServicesRequest, @@ -1092,6 +1094,12 @@ class CoreGrpcClient: request = GetConfigServicesRequest() return self.stub.GetConfigServices(request) + def get_config_service_templates( + self, name: str + ) -> GetConfigServiceTemplatesResponse: + request = GetConfigServiceTemplatesRequest(name=name) + return self.stub.GetConfigServiceTemplates(request) + def get_node_config_service( self, session_id: int, node_id: int, name: str ) -> GetNodeConfigServiceResponse: diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index ed97eb34..dd5fcff6 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -5,6 +5,7 @@ import re import tempfile import time from concurrent import futures +from typing import Type import grpc from grpc import ServicerContext @@ -14,6 +15,8 @@ from core.api.grpc.configservices_pb2 import ( ConfigService, GetConfigServicesRequest, GetConfigServicesResponse, + GetConfigServiceTemplatesRequest, + GetConfigServiceTemplatesResponse, GetNodeConfigServiceRequest, GetNodeConfigServiceResponse, GetNodeConfigServicesRequest, @@ -112,9 +115,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): except CoreError: context.abort(grpc.StatusCode.NOT_FOUND, f"node {node_id} not found") - def validate_service(self, name: str, context: ServicerContext) -> None: - if name not in self.coreemu.service_manager.services: + def validate_service( + self, name: str, context: ServicerContext + ) -> Type[ConfigService]: + service = self.coreemu.service_manager.services.get(name) + if not service: context.abort(grpc.StatusCode.NOT_FOUND, f"unknown service {name}") + return service def StartSession( self, request: core_pb2.StartSessionRequest, context: ServicerContext @@ -1456,6 +1463,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): executables=service.executables, dependencies=service.dependencies, directories=service.directories, + files=service.files, startup=service.startup, validate=service.validate, shutdown=service.shutdown, @@ -1481,6 +1489,14 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): config = {x.id: x.default for x in service.default_configs} return GetNodeConfigServiceResponse(config=config) + def GetConfigServiceTemplates( + self, request: GetConfigServiceTemplatesRequest, context: ServicerContext + ) -> GetConfigServiceTemplatesResponse: + service_class = self.validate_service(request.name, context) + service = service_class(None) + templates = service.get_templates() + return GetConfigServiceTemplatesResponse(templates=templates) + def GetNodeConfigServices( self, request: GetNodeConfigServicesRequest, context: ServicerContext ) -> GetNodeConfigServicesResponse: diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index b35307a0..ae273f77 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -55,6 +55,11 @@ class ConfigService(abc.ABC): def directories(self) -> List[str]: raise NotImplementedError + @property + @abc.abstractmethod + def files(self) -> List[str]: + raise NotImplementedError + @property @abc.abstractmethod def default_configs(self) -> List[Configuration]: @@ -120,8 +125,34 @@ class ConfigService(abc.ABC): f"failure to create service directory: {directory}" ) + def data(self) -> Dict[str, Any]: + return {} + + def get_text(self, name: str) -> str: + raise CoreError( + f"node({self.node.name} service({self.name}) unknown template({name})" + ) + + def get_templates(self) -> Dict[str, str]: + templates = {} + for name in self.files: + if self.templates.has_template(name): + template = self.templates.get_template(name).source + else: + template = self.get_text(name) + template = inspect.cleandoc(template) + templates[name] = template + return templates + def create_files(self) -> None: - raise NotImplementedError + data = self.data() + for name in self.files: + if self.templates.has_template(name): + self.render_template(name, data) + else: + text = self.get_text(name) + text = inspect.cleandoc(text) + self.render_text(name, text, data) def run_startup(self) -> None: for cmd in self.startup: @@ -176,7 +207,6 @@ class ConfigService(abc.ABC): def render_text(self, name: str, text: str, data: Dict[str, Any] = None) -> None: try: - text = inspect.cleandoc(text) template = Template(text) self._render(name, template, data) except Exception: diff --git a/daemon/core/configservices/sercurityservices/services.py b/daemon/core/configservices/sercurityservices/services.py index 6fd1fba0..a0a1aec4 100644 --- a/daemon/core/configservices/sercurityservices/services.py +++ b/daemon/core/configservices/sercurityservices/services.py @@ -7,6 +7,7 @@ class VpnClient(ConfigService): name = "VPNClient" group = GROUP_NAME directories = [] + files = ["vpnclient.sh"] executables = ["openvpn", "ip", "killall"] dependencies = [] startup = ["sh vpnclient.sh"] @@ -14,6 +15,3 @@ class VpnClient(ConfigService): shutdown = ["killall openvpn"] validation_mode = ConfigServiceMode.BLOCKING default_configs = [] - - def create_files(self): - self.render_template("vpnclient.sh") diff --git a/daemon/core/configservices/serviceutils/services.py b/daemon/core/configservices/serviceutils/services.py index d15c08c2..2b564ed4 100644 --- a/daemon/core/configservices/serviceutils/services.py +++ b/daemon/core/configservices/serviceutils/services.py @@ -1,3 +1,5 @@ +from typing import Any, Dict + import netaddr from core import utils @@ -12,6 +14,7 @@ class DefaultRoute(ConfigService): name = "DefaultRoute" group = GROUP_NAME directories = [] + files = ["defaultroute.sh"] executables = ["ip"] dependencies = [] startup = ["sh defaultroute.sh"] @@ -24,7 +27,7 @@ class DefaultRoute(ConfigService): Configuration(_id="value3", _type=ConfigDataTypes.STRING, label="Value 3"), ] - def create_files(self): + def data(self) -> Dict[str, Any]: addresses = [] for netif in self.node.netifs(): if getattr(netif, "control", False): @@ -33,14 +36,14 @@ class DefaultRoute(ConfigService): net = netaddr.IPNetwork(addr) if net[1] != net[-2]: addresses.append(net[1]) - data = dict(addresses=addresses) - self.render_template("defaultroute.sh", data) + return dict(addresses=addresses) class IpForwardService(ConfigService): name = "IPForward" group = GROUP_NAME directories = [] + files = ["ipforward.sh"] executables = ["sysctl"] dependencies = [] startup = ["sh ipforward.sh"] @@ -49,10 +52,9 @@ class IpForwardService(ConfigService): validation_mode = ConfigServiceMode.BLOCKING default_configs = [] - def create_files(self) -> None: + def data(self) -> Dict[str, Any]: devnames = [] for ifc in self.node.netifs(): devname = utils.sysctl_devname(ifc.name) devnames.append(devname) - data = dict(devnames=devnames) - self.render_template("ipforward.sh", data) + return dict(devnames=devnames) diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py index 8fb40938..efee5275 100644 --- a/daemon/core/configservices/simpleservice.py +++ b/daemon/core/configservices/simpleservice.py @@ -5,6 +5,7 @@ class SimpleService(ConfigService): name = "Simple" group = "SimpleGroup" directories = [] + files = ["test1.sh"] executables = [] dependencies = [] startup = [] @@ -13,10 +14,10 @@ class SimpleService(ConfigService): validation_mode = ConfigServiceMode.BLOCKING default_configs = [] - def create_files(self): - text = """ - # sample script - # node id(${node.id}) name(${node.name}) - echo hello - """ - self.render_text("test1.sh", text) + def get_text(self, name: str) -> str: + if name == "test1.sh": + return """ + # sample script + # node id(${node.id}) name(${node.name}) + echo hello + """ diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index 1e44715a..78eaecf3 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -63,6 +63,7 @@ class CoreClient: self.app = app self.master = app.master self.services = {} + self.config_services_groups = {} self.config_services = {} self.default_services = {} self.emane_models = [] @@ -417,7 +418,10 @@ class CoreClient: # get config service informations response = self.client.get_config_services() for service in response.services: - group_services = self.config_services.setdefault(service.group, set()) + self.config_services[service.name] = service + group_services = self.config_services_groups.setdefault( + service.group, set() + ) group_services.add(service.name) # if there are no sessions, create a new session, else join a session diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index b017212b..4c86dd3b 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -46,7 +46,7 @@ class ConfigServiceConfigDialog(Dialog): self.default_shutdown = [] self.validation_mode = None self.validation_time = None - self.validation_period = None + self.validation_period = tk.StringVar() self.documentnew_img = Images.get(ImageEnum.DOCUMENTNEW, 16) self.editdelete_img = Images.get(ImageEnum.EDITDELETE, 16) @@ -68,35 +68,33 @@ class ConfigServiceConfigDialog(Dialog): def load(self): try: self.app.core.create_nodes_and_links() - default_config = self.app.core.get_node_service( - self.node_id, self.service_name - ) - self.default_startup = default_config.startup[:] - self.default_validate = default_config.validate[:] - self.default_shutdown = default_config.shutdown[:] - custom_configs = self.service_configs - if ( - self.node_id in custom_configs - and self.service_name in custom_configs[self.node_id] - ): - service_config = custom_configs[self.node_id][self.service_name] - else: - service_config = default_config + # default_config = self.app.core.get_node_service( + # self.node_id, self.service_name + # ) + # self.default_startup = default_config.startup[:] + # self.default_validate = default_config.validate[:] + # self.default_shutdown = default_config.shutdown[:] + # custom_configs = self.service_configs + # if ( + # self.node_id in custom_configs + # and self.service_name in custom_configs[self.node_id] + # ): + # service_config = custom_configs[self.node_id][self.service_name] + # else: + # service_config = default_config + service_config = self.core.config_services[self.service_name] self.dependencies = service_config.dependencies[:] self.executables = service_config.executables[:] - self.filenames = service_config.configs[:] + self.filenames = service_config.files[:] self.startup_commands = service_config.startup[:] self.validation_commands = service_config.validate[:] self.shutdown_commands = service_config.shutdown[:] self.validation_mode = service_config.validation_mode self.validation_time = service_config.validation_timer - self.original_service_files = { - x: self.app.core.get_node_service_file( - self.node_id, self.service_name, x - ) - for x in self.filenames - } + self.validation_period.set(service_config.validation_period) + response = self.core.client.get_config_service_templates(self.service_name) + self.original_service_files = response.templates self.temp_service_files = dict(self.original_service_files) file_configs = self.file_configs if ( @@ -303,7 +301,7 @@ class ConfigServiceConfigDialog(Dialog): label = ttk.Label(frame, text="Validation Period") label.grid(row=2, column=0, sticky="w", padx=PADX) self.validation_period_entry = ttk.Entry( - frame, state=tk.DISABLED, textvariable=tk.StringVar() + frame, state=tk.DISABLED, textvariable=self.validation_period ) self.validation_period_entry.grid(row=2, column=1, sticky="ew", pady=PADY) diff --git a/daemon/core/gui/dialogs/nodeconfigservice.py b/daemon/core/gui/dialogs/nodeconfigservice.py index 46897d60..e4aefe25 100644 --- a/daemon/core/gui/dialogs/nodeconfigservice.py +++ b/daemon/core/gui/dialogs/nodeconfigservice.py @@ -51,7 +51,7 @@ class NodeConfigServiceDialog(Dialog): label_frame.columnconfigure(0, weight=1) self.groups = ListboxScroll(label_frame) self.groups.grid(sticky="nsew") - for group in sorted(self.app.core.config_services): + for group in sorted(self.app.core.config_services_groups): self.groups.listbox.insert(tk.END, group) self.groups.listbox.bind("<>", self.handle_group_change) self.groups.listbox.selection_set(0) @@ -98,7 +98,7 @@ class NodeConfigServiceDialog(Dialog): index = selection[0] group = self.groups.listbox.get(index) self.services.clear() - for name in sorted(self.app.core.config_services[group]): + for name in sorted(self.app.core.config_services_groups[group]): checked = name in self.current_services self.services.add(name, checked) diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto index a6d3e95a..4b744944 100644 --- a/daemon/proto/core/api/grpc/configservices.proto +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -16,12 +16,13 @@ message ConfigService { repeated string executables = 3; repeated string dependencies = 4; repeated string directories = 5; - repeated string startup = 6; - repeated string validate = 7; - repeated string shutdown = 8; - ConfigServiceValidationMode.Enum validation_mode = 9; - int32 validation_timer = 10; - float validation_period = 11; + repeated string files = 6; + repeated string startup = 7; + repeated string validate = 8; + repeated string shutdown = 9; + ConfigServiceValidationMode.Enum validation_mode = 10; + int32 validation_timer = 11; + float validation_period = 12; } message GetConfigServicesRequest { @@ -32,6 +33,14 @@ message GetConfigServicesResponse { repeated ConfigService services = 1; } +message GetConfigServiceTemplatesRequest { + string name = 1; +} + +message GetConfigServiceTemplatesResponse { + map templates = 1; +} + message GetNodeConfigServiceRequest { int32 session_id = 1; int32 node_id = 2; diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 93435077..8f09317a 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -107,6 +107,8 @@ service CoreApi { // config services rpc GetConfigServices (configservices.GetConfigServicesRequest) returns (configservices.GetConfigServicesResponse) { } + rpc GetConfigServiceTemplates (configservices.GetConfigServiceTemplatesRequest) returns (configservices.GetConfigServiceTemplatesResponse) { + } rpc GetNodeConfigService (configservices.GetNodeConfigServiceRequest) returns (configservices.GetNodeConfigServiceResponse) { } rpc GetNodeConfigServices (configservices.GetNodeConfigServicesRequest) returns (configservices.GetNodeConfigServicesResponse) { From a4f3abf27ca385b6ab7395245841a953a152639d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 20 Jan 2020 15:54:47 -0800 Subject: [PATCH 12/37] updated config service grpc to return default templates and config, added logic to check for local custom templates --- daemon/core/api/grpc/client.py | 12 ++--- daemon/core/api/grpc/server.py | 16 +++--- daemon/core/configservice/base.py | 15 +++++- daemon/core/gui/coreclient.py | 1 + .../core/gui/dialogs/configserviceconfig.py | 54 +++++++------------ .../proto/core/api/grpc/configservices.proto | 5 +- daemon/proto/core/api/grpc/core.proto | 2 +- 7 files changed, 51 insertions(+), 54 deletions(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index 297fb52d..19f7a795 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -13,10 +13,10 @@ import netaddr from core import utils from core.api.grpc import core_pb2, core_pb2_grpc from core.api.grpc.configservices_pb2 import ( + GetConfigServiceDefaultsRequest, + GetConfigServiceDefaultsResponse, GetConfigServicesRequest, GetConfigServicesResponse, - GetConfigServiceTemplatesRequest, - GetConfigServiceTemplatesResponse, GetNodeConfigServiceRequest, GetNodeConfigServiceResponse, GetNodeConfigServicesRequest, @@ -1094,11 +1094,11 @@ class CoreGrpcClient: request = GetConfigServicesRequest() return self.stub.GetConfigServices(request) - def get_config_service_templates( + def get_config_service_defaults( self, name: str - ) -> GetConfigServiceTemplatesResponse: - request = GetConfigServiceTemplatesRequest(name=name) - return self.stub.GetConfigServiceTemplates(request) + ) -> GetConfigServiceDefaultsResponse: + request = GetConfigServiceDefaultsRequest(name=name) + return self.stub.GetConfigServiceDefaults(request) def get_node_config_service( self, session_id: int, node_id: int, name: str diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index dd5fcff6..b0a7609b 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -13,10 +13,10 @@ from grpc import ServicerContext from core.api.grpc import core_pb2, core_pb2_grpc, grpcutils from core.api.grpc.configservices_pb2 import ( ConfigService, + GetConfigServiceDefaultsRequest, + GetConfigServiceDefaultsResponse, GetConfigServicesRequest, GetConfigServicesResponse, - GetConfigServiceTemplatesRequest, - GetConfigServiceTemplatesResponse, GetNodeConfigServiceRequest, GetNodeConfigServiceResponse, GetNodeConfigServicesRequest, @@ -1482,20 +1482,20 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): self.validate_service(request.name, context) service = node.config_services.get(request.name) if service: - config = service.get_config() - + config = service.render_config() else: service = self.coreemu.service_manager.get_service(request.name) config = {x.id: x.default for x in service.default_configs} return GetNodeConfigServiceResponse(config=config) - def GetConfigServiceTemplates( - self, request: GetConfigServiceTemplatesRequest, context: ServicerContext - ) -> GetConfigServiceTemplatesResponse: + def GetConfigServiceDefaults( + self, request: GetConfigServiceDefaultsRequest, context: ServicerContext + ) -> GetConfigServiceDefaultsResponse: service_class = self.validate_service(request.name, context) service = service_class(None) templates = service.get_templates() - return GetConfigServiceTemplatesResponse(templates=templates) + config = service.render_config() + return GetConfigServiceDefaultsResponse(templates=templates, config=config) def GetNodeConfigServices( self, request: GetNodeConfigServicesRequest, context: ServicerContext diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index ae273f77..6ee22f14 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -37,6 +37,7 @@ class ConfigService(abc.ABC): logging.info(templates_path) self.templates = TemplateLookup(directories=templates_path) self.config = {} + self.custom_templates = {} configs = self.default_configs[:] self._define_config(configs) @@ -128,6 +129,9 @@ class ConfigService(abc.ABC): def data(self) -> Dict[str, Any]: return {} + def custom_template(self, name: str, template: str) -> None: + self.custom_templates[name] = template + def get_text(self, name: str) -> str: raise CoreError( f"node({self.node.name} service({self.name}) unknown template({name})" @@ -136,7 +140,10 @@ class ConfigService(abc.ABC): def get_templates(self) -> Dict[str, str]: templates = {} for name in self.files: - if self.templates.has_template(name): + if name in self.custom_templates: + template = self.custom_templates[name] + template = inspect.cleandoc(template) + elif self.templates.has_template(name): template = self.templates.get_template(name).source else: template = self.get_text(name) @@ -147,7 +154,11 @@ class ConfigService(abc.ABC): def create_files(self) -> None: data = self.data() for name in self.files: - if self.templates.has_template(name): + if name in self.custom_templates: + text = self.custom_templates[name] + text = inspect.cleandoc(text) + self.render_text(name, text, data) + elif self.templates.has_template(name): self.render_template(name, data) else: text = self.get_text(name) diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index 78eaecf3..b9522730 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -90,6 +90,7 @@ class CoreClient: self.emane_model_configs = {} self.emane_config = None self.service_configs = {} + self.config_service_configs = {} self.file_configs = {} self.mobility_players = {} self.handling_throughputs = None diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index 4c86dd3b..ea5a025a 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -30,8 +30,7 @@ class ConfigServiceConfigDialog(Dialog): self.core = app.core self.node_id = node_id self.service_name = service_name - self.service_configs = app.core.service_configs - self.file_configs = app.core.file_configs + self.service_configs = app.core.config_service_configs self.radiovar = tk.IntVar() self.radiovar.set(2) @@ -68,41 +67,26 @@ class ConfigServiceConfigDialog(Dialog): def load(self): try: self.app.core.create_nodes_and_links() - # default_config = self.app.core.get_node_service( - # self.node_id, self.service_name - # ) - # self.default_startup = default_config.startup[:] - # self.default_validate = default_config.validate[:] - # self.default_shutdown = default_config.shutdown[:] - # custom_configs = self.service_configs - # if ( - # self.node_id in custom_configs - # and self.service_name in custom_configs[self.node_id] - # ): - # service_config = custom_configs[self.node_id][self.service_name] - # else: - # service_config = default_config + service = self.core.config_services[self.service_name] + self.dependencies = service.dependencies[:] + self.executables = service.executables[:] + self.filenames = service.files[:] + self.startup_commands = service.startup[:] + self.validation_commands = service.validate[:] + self.shutdown_commands = service.shutdown[:] + self.validation_mode = service.validation_mode + self.validation_time = service.validation_timer + self.validation_period.set(service.validation_period) - service_config = self.core.config_services[self.service_name] - self.dependencies = service_config.dependencies[:] - self.executables = service_config.executables[:] - self.filenames = service_config.files[:] - self.startup_commands = service_config.startup[:] - self.validation_commands = service_config.validate[:] - self.shutdown_commands = service_config.shutdown[:] - self.validation_mode = service_config.validation_mode - self.validation_time = service_config.validation_timer - self.validation_period.set(service_config.validation_period) - response = self.core.client.get_config_service_templates(self.service_name) + response = self.core.client.get_config_service_defaults(self.service_name) self.original_service_files = response.templates self.temp_service_files = dict(self.original_service_files) - file_configs = self.file_configs - if ( - self.node_id in file_configs - and self.service_name in file_configs[self.node_id] - ): - for file, data in file_configs[self.node_id][self.service_name].items(): - self.temp_service_files[file] = data + + node_configs = self.service_configs.get(self.node_id, {}) + service_config = node_configs.get(self.service_name, {}) + custom_templates = service_config.get("templates", {}) + for file, data in custom_templates.items(): + self.temp_service_files[file] = data except grpc.RpcError as e: show_grpc_error(e) @@ -270,7 +254,7 @@ class ConfigServiceConfigDialog(Dialog): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) tab.grid(sticky="nsew") tab.columnconfigure(0, weight=1) - self.notebook.add(tab, text="Configuration", sticky="nsew") + self.notebook.add(tab, text="Validation", sticky="nsew") frame = ttk.Frame(tab) frame.grid(sticky="ew", pady=PADY) diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto index 4b744944..e52bd1a0 100644 --- a/daemon/proto/core/api/grpc/configservices.proto +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -33,12 +33,13 @@ message GetConfigServicesResponse { repeated ConfigService services = 1; } -message GetConfigServiceTemplatesRequest { +message GetConfigServiceDefaultsRequest { string name = 1; } -message GetConfigServiceTemplatesResponse { +message GetConfigServiceDefaultsResponse { map templates = 1; + map config = 2; } message GetNodeConfigServiceRequest { diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 8f09317a..4cea4143 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -107,7 +107,7 @@ service CoreApi { // config services rpc GetConfigServices (configservices.GetConfigServicesRequest) returns (configservices.GetConfigServicesResponse) { } - rpc GetConfigServiceTemplates (configservices.GetConfigServiceTemplatesRequest) returns (configservices.GetConfigServiceTemplatesResponse) { + rpc GetConfigServiceDefaults (configservices.GetConfigServiceDefaultsRequest) returns (configservices.GetConfigServiceDefaultsResponse) { } rpc GetNodeConfigService (configservices.GetNodeConfigServiceRequest) returns (configservices.GetNodeConfigServiceResponse) { } From 83e7853821c798cb6822e76b602661f0d534cbf7 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 20 Jan 2020 16:25:56 -0800 Subject: [PATCH 13/37] fixed protobuf generation to avoid grpc generation for files with no definitions, added service config tab display to coretk --- daemon/core/api/grpc/grpcutils.py | 6 ++--- daemon/core/api/grpc/server.py | 14 +++++++++-- daemon/core/gui/coreclient.py | 10 ++++---- .../core/gui/dialogs/configserviceconfig.py | 21 +++++++++++++--- daemon/core/gui/widgets.py | 4 ++-- daemon/proto/Makefile.am | 3 ++- daemon/proto/core/api/grpc/common.proto | 12 ++++++++++ .../proto/core/api/grpc/configservices.proto | 4 +++- daemon/proto/core/api/grpc/core.proto | 24 +++++++------------ 9 files changed, 65 insertions(+), 33 deletions(-) create mode 100644 daemon/proto/core/api/grpc/common.proto diff --git a/daemon/core/api/grpc/grpcutils.py b/daemon/core/api/grpc/grpcutils.py index d54cba27..f7e80fc1 100644 --- a/daemon/core/api/grpc/grpcutils.py +++ b/daemon/core/api/grpc/grpcutils.py @@ -3,7 +3,7 @@ import time from typing import Any, Dict, List, Tuple, Type from core import utils -from core.api.grpc import core_pb2 +from core.api.grpc import common_pb2, core_pb2 from core.config import ConfigurableOptions from core.emulator.data import LinkData from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions @@ -190,7 +190,7 @@ def convert_value(value: Any) -> str: def get_config_options( config: Dict[str, str], configurable_options: Type[ConfigurableOptions] -) -> Dict[str, core_pb2.ConfigOption]: +) -> Dict[str, common_pb2.ConfigOption]: """ Retrieve configuration options in a form that is used by the grpc server. @@ -201,7 +201,7 @@ def get_config_options( results = {} for configuration in configurable_options.configurations(): value = config[configuration.id] - config_option = core_pb2.ConfigOption( + config_option = common_pb2.ConfigOption( label=configuration.label, name=configuration.id, value=value, diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index b0a7609b..df6178b9 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,7 +10,7 @@ from typing import Type import grpc from grpc import ServicerContext -from core.api.grpc import core_pb2, core_pb2_grpc, grpcutils +from core.api.grpc import common_pb2, core_pb2, core_pb2_grpc, grpcutils from core.api.grpc.configservices_pb2 import ( ConfigService, GetConfigServiceDefaultsRequest, @@ -1494,7 +1494,17 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): service_class = self.validate_service(request.name, context) service = service_class(None) templates = service.get_templates() - config = service.render_config() + config = {} + for configuration in service.default_configs: + config_option = common_pb2.ConfigOption( + label=configuration.label, + name=configuration.id, + value=configuration.default, + type=configuration.type.value, + select=configuration.options, + group="Settings", + ) + config[configuration.id] = config_option return GetConfigServiceDefaultsResponse(templates=templates, config=config) def GetNodeConfigServices( diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index b9522730..216a50a9 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Dict, List import grpc -from core.api.grpc import client, core_pb2 +from core.api.grpc import client, common_pb2, core_pb2 from core.gui import appconfig from core.gui.dialogs.mobilityplayer import MobilityPlayer from core.gui.dialogs.sessions import SessionsDialog @@ -875,14 +875,14 @@ class CoreClient: logging.info("running node(%s) cmd: %s", node_id, self.observer) return self.client.node_command(self.session_id, node_id, self.observer).output - def get_wlan_config(self, node_id: int) -> Dict[str, core_pb2.ConfigOption]: + def get_wlan_config(self, node_id: int) -> Dict[str, common_pb2.ConfigOption]: config = self.wlan_configs.get(node_id) if not config: response = self.client.get_wlan_config(self.session_id, node_id) config = response.config return config - def get_mobility_config(self, node_id: int) -> Dict[str, core_pb2.ConfigOption]: + def get_mobility_config(self, node_id: int) -> Dict[str, common_pb2.ConfigOption]: config = self.mobility_configs.get(node_id) if not config: response = self.client.get_mobility_config(self.session_id, node_id) @@ -891,7 +891,7 @@ class CoreClient: def get_emane_model_config( self, node_id: int, model: str, interface: int = None - ) -> Dict[str, core_pb2.ConfigOption]: + ) -> Dict[str, common_pb2.ConfigOption]: logging.info("getting emane model config: %s %s %s", node_id, model, interface) config = self.emane_model_configs.get((node_id, model, interface)) if not config: @@ -907,7 +907,7 @@ class CoreClient: self, node_id: int, model: str, - config: Dict[str, core_pb2.ConfigOption], + config: Dict[str, common_pb2.ConfigOption], interface: int = None, ): logging.info("setting emane model config: %s %s %s", node_id, model, interface) diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index ea5a025a..94556a3c 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -1,6 +1,7 @@ """ Service configuration dialog """ +import logging import tkinter as tk from tkinter import ttk from typing import TYPE_CHECKING, Any, List @@ -13,7 +14,7 @@ from core.gui.dialogs.dialog import Dialog from core.gui.errors import show_grpc_error from core.gui.images import ImageEnum, Images from core.gui.themes import FRAME_PAD, PADX, PADY -from core.gui.widgets import CodeText, ListboxScroll +from core.gui.widgets import CodeText, ConfigFrame, ListboxScroll if TYPE_CHECKING: from core.gui.app import Application @@ -61,6 +62,8 @@ class ConfigServiceConfigDialog(Dialog): self.original_service_files = {} self.temp_service_files = {} self.modified_files = set() + self.config_frame = None + self.config = None self.load() self.draw() @@ -81,6 +84,7 @@ class ConfigServiceConfigDialog(Dialog): response = self.core.client.get_config_service_defaults(self.service_name) self.original_service_files = response.templates self.temp_service_files = dict(self.original_service_files) + self.config = response.config node_configs = self.service_configs.get(self.node_id, {}) service_config = node_configs.get(self.service_name, {}) @@ -98,9 +102,10 @@ class ConfigServiceConfigDialog(Dialog): self.notebook = ttk.Notebook(self.top) self.notebook.grid(sticky="nsew", pady=PADY) self.draw_tab_files() + self.draw_tab_config() self.draw_tab_directories() self.draw_tab_startstop() - self.draw_tab_configuration() + self.draw_tab_validation() self.draw_buttons() @@ -184,6 +189,16 @@ class ConfigServiceConfigDialog(Dialog): "", self.update_temp_service_file_data ) + def draw_tab_config(self): + tab = ttk.Frame(self.notebook, padding=FRAME_PAD) + tab.grid(sticky="nsew") + tab.columnconfigure(0, weight=1) + self.notebook.add(tab, text="Configuration") + logging.info("config service config: %s", self.config) + self.config_frame = ConfigFrame(tab, self.app, self.config) + self.config_frame.draw_config() + self.config_frame.grid(sticky="nsew", pady=PADY) + def draw_tab_directories(self): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) tab.grid(sticky="nsew") @@ -250,7 +265,7 @@ class ConfigServiceConfigDialog(Dialog): elif i == 2: self.validate_commands_listbox = listbox_scroll.listbox - def draw_tab_configuration(self): + def draw_tab_validation(self): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) tab.grid(sticky="nsew") tab.columnconfigure(0, weight=1) diff --git a/daemon/core/gui/widgets.py b/daemon/core/gui/widgets.py index 8dc163ab..e6d2e940 100644 --- a/daemon/core/gui/widgets.py +++ b/daemon/core/gui/widgets.py @@ -5,7 +5,7 @@ from pathlib import PosixPath from tkinter import filedialog, font, ttk from typing import TYPE_CHECKING, Dict -from core.api.grpc import core_pb2 +from core.api.grpc import common_pb2, core_pb2 from core.gui import themes from core.gui.themes import FRAME_PAD, PADX, PADY @@ -74,7 +74,7 @@ class ConfigFrame(ttk.Notebook): self, master: tk.Widget, app: "Application", - config: Dict[str, core_pb2.ConfigOption], + config: Dict[str, common_pb2.ConfigOption], **kw ): super().__init__(master, **kw) diff --git a/daemon/proto/Makefile.am b/daemon/proto/Makefile.am index e63179ca..af535c1e 100644 --- a/daemon/proto/Makefile.am +++ b/daemon/proto/Makefile.am @@ -1,5 +1,6 @@ all: - $(PYTHON) -m grpc_tools.protoc -I . --python_out=.. --grpc_python_out=.. core/api/grpc/*.proto + $(PYTHON) -m grpc_tools.protoc -I . --python_out=.. core/api/grpc/*.proto + $(PYTHON) -m grpc_tools.protoc -I . --grpc_python_out=.. core/api/grpc/core.proto clean: -rm -f ../core/api/grpc/*_pb2* diff --git a/daemon/proto/core/api/grpc/common.proto b/daemon/proto/core/api/grpc/common.proto new file mode 100644 index 00000000..590e2262 --- /dev/null +++ b/daemon/proto/core/api/grpc/common.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package common; + +message ConfigOption { + string label = 1; + string name = 2; + string value = 3; + int32 type = 4; + repeated string select = 5; + string group = 6; +} diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto index e52bd1a0..156bad6b 100644 --- a/daemon/proto/core/api/grpc/configservices.proto +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package configservices; +import "core/api/grpc/common.proto"; + message ConfigServiceValidationMode { enum Enum { BLOCKING = 0; @@ -39,7 +41,7 @@ message GetConfigServiceDefaultsRequest { message GetConfigServiceDefaultsResponse { map templates = 1; - map config = 2; + map config = 2; } message GetNodeConfigServiceRequest { diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 4cea4143..82c26b2a 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -6,6 +6,7 @@ option java_package = "com.core.client.grpc"; option java_outer_classname = "CoreProto"; import "core/api/grpc/configservices.proto"; +import "core/api/grpc/common.proto"; service CoreApi { // session rpc @@ -217,7 +218,7 @@ message GetSessionOptionsRequest { } message GetSessionOptionsResponse { - map config = 2; + map config = 2; } message SetSessionOptionsRequest { @@ -508,7 +509,7 @@ message GetMobilityConfigRequest { } message GetMobilityConfigResponse { - map config = 1; + map config = 1; } message SetMobilityConfigRequest { @@ -633,7 +634,7 @@ message GetWlanConfigRequest { } message GetWlanConfigResponse { - map config = 1; + map config = 1; } message SetWlanConfigRequest { @@ -650,7 +651,7 @@ message GetEmaneConfigRequest { } message GetEmaneConfigResponse { - map config = 1; + map config = 1; } message SetEmaneConfigRequest { @@ -678,7 +679,7 @@ message GetEmaneModelConfigRequest { } message GetEmaneModelConfigResponse { - map config = 1; + map config = 1; } message SetEmaneModelConfigRequest { @@ -699,7 +700,7 @@ message GetEmaneModelConfigsResponse { int32 node_id = 1; string model = 2; int32 interface = 3; - map config = 4; + map config = 4; } repeated ModelConfig configs = 1; } @@ -917,16 +918,7 @@ message NodeServiceData { } message MappedConfig { - map config = 1; -} - -message ConfigOption { - string label = 1; - string name = 2; - string value = 3; - int32 type = 4; - repeated string select = 5; - string group = 6; + map config = 1; } message Session { From da107cc1d985f584057e3ca132a14914b3b601b2 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 21 Jan 2020 10:35:46 -0800 Subject: [PATCH 14/37] updates to config services dialogs in coretk, initial working state for running config services from the coretk gui --- daemon/core/api/grpc/client.py | 5 +- daemon/core/api/grpc/grpcutils.py | 1 + daemon/core/api/grpc/server.py | 8 + daemon/core/configservices/simpleservice.py | 12 +- .../core/gui/dialogs/configserviceconfig.py | 268 +++++------------- daemon/core/gui/dialogs/nodeconfigservice.py | 13 +- daemon/core/gui/widgets.py | 5 + .../proto/core/api/grpc/configservices.proto | 7 + daemon/proto/core/api/grpc/core.proto | 1 + 9 files changed, 111 insertions(+), 209 deletions(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index 19f7a795..ba80ace8 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -11,7 +11,7 @@ import grpc import netaddr from core import utils -from core.api.grpc import core_pb2, core_pb2_grpc +from core.api.grpc import configservices_pb2, core_pb2, core_pb2_grpc from core.api.grpc.configservices_pb2 import ( GetConfigServiceDefaultsRequest, GetConfigServiceDefaultsResponse, @@ -175,6 +175,7 @@ class CoreGrpcClient: service_configs: List[core_pb2.ServiceConfig] = None, service_file_configs: List[core_pb2.ServiceFileConfig] = None, asymmetric_links: List[core_pb2.Link] = None, + config_service_configs: List[configservices_pb2.ConfigServiceConfig] = None, ) -> core_pb2.StartSessionResponse: """ Start a session. @@ -191,6 +192,7 @@ class CoreGrpcClient: :param service_configs: node service configurations :param service_file_configs: node service file configurations :param asymmetric_links: asymmetric links to edit + :param config_service_configs: config service configurations :return: start session response """ request = core_pb2.StartSessionRequest( @@ -206,6 +208,7 @@ class CoreGrpcClient: service_configs=service_configs, service_file_configs=service_file_configs, asymmetric_links=asymmetric_links, + config_service_configs=config_service_configs, ) return self.stub.StartSession(request) diff --git a/daemon/core/api/grpc/grpcutils.py b/daemon/core/api/grpc/grpcutils.py index f7e80fc1..94a9b98c 100644 --- a/daemon/core/api/grpc/grpcutils.py +++ b/daemon/core/api/grpc/grpcutils.py @@ -33,6 +33,7 @@ def add_node_data(node_proto: core_pb2.Node) -> Tuple[NodeTypes, int, NodeOption options.opaque = node_proto.opaque options.image = node_proto.image options.services = node_proto.services + options.config_services = node_proto.config_services if node_proto.emane: options.emane = node_proto.emane if node_proto.server: diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index df6178b9..e39f77ea 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -179,6 +179,14 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): for config in request.service_configs: grpcutils.service_configuration(session, config) + # config service configs + for config in request.config_service_configs: + node = self.get_node(session, config.node_id, context) + service = node.config_services[request.name] + service.set_config(config.config) + for name, template in config.templates.values(): + service.custom_template(name, template) + # service file configs for config in request.service_file_configs: session.services.set_service_file( diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py index efee5275..1ab0ebf0 100644 --- a/daemon/core/configservices/simpleservice.py +++ b/daemon/core/configservices/simpleservice.py @@ -4,8 +4,8 @@ from core.configservice.base import ConfigService, ConfigServiceMode class SimpleService(ConfigService): name = "Simple" group = "SimpleGroup" - directories = [] - files = ["test1.sh"] + directories = ["/etc/quagga", "/usr/local/lib"] + files = ["test1.sh", "test2.sh"] executables = [] dependencies = [] startup = [] @@ -17,7 +17,13 @@ class SimpleService(ConfigService): def get_text(self, name: str) -> str: if name == "test1.sh": return """ - # sample script + # sample script 1 # node id(${node.id}) name(${node.name}) echo hello """ + elif name == "test2.sh": + return """ + # sample script 2 + # node id(${node.id}) name(${node.name}) + echo hello2 + """ diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index 94556a3c..89a07c86 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, List import grpc from core.api.grpc import core_pb2 -from core.gui.dialogs.copyserviceconfig import CopyServiceConfigDialog from core.gui.dialogs.dialog import Dialog from core.gui.errors import show_grpc_error from core.gui.images import ImageEnum, Images @@ -35,7 +34,8 @@ class ConfigServiceConfigDialog(Dialog): self.radiovar = tk.IntVar() self.radiovar.set(2) - self.filenames = [] + self.directories = [] + self.templates = [] self.dependencies = [] self.executables = [] self.startup_commands = [] @@ -51,29 +51,31 @@ class ConfigServiceConfigDialog(Dialog): self.editdelete_img = Images.get(ImageEnum.EDITDELETE, 16) self.notebook = None - self.filename_combobox = None + self.templates_combobox = None self.startup_commands_listbox = None self.shutdown_commands_listbox = None self.validate_commands_listbox = None self.validation_time_entry = None self.validation_mode_entry = None - self.service_file_data = None + self.template_text = None self.validation_period_entry = None self.original_service_files = {} self.temp_service_files = {} self.modified_files = set() self.config_frame = None + self.default_config = None self.config = None self.load() self.draw() def load(self): try: - self.app.core.create_nodes_and_links() + self.core.create_nodes_and_links() service = self.core.config_services[self.service_name] self.dependencies = service.dependencies[:] self.executables = service.executables[:] - self.filenames = service.files[:] + self.directories = service.directories[:] + self.templates = service.files[:] self.startup_commands = service.startup[:] self.validation_commands = service.validate[:] self.shutdown_commands = service.shutdown[:] @@ -84,10 +86,17 @@ class ConfigServiceConfigDialog(Dialog): response = self.core.client.get_config_service_defaults(self.service_name) self.original_service_files = response.templates self.temp_service_files = dict(self.original_service_files) - self.config = response.config node_configs = self.service_configs.get(self.node_id, {}) service_config = node_configs.get(self.service_name, {}) + + self.default_config = response.config + custom_config = service_config.get("config") + if custom_config: + self.config = custom_config + else: + self.config = dict(self.default_config) + custom_templates = service_config.get("templates", {}) for file, data in custom_templates.items(): self.temp_service_files[file] = data @@ -96,98 +105,61 @@ class ConfigServiceConfigDialog(Dialog): def draw(self): self.top.columnconfigure(0, weight=1) - self.top.rowconfigure(1, weight=1) + self.top.rowconfigure(0, weight=1) # draw notebook self.notebook = ttk.Notebook(self.top) self.notebook.grid(sticky="nsew", pady=PADY) self.draw_tab_files() - self.draw_tab_config() - self.draw_tab_directories() + if self.config: + self.draw_tab_config() self.draw_tab_startstop() self.draw_tab_validation() - self.draw_buttons() def draw_tab_files(self): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) tab.grid(sticky="nsew") tab.columnconfigure(0, weight=1) - self.notebook.add(tab, text="Files") + self.notebook.add(tab, text="Directories/Files") label = ttk.Label( - tab, text="Config files and scripts that are generated for this service." + tab, text="Directories and templates that will be used for this service." ) - label.grid() + label.grid(pady=PADY) frame = ttk.Frame(tab) frame.grid(sticky="ew", pady=PADY) frame.columnconfigure(1, weight=1) - label = ttk.Label(frame, text="File Name") - label.grid(row=0, column=0, padx=PADX, sticky="w") - self.filename_combobox = ttk.Combobox( - frame, values=self.filenames, state="readonly" + label = ttk.Label(frame, text="Directories") + label.grid(row=0, column=0, sticky="w", padx=PADX) + directories_combobox = ttk.Combobox( + frame, values=self.directories, state="readonly" ) - self.filename_combobox.bind( - "<>", self.display_service_file_data - ) - self.filename_combobox.grid(row=0, column=1, sticky="ew", padx=PADX) - button = ttk.Button(frame, image=self.documentnew_img, state="disabled") - button.bind("", self.add_filename) - button.grid(row=0, column=2, padx=PADX) - button = ttk.Button(frame, image=self.editdelete_img, state="disabled") - button.bind("", self.delete_filename) - button.grid(row=0, column=3) + directories_combobox.grid(row=0, column=1, sticky="ew", pady=PADY) + if self.directories: + directories_combobox.current(0) - frame = ttk.Frame(tab) - frame.grid(sticky="ew", pady=PADY) - frame.columnconfigure(1, weight=1) - button = ttk.Radiobutton( - frame, - variable=self.radiovar, - text="Copy Source File", - value=1, - state=tk.DISABLED, + label = ttk.Label(frame, text="Templates") + label.grid(row=1, column=0, sticky="w", padx=PADX) + self.templates_combobox = ttk.Combobox( + frame, values=self.templates, state="readonly" ) - button.grid(row=0, column=0, sticky="w", padx=PADX) - entry = ttk.Entry(frame, state=tk.DISABLED) - entry.grid(row=0, column=1, sticky="ew", padx=PADX) - image = Images.get(ImageEnum.FILEOPEN, 16) - button = ttk.Button(frame, image=image) - button.image = image - button.grid(row=0, column=2) - - frame = ttk.Frame(tab) - frame.grid(sticky="ew", pady=PADY) - frame.columnconfigure(0, weight=1) - button = ttk.Radiobutton( - frame, - variable=self.radiovar, - text="Use text below for file contents", - value=2, + self.templates_combobox.bind( + "<>", self.handle_template_changed ) - button.grid(row=0, column=0, sticky="ew") - image = Images.get(ImageEnum.FILEOPEN, 16) - button = ttk.Button(frame, image=image) - button.image = image - button.grid(row=0, column=1) - image = Images.get(ImageEnum.DOCUMENTSAVE, 16) - button = ttk.Button(frame, image=image) - button.image = image - button.grid(row=0, column=2) + self.templates_combobox.grid(row=1, column=1, sticky="ew", pady=PADY) - self.service_file_data = CodeText(tab) - self.service_file_data.grid(sticky="nsew") - tab.rowconfigure(self.service_file_data.grid_info()["row"], weight=1) - if len(self.filenames) > 0: - self.filename_combobox.current(0) - self.service_file_data.text.delete(1.0, "end") - self.service_file_data.text.insert( - "end", self.temp_service_files[self.filenames[0]] + self.template_text = CodeText(tab) + self.template_text.grid(sticky="nsew") + tab.rowconfigure(self.template_text.grid_info()["row"], weight=1) + if self.templates: + self.templates_combobox.current(0) + self.template_text.text.delete(1.0, "end") + self.template_text.text.insert( + "end", self.temp_service_files[self.templates[0]] ) - self.service_file_data.text.bind( - "", self.update_temp_service_file_data - ) + self.template_text.text.bind("", self.update_template_file_data) def draw_tab_config(self): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) @@ -199,18 +171,6 @@ class ConfigServiceConfigDialog(Dialog): self.config_frame.draw_config() self.config_frame.grid(sticky="nsew", pady=PADY) - def draw_tab_directories(self): - tab = ttk.Frame(self.notebook, padding=FRAME_PAD) - tab.grid(sticky="nsew") - tab.columnconfigure(0, weight=1) - self.notebook.add(tab, text="Directories") - - label = ttk.Label( - tab, - text="Directories required by this service that are unique for each node.", - ) - label.grid() - def draw_tab_startstop(self): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) tab.grid(sticky="nsew") @@ -238,26 +198,13 @@ class ConfigServiceConfigDialog(Dialog): ) commands = self.validation_commands label_frame.columnconfigure(0, weight=1) - label_frame.rowconfigure(1, weight=1) + label_frame.rowconfigure(0, weight=1) label_frame.grid(row=i, column=0, sticky="nsew", pady=PADY) - - frame = ttk.Frame(label_frame) - frame.grid(row=0, column=0, sticky="nsew", pady=PADY) - frame.columnconfigure(0, weight=1) - entry = ttk.Entry(frame, textvariable=tk.StringVar()) - entry.grid(row=0, column=0, stick="ew", padx=PADX) - button = ttk.Button(frame, image=self.documentnew_img) - button.bind("", self.add_command) - button.grid(row=0, column=1, sticky="ew", padx=PADX) - button = ttk.Button(frame, image=self.editdelete_img) - button.grid(row=0, column=2, sticky="ew") - button.bind("", self.delete_command) listbox_scroll = ListboxScroll(label_frame) - listbox_scroll.listbox.bind("<>", self.update_entry) for command in commands: listbox_scroll.listbox.insert("end", command) listbox_scroll.listbox.config(height=4) - listbox_scroll.grid(row=1, column=0, sticky="nsew") + listbox_scroll.grid(sticky="nsew") if i == 0: self.startup_commands_listbox = listbox_scroll.listbox elif i == 1: @@ -267,7 +214,7 @@ class ConfigServiceConfigDialog(Dialog): def draw_tab_validation(self): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) - tab.grid(sticky="nsew") + tab.grid(sticky="ew") tab.columnconfigure(0, weight=1) self.notebook.add(tab, text="Validation", sticky="nsew") @@ -338,57 +285,6 @@ class ConfigServiceConfigDialog(Dialog): button = ttk.Button(frame, text="Cancel", command=self.destroy) button.grid(row=0, column=3, sticky="ew") - def add_filename(self, event: tk.Event): - # not worry about it for now - return - frame_contains_button = event.widget.master - combobox = frame_contains_button.grid_slaves(row=0, column=1)[0] - filename = combobox.get() - if filename not in combobox["values"]: - combobox["values"] += (filename,) - - def delete_filename(self, event: tk.Event): - # not worry about it for now - return - frame_comntains_button = event.widget.master - combobox = frame_comntains_button.grid_slaves(row=0, column=1)[0] - filename = combobox.get() - if filename in combobox["values"]: - combobox["values"] = tuple([x for x in combobox["values"] if x != filename]) - combobox.set("") - - def add_command(self, event: tk.Event): - frame_contains_button = event.widget.master - listbox = frame_contains_button.master.grid_slaves(row=1, column=0)[0].listbox - command_to_add = frame_contains_button.grid_slaves(row=0, column=0)[0].get() - if command_to_add == "": - return - for cmd in listbox.get(0, tk.END): - if cmd == command_to_add: - return - listbox.insert(tk.END, command_to_add) - - def update_entry(self, event: tk.Event): - listbox = event.widget - current_selection = listbox.curselection() - if len(current_selection) > 0: - cmd = listbox.get(current_selection[0]) - entry = listbox.master.master.grid_slaves(row=0, column=0)[0].grid_slaves( - row=0, column=0 - )[0] - entry.delete(0, "end") - entry.insert(0, cmd) - - def delete_command(self, event: tk.Event): - button = event.widget - frame_contains_button = button.master - listbox = frame_contains_button.master.grid_slaves(row=1, column=0)[0].listbox - current_selection = listbox.curselection() - if len(current_selection) > 0: - listbox.delete(current_selection[0]) - entry = frame_contains_button.grid_slaves(row=0, column=0)[0] - entry.delete(0, tk.END) - def click_apply(self): current_listbox = self.master.current.listbox if not self.is_custom_service_config() and not self.is_custom_service_file(): @@ -399,53 +295,32 @@ class ConfigServiceConfigDialog(Dialog): return try: - if self.is_custom_service_config(): - startup_commands = self.startup_commands_listbox.get(0, "end") - shutdown_commands = self.shutdown_commands_listbox.get(0, "end") - validate_commands = self.validate_commands_listbox.get(0, "end") - config = self.core.set_node_service( - self.node_id, - self.service_name, - startup_commands, - validate_commands, - shutdown_commands, - ) - if self.node_id not in self.service_configs: - self.service_configs[self.node_id] = {} - self.service_configs[self.node_id][self.service_name] = config - + node_config = self.service_configs.setdefault(self.node_id, {}) + service_config = node_config.setdefault(self.service_name, {}) + self.config_frame.parse_config() + service_config["config"] = self.config + templates_config = service_config.setdefault("templates", {}) for file in self.modified_files: - if self.node_id not in self.file_configs: - self.file_configs[self.node_id] = {} - if self.service_name not in self.file_configs[self.node_id]: - self.file_configs[self.node_id][self.service_name] = {} - self.file_configs[self.node_id][self.service_name][ - file - ] = self.temp_service_files[file] - - self.app.core.set_node_service_file( - self.node_id, self.service_name, file, self.temp_service_files[file] - ) + templates_config[file] = self.temp_service_files[file] all_current = current_listbox.get(0, tk.END) current_listbox.itemconfig(all_current.index(self.service_name), bg="green") except grpc.RpcError as e: show_grpc_error(e) self.destroy() - def display_service_file_data(self, event: tk.Event): - combobox = event.widget - filename = combobox.get() - self.service_file_data.text.delete(1.0, "end") - self.service_file_data.text.insert("end", self.temp_service_files[filename]) + def handle_template_changed(self, event: tk.Event): + template = self.templates_combobox.get() + self.template_text.text.delete(1.0, "end") + self.template_text.text.insert("end", self.temp_service_files[template]) - def update_temp_service_file_data(self, event: tk.Event): + def update_template_file_data(self, event: tk.Event): scrolledtext = event.widget - filename = self.filename_combobox.get() - self.temp_service_files[filename] = scrolledtext.get(1.0, "end") - if self.temp_service_files[filename] != self.original_service_files[filename]: - self.modified_files.add(filename) + template = self.templates_combobox.get() + self.temp_service_files[template] = scrolledtext.get(1.0, "end") + if self.temp_service_files[template] != self.original_service_files[template]: + self.modified_files.add(template) else: - self.modified_files.discard(filename) + self.modified_files.discard(template) def is_custom_service_config(self): startup_commands = self.startup_commands_listbox.get(0, "end") @@ -462,13 +337,13 @@ class ConfigServiceConfigDialog(Dialog): def click_defaults(self): if self.node_id in self.service_configs: - self.service_configs[self.node_id].pop(self.service_name, None) - if self.node_id in self.file_configs: - self.file_configs[self.node_id].pop(self.service_name, None) + node_config = self.service_configs.get(self.node_id, {}) + node_config.pop(self.service_name, None) self.temp_service_files = dict(self.original_service_files) - filename = self.filename_combobox.get() - self.service_file_data.text.delete(1.0, "end") - self.service_file_data.text.insert("end", self.temp_service_files[filename]) + filename = self.templates_combobox.get() + self.template_text.text.delete(1.0, "end") + self.template_text.text.insert("end", self.temp_service_files[filename]) + self.config_frame.set_values(self.default_config) self.startup_commands_listbox.delete(0, tk.END) self.validate_commands_listbox.delete(0, tk.END) self.shutdown_commands_listbox.delete(0, tk.END) @@ -480,8 +355,7 @@ class ConfigServiceConfigDialog(Dialog): self.shutdown_commands_listbox.insert(tk.END, cmd) def click_copy(self): - dialog = CopyServiceConfigDialog(self, self.app, self.node_id) - dialog.show() + pass def append_commands( self, commands: List[str], listbox: tk.Listbox, to_add: List[str] diff --git a/daemon/core/gui/dialogs/nodeconfigservice.py b/daemon/core/gui/dialogs/nodeconfigservice.py index e4aefe25..bd9ec784 100644 --- a/daemon/core/gui/dialogs/nodeconfigservice.py +++ b/daemon/core/gui/dialogs/nodeconfigservice.py @@ -1,6 +1,7 @@ """ core node services """ +import logging import tkinter as tk from tkinter import messagebox, ttk from typing import TYPE_CHECKING, Any, Set @@ -130,14 +131,10 @@ class NodeConfigServiceDialog(Dialog): ) def click_save(self): - if ( - self.current_services - != self.app.core.default_services[self.canvas_node.core_node.model] - ): - self.canvas_node.core_node.config_services[:] = self.current_services - else: - if len(self.canvas_node.core_node.config_services) > 0: - self.canvas_node.core_node.config_services[:] = [] + self.canvas_node.core_node.config_services[:] = self.current_services + logging.info( + "saved node config services: %s", self.canvas_node.core_node.config_services + ) self.destroy() def click_cancel(self): diff --git a/daemon/core/gui/widgets.py b/daemon/core/gui/widgets.py index e6d2e940..b50959dc 100644 --- a/daemon/core/gui/widgets.py +++ b/daemon/core/gui/widgets.py @@ -184,6 +184,11 @@ class ConfigFrame(ttk.Notebook): return {x: self.config[x].value for x in self.config} + def set_values(self, config: Dict[str, common_pb2.ConfigOption]) -> None: + for name, option in config.items(): + value = self.values[name] + value.set(option.value) + class ListboxScroll(ttk.Frame): def __init__(self, master: tk.Widget = None, **kw): diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto index 156bad6b..9cc40b7a 100644 --- a/daemon/proto/core/api/grpc/configservices.proto +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -4,6 +4,13 @@ package configservices; import "core/api/grpc/common.proto"; +message ConfigServiceConfig { + int32 node_id = 1; + string name = 2; + map templates = 3; + map config = 4; +} + message ConfigServiceValidationMode { enum Enum { BLOCKING = 0; diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index 82c26b2a..f7e9cf99 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -166,6 +166,7 @@ message StartSessionRequest { repeated ServiceConfig service_configs = 10; repeated ServiceFileConfig service_file_configs = 11; repeated Link asymmetric_links = 12; + repeated configservices.ConfigServiceConfig config_service_configs = 13; } message StartSessionResponse { From 80a4955bd4b9d485a0cce24c90f09526c80fcd40 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 21 Jan 2020 11:58:43 -0800 Subject: [PATCH 15/37] updated coretk to working order for configuring config services from coretk --- daemon/core/api/grpc/server.py | 7 ++++--- daemon/core/emulator/session.py | 1 + daemon/core/gui/coreclient.py | 19 ++++++++++++++++++- .../core/gui/dialogs/configserviceconfig.py | 8 +++++--- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index e39f77ea..763d5958 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -182,9 +182,10 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): # config service configs for config in request.config_service_configs: node = self.get_node(session, config.node_id, context) - service = node.config_services[request.name] - service.set_config(config.config) - for name, template in config.templates.values(): + service = node.config_services[config.name] + if config.config: + service.set_config(config.config) + for name, template in config.templates.items(): service.custom_template(name, template) # service file configs diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index c76cbae1..da678211 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -730,6 +730,7 @@ class Session: self.services.add_services(node, node.type, options.services) # add config services + logging.info("setting node config services: %s", options.config_services) for name in options.config_services: service_class = self.service_manager.get_service(name) node.add_config_service(service_class) diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index 216a50a9..2c364b58 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Dict, List import grpc -from core.api.grpc import client, common_pb2, core_pb2 +from core.api.grpc import client, common_pb2, configservices_pb2, core_pb2 from core.gui import appconfig from core.gui.dialogs.mobilityplayer import MobilityPlayer from core.gui.dialogs.sessions import SessionsDialog @@ -462,6 +462,7 @@ class CoreClient: asymmetric_links = [ x.asymmetric_link for x in self.links.values() if x.asymmetric_link ] + config_service_configs = self.get_config_service_configs_proto() if self.emane_config: emane_config = {x: self.emane_config[x].value for x in self.emane_config} else: @@ -482,6 +483,7 @@ class CoreClient: service_configs, file_configs, asymmetric_links, + config_service_configs, ) logging.debug( "start session(%s), result: %s", self.session_id, response.result @@ -871,6 +873,21 @@ class CoreClient: configs.append(config_proto) return configs + def get_config_service_configs_proto( + self + ) -> List[configservices_pb2.ConfigServiceConfig]: + config_service_protos = [] + for node_id, node_config in self.config_service_configs.items(): + for name, service_config in node_config.items(): + config_proto = configservices_pb2.ConfigServiceConfig( + node_id=node_id, + name=name, + templates=service_config["templates"], + config=service_config.get("config"), + ) + config_service_protos.append(config_proto) + return config_service_protos + def run(self, node_id: int) -> str: logging.info("running node(%s) cmd: %s", node_id, self.observer) return self.client.node_command(self.session_id, node_id, self.observer).output diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index 89a07c86..abbb26fc 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -297,8 +297,9 @@ class ConfigServiceConfigDialog(Dialog): try: node_config = self.service_configs.setdefault(self.node_id, {}) service_config = node_config.setdefault(self.service_name, {}) - self.config_frame.parse_config() - service_config["config"] = self.config + if self.config_frame: + self.config_frame.parse_config() + service_config["config"] = self.config templates_config = service_config.setdefault("templates", {}) for file in self.modified_files: templates_config[file] = self.temp_service_files[file] @@ -343,7 +344,8 @@ class ConfigServiceConfigDialog(Dialog): filename = self.templates_combobox.get() self.template_text.text.delete(1.0, "end") self.template_text.text.insert("end", self.temp_service_files[filename]) - self.config_frame.set_values(self.default_config) + if self.config_frame: + self.config_frame.set_values(self.default_config) self.startup_commands_listbox.delete(0, tk.END) self.validate_commands_listbox.delete(0, tk.END) self.shutdown_commands_listbox.delete(0, tk.END) From 1ca3b0e3f42f909511bc66b52a74d27a655eddfd Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 21 Jan 2020 12:04:22 -0800 Subject: [PATCH 16/37] moved config values to simple service from default route, fixed coretk issue with config service config data --- daemon/core/configservices/serviceutils/services.py | 8 +------- daemon/core/configservices/simpleservice.py | 10 +++++++++- daemon/core/gui/coreclient.py | 6 +++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/daemon/core/configservices/serviceutils/services.py b/daemon/core/configservices/serviceutils/services.py index 2b564ed4..a770926d 100644 --- a/daemon/core/configservices/serviceutils/services.py +++ b/daemon/core/configservices/serviceutils/services.py @@ -3,9 +3,7 @@ from typing import Any, Dict import netaddr from core import utils -from core.config import Configuration from core.configservice.base import ConfigService, ConfigServiceMode -from core.emulator.enumerations import ConfigDataTypes GROUP_NAME = "Utility" @@ -21,11 +19,7 @@ class DefaultRoute(ConfigService): validate = [] shutdown = [] validation_mode = ConfigServiceMode.BLOCKING - default_configs = [ - Configuration(_id="value1", _type=ConfigDataTypes.STRING, label="Value 1"), - Configuration(_id="value2", _type=ConfigDataTypes.STRING, label="Value 2"), - Configuration(_id="value3", _type=ConfigDataTypes.STRING, label="Value 3"), - ] + default_configs = [] def data(self) -> Dict[str, Any]: addresses = [] diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py index 1ab0ebf0..ae9aa8c4 100644 --- a/daemon/core/configservices/simpleservice.py +++ b/daemon/core/configservices/simpleservice.py @@ -1,4 +1,6 @@ +from core.config import Configuration from core.configservice.base import ConfigService, ConfigServiceMode +from core.emulator.enumerations import ConfigDataTypes class SimpleService(ConfigService): @@ -12,18 +14,24 @@ class SimpleService(ConfigService): validate = [] shutdown = [] validation_mode = ConfigServiceMode.BLOCKING - default_configs = [] + default_configs = [ + Configuration(_id="value1", _type=ConfigDataTypes.STRING, label="Value 1"), + Configuration(_id="value2", _type=ConfigDataTypes.STRING, label="Value 2"), + Configuration(_id="value3", _type=ConfigDataTypes.STRING, label="Value 3"), + ] def get_text(self, name: str) -> str: if name == "test1.sh": return """ # sample script 1 # node id(${node.id}) name(${node.name}) + # config: ${config} echo hello """ elif name == "test2.sh": return """ # sample script 2 # node id(${node.id}) name(${node.name}) + # config: ${config} echo hello2 """ diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index 2c364b58..2748cf85 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -879,11 +879,15 @@ class CoreClient: config_service_protos = [] for node_id, node_config in self.config_service_configs.items(): for name, service_config in node_config.items(): + config = service_config.get("config", {}) + config_values = {} + for key, option in config.items(): + config_values[key] = option.value config_proto = configservices_pb2.ConfigServiceConfig( node_id=node_id, name=name, templates=service_config["templates"], - config=service_config.get("config"), + config=config_values, ) config_service_protos.append(config_proto) return config_service_protos From 0ea2f73a8045b819871e2f389715d3e6b65a335b Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 21 Jan 2020 14:00:55 -0800 Subject: [PATCH 17/37] added modes to config services that allows them to decide sets of configurations --- daemon/core/api/grpc/server.py | 16 +++++++- daemon/core/configservice/base.py | 5 +++ .../sercurityservices/services.py | 1 + .../configservices/serviceutils/services.py | 2 + daemon/core/configservices/simpleservice.py | 5 +++ .../core/gui/dialogs/configserviceconfig.py | 41 +++++++++++++------ daemon/core/gui/widgets.py | 6 +-- .../proto/core/api/grpc/configservices.proto | 6 +++ 8 files changed, 64 insertions(+), 18 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 763d5958..9b2634a9 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -10,7 +10,13 @@ from typing import Type import grpc from grpc import ServicerContext -from core.api.grpc import common_pb2, core_pb2, core_pb2_grpc, grpcutils +from core.api.grpc import ( + common_pb2, + configservices_pb2, + core_pb2, + core_pb2_grpc, + grpcutils, +) from core.api.grpc.configservices_pb2 import ( ConfigService, GetConfigServiceDefaultsRequest, @@ -1514,7 +1520,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): group="Settings", ) config[configuration.id] = config_option - return GetConfigServiceDefaultsResponse(templates=templates, config=config) + modes = [] + for name, mode_config in service.modes.items(): + mode = configservices_pb2.ConfigMode(name=name, config=mode_config) + modes.append(mode) + return GetConfigServiceDefaultsResponse( + templates=templates, config=config, modes=modes + ) def GetNodeConfigServices( self, request: GetNodeConfigServicesRequest, context: ServicerContext diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 6ee22f14..c14aec7e 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -66,6 +66,11 @@ class ConfigService(abc.ABC): def default_configs(self) -> List[Configuration]: raise NotImplementedError + @property + @abc.abstractmethod + def modes(self) -> Dict[str, Dict[str, str]]: + raise NotImplementedError + @property @abc.abstractmethod def executables(self) -> List[str]: diff --git a/daemon/core/configservices/sercurityservices/services.py b/daemon/core/configservices/sercurityservices/services.py index a0a1aec4..9c8d416a 100644 --- a/daemon/core/configservices/sercurityservices/services.py +++ b/daemon/core/configservices/sercurityservices/services.py @@ -15,3 +15,4 @@ class VpnClient(ConfigService): shutdown = ["killall openvpn"] validation_mode = ConfigServiceMode.BLOCKING default_configs = [] + modes = {} diff --git a/daemon/core/configservices/serviceutils/services.py b/daemon/core/configservices/serviceutils/services.py index a770926d..51fb5828 100644 --- a/daemon/core/configservices/serviceutils/services.py +++ b/daemon/core/configservices/serviceutils/services.py @@ -20,6 +20,7 @@ class DefaultRoute(ConfigService): shutdown = [] validation_mode = ConfigServiceMode.BLOCKING default_configs = [] + modes = {} def data(self) -> Dict[str, Any]: addresses = [] @@ -45,6 +46,7 @@ class IpForwardService(ConfigService): shutdown = [] validation_mode = ConfigServiceMode.BLOCKING default_configs = [] + modes = {} def data(self) -> Dict[str, Any]: devnames = [] diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py index ae9aa8c4..ab2dcb5b 100644 --- a/daemon/core/configservices/simpleservice.py +++ b/daemon/core/configservices/simpleservice.py @@ -19,6 +19,11 @@ class SimpleService(ConfigService): Configuration(_id="value2", _type=ConfigDataTypes.STRING, label="Value 2"), Configuration(_id="value3", _type=ConfigDataTypes.STRING, label="Value 3"), ] + modes = { + "mode1": {"value1": "m1", "value2": "m1", "value3": "m1"}, + "mode2": {"value1": "m2", "value2": "m2", "value3": "m2"}, + "mode3": {"value1": "m3", "value2": "m3", "value3": "m3"}, + } def get_text(self, name: str) -> str: if name == "test1.sh": diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index abbb26fc..8a4a0a0c 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -11,7 +11,6 @@ import grpc from core.api.grpc import core_pb2 from core.gui.dialogs.dialog import Dialog from core.gui.errors import show_grpc_error -from core.gui.images import ImageEnum, Images from core.gui.themes import FRAME_PAD, PADX, PADY from core.gui.widgets import CodeText, ConfigFrame, ListboxScroll @@ -47,11 +46,12 @@ class ConfigServiceConfigDialog(Dialog): self.validation_mode = None self.validation_time = None self.validation_period = tk.StringVar() - self.documentnew_img = Images.get(ImageEnum.DOCUMENTNEW, 16) - self.editdelete_img = Images.get(ImageEnum.EDITDELETE, 16) + self.modes = [] + self.mode_configs = {} self.notebook = None self.templates_combobox = None + self.modes_combobox = None self.startup_commands_listbox = None self.shutdown_commands_listbox = None self.validate_commands_listbox = None @@ -87,6 +87,9 @@ class ConfigServiceConfigDialog(Dialog): self.original_service_files = response.templates self.temp_service_files = dict(self.original_service_files) + self.modes = sorted(x.name for x in response.modes) + self.mode_configs = {x.name: x.config for x in response.modes} + node_configs = self.service_configs.get(self.node_id, {}) service_config = node_configs.get(self.service_name, {}) @@ -166,10 +169,24 @@ class ConfigServiceConfigDialog(Dialog): tab.grid(sticky="nsew") tab.columnconfigure(0, weight=1) self.notebook.add(tab, text="Configuration") + + if self.modes: + frame = ttk.Frame(tab) + frame.grid(sticky="ew", pady=PADY) + frame.columnconfigure(1, weight=1) + label = ttk.Label(frame, text="Modes") + label.grid(row=0, column=0, padx=PADX) + self.modes_combobox = ttk.Combobox( + frame, values=self.modes, state="readonly" + ) + self.modes_combobox.bind("<>", self.handle_mode_changed) + self.modes_combobox.grid(row=0, column=1, sticky="ew", pady=PADY) + logging.info("config service config: %s", self.config) self.config_frame = ConfigFrame(tab, self.app, self.config) self.config_frame.draw_config() self.config_frame.grid(sticky="nsew", pady=PADY) + tab.rowconfigure(self.config_frame.grid_info()["row"], weight=1) def draw_tab_startstop(self): tab = ttk.Frame(self.notebook, padding=FRAME_PAD) @@ -314,6 +331,12 @@ class ConfigServiceConfigDialog(Dialog): self.template_text.text.delete(1.0, "end") self.template_text.text.insert("end", self.temp_service_files[template]) + def handle_mode_changed(self, event: tk.Event): + mode = self.modes_combobox.get() + config = self.mode_configs[mode] + logging.info("mode config: %s", config) + self.config_frame.set_values(config) + def update_template_file_data(self, event: tk.Event): scrolledtext = event.widget template = self.templates_combobox.get() @@ -345,16 +368,8 @@ class ConfigServiceConfigDialog(Dialog): self.template_text.text.delete(1.0, "end") self.template_text.text.insert("end", self.temp_service_files[filename]) if self.config_frame: - self.config_frame.set_values(self.default_config) - self.startup_commands_listbox.delete(0, tk.END) - self.validate_commands_listbox.delete(0, tk.END) - self.shutdown_commands_listbox.delete(0, tk.END) - for cmd in self.default_startup: - self.startup_commands_listbox.insert(tk.END, cmd) - for cmd in self.default_validate: - self.validate_commands_listbox.insert(tk.END, cmd) - for cmd in self.default_shutdown: - self.shutdown_commands_listbox.insert(tk.END, cmd) + defaults = {x.id: x.value for x in self.default_config.values()} + self.config_frame.set_values(defaults) def click_copy(self): pass diff --git a/daemon/core/gui/widgets.py b/daemon/core/gui/widgets.py index b50959dc..e0dc4b25 100644 --- a/daemon/core/gui/widgets.py +++ b/daemon/core/gui/widgets.py @@ -184,10 +184,10 @@ class ConfigFrame(ttk.Notebook): return {x: self.config[x].value for x in self.config} - def set_values(self, config: Dict[str, common_pb2.ConfigOption]) -> None: - for name, option in config.items(): + def set_values(self, config: Dict[str, str]) -> None: + for name, data in config.items(): value = self.values[name] - value.set(option.value) + value.set(data) class ListboxScroll(ttk.Frame): diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto index 9cc40b7a..e8d93fb0 100644 --- a/daemon/proto/core/api/grpc/configservices.proto +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -34,6 +34,11 @@ message ConfigService { float validation_period = 12; } +message ConfigMode { + string name = 1; + map config = 2; +} + message GetConfigServicesRequest { int32 session_id = 1; } @@ -49,6 +54,7 @@ message GetConfigServiceDefaultsRequest { message GetConfigServiceDefaultsResponse { map templates = 1; map config = 2; + repeated ConfigMode modes = 3; } message GetNodeConfigServiceRequest { From d7d0a55fd24f25193b083a8a0a85d79cb50def9a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 21 Jan 2020 17:32:12 -0800 Subject: [PATCH 18/37] updates to support saving config services to xml, loading config services from xml, retrieving config services from coretk when joining a session --- daemon/core/api/grpc/client.py | 8 ++ daemon/core/api/grpc/server.py | 30 +++++++- daemon/core/configservice/base.py | 13 ++-- daemon/core/gui/coreclient.py | 17 ++++- .../core/gui/dialogs/configserviceconfig.py | 39 +++++----- daemon/core/gui/dialogs/nodeconfigservice.py | 15 ++-- daemon/core/xml/corexml.py | 73 ++++++++++++++++++- .../proto/core/api/grpc/configservices.proto | 8 ++ daemon/proto/core/api/grpc/core.proto | 2 + 9 files changed, 162 insertions(+), 43 deletions(-) diff --git a/daemon/core/api/grpc/client.py b/daemon/core/api/grpc/client.py index ba80ace8..0f939921 100644 --- a/daemon/core/api/grpc/client.py +++ b/daemon/core/api/grpc/client.py @@ -17,6 +17,8 @@ from core.api.grpc.configservices_pb2 import ( GetConfigServiceDefaultsResponse, GetConfigServicesRequest, GetConfigServicesResponse, + GetNodeConfigServiceConfigsRequest, + GetNodeConfigServiceConfigsResponse, GetNodeConfigServiceRequest, GetNodeConfigServiceResponse, GetNodeConfigServicesRequest, @@ -1103,6 +1105,12 @@ class CoreGrpcClient: request = GetConfigServiceDefaultsRequest(name=name) return self.stub.GetConfigServiceDefaults(request) + def get_node_config_service_configs( + self, session_id: int + ) -> GetNodeConfigServiceConfigsResponse: + request = GetNodeConfigServiceConfigsRequest(session_id=session_id) + return self.stub.GetNodeConfigServiceConfigs(request) + def get_node_config_service( self, session_id: int, node_id: int, name: str ) -> GetNodeConfigServiceResponse: diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 9b2634a9..eee6a446 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -23,6 +23,8 @@ from core.api.grpc.configservices_pb2 import ( GetConfigServiceDefaultsResponse, GetConfigServicesRequest, GetConfigServicesResponse, + GetNodeConfigServiceConfigsRequest, + GetNodeConfigServiceConfigsResponse, GetNodeConfigServiceRequest, GetNodeConfigServiceResponse, GetNodeConfigServicesRequest, @@ -45,7 +47,7 @@ from core.emulator.enumerations import EventTypes, LinkTypes, MessageFlags from core.emulator.session import Session from core.errors import CoreCommandError, CoreError from core.location.mobility import BasicRangeModel, Ns2ScriptedMobility -from core.nodes.base import NodeBase +from core.nodes.base import CoreNodeBase, NodeBase from core.nodes.docker import DockerNode from core.nodes.lxd import LxcNode from core.services.coreservices import ServiceManager @@ -192,7 +194,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): if config.config: service.set_config(config.config) for name, template in config.templates.items(): - service.custom_template(name, template) + service.set_template(name, template) # service file configs for config in request.service_file_configs: @@ -463,6 +465,8 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): if services is None: services = [] services = [x.name for x in services] + config_services = getattr(node, "config_services", {}) + config_services = [x for x in config_services] emane_model = None if isinstance(node, EmaneNet): emane_model = node.model.name @@ -478,6 +482,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): services=services, icon=node.icon, image=image, + config_services=config_services, ) if isinstance(node, (DockerNode, LxcNode)): node_proto.image = node.image @@ -1528,6 +1533,27 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): templates=templates, config=config, modes=modes ) + def GetNodeConfigServiceConfigs( + self, request: GetNodeConfigServiceConfigsRequest, context: ServicerContext + ) -> GetNodeConfigServiceConfigsResponse: + session = self.get_session(request.session_id, context) + configs = [] + for node in session.nodes.values(): + if not isinstance(node, CoreNodeBase): + continue + + for name, service in node.config_services.items(): + if not service.custom_templates and not service.custom_config: + continue + config_proto = configservices_pb2.ConfigServiceConfig( + node_id=node.id, + name=name, + templates=service.custom_templates, + config=service.custom_config, + ) + configs.append(config_proto) + return GetNodeConfigServiceConfigsResponse(configs=configs) + def GetNodeConfigServices( self, request: GetNodeConfigServicesRequest, context: ServicerContext ) -> GetNodeConfigServicesResponse: diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index c14aec7e..990452cd 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -38,6 +38,7 @@ class ConfigService(abc.ABC): self.templates = TemplateLookup(directories=templates_path) self.config = {} self.custom_templates = {} + self.custom_config = {} configs = self.default_configs[:] self._define_config(configs) @@ -134,7 +135,7 @@ class ConfigService(abc.ABC): def data(self) -> Dict[str, Any]: return {} - def custom_template(self, name: str, template: str) -> None: + def set_template(self, name: str, template: str) -> None: self.custom_templates[name] = template def get_text(self, name: str) -> str: @@ -248,11 +249,13 @@ class ConfigService(abc.ABC): self.config[config.id] = config def render_config(self) -> Dict[str, str]: - return {k: v.default for k, v in self.config.items()} + if self.custom_config: + return self.custom_config + else: + return {k: v.default for k, v in self.config.items()} def set_config(self, data: Dict[str, str]) -> None: for key, value in data.items(): - config = self.config.get(key) - if config is None: + if key not in self.config: raise CoreError(f"unknown config: {key}") - config.default = value + self.custom_config[key] = value diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index 2748cf85..8acf9475 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -297,6 +297,18 @@ class CoreClient: data = config.files[file_name] files[file_name] = data + # get config service configurations + response = self.client.get_node_config_service_configs(self.session_id) + for config in response.configs: + node_configs = self.config_service_configs.setdefault( + config.node_id, {} + ) + service_config = node_configs.setdefault(config.name, {}) + if config.templates: + service_config["templates"] = config.templates + if config.config: + service_config["config"] = config.config + # draw session self.app.canvas.reset_and_redraw(session) @@ -880,14 +892,11 @@ class CoreClient: for node_id, node_config in self.config_service_configs.items(): for name, service_config in node_config.items(): config = service_config.get("config", {}) - config_values = {} - for key, option in config.items(): - config_values[key] = option.value config_proto = configservices_pb2.ConfigServiceConfig( node_id=node_id, name=name, templates=service_config["templates"], - config=config_values, + config=config, ) config_service_protos.append(config_proto) return config_service_protos diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index 8a4a0a0c..f92d23bb 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -93,15 +93,17 @@ class ConfigServiceConfigDialog(Dialog): node_configs = self.service_configs.get(self.node_id, {}) service_config = node_configs.get(self.service_name, {}) - self.default_config = response.config + self.config = response.config + self.default_config = {x.name: x.value for x in self.config.values()} custom_config = service_config.get("config") if custom_config: - self.config = custom_config - else: - self.config = dict(self.default_config) + for key, value in custom_config.items(): + self.config[key].value = value + logging.info("default config: %s", self.default_config) custom_templates = service_config.get("templates", {}) for file, data in custom_templates.items(): + self.modified_files.add(file) self.temp_service_files[file] = data except grpc.RpcError as e: show_grpc_error(e) @@ -304,7 +306,7 @@ class ConfigServiceConfigDialog(Dialog): def click_apply(self): current_listbox = self.master.current.listbox - if not self.is_custom_service_config() and not self.is_custom_service_file(): + if not self.is_custom(): if self.node_id in self.service_configs: self.service_configs[self.node_id].pop(self.service_name, None) current_listbox.itemconfig(current_listbox.curselection()[0], bg="") @@ -316,7 +318,9 @@ class ConfigServiceConfigDialog(Dialog): service_config = node_config.setdefault(self.service_name, {}) if self.config_frame: self.config_frame.parse_config() - service_config["config"] = self.config + service_config["config"] = { + x.name: x.value for x in self.config.values() + } templates_config = service_config.setdefault("templates", {}) for file in self.modified_files: templates_config[file] = self.temp_service_files[file] @@ -346,18 +350,13 @@ class ConfigServiceConfigDialog(Dialog): else: self.modified_files.discard(template) - def is_custom_service_config(self): - startup_commands = self.startup_commands_listbox.get(0, "end") - shutdown_commands = self.shutdown_commands_listbox.get(0, "end") - validate_commands = self.validate_commands_listbox.get(0, "end") - return ( - set(self.default_startup) != set(startup_commands) - or set(self.default_validate) != set(validate_commands) - or set(self.default_shutdown) != set(shutdown_commands) - ) - - def is_custom_service_file(self): - return len(self.modified_files) > 0 + def is_custom(self): + has_custom_templates = len(self.modified_files) > 0 + has_custom_config = False + if self.config_frame: + current = self.config_frame.parse_config() + has_custom_config = self.default_config != current + return has_custom_templates or has_custom_config def click_defaults(self): if self.node_id in self.service_configs: @@ -368,8 +367,8 @@ class ConfigServiceConfigDialog(Dialog): self.template_text.text.delete(1.0, "end") self.template_text.text.insert("end", self.temp_service_files[filename]) if self.config_frame: - defaults = {x.id: x.value for x in self.default_config.values()} - self.config_frame.set_values(defaults) + logging.info("resetting defaults: %s", self.default_config) + self.config_frame.set_values(self.default_config) def click_copy(self): pass diff --git a/daemon/core/gui/dialogs/nodeconfigservice.py b/daemon/core/gui/dialogs/nodeconfigservice.py index bd9ec784..1230cede 100644 --- a/daemon/core/gui/dialogs/nodeconfigservice.py +++ b/daemon/core/gui/dialogs/nodeconfigservice.py @@ -153,14 +153,9 @@ class NodeConfigServiceDialog(Dialog): return def is_custom_service(self, service: str) -> bool: - service_configs = self.app.core.service_configs - file_configs = self.app.core.file_configs - if self.node_id in service_configs and service in service_configs[self.node_id]: + node_configs = self.app.core.config_service_configs.get(self.node_id, {}) + service_config = node_configs.get(service) + if node_configs and service_config: return True - if ( - self.node_id in file_configs - and service in file_configs[self.node_id] - and file_configs[self.node_id][service] - ): - return True - return False + else: + return False diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index df73901f..4f51f4ce 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -9,7 +9,7 @@ from core.emane.nodes import EmaneNet from core.emulator.data import LinkData from core.emulator.emudata import InterfaceData, LinkOptions, NodeOptions from core.emulator.enumerations import NodeTypes -from core.nodes.base import CoreNetworkBase, NodeBase +from core.nodes.base import CoreNetworkBase, CoreNodeBase, NodeBase from core.nodes.network import CtrlNet from core.services.coreservices import CoreService @@ -219,10 +219,15 @@ class DeviceElement(NodeElement): service_elements = etree.Element("services") for service in self.node.services: etree.SubElement(service_elements, "service", name=service.name) - if service_elements.getchildren(): self.element.append(service_elements) + config_service_elements = etree.Element("configservices") + for name, service in self.node.config_services.items(): + etree.SubElement(config_service_elements, "service", name=name) + if config_service_elements.getchildren(): + self.element.append(config_service_elements) + class NetworkElement(NodeElement): def __init__(self, session: "Session", node: NodeBase) -> None: @@ -261,6 +266,7 @@ class CoreXmlWriter: self.write_mobility_configs() self.write_emane_configs() self.write_service_configs() + self.write_configservice_configs() self.write_session_origin() self.write_session_hooks() self.write_session_options() @@ -399,6 +405,32 @@ class CoreXmlWriter: if service_configurations.getchildren(): self.scenario.append(service_configurations) + def write_configservice_configs(self) -> None: + service_configurations = etree.Element("configservice_configurations") + for node in self.session.nodes.values(): + if not isinstance(node, CoreNodeBase): + continue + for name, service in node.config_services.items(): + service_element = etree.SubElement( + service_configurations, "service", name=name + ) + add_attribute(service_element, "node", node.id) + if service.custom_config: + configs_element = etree.SubElement(service_element, "configs") + for key, value in service.custom_config.items(): + etree.SubElement( + configs_element, "config", key=key, value=value + ) + if service.custom_templates: + templates_element = etree.SubElement(service_element, "templates") + for template_name, template in service.custom_templates.items(): + template_element = etree.SubElement( + templates_element, "template", name=template_name + ) + template_element.text = etree.CDATA(template) + if service_configurations.getchildren(): + self.scenario.append(service_configurations) + def write_default_services(self) -> None: node_types = etree.Element("default_services") for node_type in self.session.services.default_services: @@ -568,6 +600,7 @@ class CoreXmlReader: self.read_mobility_configs() self.read_emane_configs() self.read_nodes() + self.read_configservice_configs() self.read_links() def read_default_services(self) -> None: @@ -768,6 +801,12 @@ class CoreXmlReader: if service_elements is not None: options.services = [x.get("name") for x in service_elements.iterchildren()] + config_service_elements = device_element.find("configservices") + if config_service_elements is not None: + options.config_services = [ + x.get("name") for x in config_service_elements.iterchildren() + ] + position_element = device_element.find("position") if position_element is not None: x = get_float(position_element, "x") @@ -808,6 +847,36 @@ class CoreXmlReader: ) self.session.add_node(_type=node_type, _id=node_id, options=options) + def read_configservice_configs(self) -> None: + configservice_configs = self.scenario.find("configservice_configurations") + if configservice_configs is None: + return + + for configservice_element in configservice_configs.iterchildren(): + name = configservice_element.get("name") + node_id = get_int(configservice_element, "node") + node = self.session.get_node(node_id) + service = node.config_services[name] + + configs_element = configservice_element.find("configs") + if configs_element is not None: + config = {} + for config_element in configs_element.iterchildren(): + key = config_element.get("key") + value = config_element.get("value") + config[key] = value + service.set_config(config) + + templates_element = configservice_element.find("templates") + if templates_element is not None: + for template_element in templates_element.iterchildren(): + name = template_element.get("name") + template = template_element.text + logging.info( + "loading xml template(%s): %s", type(template), template + ) + service.set_template(name, template) + def read_links(self) -> None: link_elements = self.scenario.find("links") if link_elements is None: diff --git a/daemon/proto/core/api/grpc/configservices.proto b/daemon/proto/core/api/grpc/configservices.proto index e8d93fb0..f1272df8 100644 --- a/daemon/proto/core/api/grpc/configservices.proto +++ b/daemon/proto/core/api/grpc/configservices.proto @@ -57,6 +57,14 @@ message GetConfigServiceDefaultsResponse { repeated ConfigMode modes = 3; } +message GetNodeConfigServiceConfigsRequest { + int32 session_id = 1; +} + +message GetNodeConfigServiceConfigsResponse { + repeated ConfigServiceConfig configs = 1; +} + message GetNodeConfigServiceRequest { int32 session_id = 1; int32 node_id = 2; diff --git a/daemon/proto/core/api/grpc/core.proto b/daemon/proto/core/api/grpc/core.proto index f7e9cf99..aa9bde24 100644 --- a/daemon/proto/core/api/grpc/core.proto +++ b/daemon/proto/core/api/grpc/core.proto @@ -110,6 +110,8 @@ service CoreApi { } rpc GetConfigServiceDefaults (configservices.GetConfigServiceDefaultsRequest) returns (configservices.GetConfigServiceDefaultsResponse) { } + rpc GetNodeConfigServiceConfigs (configservices.GetNodeConfigServiceConfigsRequest) returns (configservices.GetNodeConfigServiceConfigsResponse) { + } rpc GetNodeConfigService (configservices.GetNodeConfigServiceRequest) returns (configservices.GetNodeConfigServiceResponse) { } rpc GetNodeConfigServices (configservices.GetNodeConfigServicesRequest) returns (configservices.GetNodeConfigServicesResponse) { From 640b2c7d5b69294846f2ea7deda178c7e00b8633 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 22 Jan 2020 16:54:45 -0800 Subject: [PATCH 19/37] updated config services to account for files that have a path, basename will be used for template rendering, converted all current utility/security services to config service format --- daemon/core/configservice/base.py | 20 +- .../sercurityservices/services.py | 70 +++++ .../sercurityservices/templates/firewall.sh | 30 +++ .../sercurityservices/templates/ipsec.sh | 114 ++++++++ .../sercurityservices/templates/nat.sh | 14 + .../sercurityservices/templates/vpnclient.sh | 2 +- .../sercurityservices/templates/vpnserver.sh | 147 +++++++++++ .../configservices/serviceutils/services.py | 246 +++++++++++++++++- .../serviceutils/templates/apache2.conf | 102 ++++++++ .../serviceutils/templates/defaultmroute.sh | 4 + .../serviceutils/templates/dhcpd.conf | 22 ++ .../serviceutils/templates/envvars | 10 + .../serviceutils/templates/index.html | 13 + .../serviceutils/templates/pcap.sh | 11 + .../serviceutils/templates/radvd.conf | 19 ++ .../serviceutils/templates/sshd_config | 37 +++ .../serviceutils/templates/startatd.sh | 5 + .../serviceutils/templates/startdhcpclient.sh | 8 + .../serviceutils/templates/startsshd.sh | 6 + .../serviceutils/templates/staticroute.sh | 7 + .../serviceutils/templates/vsftpd.conf | 12 + 21 files changed, 888 insertions(+), 11 deletions(-) create mode 100644 daemon/core/configservices/sercurityservices/templates/firewall.sh create mode 100644 daemon/core/configservices/sercurityservices/templates/ipsec.sh create mode 100644 daemon/core/configservices/sercurityservices/templates/nat.sh create mode 100644 daemon/core/configservices/sercurityservices/templates/vpnserver.sh create mode 100644 daemon/core/configservices/serviceutils/templates/apache2.conf create mode 100644 daemon/core/configservices/serviceutils/templates/defaultmroute.sh create mode 100644 daemon/core/configservices/serviceutils/templates/dhcpd.conf create mode 100644 daemon/core/configservices/serviceutils/templates/envvars create mode 100644 daemon/core/configservices/serviceutils/templates/index.html create mode 100644 daemon/core/configservices/serviceutils/templates/pcap.sh create mode 100644 daemon/core/configservices/serviceutils/templates/radvd.conf create mode 100644 daemon/core/configservices/serviceutils/templates/sshd_config create mode 100644 daemon/core/configservices/serviceutils/templates/startatd.sh create mode 100644 daemon/core/configservices/serviceutils/templates/startdhcpclient.sh create mode 100644 daemon/core/configservices/serviceutils/templates/startsshd.sh create mode 100644 daemon/core/configservices/serviceutils/templates/staticroute.sh create mode 100644 daemon/core/configservices/serviceutils/templates/vsftpd.conf diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 990452cd..fa8f0ad3 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -139,18 +139,17 @@ class ConfigService(abc.ABC): self.custom_templates[name] = template def get_text(self, name: str) -> str: - raise CoreError( - f"node({self.node.name} service({self.name}) unknown template({name})" - ) + raise CoreError(f"service({self.name}) unknown template({name})") def get_templates(self) -> Dict[str, str]: templates = {} for name in self.files: + basename = pathlib.Path(name).name if name in self.custom_templates: template = self.custom_templates[name] template = inspect.cleandoc(template) - elif self.templates.has_template(name): - template = self.templates.get_template(name).source + elif self.templates.has_template(basename): + template = self.templates.get_template(basename).source else: template = self.get_text(name) template = inspect.cleandoc(template) @@ -160,12 +159,13 @@ class ConfigService(abc.ABC): def create_files(self) -> None: data = self.data() for name in self.files: + basename = pathlib.Path(name).name if name in self.custom_templates: text = self.custom_templates[name] text = inspect.cleandoc(text) self.render_text(name, text, data) - elif self.templates.has_template(name): - self.render_template(name, data) + elif self.templates.has_template(basename): + self.render_template(name, basename, data) else: text = self.get_text(name) text = inspect.cleandoc(text) @@ -233,9 +233,11 @@ class ConfigService(abc.ABC): f"{exceptions.text_error_template().render_unicode()}" ) - def render_template(self, name: str, data: Dict[str, Any] = None) -> None: + def render_template( + self, name: str, basename: str, data: Dict[str, Any] = None + ) -> None: try: - template = self.templates.get_template(name) + template = self.templates.get_template(basename) self._render(name, template, data) except Exception: raise CoreError( diff --git a/daemon/core/configservices/sercurityservices/services.py b/daemon/core/configservices/sercurityservices/services.py index 9c8d416a..cc95265d 100644 --- a/daemon/core/configservices/sercurityservices/services.py +++ b/daemon/core/configservices/sercurityservices/services.py @@ -1,3 +1,5 @@ +from typing import Any, Dict + from core.configservice.base import ConfigService, ConfigServiceMode GROUP_NAME = "Security" @@ -16,3 +18,71 @@ class VpnClient(ConfigService): validation_mode = ConfigServiceMode.BLOCKING default_configs = [] modes = {} + + +class VPNServer(ConfigService): + name = "VPNServer" + group = GROUP_NAME + directories = [] + files = ["vpnserver.sh"] + executables = ["openvpn", "ip", "killall"] + dependencies = [] + startup = ["sh vpnserver.sh"] + validate = ["pidof openvpn"] + shutdown = ["killall openvpn"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + +class IPsec(ConfigService): + name = "IPsec" + group = GROUP_NAME + directories = [] + files = ["ipsec.sh"] + executables = ["racoon", "ip", "setkey", "killall"] + dependencies = [] + startup = ["sh ipsec.sh"] + validate = ["pidof racoon"] + shutdown = ["killall racoon"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + +class Firewall(ConfigService): + name = "Firewall" + group = GROUP_NAME + directories = [] + files = ["firewall.sh"] + executables = ["iptables"] + dependencies = [] + startup = ["sh firewall.sh"] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + +class Nat(ConfigService): + name = "NAT" + group = GROUP_NAME + directories = [] + files = ["nat.sh"] + executables = ["iptables"] + dependencies = [] + startup = ["sh nat.sh"] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + ifnames = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifnames.append(ifc.name) + return dict(ifnames=ifnames) diff --git a/daemon/core/configservices/sercurityservices/templates/firewall.sh b/daemon/core/configservices/sercurityservices/templates/firewall.sh new file mode 100644 index 00000000..a445d133 --- /dev/null +++ b/daemon/core/configservices/sercurityservices/templates/firewall.sh @@ -0,0 +1,30 @@ +# -------- CUSTOMIZATION REQUIRED -------- +# +# Below are sample iptables firewall rules that you can uncomment and edit. +# You can also use ip6tables rules for IPv6. +# + +# start by flushing all firewall rules (so this script may be re-run) +#iptables -F + +# allow traffic related to established connections +#iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + +# allow TCP packets from any source destined for 192.168.1.1 +#iptables -A INPUT -s 0/0 -i eth0 -d 192.168.1.1 -p TCP -j ACCEPT + +# allow OpenVPN server traffic from eth0 +#iptables -A INPUT -p udp --dport 1194 -j ACCEPT +#iptables -A INPUT -i eth0 -j DROP +#iptables -A OUTPUT -p udp --sport 1194 -j ACCEPT +#iptables -A OUTPUT -o eth0 -j DROP + +# allow ICMP ping traffic +#iptables -A OUTPUT -p icmp --icmp-type echo-request -j ACCEPT +#iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT + +# allow SSH traffic +#iptables -A -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT + +# drop all other traffic coming in eth0 +#iptables -A INPUT -i eth0 -j DROP diff --git a/daemon/core/configservices/sercurityservices/templates/ipsec.sh b/daemon/core/configservices/sercurityservices/templates/ipsec.sh new file mode 100644 index 00000000..e8fde77e --- /dev/null +++ b/daemon/core/configservices/sercurityservices/templates/ipsec.sh @@ -0,0 +1,114 @@ +# -------- CUSTOMIZATION REQUIRED -------- +# +# The IPsec service builds ESP tunnels between the specified peers using the +# racoon IKEv2 keying daemon. You need to provide keys and the addresses of +# peers, along with subnets to tunnel. + +# directory containing the certificate and key described below +keydir=/etc/core/keys + +# the name used for the "$certname.pem" x509 certificate and +# "$certname.key" RSA private key, which can be generated using openssl +certname=ipsec1 + +# list the public-facing IP addresses, starting with the localhost and followed +# by each tunnel peer, separated with a single space +tunnelhosts="172.16.0.1AND172.16.0.2 172.16.0.1AND172.16.2.1" + +# Define T where i is the index for each tunnel peer host from +# the tunnel_hosts list above (0 is localhost). +# T is a list of IPsec tunnels with peer i, with a local subnet address +# followed by the remote subnet address: +# T="AND AND" +# For example, 172.16.0.0/24 is a local network (behind this node) to be +# tunneled and 172.16.2.0/24 is a remote network (behind peer 1) +T1="172.16.3.0/24AND172.16.5.0/24" +T2="172.16.4.0/24AND172.16.5.0/24 172.16.4.0/24AND172.16.6.0/24" + +# -------- END CUSTOMIZATION -------- + +echo "building config $PWD/ipsec.conf..." +echo "building config $PWD/ipsec.conf..." > $PWD/ipsec.log + +checkip=0 +if [ "$(dpkg -l | grep " sipcalc ")" = "" ]; then + echo "WARNING: ip validation disabled because package sipcalc not installed + " >> $PWD/ipsec.log + checkip=1 +fi + +echo "#!/usr/sbin/setkey -f + # Flush the SAD and SPD + flush; + spdflush; + + # Security policies " > $PWD/ipsec.conf +i=0 +for hostpair in $tunnelhosts; do + i=`expr $i + 1` + # parse tunnel host IP + thishost=$${}{hostpair%%AND*} + peerhost=$${}{hostpair##*AND} + if [ $checkip = "0" ] && + [ "$(sipcalc "$thishost" "$peerhost" | grep ERR)" != "" ]; then + echo "ERROR: invalid host address $thishost or $peerhost " >> $PWD/ipsec.log + fi + # parse each tunnel addresses + tunnel_list_var_name=T$i + eval tunnels="$"$tunnel_list_var_name"" + for ttunnel in $tunnels; do + lclnet=$${}{ttunnel%%AND*} + rmtnet=$${}{ttunnel##*AND} + if [ $checkip = "0" ] && + [ "$(sipcalc "$lclnet" "$rmtnet"| grep ERR)" != "" ]; then + echo "ERROR: invalid tunnel address $lclnet and $rmtnet " >> $PWD/ipsec.log + fi + # add tunnel policies + echo " + spdadd $lclnet $rmtnet any -P out ipsec + esp/tunnel/$thishost-$peerhost/require; + spdadd $rmtnet $lclnet any -P in ipsec + esp/tunnel/$peerhost-$thishost/require; " >> $PWD/ipsec.conf + done +done + +echo "building config $PWD/racoon.conf..." +if [ ! -e $keydir\/$certname.key ] || [ ! -e $keydir\/$certname.pem ]; then + echo "ERROR: missing certification files under $keydir $certname.key or $certname.pem " >> $PWD/ipsec.log +fi +echo " + path certificate \"$keydir\"; + listen { + adminsock disabled; + } + remote anonymous + { + exchange_mode main; + certificate_type x509 \"$certname.pem\" \"$certname.key\"; + ca_type x509 \"ca-cert.pem\"; + my_identifier asn1dn; + peers_identifier asn1dn; + + proposal { + encryption_algorithm 3des ; + hash_algorithm sha1; + authentication_method rsasig ; + dh_group modp768; + } + } + sainfo anonymous + { + pfs_group modp768; + lifetime time 1 hour ; + encryption_algorithm 3des, blowfish 448, rijndael ; + authentication_algorithm hmac_sha1, hmac_md5 ; + compression_algorithm deflate ; + } + " > $PWD/racoon.conf + +# the setkey program is required from the ipsec-tools package +echo "running setkey -f $PWD/ipsec.conf..." +setkey -f $PWD/ipsec.conf + +echo "running racoon -d -f $PWD/racoon.conf..." +racoon -d -f $PWD/racoon.conf -l racoon.log diff --git a/daemon/core/configservices/sercurityservices/templates/nat.sh b/daemon/core/configservices/sercurityservices/templates/nat.sh new file mode 100644 index 00000000..80b96a08 --- /dev/null +++ b/daemon/core/configservices/sercurityservices/templates/nat.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# generated by security.py +# NAT out the first interface by default +% for index, ifname in enumerate(ifnames): +% if index == 0: +iptables -t nat -A POSTROUTING -o ${ifname} -j MASQUERADE +iptables -A FORWARD -i ${ifname} -m state --state RELATED,ESTABLISHED -j ACCEPT +iptables -A FORWARD -i ${ifname} -j DROP +% else: +# iptables -t nat -A POSTROUTING -o ${ifname} -j MASQUERADE +# iptables -A FORWARD -i ${ifname} -m state --state RELATED,ESTABLISHED -j ACCEPT +# iptables -A FORWARD -i ${ifname} -j DROP +% endif +% endfor diff --git a/daemon/core/configservices/sercurityservices/templates/vpnclient.sh b/daemon/core/configservices/sercurityservices/templates/vpnclient.sh index 7d620b46..9e2a5d10 100644 --- a/daemon/core/configservices/sercurityservices/templates/vpnclient.sh +++ b/daemon/core/configservices/sercurityservices/templates/vpnclient.sh @@ -34,7 +34,7 @@ if [ ! -e $keydir\/$keyname.key ] || [ ! -e $keydir\/$keyname.crt ] \ fi # if necessary, add a static route for reaching the VPN server IP via the IF -vpnservernet=${vpnserver%.*}.0/24 +vpnservernet=$${}{vpnserver%.*}.0/24 if [ "$nexthop" != "" ]; then ip route add $vpnservernet via $nexthop fi diff --git a/daemon/core/configservices/sercurityservices/templates/vpnserver.sh b/daemon/core/configservices/sercurityservices/templates/vpnserver.sh new file mode 100644 index 00000000..c46812e2 --- /dev/null +++ b/daemon/core/configservices/sercurityservices/templates/vpnserver.sh @@ -0,0 +1,147 @@ +# -------- CUSTOMIZATION REQUIRED -------- +# +# The VPNServer service sets up the OpenVPN server for building VPN tunnels +# that allow access via TUN/TAP device to private networks. +# +# note that the IPForward and DefaultRoute services should be enabled + +# directory containing the certificate and key described below, in addition to +# a CA certificate and DH key +keydir=/etc/core/keys + +# the name used for a "$keyname.crt" certificate and "$keyname.key" private key. +keyname=server2 + +# the VPN subnet address from which the client VPN IP (for the TUN/TAP) +# will be allocated +vpnsubnet=10.0.200.0 + +# public IP address of this vpn server (same as VPNClient vpnserver= setting) +vpnserver=10.0.2.10 + +# optional list of private subnets reachable behind this VPN server +# each subnet and next hop is separated by a space +# ", , ..." +privatenets="10.0.11.0,10.0.10.1 10.0.12.0,10.0.10.1" + +# optional list of VPN clients, for statically assigning IP addresses to +# clients; also, an optional client subnet can be specified for adding static +# routes via the client +# Note: VPN addresses x.x.x.0-3 are reserved +# ",, ,, ..." +vpnclients="client1KeyFilename,10.0.200.5,10.0.0.0 client2KeyFilename,," + +# NOTE: you may need to enable the StaticRoutes service on nodes within the +# private subnet, in order to have routes back to the client. +# /sbin/ip ro add /24 via +# /sbin/ip ro add /24 via + +# -------- END CUSTOMIZATION -------- + +echo > $PWD/vpnserver.log +rm -f -r $PWD/ccd + +# validate key and certification files +if [ ! -e $keydir\/$keyname.key ] || [ ! -e $keydir\/$keyname.crt ] \ + || [ ! -e $keydir\/ca.crt ] || [ ! -e $keydir\/dh1024.pem ]; then + echo "ERROR: missing certification or key files under $keydir \ +$keyname.key or $keyname.crt or ca.crt or dh1024.pem" >> $PWD/vpnserver.log +fi + +# validate configuration IP addresses +checkip=0 +if [ "$(dpkg -l | grep " sipcalc ")" = "" ]; then + echo "WARNING: ip validation disabled because package sipcalc not installed\ + " >> $PWD/vpnserver.log + checkip=1 +else + if [ "$(sipcalc "$vpnsubnet" "$vpnserver" | grep ERR)" != "" ]; then + echo "ERROR: invalid vpn subnet or server address \ +$vpnsubnet or $vpnserver " >> $PWD/vpnserver.log + fi +fi + +# create client vpn ip pool file +( +cat << EOF +EOF +)> $PWD/ippool.txt + +# create server.conf file +( +cat << EOF +# openvpn server config +local $vpnserver +server $vpnsubnet 255.255.255.0 +push redirect-gateway def1 +EOF +)> $PWD/server.conf + +# add routes to VPN server private subnets, and push these routes to clients +for privatenet in $privatenets; do + if [ $privatenet != "" ]; then + net=$${}{privatenet%%,*} + nexthop=$${}{privatenet##*,} + if [ $checkip = "0" ] && + [ "$(sipcalc "$net" "$nexthop" | grep ERR)" != "" ]; then + echo "ERROR: invalid vpn server private net address \ +$net or $nexthop " >> $PWD/vpnserver.log + fi + echo push route $net 255.255.255.0 >> $PWD/server.conf + ip ro add $net/24 via $nexthop + ip ro add $vpnsubnet/24 via $nexthop + fi +done + +# allow subnet through this VPN, one route for each client subnet +for client in $vpnclients; do + if [ $client != "" ]; then + cSubnetIP=$${}{client##*,} + cVpnIP=$${}{client#*,} + cVpnIP=$${}{cVpnIP%%,*} + cKeyFilename=$${}{client%%,*} + if [ "$cSubnetIP" != "" ]; then + if [ $checkip = "0" ] && + [ "$(sipcalc "$cSubnetIP" "$cVpnIP" | grep ERR)" != "" ]; then + echo "ERROR: invalid vpn client and subnet address \ +$cSubnetIP or $cVpnIP " >> $PWD/vpnserver.log + fi + echo route $cSubnetIP 255.255.255.0 >> $PWD/server.conf + if ! test -d $PWD/ccd; then + mkdir -p $PWD/ccd + echo client-config-dir $PWD/ccd >> $PWD/server.conf + fi + if test -e $PWD/ccd/$cKeyFilename; then + echo iroute $cSubnetIP 255.255.255.0 >> $PWD/ccd/$cKeyFilename + else + echo iroute $cSubnetIP 255.255.255.0 > $PWD/ccd/$cKeyFilename + fi + fi + if [ "$cVpnIP" != "" ]; then + echo $cKeyFilename,$cVpnIP >> $PWD/ippool.txt + fi + fi +done + +( +cat << EOF +keepalive 10 120 +ca $keydir/ca.crt +cert $keydir/$keyname.crt +key $keydir/$keyname.key +dh $keydir/dh1024.pem +cipher AES-256-CBC +status /var/log/openvpn-status.log +log /var/log/openvpn-server.log +ifconfig-pool-linear +ifconfig-pool-persist $PWD/ippool.txt +port 1194 +proto udp +dev tun +verb 4 +daemon +EOF +)>> $PWD/server.conf + +# start vpn server +openvpn --config server.conf diff --git a/daemon/core/configservices/serviceutils/services.py b/daemon/core/configservices/serviceutils/services.py index 51fb5828..1165b149 100644 --- a/daemon/core/configservices/serviceutils/services.py +++ b/daemon/core/configservices/serviceutils/services.py @@ -8,7 +8,7 @@ from core.configservice.base import ConfigService, ConfigServiceMode GROUP_NAME = "Utility" -class DefaultRoute(ConfigService): +class DefaultRouteService(ConfigService): name = "DefaultRoute" group = GROUP_NAME directories = [] @@ -34,6 +34,61 @@ class DefaultRoute(ConfigService): return dict(addresses=addresses) +class DefaultMulticastRouteService(ConfigService): + name = "DefaultMulticastRoute" + group = GROUP_NAME + directories = [] + files = ["defaultmroute.sh"] + executables = [] + dependencies = [] + startup = ["sh defaultmroute.sh"] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + ifname = None + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifname = ifc.name + break + return dict(ifname=ifname) + + +class StaticRouteService(ConfigService): + name = "StaticRoute" + group = GROUP_NAME + directories = [] + files = ["staticroute.sh"] + executables = [] + dependencies = [] + startup = ["sh staticroute.sh"] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + routes = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + for x in ifc.addrlist: + addr = x.split("/")[0] + if netaddr.valid_ipv6(addr): + dst = "3ffe:4::/64" + else: + dst = "10.9.8.0/24" + net = netaddr.IPNetwork(x) + if net[-2] != net[1]: + routes.append((dst, net[1])) + return dict(routes=routes) + + class IpForwardService(ConfigService): name = "IPForward" group = GROUP_NAME @@ -54,3 +109,192 @@ class IpForwardService(ConfigService): devname = utils.sysctl_devname(ifc.name) devnames.append(devname) return dict(devnames=devnames) + + +class SshService(ConfigService): + name = "SSH" + group = GROUP_NAME + directories = ["/etc/ssh", "/var/run/sshd"] + files = ["startsshd.sh", "/etc/ssh/sshd_config"] + executables = ["sshd"] + dependencies = [] + startup = ["sh startsshd.sh"] + validate = [] + shutdown = ["killall sshd"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + return dict( + sshcfgdir=self.directories[0], + sshstatedir=self.directories[1], + sshlibdir="/usr/lib/openssh", + ) + + +class DhcpService(ConfigService): + name = "DHCP" + group = GROUP_NAME + directories = ["/etc/dhcp", "/var/lib/dhcp"] + files = ["/etc/dhcp/dhcpd.conf"] + executables = ["dhcpd"] + dependencies = [] + startup = ["touch /var/lib/dhcp/dhcpd.leases", "dhcpd"] + validate = ["pidof dhcpd"] + shutdown = ["killall dhcpd"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + subnets = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + for x in ifc.addrlist: + addr = x.split("/")[0] + if netaddr.valid_ipv4(addr): + net = netaddr.IPNetwork(x) + # divide the address space in half + index = (net.size - 2) / 2 + rangelow = net[index] + rangehigh = net[-2] + subnets.append((net.ip, net.netmask, rangelow, rangehigh, addr)) + return dict(subnets=subnets) + + +class DhcpClientService(ConfigService): + name = "DHCPClient" + group = GROUP_NAME + directories = [] + files = ["startdhcpclient.sh"] + executables = ["dhclient"] + dependencies = [] + startup = ["sh startdhcpclient.sh"] + validate = ["pidof dhclient"] + shutdown = ["killall dhclient"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + ifnames = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifnames.append(ifc.name) + return dict(ifnames=ifnames) + + +class FtpService(ConfigService): + name = "FTP" + group = GROUP_NAME + directories = ["/var/run/vsftpd/empty", "/var/ftp"] + files = ["vsftpd.conf"] + executables = ["vsftpd"] + dependencies = [] + startup = ["vsftpd ./vsftpd.conf"] + validate = ["pidof vsftpd"] + shutdown = ["killall vsftpd"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + +class PcapService(ConfigService): + name = "pcap" + group = GROUP_NAME + directories = [] + files = ["pcap.sh"] + executables = ["tcpdump"] + dependencies = [] + startup = ["sh pcap.sh start"] + validate = ["pidof tcpdump"] + shutdown = ["sh pcap.sh stop"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + ifnames = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifnames.append(ifc.name) + return dict() + + +class RadvdService(ConfigService): + name = "radvd" + group = GROUP_NAME + directories = ["/etc/radvd"] + files = ["/etc/radvd/radvd.conf"] + executables = ["radvd"] + dependencies = [] + startup = ["radvd -C /etc/radvd/radvd.conf -m logfile -l /var/log/radvd.log"] + validate = ["pidof radvd"] + shutdown = ["pkill radvd"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + interfaces = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + prefixes = [] + for x in ifc.addrlist: + addr = x.split("/")[0] + if netaddr.valid_ipv6(addr): + prefixes.append(x) + if not prefixes: + continue + interfaces.append((ifc.name, prefixes)) + return dict(interfaces=interfaces) + + +class AtdService(ConfigService): + name = "atd" + group = GROUP_NAME + directories = ["/var/spool/cron/atjobs", "/var/spool/cron/atspool"] + files = ["startatd.sh"] + executables = ["atd"] + dependencies = [] + startup = ["sh startatd.sh"] + validate = ["pidof atd"] + shutdown = ["pkill atd"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + +class HttpService(ConfigService): + name = "HTTP" + group = GROUP_NAME + directories = [ + "/etc/apache2", + "/var/run/apache2", + "/var/log/apache2", + "/run/lock", + "/var/lock/apache2", + "/var/www", + ] + files = ["/etc/apache2/apache2.conf", "/etc/apache2/envvars", "/var/www/index.html"] + executables = ["apache2ctl"] + dependencies = [] + startup = ["chown www-data /var/lock/apache2", "apache2ctl start"] + validate = ["pidof apache2"] + shutdown = ["apache2ctl stop"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + interfaces = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + interfaces.append(ifc) + return dict(interfaces=interfaces) diff --git a/daemon/core/configservices/serviceutils/templates/apache2.conf b/daemon/core/configservices/serviceutils/templates/apache2.conf new file mode 100644 index 00000000..c53e48af --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/apache2.conf @@ -0,0 +1,102 @@ +# apache2.conf generated by utility.py:HttpService +Mutex file:$APACHE_LOCK_DIR default + +PidFile $APACHE_PID_FILE +Timeout 300 +KeepAlive On +MaxKeepAliveRequests 100 +KeepAliveTimeout 5 + +LoadModule mpm_worker_module /usr/lib/apache2/modules/mod_mpm_worker.so + + + StartServers 5 + MinSpareServers 5 + MaxSpareServers 10 + MaxClients 150 + MaxRequestsPerChild 0 + + + + StartServers 2 + MinSpareThreads 25 + MaxSpareThreads 75 + ThreadLimit 64 + ThreadsPerChild 25 + MaxClients 150 + MaxRequestsPerChild 0 + + + + StartServers 2 + MinSpareThreads 25 + MaxSpareThreads 75 + ThreadLimit 64 + ThreadsPerChild 25 + MaxClients 150 + MaxRequestsPerChild 0 + + +User $APACHE_RUN_USER +Group $APACHE_RUN_GROUP + +AccessFileName .htaccess + + + Require all denied + + +DefaultType None + +HostnameLookups Off + +ErrorLog $APACHE_LOG_DIR/error.log +LogLevel warn + +#Include mods-enabled/*.load +#Include mods-enabled/*.conf +LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so +LoadModule auth_basic_module /usr/lib/apache2/modules/mod_auth_basic.so +LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so +LoadModule authz_host_module /usr/lib/apache2/modules/mod_authz_host.so +LoadModule authz_user_module /usr/lib/apache2/modules/mod_authz_user.so +LoadModule autoindex_module /usr/lib/apache2/modules/mod_autoindex.so +LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so +LoadModule env_module /usr/lib/apache2/modules/mod_env.so + +NameVirtualHost *:80 +Listen 80 + + + Listen 443 + + + Listen 443 + + +LogFormat "%v:%p %h %l %u %t \\"%r\\" %>s %O \\"%{Referer}i\\" \\"%{User-Agent}i\\"" vhost_combined +LogFormat "%h %l %u %t \\"%r\\" %>s %O \\"%{Referer}i\\" \\"%{User-Agent}i\\"" combined +LogFormat "%h %l %u %t \\"%r\\" %>s %O" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent + +ServerTokens OS +ServerSignature On +TraceEnable Off + + + ServerAdmin webmaster@localhost + DocumentRoot /var/www + + Options FollowSymLinks + AllowOverride None + + + Options Indexes FollowSymLinks MultiViews + AllowOverride None + Require all granted + + ErrorLog $APACHE_LOG_DIR/error.log + LogLevel warn + CustomLog $APACHE_LOG_DIR/access.log combined + diff --git a/daemon/core/configservices/serviceutils/templates/defaultmroute.sh b/daemon/core/configservices/serviceutils/templates/defaultmroute.sh new file mode 100644 index 00000000..4a8d9403 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/defaultmroute.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# auto-generated by DefaultMulticastRoute service (utility.py) +# the first interface is chosen below; please change it as needed +ip route add 224.0.0.0/4 dev ${ifname} diff --git a/daemon/core/configservices/serviceutils/templates/dhcpd.conf b/daemon/core/configservices/serviceutils/templates/dhcpd.conf new file mode 100644 index 00000000..7be7f4e8 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/dhcpd.conf @@ -0,0 +1,22 @@ +# auto-generated by DHCP service (utility.py) +# NOTE: move these option lines into the desired pool { } block(s) below +#option domain-name "test.com"; +#option domain-name-servers 10.0.0.1; +#option routers 10.0.0.1; + +log-facility local6; + +default-lease-time 600; +max-lease-time 7200; + +ddns-update-style none; + +% for subnet, netmask, rangelow, rangehigh, addr in subnets: +subnet ${subnet} netmask ${netmask} { + pool { + range ${rangelow} ${rangehigh}; + default-lease-time 600; + option routers ${addr}; + } +} +% endfor diff --git a/daemon/core/configservices/serviceutils/templates/envvars b/daemon/core/configservices/serviceutils/templates/envvars new file mode 100644 index 00000000..fcfc4d9e --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/envvars @@ -0,0 +1,10 @@ +# this file is used by apache2ctl - generated by utility.py:HttpService +# these settings come from a default Ubuntu apache2 installation +export APACHE_RUN_USER=www-data +export APACHE_RUN_GROUP=www-data +export APACHE_PID_FILE=/var/run/apache2.pid +export APACHE_RUN_DIR=/var/run/apache2 +export APACHE_LOCK_DIR=/var/lock/apache2 +export APACHE_LOG_DIR=/var/log/apache2 +export LANG=C +export LANG diff --git a/daemon/core/configservices/serviceutils/templates/index.html b/daemon/core/configservices/serviceutils/templates/index.html new file mode 100644 index 00000000..aaf9d9fa --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/index.html @@ -0,0 +1,13 @@ + + + +

${node.name} web server

+

This is the default web page for this server.

+

The web server software is running but no content has been added, yet.

+
    +% for ifc in interfaces: +
  • ${ifc.name} - ${ifc.addrlist}
    • +% endfor +
+ + diff --git a/daemon/core/configservices/serviceutils/templates/pcap.sh b/daemon/core/configservices/serviceutils/templates/pcap.sh new file mode 100644 index 00000000..6a099f8c --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/pcap.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# set tcpdump options here (see 'man tcpdump' for help) +# (-s snap length, -C limit pcap file length, -n disable name resolution) +if [ "x$1" = "xstart" ]; then +% for ifname in ifnames: + tcpdump -s 12288 -C 10 -n -w ${node.name}.${ifname}.pcap -i ${ifname} < /dev/null & +% endfor +elif [ "x$1" = "xstop" ]; then + mkdir -p $SESSION_DIR/pcap + mv *.pcap $SESSION_DIR/pcap +fi; diff --git a/daemon/core/configservices/serviceutils/templates/radvd.conf b/daemon/core/configservices/serviceutils/templates/radvd.conf new file mode 100644 index 00000000..1436f068 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/radvd.conf @@ -0,0 +1,19 @@ +# auto-generated by RADVD service (utility.py) +% for ifname, prefixes in values: +interface ${ifname} +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 10; + AdvDefaultPreference low; + AdvHomeAgentFlag off; +% for prefix in prefixes: + prefix ${prefix} + { + AdvOnLink on; + AdvAutonomous on; + AdvRouterAddr on; + }; +% endfor +}; +% endfor diff --git a/daemon/core/configservices/serviceutils/templates/sshd_config b/daemon/core/configservices/serviceutils/templates/sshd_config new file mode 100644 index 00000000..826dd098 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/sshd_config @@ -0,0 +1,37 @@ +# auto-generated by SSH service (utility.py) +Port 22 +Protocol 2 +HostKey ${sshcfgdir}/ssh_host_rsa_key +UsePrivilegeSeparation yes +PidFile ${sshstatedir}/sshd.pid + +KeyRegenerationInterval 3600 +ServerKeyBits 768 + +SyslogFacility AUTH +LogLevel INFO + +LoginGraceTime 120 +PermitRootLogin yes +StrictModes yes + +RSAAuthentication yes +PubkeyAuthentication yes + +IgnoreRhosts yes +RhostsRSAAuthentication no +HostbasedAuthentication no + +PermitEmptyPasswords no +ChallengeResponseAuthentication no + +X11Forwarding yes +X11DisplayOffset 10 +PrintMotd no +PrintLastLog yes +TCPKeepAlive yes + +AcceptEnv LANG LC_* +Subsystem sftp ${sshlibdir}/sftp-server +UsePAM yes +UseDNS no diff --git a/daemon/core/configservices/serviceutils/templates/startatd.sh b/daemon/core/configservices/serviceutils/templates/startatd.sh new file mode 100644 index 00000000..6d9d2949 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/startatd.sh @@ -0,0 +1,5 @@ +#!/bin/sh +echo 00001 > /var/spool/cron/atjobs/.SEQ +chown -R daemon /var/spool/cron/* +chmod -R 700 /var/spool/cron/* +atd diff --git a/daemon/core/configservices/serviceutils/templates/startdhcpclient.sh b/daemon/core/configservices/serviceutils/templates/startdhcpclient.sh new file mode 100644 index 00000000..061e66d7 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/startdhcpclient.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# auto-generated by DHCPClient service (utility.py) +# uncomment this mkdir line and symlink line to enable client-side DNS\n# resolution based on the DHCP server response. +#mkdir -p /var/run/resolvconf/interface +% for ifname in ifnames: +#ln -s /var/run/resolvconf/interface/${ifname}.dhclient /var/run/resolvconf/resolv.conf +dhclient -nw -pf /var/run/dhclient-${ifname}.pid -lf /var/run/dhclient-${ifname}.lease ${ifname} +% endfor diff --git a/daemon/core/configservices/serviceutils/templates/startsshd.sh b/daemon/core/configservices/serviceutils/templates/startsshd.sh new file mode 100644 index 00000000..b35fdb07 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/startsshd.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# auto-generated by SSH service (utility.py) +ssh-keygen -q -t rsa -N "" -f ${sshcfgdir}/ssh_host_rsa_key +chmod 655 ${sshstatedir} +# wait until RSA host key has been generated to launch sshd +$(which sshd) -f ${sshcfgdir}/sshd_config diff --git a/daemon/core/configservices/serviceutils/templates/staticroute.sh b/daemon/core/configservices/serviceutils/templates/staticroute.sh new file mode 100644 index 00000000..c47c09fd --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/staticroute.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# auto-generated by StaticRoute service (utility.py) +# NOTE: this service must be customized to be of any use +# Below are samples that you can uncomment and edit. +% for dest, addr in routes: +#ip route add ${dest} via ${addr} +% endfor diff --git a/daemon/core/configservices/serviceutils/templates/vsftpd.conf b/daemon/core/configservices/serviceutils/templates/vsftpd.conf new file mode 100644 index 00000000..988b8727 --- /dev/null +++ b/daemon/core/configservices/serviceutils/templates/vsftpd.conf @@ -0,0 +1,12 @@ +# vsftpd.conf auto-generated by FTP service (utility.py) +listen=YES +anonymous_enable=YES +local_enable=YES +dirmessage_enable=YES +use_localtime=YES +xferlog_enable=YES +connect_from_port_20=YES +xferlog_file=/var/log/vsftpd.log +ftpd_banner=Welcome to the CORE FTP service +secure_chroot_dir=/var/run/vsftpd/empty +anon_root=/var/ftp From 062273a99b31bbf497b9e2b77c0dfd25b6e10d85 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 22 Jan 2020 22:10:30 -0800 Subject: [PATCH 20/37] added several nrl services updated to config service format --- .../{serviceutils => nrlservices}/__init__.py | 0 .../configservices/nrlservices/services.py | 156 ++++++++++++++++++ .../nrlservices/templates/mgensink.sh | 1 + .../nrlservices/templates/nrlnhdp.sh | 7 + .../nrlservices/templates/nrlolsrd.sh | 9 + .../nrlservices/templates/sink.mgen | 4 + .../nrlservices/templates/startsmf.sh | 19 +++ .../configservices/utilservices/__init__.py | 0 .../services.py | 0 .../templates/apache2.conf | 0 .../templates/defaultmroute.sh | 0 .../templates/defaultroute.sh | 0 .../templates/dhcpd.conf | 0 .../templates/envvars | 0 .../templates/index.html | 0 .../templates/ipforward.sh | 0 .../templates/pcap.sh | 0 .../templates/radvd.conf | 0 .../templates/sshd_config | 0 .../templates/startatd.sh | 0 .../templates/startdhcpclient.sh | 0 .../templates/startsshd.sh | 0 .../templates/staticroute.sh | 0 .../templates/vsftpd.conf | 0 24 files changed, 196 insertions(+) rename daemon/core/configservices/{serviceutils => nrlservices}/__init__.py (100%) create mode 100644 daemon/core/configservices/nrlservices/services.py create mode 100644 daemon/core/configservices/nrlservices/templates/mgensink.sh create mode 100644 daemon/core/configservices/nrlservices/templates/nrlnhdp.sh create mode 100644 daemon/core/configservices/nrlservices/templates/nrlolsrd.sh create mode 100644 daemon/core/configservices/nrlservices/templates/sink.mgen create mode 100644 daemon/core/configservices/nrlservices/templates/startsmf.sh create mode 100644 daemon/core/configservices/utilservices/__init__.py rename daemon/core/configservices/{serviceutils => utilservices}/services.py (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/apache2.conf (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/defaultmroute.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/defaultroute.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/dhcpd.conf (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/envvars (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/index.html (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/ipforward.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/pcap.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/radvd.conf (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/sshd_config (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/startatd.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/startdhcpclient.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/startsshd.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/staticroute.sh (100%) rename daemon/core/configservices/{serviceutils => utilservices}/templates/vsftpd.conf (100%) diff --git a/daemon/core/configservices/serviceutils/__init__.py b/daemon/core/configservices/nrlservices/__init__.py similarity index 100% rename from daemon/core/configservices/serviceutils/__init__.py rename to daemon/core/configservices/nrlservices/__init__.py diff --git a/daemon/core/configservices/nrlservices/services.py b/daemon/core/configservices/nrlservices/services.py new file mode 100644 index 00000000..2845dc8c --- /dev/null +++ b/daemon/core/configservices/nrlservices/services.py @@ -0,0 +1,156 @@ +from typing import Any, Dict + +import netaddr + +from core import utils +from core.configservice.base import ConfigService, ConfigServiceMode + +GROUP = "ProtoSvc" + + +class NrlService(ConfigService): + name = "NrlBase" + group = GROUP + directories = [] + files = [] + executables = [] + dependencies = [] + startup = [] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + @classmethod + def generate_config(cls, node, filename): + return "" + + @staticmethod + def firstipv4prefix(node, prefixlen=24): + """ + Similar to QuaggaService.routerid(). Helper to return the first IPv4 + prefix of a node, using the supplied prefix length. This ignores the + interface's prefix length, so e.g. '/32' can turn into '/24'. + """ + for ifc in node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + for a in ifc.addrlist: + a = a.split("/")[0] + if netaddr.valid_ipv4(a): + return f"{a}/{prefixlen}" + # raise ValueError, "no IPv4 address found" + return "0.0.0.0/%s" % prefixlen + + +class MgenSinkService(ConfigService): + name = "MGEN_Sink" + group = GROUP + directories = [] + files = ["mgensink.sh", "sink.mgen"] + executables = ["mgen"] + dependencies = [] + startup = ["sh mgensink.sh"] + validate = ["pidof mgen"] + shutdown = ["killall mgen"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + ifnames = [] + for ifc in self.node.netifs(): + name = utils.sysctl_devname(ifc.name) + ifnames.append(name) + return dict(ifnames=ifnames) + + +class NrlNhdp(NrlService): + name = "NHDP" + group = GROUP + directories = [] + files = ["nrlnhdp.sh"] + executables = ["nrlnhdp"] + dependencies = [] + startup = ["sh nrlnhdp.sh"] + validate = ["pidof nrlnhdp"] + shutdown = ["killall nrlnhdp"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + has_smf = "SMF" in self.node.config_services + ifnames = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifnames.append(ifc.name) + return dict(has_smf=has_smf, ifnames=ifnames) + + +class NrlSmf(ConfigService): + name = "SMF" + group = GROUP + directories = [] + files = ["startsmf.sh"] + executables = ["nrlsmf", "killall"] + dependencies = [] + startup = ["sh startsmf.sh"] + validate = ["pidof nrlsmf"] + shutdown = ["killall nrlsmf"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + has_arouted = "arouted" in self.node.config_services + has_nhdp = "NHDP" in self.node.config_services + has_olsr = "OLSR" in self.node.config_services + ifnames = [] + ip4_prefix = None + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifnames.append(ifc.name) + if ip4_prefix: + continue + for a in ifc.addrlist: + a = a.split("/")[0] + if netaddr.valid_ipv4(a): + ip4_prefix = f"{a}/{24}" + break + return dict( + has_arouted=has_arouted, + has_nhdp=has_nhdp, + has_olsr=has_olsr, + ifnames=ifnames, + ip4_prefix=ip4_prefix, + ) + + +class NrlOlsr(ConfigService): + name = "OLSR" + group = GROUP + directories = [] + files = ["nrlolsrd.sh"] + executables = ["nrlolsrd"] + dependencies = [] + startup = ["sh nrlolsrd.sh"] + validate = ["pidof nrlolsrd"] + shutdown = ["killall nrlolsrd"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + has_smf = "SMF" in self.node.config_services + has_zebra = "zebra" in self.node.config_services + ifname = None + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifname = ifc.name + break + return dict(has_smf=has_smf, has_zebra=has_zebra, ifname=ifname) diff --git a/daemon/core/configservices/nrlservices/templates/mgensink.sh b/daemon/core/configservices/nrlservices/templates/mgensink.sh new file mode 100644 index 00000000..bdbd0a8d --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/mgensink.sh @@ -0,0 +1 @@ +mgen input sink.mgen output mgen_${node.name}.log diff --git a/daemon/core/configservices/nrlservices/templates/nrlnhdp.sh b/daemon/core/configservices/nrlservices/templates/nrlnhdp.sh new file mode 100644 index 00000000..00b7e11d --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/nrlnhdp.sh @@ -0,0 +1,7 @@ +<% + interfaces = "-i " + " -i ".join(ifnames) + smf = "" + if has_smf: + smf = "-flooding ecds -smfClient %s_smf" % node.name +%> +nrlnhdp -l /var/log/nrlnhdp.log -rpipe ${node.name}_nhdp ${smf} ${interfaces} diff --git a/daemon/core/configservices/nrlservices/templates/nrlolsrd.sh b/daemon/core/configservices/nrlservices/templates/nrlolsrd.sh new file mode 100644 index 00000000..4072d189 --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/nrlolsrd.sh @@ -0,0 +1,9 @@ +<% + smf = "" + if has_smf: + smf = "-flooding s-mpr -smfClient %s_smf" % node.name + zebra = "" + if has_zebra: + zebra = "-z" +%> +nrlolsrd -i ${ifname} -l /var/log/nrlolsrd.log -rpipe ${node.name}_olsr ${smf} ${zebra} diff --git a/daemon/core/configservices/nrlservices/templates/sink.mgen b/daemon/core/configservices/nrlservices/templates/sink.mgen new file mode 100644 index 00000000..21d4fde6 --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/sink.mgen @@ -0,0 +1,4 @@ +0.0 LISTEN UDP 5000 +% for ifname in ifnames: +0.0 Join 224.225.1.2 INTERFACE ${ifname} +% endfor diff --git a/daemon/core/configservices/nrlservices/templates/startsmf.sh b/daemon/core/configservices/nrlservices/templates/startsmf.sh new file mode 100644 index 00000000..2ba20f27 --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/startsmf.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# auto-generated by nrl.py:NrlSmf.generateconfig() +% if ifnames: +nrlsmf instance ${node.name}_smf \\ +% if has_arouted: + tap ${node.name}_tap unicast %s push lo,${ifnames[0]} resequence on \\ +% endif +% if has_nhdp: + ecds \\ +% elif has_olsr: + smpr \\ +% else: + cf \\ +% endif +<% + interfaces = ",".join(ifnames) +%> + ${interfaces} hash MD5 log /var/log/nrlsmf.log < /dev/null > /dev/null 2>&1 & +% endif diff --git a/daemon/core/configservices/utilservices/__init__.py b/daemon/core/configservices/utilservices/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/core/configservices/serviceutils/services.py b/daemon/core/configservices/utilservices/services.py similarity index 100% rename from daemon/core/configservices/serviceutils/services.py rename to daemon/core/configservices/utilservices/services.py diff --git a/daemon/core/configservices/serviceutils/templates/apache2.conf b/daemon/core/configservices/utilservices/templates/apache2.conf similarity index 100% rename from daemon/core/configservices/serviceutils/templates/apache2.conf rename to daemon/core/configservices/utilservices/templates/apache2.conf diff --git a/daemon/core/configservices/serviceutils/templates/defaultmroute.sh b/daemon/core/configservices/utilservices/templates/defaultmroute.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/defaultmroute.sh rename to daemon/core/configservices/utilservices/templates/defaultmroute.sh diff --git a/daemon/core/configservices/serviceutils/templates/defaultroute.sh b/daemon/core/configservices/utilservices/templates/defaultroute.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/defaultroute.sh rename to daemon/core/configservices/utilservices/templates/defaultroute.sh diff --git a/daemon/core/configservices/serviceutils/templates/dhcpd.conf b/daemon/core/configservices/utilservices/templates/dhcpd.conf similarity index 100% rename from daemon/core/configservices/serviceutils/templates/dhcpd.conf rename to daemon/core/configservices/utilservices/templates/dhcpd.conf diff --git a/daemon/core/configservices/serviceutils/templates/envvars b/daemon/core/configservices/utilservices/templates/envvars similarity index 100% rename from daemon/core/configservices/serviceutils/templates/envvars rename to daemon/core/configservices/utilservices/templates/envvars diff --git a/daemon/core/configservices/serviceutils/templates/index.html b/daemon/core/configservices/utilservices/templates/index.html similarity index 100% rename from daemon/core/configservices/serviceutils/templates/index.html rename to daemon/core/configservices/utilservices/templates/index.html diff --git a/daemon/core/configservices/serviceutils/templates/ipforward.sh b/daemon/core/configservices/utilservices/templates/ipforward.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/ipforward.sh rename to daemon/core/configservices/utilservices/templates/ipforward.sh diff --git a/daemon/core/configservices/serviceutils/templates/pcap.sh b/daemon/core/configservices/utilservices/templates/pcap.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/pcap.sh rename to daemon/core/configservices/utilservices/templates/pcap.sh diff --git a/daemon/core/configservices/serviceutils/templates/radvd.conf b/daemon/core/configservices/utilservices/templates/radvd.conf similarity index 100% rename from daemon/core/configservices/serviceutils/templates/radvd.conf rename to daemon/core/configservices/utilservices/templates/radvd.conf diff --git a/daemon/core/configservices/serviceutils/templates/sshd_config b/daemon/core/configservices/utilservices/templates/sshd_config similarity index 100% rename from daemon/core/configservices/serviceutils/templates/sshd_config rename to daemon/core/configservices/utilservices/templates/sshd_config diff --git a/daemon/core/configservices/serviceutils/templates/startatd.sh b/daemon/core/configservices/utilservices/templates/startatd.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/startatd.sh rename to daemon/core/configservices/utilservices/templates/startatd.sh diff --git a/daemon/core/configservices/serviceutils/templates/startdhcpclient.sh b/daemon/core/configservices/utilservices/templates/startdhcpclient.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/startdhcpclient.sh rename to daemon/core/configservices/utilservices/templates/startdhcpclient.sh diff --git a/daemon/core/configservices/serviceutils/templates/startsshd.sh b/daemon/core/configservices/utilservices/templates/startsshd.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/startsshd.sh rename to daemon/core/configservices/utilservices/templates/startsshd.sh diff --git a/daemon/core/configservices/serviceutils/templates/staticroute.sh b/daemon/core/configservices/utilservices/templates/staticroute.sh similarity index 100% rename from daemon/core/configservices/serviceutils/templates/staticroute.sh rename to daemon/core/configservices/utilservices/templates/staticroute.sh diff --git a/daemon/core/configservices/serviceutils/templates/vsftpd.conf b/daemon/core/configservices/utilservices/templates/vsftpd.conf similarity index 100% rename from daemon/core/configservices/serviceutils/templates/vsftpd.conf rename to daemon/core/configservices/utilservices/templates/vsftpd.conf From 8f03c9c975e4b404b1a87bf1163e75de0b3451e6 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 23 Jan 2020 08:53:20 -0800 Subject: [PATCH 21/37] finished converting nrl services to config service format --- .../configservices/nrlservices/services.py | 92 ++++++ .../nrlservices/templates/nrlolsrv2.sh | 7 + .../nrlservices/templates/olsrd.conf | 312 ++++++++++++++++++ .../nrlservices/templates/olsrd.sh | 4 + .../nrlservices/templates/start_mgen_actor.sh | 3 + .../nrlservices/templates/startarouted.sh | 15 + .../nrlservices/templates/startsmf.sh | 28 +- 7 files changed, 445 insertions(+), 16 deletions(-) create mode 100644 daemon/core/configservices/nrlservices/templates/nrlolsrv2.sh create mode 100644 daemon/core/configservices/nrlservices/templates/olsrd.conf create mode 100644 daemon/core/configservices/nrlservices/templates/olsrd.sh create mode 100644 daemon/core/configservices/nrlservices/templates/start_mgen_actor.sh create mode 100644 daemon/core/configservices/nrlservices/templates/startarouted.sh diff --git a/daemon/core/configservices/nrlservices/services.py b/daemon/core/configservices/nrlservices/services.py index 2845dc8c..98df2ceb 100644 --- a/daemon/core/configservices/nrlservices/services.py +++ b/daemon/core/configservices/nrlservices/services.py @@ -154,3 +154,95 @@ class NrlOlsr(ConfigService): ifname = ifc.name break return dict(has_smf=has_smf, has_zebra=has_zebra, ifname=ifname) + + +class NrlOlsrv2(ConfigService): + name = "OLSRv2" + group = GROUP + directories = [] + files = ["nrlolsrv2.sh"] + executables = ["nrlolsrv2"] + dependencies = [] + startup = ["sh nrlolsrv2.sh"] + validate = ["pidof nrlolsrv2"] + shutdown = ["killall nrlolsrv2"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + has_smf = "SMF" in self.node.config_services + ifnames = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifnames.append(ifc.name) + return dict(has_smf=has_smf, ifnames=ifnames) + + +class OlsrOrg(ConfigService): + name = "OLSRORG" + group = GROUP + directories = ["/etc/olsrd"] + files = ["olsrd.sh", "/etc/olsrd/olsrd.conf"] + executables = ["olsrd"] + dependencies = [] + startup = ["sh olsrd.sh"] + validate = ["pidof olsrd"] + shutdown = ["killall olsrd"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + has_smf = "SMF" in self.node.config_services + ifnames = [] + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + ifnames.append(ifc.name) + return dict(has_smf=has_smf, ifnames=ifnames) + + +class MgenActor(NrlService): + name = "MgenActor" + group = GROUP + directories = [] + files = ["start_mgen_actor.sh"] + executables = ["mgen"] + dependencies = [] + startup = ["sh start_mgen_actor.sh"] + validate = ["pidof mgen"] + shutdown = ["killall mgen"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + +class Arouted(ConfigService): + name = "arouted" + group = GROUP + directories = [] + files = ["startarouted.sh"] + executables = ["arouted"] + dependencies = [] + startup = ["sh startarouted.sh"] + validate = ["pidof arouted"] + shutdown = ["pkill arouted"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + ip4_prefix = None + for ifc in self.node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + if ip4_prefix: + continue + for a in ifc.addrlist: + a = a.split("/")[0] + if netaddr.valid_ipv4(a): + ip4_prefix = f"{a}/{24}" + break + return dict(ip4_prefix=ip4_prefix) diff --git a/daemon/core/configservices/nrlservices/templates/nrlolsrv2.sh b/daemon/core/configservices/nrlservices/templates/nrlolsrv2.sh new file mode 100644 index 00000000..d7a8d3b6 --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/nrlolsrv2.sh @@ -0,0 +1,7 @@ +<% + interfaces = "-i " + " -i ".join(ifnames) + smf = "" + if has_smf: + smf = "-flooding ecds -smfClient %s_smf" % node.name +%> +nrlolsrv2 -l /var/log/nrlolsrv2.log -rpipe ${node.name}_olsrv2 -p olsr ${smf} ${interfaces} diff --git a/daemon/core/configservices/nrlservices/templates/olsrd.conf b/daemon/core/configservices/nrlservices/templates/olsrd.conf new file mode 100644 index 00000000..a5716a98 --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/olsrd.conf @@ -0,0 +1,312 @@ +# +# OLSR.org routing daemon config file +# This file contains the usual options for an ETX based +# stationary network without fisheye +# (for other options see olsrd.conf.default.full) +# +# Lines starting with a # are discarded +# + +#### ATTENTION for IPv6 users #### +# Because of limitations in the parser IPv6 addresses must NOT +# begin with a ":", so please add a "0" as a prefix. + +########################### +### Basic configuration ### +########################### +# keep this settings at the beginning of your first configuration file + +# Debug level (0-9) +# If set to 0 the daemon runs in the background, unless "NoFork" is set to true +# (Default is 1) + +# DebugLevel 1 + +# IP version to use (4 or 6) +# (Default is 4) + +# IpVersion 4 + +################################# +### OLSRd agent configuration ### +################################# +# this parameters control the settings of the routing agent which are not +# related to the OLSR protocol and it's extensions + +# FIBMetric controls the metric value of the host-routes OLSRd sets. +# - "flat" means that the metric value is always 2. This is the preferred value +# because it helps the linux kernel routing to clean up older routes +# - "correct" use the hopcount as the metric value. +# - "approx" use the hopcount as the metric value too, but does only update the +# hopcount if the nexthop changes too +# (Default is "flat") + +# FIBMetric "flat" + +####################################### +### Linux specific OLSRd extensions ### +####################################### +# these parameters are only working on linux at the moment + +# SrcIpRoutes tells OLSRd to set the Src flag of host routes to the originator-ip +# of the node. In addition to this an additional localhost device is created +# to make sure the returning traffic can be received. +# (Default is "no") + +# SrcIpRoutes no + +# Specify the proto tag to be used for routes olsr inserts into kernel +# currently only implemented for linux +# valid values under linux are 1 .. 254 +# 1 gets remapped by olsrd to 0 UNSPECIFIED (1 is reserved for ICMP redirects) +# 2 KERNEL routes (not very wise to use) +# 3 BOOT (should in fact not be used by routing daemons) +# 4 STATIC +# 8 .. 15 various routing daemons (gated, zebra, bird, & co) +# (defaults to 0 which gets replaced by an OS-specific default value +# under linux 3 (BOOT) (for backward compatibility) + +# RtProto 0 + +# Activates (in IPv6 mode) the automatic use of NIIT +# (see README-Olsr-Extensions) +# (default is "yes") + +# UseNiit yes + +# Activates the smartgateway ipip tunnel feature. +# See README-Olsr-Extensions for a description of smartgateways. +# (default is "no") + +# SmartGateway no + +# Signals that the server tunnel must always be removed on shutdown, +# irrespective of the interface up/down state during startup. +# (default is "no") + +# SmartGatewayAlwaysRemoveServerTunnel no + +# Determines the maximum number of gateways that can be in use at any given +# time. This setting is used to mitigate the effects of breaking connections +# (due to the selection of a new gateway) on a dynamic network. +# (default is 1) + +# SmartGatewayUseCount 1 + +# Determines the take-down percentage for a non-current smart gateway tunnel. +# If the cost of the current smart gateway tunnel is less than this percentage +# of the cost of the non-current smart gateway tunnel, then the non-current smart +# gateway tunnel is taken down because it is then presumed to be 'too expensive'. +# This setting is only relevant when SmartGatewayUseCount is larger than 1; +# a value of 0 will result in the tunnels not being taken down proactively. +# (default is 0) + +# SmartGatewayTakeDownPercentage 0 + +# Determines the policy routing script that is executed during startup and +# shutdown of olsrd. The script is only executed when SmartGatewayUseCount +# is set to a value larger than 1. The script must setup policy routing +# rules such that multi-gateway mode works. A sample script is included. +# (default is not set) + +# SmartGatewayPolicyRoutingScript "" + +# Determines the egress interfaces that are part of the multi-gateway setup and +# therefore only relevant when SmartGatewayUseCount is larger than 1 (in which +# case it must be explicitly set). +# (default is not set) + +# SmartGatewayEgressInterfaces "" + +# Determines the routing tables offset for multi-gateway policy routing tables +# See the policy routing script for an explanation. +# (default is 90) + +# SmartGatewayTablesOffset 90 + +# Determines the policy routing rules offset for multi-gateway policy routing +# rules. See the policy routing script for an explanation. +# (default is 0, which indicates that the rules and tables should be aligned and +# puts this value at SmartGatewayTablesOffset - # egress interfaces - +# # olsr interfaces) + +# SmartGatewayRulesOffset 87 + +# Allows the selection of a smartgateway with NAT (only for IPv4) +# (default is "yes") + +# SmartGatewayAllowNAT yes + +# Determines the period (in milliseconds) on which a new smart gateway +# selection is performed. +# (default is 10000 milliseconds) + +# SmartGatewayPeriod 10000 + +# Determines the number of times the link state database must be stable +# before a new smart gateway is selected. +# (default is 6) + +# SmartGatewayStableCount 6 + +# When another gateway than the current one has a cost of less than the cost +# of the current gateway multiplied by SmartGatewayThreshold then the smart +# gateway is switched to the other gateway. The unit is percentage. +# (defaults to 0) + +# SmartGatewayThreshold 0 + +# The weighing factor for the gateway uplink bandwidth (exit link, uplink). +# See README-Olsr-Extensions for a description of smart gateways. +# (default is 1) + +# SmartGatewayWeightExitLinkUp 1 + +# The weighing factor for the gateway downlink bandwidth (exit link, downlink). +# See README-Olsr-Extensions for a description of smart gateways. +# (default is 1) + +# SmartGatewayWeightExitLinkDown 1 + +# The weighing factor for the ETX costs. +# See README-Olsr-Extensions for a description of smart gateways. +# (default is 1) + +# SmartGatewayWeightEtx 1 + +# The divider for the ETX costs. +# See README-Olsr-Extensions for a description of smart gateways. +# (default is 0) + +# SmartGatewayDividerEtx 0 + +# Defines what kind of Uplink this node will publish as a +# smartgateway. The existence of the uplink is detected by +# a route to 0.0.0.0/0, ::ffff:0:0/96 and/or 2000::/3. +# possible values are "none", "ipv4", "ipv6", "both" +# (default is "both") + +# SmartGatewayUplink "both" + +# Specifies if the local ipv4 uplink use NAT +# (default is "yes") + +# SmartGatewayUplinkNAT yes + +# Specifies the speed of the uplink in kilobit/s. +# First parameter is upstream, second parameter is downstream +# (default is 128/1024) + +# SmartGatewaySpeed 128 1024 + +# Specifies the EXTERNAL ipv6 prefix of the uplink. A prefix +# length of more than 64 is not allowed. +# (default is 0::/0 + +# SmartGatewayPrefix 0::/0 + +############################## +### OLSR protocol settings ### +############################## + +# HNA (Host network association) allows the OLSR to announce +# additional IPs or IP subnets to the net that are reachable +# through this node. +# Syntax for HNA4 is "network-address network-mask" +# Syntax for HNA6 is "network-address prefix-length" +# (default is no HNA) +Hna4 +{ +# Internet gateway +# 0.0.0.0 0.0.0.0 +# specific small networks reachable through this node +# 15.15.0.0 255.255.255.0 +} +Hna6 +{ +# Internet gateway +# 0:: 0 +# specific small networks reachable through this node +# fec0:2200:106:0:0:0:0:0 48 +} + +################################ +### OLSR protocol extensions ### +################################ + +# Link quality algorithm (only for lq level 2) +# (see README-Olsr-Extensions) +# - "etx_float", a floating point ETX with exponential aging +# - "etx_fpm", same as ext_float, but with integer arithmetic +# - "etx_ff" (ETX freifunk), an etx variant which use all OLSR +# traffic (instead of only hellos) for ETX calculation +# - "etx_ffeth", an incompatible variant of etx_ff that allows +# ethernet links with ETX 0.1. +# (defaults to "etx_ff") + +# LinkQualityAlgorithm "etx_ff" + +# Fisheye mechanism for TCs (0 meansoff, 1 means on) +# (default is 1) + +LinkQualityFishEye 0 + +##################################### +### Example plugin configurations ### +##################################### +# Olsrd plugins to load +# This must be the absolute path to the file +# or the loader will use the following scheme: +# - Try the paths in the LD_LIBRARY_PATH +# environment variable. +# - The list of libraries cached in /etc/ld.so.cache +# - /lib, followed by /usr/lib +# +# the examples in this list are for linux, so check if the plugin is +# available if you use windows. +# each plugin should have a README file in it's lib subfolder + +# LoadPlugin "olsrd_txtinfo.dll" +#LoadPlugin "olsrd_txtinfo.so.0.1" +#{ + # the default port is 2006 but you can change it like this: + #PlParam "port" "8080" + + # You can set a "accept" single address to allow to connect to + # txtinfo. If no address is specified, then localhost (127.0.0.1) + # is allowed by default. txtinfo will only use the first "accept" + # parameter specified and will ignore the rest. + + # to allow a specific host: + #PlParam "accept" "172.29.44.23" + # if you set it to 0.0.0.0, it will accept all connections + #PlParam "accept" "0.0.0.0" +#} + +############################################# +### OLSRD default interface configuration ### +############################################# +# the default interface section can have the same values as the following +# interface configuration. It will allow you so set common options for all +# interfaces. + +InterfaceDefaults { + Ip4Broadcast 255.255.255.255 +} + +###################################### +### OLSRd Interfaces configuration ### +###################################### +# multiple interfaces can be specified for a single configuration block +# multiple configuration blocks can be specified + +# WARNING, don't forget to insert your interface names here ! +#Interface "" "" +#{ + # Interface Mode is used to prevent unnecessary + # packet forwarding on switched ethernet interfaces + # valid Modes are "mesh" and "ether" + # (default is "mesh") + + # Mode "mesh" +#} diff --git a/daemon/core/configservices/nrlservices/templates/olsrd.sh b/daemon/core/configservices/nrlservices/templates/olsrd.sh new file mode 100644 index 00000000..076f049b --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/olsrd.sh @@ -0,0 +1,4 @@ +<% + interfaces = "-i " + " -i ".join(ifnames) +%> +olsrd ${interfaces} diff --git a/daemon/core/configservices/nrlservices/templates/start_mgen_actor.sh b/daemon/core/configservices/nrlservices/templates/start_mgen_actor.sh new file mode 100644 index 00000000..12630442 --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/start_mgen_actor.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# auto-generated by MgenActor service +mgenBasicActor.py -n ${node.name} -a 0.0.0.0 < /dev/null > /dev/null 2>&1 & diff --git a/daemon/core/configservices/nrlservices/templates/startarouted.sh b/daemon/core/configservices/nrlservices/templates/startarouted.sh new file mode 100644 index 00000000..20bcc45e --- /dev/null +++ b/daemon/core/configservices/nrlservices/templates/startarouted.sh @@ -0,0 +1,15 @@ +#!/bin/sh +for f in "/tmp/${node.name}_smf"; do + count=1 + until [ -e "$f" ]; do + if [ $count -eq 10 ]; then + echo "ERROR: nrlmsf pipe not found: $f" >&2 + exit 1 + fi + sleep 0.1 + count=$(($count + 1)) + done +done + +ip route add ${ip4_prefix} dev lo +arouted instance ${node.name}_smf tap ${node.name}_tap stability 10 2>&1 > /var/log/arouted.log & diff --git a/daemon/core/configservices/nrlservices/templates/startsmf.sh b/daemon/core/configservices/nrlservices/templates/startsmf.sh index 2ba20f27..67fc0fe6 100644 --- a/daemon/core/configservices/nrlservices/templates/startsmf.sh +++ b/daemon/core/configservices/nrlservices/templates/startsmf.sh @@ -1,19 +1,15 @@ -#!/bin/sh -# auto-generated by nrl.py:NrlSmf.generateconfig() -% if ifnames: -nrlsmf instance ${node.name}_smf \\ -% if has_arouted: - tap ${node.name}_tap unicast %s push lo,${ifnames[0]} resequence on \\ -% endif -% if has_nhdp: - ecds \\ -% elif has_olsr: - smpr \\ -% else: - cf \\ -% endif <% interfaces = ",".join(ifnames) + arouted = "" + if has_arouted: + arouted = "tap %s_tap unicast %s push lo,%s resequence on" % (node.name, ip4_prefix, ifnames[0]) + if has_nhdp: + flood = "ecds" + elif has_olsr: + flood = "smpr" + else: + flood = "cf" %> - ${interfaces} hash MD5 log /var/log/nrlsmf.log < /dev/null > /dev/null 2>&1 & -% endif +#!/bin/sh +# auto-generated by NrlSmf service +nrlsmf instance ${node.name}_smf ${interfaces} ${arouted} ${flood} hash MD5 log /var/log/nrlsmf.log < /dev/null > /dev/null 2>&1 & From 422bf9ac15a12572c9f8384010e2b4d8240b55b0 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 23 Jan 2020 11:03:56 -0800 Subject: [PATCH 22/37] initial work to add support for quagga services as config services --- daemon/core/configservice/base.py | 52 +++--- .../configservices/nrlservices/services.py | 40 +---- .../configservices/quaggaservices/__init__.py | 0 .../configservices/quaggaservices/services.py | 169 ++++++++++++++++++ .../quaggaservices/templates/Quagga.conf | 23 +++ .../quaggaservices/templates/quaggaboot.sh | 92 ++++++++++ .../quaggaservices/templates/vtysh.conf | 1 + 7 files changed, 312 insertions(+), 65 deletions(-) create mode 100644 daemon/core/configservices/quaggaservices/__init__.py create mode 100644 daemon/core/configservices/quaggaservices/services.py create mode 100644 daemon/core/configservices/quaggaservices/templates/Quagga.conf create mode 100644 daemon/core/configservices/quaggaservices/templates/quaggaboot.sh create mode 100644 daemon/core/configservices/quaggaservices/templates/vtysh.conf diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index fa8f0ad3..8db4d7f5 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -42,6 +42,10 @@ class ConfigService(abc.ABC): configs = self.default_configs[:] self._define_config(configs) + @staticmethod + def clean_text(text: str) -> str: + return inspect.cleandoc(text) + @property @abc.abstractmethod def name(self) -> str: @@ -147,12 +151,12 @@ class ConfigService(abc.ABC): basename = pathlib.Path(name).name if name in self.custom_templates: template = self.custom_templates[name] - template = inspect.cleandoc(template) + template = self.clean_text(template) elif self.templates.has_template(basename): template = self.templates.get_template(basename).source else: template = self.get_text(name) - template = inspect.cleandoc(template) + template = self.clean_text(template) templates[name] = template return templates @@ -162,14 +166,22 @@ class ConfigService(abc.ABC): basename = pathlib.Path(name).name if name in self.custom_templates: text = self.custom_templates[name] - text = inspect.cleandoc(text) - self.render_text(name, text, data) + text = self.clean_text(text) + rendered = self.render_text(text, data) elif self.templates.has_template(basename): - self.render_template(name, basename, data) + rendered = self.render_template(basename, data) else: text = self.get_text(name) - text = inspect.cleandoc(text) - self.render_text(name, text, data) + text = self.clean_text(text) + rendered = self.render_text(text, data) + logging.info( + "node(%s) service(%s) template(%s): \n%s", + self.node.name, + self.name, + name, + rendered, + ) + self.node.nodefile(name, rendered) def run_startup(self) -> None: for cmd in self.startup: @@ -205,44 +217,30 @@ class ConfigService(abc.ABC): f"failed to validate" ) - def _render( - self, name: str, template: Template, data: Dict[str, Any] = None - ) -> None: + def _render(self, template: Template, data: Dict[str, Any] = None) -> str: if data is None: data = {} - rendered = template.render_unicode( + return template.render_unicode( node=self.node, config=self.render_config(), **data ) - logging.info( - "node(%s) service(%s) template(%s): \n%s", - self.node.name, - self.name, - name, - rendered, - ) - self.node.nodefile(name, rendered) - def render_text(self, name: str, text: str, data: Dict[str, Any] = None) -> None: + def render_text(self, text: str, data: Dict[str, Any] = None) -> str: try: template = Template(text) - self._render(name, template, data) + return self._render(template, data) except Exception: raise CoreError( f"node({self.node.name}) service({self.name}) " - f"error rendering template({name}): " f"{exceptions.text_error_template().render_unicode()}" ) - def render_template( - self, name: str, basename: str, data: Dict[str, Any] = None - ) -> None: + def render_template(self, basename: str, data: Dict[str, Any] = None) -> str: try: template = self.templates.get_template(basename) - self._render(name, template, data) + return self._render(template, data) except Exception: raise CoreError( f"node({self.node.name}) service({self.name}) " - f"error rendering template({name}): " f"{exceptions.text_error_template().render_template()}" ) diff --git a/daemon/core/configservices/nrlservices/services.py b/daemon/core/configservices/nrlservices/services.py index 98df2ceb..a67fac53 100644 --- a/daemon/core/configservices/nrlservices/services.py +++ b/daemon/core/configservices/nrlservices/services.py @@ -8,42 +8,6 @@ from core.configservice.base import ConfigService, ConfigServiceMode GROUP = "ProtoSvc" -class NrlService(ConfigService): - name = "NrlBase" - group = GROUP - directories = [] - files = [] - executables = [] - dependencies = [] - startup = [] - validate = [] - shutdown = [] - validation_mode = ConfigServiceMode.BLOCKING - default_configs = [] - modes = {} - - @classmethod - def generate_config(cls, node, filename): - return "" - - @staticmethod - def firstipv4prefix(node, prefixlen=24): - """ - Similar to QuaggaService.routerid(). Helper to return the first IPv4 - prefix of a node, using the supplied prefix length. This ignores the - interface's prefix length, so e.g. '/32' can turn into '/24'. - """ - for ifc in node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: - continue - for a in ifc.addrlist: - a = a.split("/")[0] - if netaddr.valid_ipv4(a): - return f"{a}/{prefixlen}" - # raise ValueError, "no IPv4 address found" - return "0.0.0.0/%s" % prefixlen - - class MgenSinkService(ConfigService): name = "MGEN_Sink" group = GROUP @@ -66,7 +30,7 @@ class MgenSinkService(ConfigService): return dict(ifnames=ifnames) -class NrlNhdp(NrlService): +class NrlNhdp(ConfigService): name = "NHDP" group = GROUP directories = [] @@ -204,7 +168,7 @@ class OlsrOrg(ConfigService): return dict(has_smf=has_smf, ifnames=ifnames) -class MgenActor(NrlService): +class MgenActor(ConfigService): name = "MgenActor" group = GROUP directories = [] diff --git a/daemon/core/configservices/quaggaservices/__init__.py b/daemon/core/configservices/quaggaservices/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/core/configservices/quaggaservices/services.py b/daemon/core/configservices/quaggaservices/services.py new file mode 100644 index 00000000..ef36b932 --- /dev/null +++ b/daemon/core/configservices/quaggaservices/services.py @@ -0,0 +1,169 @@ +import abc +from typing import Any, Dict + +import netaddr + +from core import constants +from core.configservice.base import ConfigService, ConfigServiceMode +from core.nodes.base import CoreNodeBase +from core.nodes.interface import CoreInterface + +GROUP = "Quagga" + + +def has_mtu_mismatch(ifc: CoreInterface) -> bool: + """ + Helper to detect MTU mismatch and add the appropriate OSPF + mtu-ignore command. This is needed when e.g. a node is linked via a + GreTap device. + """ + if ifc.mtu != 1500: + return True + if not ifc.net: + return False + for i in ifc.net.netifs(): + if i.mtu != ifc.mtu: + return True + return False + + +def get_router_id(node: CoreNodeBase) -> str: + """ + Helper to return the first IPv4 address of a node as its router ID. + """ + for ifc in node.netifs(): + if hasattr(ifc, "control") and ifc.control is True: + continue + for a in ifc.addrlist: + a = a.split("/")[0] + if netaddr.valid_ipv4(a): + return a + return "0.0.0.0" + + +class Zebra(ConfigService): + name = "zebra" + group = GROUP + directories = ["/usr/local/etc/quagga", "/var/run/quagga"] + files = [ + "/usr/local/etc/quagga/Quagga.conf", + "quaggaboot.sh", + "/usr/local/etc/quagga/vtysh.conf", + ] + executables = ["zebra"] + dependencies = [] + startup = ["sh quaggaboot.sh zebra"] + validate = ["pidof zebra"] + shutdown = ["killall zebra"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + quagga_bin_search = self.node.session.options.get_config( + "quagga_bin_search", default="/usr/local/bin /usr/bin /usr/lib/quagga" + ) + quagga_sbin_search = self.node.session.options.get_config( + "quagga_sbin_search", default="/usr/local/sbin /usr/sbin /usr/lib/quagga" + ) + quagga_state_dir = constants.QUAGGA_STATE_DIR + quagga_conf = self.files[0] + + services = [] + want_ip4 = False + want_ip6 = False + for service in self.node.config_services.values(): + if self.name not in service.dependencies: + continue + if service.ipv4_routing: + want_ip4 = True + if service.ipv6_routing: + want_ip6 = True + services.append(service) + + interfaces = [] + for ifc in self.node.netifs(): + ip4s = [] + ip6s = [] + for x in ifc.addrlist: + addr = x.split("/")[0] + if netaddr.valid_ipv4(addr): + ip4s.append(x) + else: + ip6s.append(x) + is_control = getattr(ifc, "control", False) + interfaces.append((ifc, ip4s, ip6s, is_control)) + + return dict( + quagga_bin_search=quagga_bin_search, + quagga_sbin_search=quagga_sbin_search, + quagga_state_dir=quagga_state_dir, + quagga_conf=quagga_conf, + interfaces=interfaces, + want_ip4=want_ip4, + want_ip6=want_ip6, + ) + + +class QuaggaService(abc.ABC): + group = GROUP + directories = [] + files = [] + executables = [] + dependencies = ["zebra"] + startup = [] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + ipv4_routing = False + ipv6_routing = False + + @abc.abstractmethod + def quagga_interface_config(self, ifc: CoreInterface) -> str: + raise NotImplementedError + + @abc.abstractmethod + def quagga_config(self) -> str: + raise NotImplementedError + + +class Ospfv2(QuaggaService, ConfigService): + """ + The OSPFv2 service provides IPv4 routing for wired networks. It does + not build its own configuration file but has hooks for adding to the + unified Quagga.conf file. + """ + + name = "OSPFv2" + validate = ["pidof ospfd"] + shutdown = ["killall ospfd"] + ipv4_routing = True + + def quagga_interface_config(self, ifc: CoreInterface) -> str: + if has_mtu_mismatch(ifc): + return "ip ospf mtu-ignore" + else: + return "" + + def quagga_config(self) -> str: + router_id = get_router_id(self.node) + addresses = [] + for ifc in self.node.netifs(): + if getattr(ifc, "control", False): + continue + for a in ifc.addrlist: + addr = a.split("/")[0] + if netaddr.valid_ipv4(addr): + addresses.append(a) + data = dict(router_id=router_id, addresses=addresses) + text = """ + router ospf + router-id ${router_id} + % for addr in addresses: + network ${addr} area 0 + % endfor + ! + """ + return self.render_text(text, data) diff --git a/daemon/core/configservices/quaggaservices/templates/Quagga.conf b/daemon/core/configservices/quaggaservices/templates/Quagga.conf new file mode 100644 index 00000000..138f4288 --- /dev/null +++ b/daemon/core/configservices/quaggaservices/templates/Quagga.conf @@ -0,0 +1,23 @@ +% for ifc, ip4s, ip6s in interfaces: +interface ${ifc.name} + % if want_ip4: + % for addr in ip4s: + ip address ${addr} + % endfor + % endif + % if want_ip6: + % for addr in ip6s: + ipv6 address ${addr} + % endfor + % endif + % if is_control: + % for service in services: + ${service.quagga_interface_config(ifc)} + % endfor + % endif +! +% endfor + +% for service in services: + ${service.quagga_config()} +% endfor diff --git a/daemon/core/configservices/quaggaservices/templates/quaggaboot.sh b/daemon/core/configservices/quaggaservices/templates/quaggaboot.sh new file mode 100644 index 00000000..c22fdd5f --- /dev/null +++ b/daemon/core/configservices/quaggaservices/templates/quaggaboot.sh @@ -0,0 +1,92 @@ +#!/bin/sh +# auto-generated by zebra service (quagga.py) +QUAGGA_CONF="${quagga_conf}" +QUAGGA_SBIN_SEARCH="${quagga_sbin_search}" +QUAGGA_BIN_SEARCH="${quagga_bin_search}" +QUAGGA_STATE_DIR="${quagga_state_dir}" + +searchforprog() +{ + prog=$1 + searchpath=$@ + ret= + for p in $searchpath; do + if [ -x $p/$prog ]; then + ret=$p + break + fi + done + echo $ret +} + +confcheck() +{ + CONF_DIR=`dirname $QUAGGA_CONF` + # if /etc/quagga exists, point /etc/quagga/Quagga.conf -> CONF_DIR + if [ "$CONF_DIR" != "/etc/quagga" ] && [ -d /etc/quagga ] && [ ! -e /etc/quagga/Quagga.conf ]; then + ln -s $CONF_DIR/Quagga.conf /etc/quagga/Quagga.conf + fi + # if /etc/quagga exists, point /etc/quagga/vtysh.conf -> CONF_DIR + if [ "$CONF_DIR" != "/etc/quagga" ] && [ -d /etc/quagga ] && [ ! -e /etc/quagga/vtysh.conf ]; then + ln -s $CONF_DIR/vtysh.conf /etc/quagga/vtysh.conf + fi +} + +bootdaemon() +{ + QUAGGA_SBIN_DIR=$(searchforprog $1 $QUAGGA_SBIN_SEARCH) + if [ "z$QUAGGA_SBIN_DIR" = "z" ]; then + echo "ERROR: Quagga's '$1' daemon not found in search path:" + echo " $QUAGGA_SBIN_SEARCH" + return 1 + fi + + flags="" + + if [ "$1" = "xpimd" ] && \\ + grep -E -q '^[[:space:]]*router[[:space:]]+pim6[[:space:]]*$' $QUAGGA_CONF; then + flags="$flags -6" + fi + + $QUAGGA_SBIN_DIR/$1 $flags -d + if [ "$?" != "0" ]; then + echo "ERROR: Quagga's '$1' daemon failed to start!:" + return 1 + fi +} + +bootquagga() +{ + QUAGGA_BIN_DIR=$(searchforprog 'vtysh' $QUAGGA_BIN_SEARCH) + if [ "z$QUAGGA_BIN_DIR" = "z" ]; then + echo "ERROR: Quagga's 'vtysh' program not found in search path:" + echo " $QUAGGA_BIN_SEARCH" + return 1 + fi + + # fix /var/run/quagga permissions + id -u quagga 2>/dev/null >/dev/null + if [ "$?" = "0" ]; then + chown quagga $QUAGGA_STATE_DIR + fi + + bootdaemon "zebra" + for r in rip ripng ospf6 ospf bgp babel; do + if grep -q "^router \\<$${}{r}\\>" $QUAGGA_CONF; then + bootdaemon "$${}{r}d" + fi + done + + if grep -E -q '^[[:space:]]*router[[:space:]]+pim6?[[:space:]]*$' $QUAGGA_CONF; then + bootdaemon "xpimd" + fi + + $QUAGGA_BIN_DIR/vtysh -b +} + +if [ "$1" != "zebra" ]; then + echo "WARNING: '$1': all Quagga daemons are launched by the 'zebra' service!" + exit 1 +fi +confcheck +bootquagga diff --git a/daemon/core/configservices/quaggaservices/templates/vtysh.conf b/daemon/core/configservices/quaggaservices/templates/vtysh.conf new file mode 100644 index 00000000..e0ab9cb6 --- /dev/null +++ b/daemon/core/configservices/quaggaservices/templates/vtysh.conf @@ -0,0 +1 @@ +service integrated-vtysh-config From 852eb60ab9dfc25c22e52518a7ea8bd2ac2932f7 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 23 Jan 2020 11:06:09 -0800 Subject: [PATCH 23/37] renamed config service get_text to get_text_template to be more obvious to the purpose --- daemon/core/configservice/base.py | 6 +++--- daemon/core/configservices/simpleservice.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 8db4d7f5..bc788a2c 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -142,7 +142,7 @@ class ConfigService(abc.ABC): def set_template(self, name: str, template: str) -> None: self.custom_templates[name] = template - def get_text(self, name: str) -> str: + def get_text_template(self, name: str) -> str: raise CoreError(f"service({self.name}) unknown template({name})") def get_templates(self) -> Dict[str, str]: @@ -155,7 +155,7 @@ class ConfigService(abc.ABC): elif self.templates.has_template(basename): template = self.templates.get_template(basename).source else: - template = self.get_text(name) + template = self.get_text_template(name) template = self.clean_text(template) templates[name] = template return templates @@ -171,7 +171,7 @@ class ConfigService(abc.ABC): elif self.templates.has_template(basename): rendered = self.render_template(basename, data) else: - text = self.get_text(name) + text = self.get_text_template(name) text = self.clean_text(text) rendered = self.render_text(text, data) logging.info( diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py index ab2dcb5b..3284471d 100644 --- a/daemon/core/configservices/simpleservice.py +++ b/daemon/core/configservices/simpleservice.py @@ -25,7 +25,7 @@ class SimpleService(ConfigService): "mode3": {"value1": "m3", "value2": "m3", "value3": "m3"}, } - def get_text(self, name: str) -> str: + def get_text_template(self, name: str) -> str: if name == "test1.sh": return """ # sample script 1 From 0749dcacb20e72a49fe5c30dc11f11687a4436e6 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 23 Jan 2020 11:27:05 -0800 Subject: [PATCH 24/37] updated config services to use getattr instead of hasattr to simplify code --- .../core/configservices/nrlservices/services.py | 12 ++++++------ .../configservices/quaggaservices/services.py | 2 +- .../configservices/sercurityservices/services.py | 2 +- .../core/configservices/utilservices/services.py | 16 +++++++++------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/daemon/core/configservices/nrlservices/services.py b/daemon/core/configservices/nrlservices/services.py index a67fac53..3dddf1ba 100644 --- a/daemon/core/configservices/nrlservices/services.py +++ b/daemon/core/configservices/nrlservices/services.py @@ -48,7 +48,7 @@ class NrlNhdp(ConfigService): has_smf = "SMF" in self.node.config_services ifnames = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifnames.append(ifc.name) return dict(has_smf=has_smf, ifnames=ifnames) @@ -75,7 +75,7 @@ class NrlSmf(ConfigService): ifnames = [] ip4_prefix = None for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifnames.append(ifc.name) if ip4_prefix: @@ -113,7 +113,7 @@ class NrlOlsr(ConfigService): has_zebra = "zebra" in self.node.config_services ifname = None for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifname = ifc.name break @@ -138,7 +138,7 @@ class NrlOlsrv2(ConfigService): has_smf = "SMF" in self.node.config_services ifnames = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifnames.append(ifc.name) return dict(has_smf=has_smf, ifnames=ifnames) @@ -162,7 +162,7 @@ class OlsrOrg(ConfigService): has_smf = "SMF" in self.node.config_services ifnames = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifnames.append(ifc.name) return dict(has_smf=has_smf, ifnames=ifnames) @@ -200,7 +200,7 @@ class Arouted(ConfigService): def data(self) -> Dict[str, Any]: ip4_prefix = None for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue if ip4_prefix: continue diff --git a/daemon/core/configservices/quaggaservices/services.py b/daemon/core/configservices/quaggaservices/services.py index ef36b932..5a2c869f 100644 --- a/daemon/core/configservices/quaggaservices/services.py +++ b/daemon/core/configservices/quaggaservices/services.py @@ -32,7 +32,7 @@ def get_router_id(node: CoreNodeBase) -> str: Helper to return the first IPv4 address of a node as its router ID. """ for ifc in node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue for a in ifc.addrlist: a = a.split("/")[0] diff --git a/daemon/core/configservices/sercurityservices/services.py b/daemon/core/configservices/sercurityservices/services.py index cc95265d..fca86373 100644 --- a/daemon/core/configservices/sercurityservices/services.py +++ b/daemon/core/configservices/sercurityservices/services.py @@ -82,7 +82,7 @@ class Nat(ConfigService): def data(self) -> Dict[str, Any]: ifnames = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifnames.append(ifc.name) return dict(ifnames=ifnames) diff --git a/daemon/core/configservices/utilservices/services.py b/daemon/core/configservices/utilservices/services.py index 1165b149..671f12f1 100644 --- a/daemon/core/configservices/utilservices/services.py +++ b/daemon/core/configservices/utilservices/services.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Dict import netaddr @@ -28,6 +29,7 @@ class DefaultRouteService(ConfigService): if getattr(netif, "control", False): continue for addr in netif.addrlist: + logging.info("default route address: %s", addr) net = netaddr.IPNetwork(addr) if net[1] != net[-2]: addresses.append(net[1]) @@ -51,7 +53,7 @@ class DefaultMulticastRouteService(ConfigService): def data(self) -> Dict[str, Any]: ifname = None for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifname = ifc.name break @@ -75,7 +77,7 @@ class StaticRouteService(ConfigService): def data(self) -> Dict[str, Any]: routes = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue for x in ifc.addrlist: addr = x.split("/")[0] @@ -150,7 +152,7 @@ class DhcpService(ConfigService): def data(self) -> Dict[str, Any]: subnets = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue for x in ifc.addrlist: addr = x.split("/")[0] @@ -181,7 +183,7 @@ class DhcpClientService(ConfigService): def data(self) -> Dict[str, Any]: ifnames = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifnames.append(ifc.name) return dict(ifnames=ifnames) @@ -219,7 +221,7 @@ class PcapService(ConfigService): def data(self) -> Dict[str, Any]: ifnames = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue ifnames.append(ifc.name) return dict() @@ -242,7 +244,7 @@ class RadvdService(ConfigService): def data(self) -> Dict[str, Any]: interfaces = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue prefixes = [] for x in ifc.addrlist: @@ -294,7 +296,7 @@ class HttpService(ConfigService): def data(self) -> Dict[str, Any]: interfaces = [] for ifc in self.node.netifs(): - if hasattr(ifc, "control") and ifc.control is True: + if getattr(ifc, "control", False): continue interfaces.append(ifc) return dict(interfaces=interfaces) From fcc445bb72eb4e49d56f1bd51987b9cd6d9f8a00 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 23 Jan 2020 13:22:47 -0800 Subject: [PATCH 25/37] fixed issues in zebra config service, updated config services to start and validate different modes appropriately, added service dependency startup for config services --- daemon/core/configservice/base.py | 47 +++++---- daemon/core/configservice/dependencies.py | 97 +++++++++++++++++++ .../configservices/quaggaservices/services.py | 5 +- .../quaggaservices/templates/Quagga.conf | 4 +- daemon/core/emulator/session.py | 3 +- daemon/core/nodes/base.py | 7 ++ 6 files changed, 140 insertions(+), 23 deletions(-) create mode 100644 daemon/core/configservice/dependencies.py diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index bc788a2c..39685768 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -23,6 +23,10 @@ class ConfigServiceMode(enum.Enum): TIMER = 2 +class ConfigServiceBootError(Exception): + pass + + class ConfigService(abc.ABC): # validation period in seconds, how frequent validation is attempted validation_period = 0.5 @@ -107,10 +111,16 @@ class ConfigService(abc.ABC): raise NotImplementedError def start(self) -> None: + logging.info("node(%s) service(%s) starting...", self.node.name, self.name) self.create_dirs() self.create_files() - self.run_startup() - self.run_validation() + wait = self.validation_mode == ConfigServiceMode.BLOCKING + self.run_startup(wait) + if not wait: + if self.validation_mode == ConfigServiceMode.TIMER: + time.sleep(self.validation_timer) + else: + self.run_validation() def stop(self) -> None: for cmd in self.shutdown: @@ -166,13 +176,11 @@ class ConfigService(abc.ABC): basename = pathlib.Path(name).name if name in self.custom_templates: text = self.custom_templates[name] - text = self.clean_text(text) rendered = self.render_text(text, data) elif self.templates.has_template(basename): rendered = self.render_template(basename, data) else: text = self.get_text_template(name) - text = self.clean_text(text) rendered = self.render_text(text, data) logging.info( "node(%s) service(%s) template(%s): \n%s", @@ -183,25 +191,23 @@ class ConfigService(abc.ABC): ) self.node.nodefile(name, rendered) - def run_startup(self) -> None: + def run_startup(self, wait: bool) -> None: for cmd in self.startup: try: - self.node.cmd(cmd) - except CoreCommandError: - raise CoreError( - f"node({self.node.name}) service({self.name}) " - f"failed startup: {cmd}" + self.node.cmd(cmd, wait=wait) + except CoreCommandError as e: + raise ConfigServiceBootError( + f"node({self.node.name}) service({self.name}) failed startup: {e}" ) def run_validation(self) -> None: - wait = self.validation_mode == ConfigServiceMode.BLOCKING start = time.monotonic() - index = 0 cmds = self.validate[:] + index = 0 while cmds: cmd = cmds[index] try: - self.node.cmd(cmd, wait=wait) + self.node.cmd(cmd) del cmds[index] index += 1 except CoreCommandError: @@ -211,10 +217,9 @@ class ConfigService(abc.ABC): ) time.sleep(self.validation_period) - if time.monotonic() - start > 0: - raise CoreError( - f"node({self.node.name}) service({self.name}) " - f"failed to validate" + if cmds and time.monotonic() - start > 0: + raise ConfigServiceBootError( + f"node({self.node.name}) service({self.name}) failed to validate" ) def _render(self, template: Template, data: Dict[str, Any] = None) -> str: @@ -225,6 +230,7 @@ class ConfigService(abc.ABC): ) def render_text(self, text: str, data: Dict[str, Any] = None) -> str: + text = self.clean_text(text) try: template = Template(text) return self._render(template, data) @@ -235,6 +241,13 @@ class ConfigService(abc.ABC): ) def render_template(self, basename: str, data: Dict[str, Any] = None) -> str: + logging.info( + "node(%s) service(%s) rendering template(%s): %s", + self.node.name, + self.name, + basename, + data, + ) try: template = self.templates.get_template(basename) return self._render(template, data) diff --git a/daemon/core/configservice/dependencies.py b/daemon/core/configservice/dependencies.py new file mode 100644 index 00000000..7f62a563 --- /dev/null +++ b/daemon/core/configservice/dependencies.py @@ -0,0 +1,97 @@ +import logging +from typing import TYPE_CHECKING, Dict, List + +if TYPE_CHECKING: + from core.configservice.base import ConfigService + + +class ConfigServiceDependencies: + """ + Can generate boot paths for services, based on their dependencies. Will validate + that all services will be booted and that all dependencies exist within the services provided. + """ + + def __init__(self, services: Dict[str, "ConfigService"]) -> None: + # helpers to check validity + self.dependents = {} + self.booted = set() + self.node_services = {} + for service in services.values(): + self.node_services[service.name] = service + for dependency in service.dependencies: + dependents = self.dependents.setdefault(dependency, set()) + dependents.add(service.name) + + # used to find paths + self.path = [] + self.visited = set() + self.visiting = set() + + def boot_paths(self) -> List[List["ConfigService"]]: + paths = [] + for name in self.node_services: + service = self.node_services[name] + if service.name in self.booted: + logging.debug( + "skipping service that will already be booted: %s", service.name + ) + continue + + path = self._start(service) + if path: + paths.append(path) + + if self.booted != set(self.node_services): + raise ValueError( + "failure to boot all services: %s != %s" + % (self.booted, self.node_services.keys()) + ) + + return paths + + def _reset(self) -> None: + self.path = [] + self.visited.clear() + self.visiting.clear() + + def _start(self, service: "ConfigService") -> List["ConfigService"]: + logging.debug("starting service dependency check: %s", service.name) + self._reset() + return self._visit(service) + + def _visit(self, current_service: "ConfigService") -> List["ConfigService"]: + logging.debug("visiting service(%s): %s", current_service.name, self.path) + self.visited.add(current_service.name) + self.visiting.add(current_service.name) + + # dive down + for service_name in current_service.dependencies: + if service_name not in self.node_services: + raise ValueError( + "required dependency was not included in node services: %s" + % service_name + ) + + if service_name in self.visiting: + raise ValueError( + "cyclic dependency at service(%s): %s" + % (current_service.name, service_name) + ) + + if service_name not in self.visited: + service = self.node_services[service_name] + self._visit(service) + + # add service when bottom is found + logging.debug("adding service to boot path: %s", current_service.name) + self.booted.add(current_service.name) + self.path.append(current_service) + self.visiting.remove(current_service.name) + + # rise back up + for service_name in self.dependents.get(current_service.name, []): + if service_name not in self.visited: + service = self.node_services[service_name] + self._visit(service) + + return self.path diff --git a/daemon/core/configservices/quaggaservices/services.py b/daemon/core/configservices/quaggaservices/services.py index 5a2c869f..604c98a6 100644 --- a/daemon/core/configservices/quaggaservices/services.py +++ b/daemon/core/configservices/quaggaservices/services.py @@ -62,10 +62,10 @@ class Zebra(ConfigService): def data(self) -> Dict[str, Any]: quagga_bin_search = self.node.session.options.get_config( "quagga_bin_search", default="/usr/local/bin /usr/bin /usr/lib/quagga" - ) + ).strip('"') quagga_sbin_search = self.node.session.options.get_config( "quagga_sbin_search", default="/usr/local/sbin /usr/sbin /usr/lib/quagga" - ) + ).strip('"') quagga_state_dir = constants.QUAGGA_STATE_DIR quagga_conf = self.files[0] @@ -102,6 +102,7 @@ class Zebra(ConfigService): interfaces=interfaces, want_ip4=want_ip4, want_ip6=want_ip6, + services=services, ) diff --git a/daemon/core/configservices/quaggaservices/templates/Quagga.conf b/daemon/core/configservices/quaggaservices/templates/Quagga.conf index 138f4288..282fc63d 100644 --- a/daemon/core/configservices/quaggaservices/templates/Quagga.conf +++ b/daemon/core/configservices/quaggaservices/templates/Quagga.conf @@ -1,4 +1,4 @@ -% for ifc, ip4s, ip6s in interfaces: +% for ifc, ip4s, ip6s, is_control in interfaces: interface ${ifc.name} % if want_ip4: % for addr in ip4s: @@ -19,5 +19,5 @@ interface ${ifc.name} % endfor % for service in services: - ${service.quagga_config()} +${service.quagga_config()} % endfor diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index da678211..a686ce9e 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -1611,8 +1611,7 @@ class Session: logging.info("booting node(%s): %s", node.name, [x.name for x in node.services]) self.add_remove_control_interface(node=node, remove=False) self.services.boot_services(node) - for service in node.config_services.values(): - service.start() + node.start_config_services() def boot_nodes(self) -> List[Exception]: """ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 5e7a72dc..af2854f7 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type import netaddr from core import utils +from core.configservice.dependencies import ConfigServiceDependencies from core.constants import MOUNT_BIN, VNODED_BIN from core.emulator.data import LinkData, NodeData from core.emulator.enumerations import LinkTypes, NodeTypes @@ -296,6 +297,12 @@ class CoreNodeBase(NodeBase): raise CoreError(f"node({self.name}) does not have service({name})") service.set_config(data) + def start_config_services(self) -> None: + boot_paths = ConfigServiceDependencies(self.config_services).boot_paths() + for boot_path in boot_paths: + for service in boot_path: + service.start() + def makenodedir(self) -> None: """ Create the node directory. From b9cbbf5709e6324a6ffc42f5a83cb1fc9caaf0e3 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 23 Jan 2020 21:08:40 -0800 Subject: [PATCH 26/37] finished converting quagga services to config services --- .../configservices/quaggaservices/services.py | 245 ++++++++++++++++++ .../quaggaservices/templates/Quagga.conf | 6 +- 2 files changed, 249 insertions(+), 2 deletions(-) diff --git a/daemon/core/configservices/quaggaservices/services.py b/daemon/core/configservices/quaggaservices/services.py index 604c98a6..bc937038 100644 --- a/daemon/core/configservices/quaggaservices/services.py +++ b/daemon/core/configservices/quaggaservices/services.py @@ -1,12 +1,15 @@ import abc +import logging from typing import Any, Dict import netaddr from core import constants from core.configservice.base import ConfigService, ConfigServiceMode +from core.emane.nodes import EmaneNet from core.nodes.base import CoreNodeBase from core.nodes.interface import CoreInterface +from core.nodes.network import WlanNode GROUP = "Quagga" @@ -27,6 +30,20 @@ def has_mtu_mismatch(ifc: CoreInterface) -> bool: return False +def get_min_mtu(ifc): + """ + Helper to discover the minimum MTU of interfaces linked with the + given interface. + """ + mtu = ifc.mtu + if not ifc.net: + return mtu + for i in ifc.net.netifs(): + if i.mtu < mtu: + mtu = i.mtu + return mtu + + def get_router_id(node: CoreNodeBase) -> str: """ Helper to return the first IPv4 address of a node as its router ID. @@ -168,3 +185,231 @@ class Ospfv2(QuaggaService, ConfigService): ! """ return self.render_text(text, data) + + +class Ospfv3(QuaggaService, ConfigService): + """ + The OSPFv3 service provides IPv6 routing for wired networks. It does + not build its own configuration file but has hooks for adding to the + unified Quagga.conf file. + """ + + name = "OSPFv3" + shutdown = ("killall ospf6d",) + validate = ("pidof ospf6d",) + ipv4_routing = True + ipv6_routing = True + + def quagga_interface_config(self, ifc: CoreInterface) -> str: + mtu = get_min_mtu(ifc) + if mtu < ifc.mtu: + return f"ipv6 ospf6 ifmtu {mtu}" + else: + return "" + + def quagga_config(self) -> str: + router_id = get_router_id(self.node) + ifnames = [] + for ifc in self.node.netifs(): + if getattr(ifc, "control", False): + continue + ifnames.append(ifc.name) + data = dict(router_id=router_id, ifnames=ifnames) + text = """ + router ospf6 + instance-id 65 + router-id ${router_id} + % for ifname in ifnames: + interface ${ifname} area 0.0.0.0 + % endfor + ! + """ + return self.render_text(text, data) + + +class Ospfv3mdr(Ospfv3): + """ + The OSPFv3 MANET Designated Router (MDR) service provides IPv6 + routing for wireless networks. It does not build its own + configuration file but has hooks for adding to the + unified Quagga.conf file. + """ + + name = "OSPFv3MDR" + + def data(self) -> Dict[str, Any]: + for ifc in self.node.netifs(): + is_wireless = isinstance(ifc.net, (WlanNode, EmaneNet)) + logging.info("MDR wireless: %s", is_wireless) + return dict() + + def quagga_interface_config(self, ifc: CoreInterface) -> str: + config = super().quagga_interface_config(ifc) + if isinstance(ifc.net, (WlanNode, EmaneNet)): + config = self.clean_text( + f""" + {config} + ipv6 ospf6 hello-interval 2 + ipv6 ospf6 dead-interval 6 + ipv6 ospf6 retransmit-interval 5 + ipv6 ospf6 network manet-designated-router + ipv6 ospf6 twohoprefresh 3 + ipv6 ospf6 adjacencyconnectivity uniconnected + ipv6 ospf6 lsafullness mincostlsa + """ + ) + return config + + +class Bgp(QuaggaService, ConfigService): + """ + The BGP service provides interdomain routing. + Peers must be manually configured, with a full mesh for those + having the same AS number. + """ + + name = "BGP" + shutdown = ["killall bgpd"] + validate = ["pidof bgpd"] + ipv4_routing = True + ipv6_routing = True + + def quagga_interface_config(self, ifc: CoreInterface) -> str: + router_id = get_router_id(self.node) + text = f""" + ! BGP configuration + ! You should configure the AS number below + ! along with this router's peers. + router bgp {self.node.id} + bgp router-id {router_id} + redistribute connected + !neighbor 1.2.3.4 remote-as 555 + ! + """ + return self.clean_text(text) + + +class Rip(QuaggaService, ConfigService): + """ + The RIP service provides IPv4 routing for wired networks. + """ + + name = "RIP" + shutdown = ["killall ripd"] + validate = ["pidof ripd"] + ipv4_routing = True + + def quagga_config(self) -> str: + text = """ + router rip + redistribute static + redistribute connected + redistribute ospf + network 0.0.0.0/0 + ! + """ + return self.clean_text(text) + + +class Ripng(QuaggaService, ConfigService): + """ + The RIP NG service provides IPv6 routing for wired networks. + """ + + name = "RIPNG" + shutdown = ["killall ripngd"] + validate = ["pidof ripngd"] + ipv6_routing = True + + def quagga_config(self) -> str: + text = """ + router ripng + redistribute static + redistribute connected + redistribute ospf6 + network ::/0 + ! + """ + return self.clean_text(text) + + +class Babel(QuaggaService, ConfigService): + """ + The Babel service provides a loop-avoiding distance-vector routing + protocol for IPv6 and IPv4 with fast convergence properties. + """ + + name = "Babel" + shutdown = ["killall babeld"] + validate = ["pidof babeld"] + ipv6_routing = True + + def quagga_config(self) -> str: + ifnames = [] + for ifc in self.node.netifs(): + if getattr(ifc, "control", False): + continue + ifnames.append(ifc.name) + text = """ + router babel + % for ifname in ifnames: + network ${ifname} + % endfor + redistribute static + redistribute connected + ! + """ + data = dict(ifnames=ifnames) + return self.render_text(text, data) + + def quagga_interface_config(self, ifc: CoreInterface) -> str: + if isinstance(ifc.net, (WlanNode, EmaneNet)): + text = """ + babel wireless + no babel split-horizon + """ + else: + text = """ + babel wired + babel split-horizon + """ + return self.clean_text(text) + + +class Xpimd(QuaggaService, ConfigService): + """ + PIM multicast routing based on XORP. + """ + + name = "Xpimd" + shutdown = ["killall xpimd"] + validate = ["pidof xpimd"] + ipv4_routing = True + + def quagga_config(self) -> str: + ifname = "eth0" + for ifc in self.node.netifs(): + if ifc.name != "lo": + ifname = ifc.name + break + + text = f""" + router mfea + ! + router igmp + ! + router pim + !ip pim rp-address 10.0.0.1 + ip pim bsr-candidate {ifname} + ip pim rp-candidate {ifname} + !ip pim spt-threshold interval 10 bytes 80000 + ! + """ + return self.clean_text(text) + + def quagga_interface_config(self, ifc: CoreInterface) -> str: + text = """ + ip mfea + ip pim + """ + return self.clean_text(text) diff --git a/daemon/core/configservices/quaggaservices/templates/Quagga.conf b/daemon/core/configservices/quaggaservices/templates/Quagga.conf index 282fc63d..853b1707 100644 --- a/daemon/core/configservices/quaggaservices/templates/Quagga.conf +++ b/daemon/core/configservices/quaggaservices/templates/Quagga.conf @@ -10,9 +10,11 @@ interface ${ifc.name} ipv6 address ${addr} % endfor % endif - % if is_control: + % if not is_control: % for service in services: - ${service.quagga_interface_config(ifc)} + % for line in service.quagga_interface_config(ifc).split("\n"): + ${line} + % endfor % endfor % endif ! From 531b55e1e7f998a0a205fde1976453cab972fb3d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 23 Jan 2020 22:06:30 -0800 Subject: [PATCH 27/37] small update to quagga config services, converted frr services to config services --- .../configservices/frrservices/__init__.py | 0 .../configservices/frrservices/services.py | 391 ++++++++++++++++++ .../frrservices/templates/daemons | 59 +++ .../frrservices/templates/frr.conf | 25 ++ .../frrservices/templates/frrboot.sh | 95 +++++ .../frrservices/templates/vtysh.conf | 1 + .../configservices/quaggaservices/services.py | 9 + 7 files changed, 580 insertions(+) create mode 100644 daemon/core/configservices/frrservices/__init__.py create mode 100644 daemon/core/configservices/frrservices/services.py create mode 100644 daemon/core/configservices/frrservices/templates/daemons create mode 100644 daemon/core/configservices/frrservices/templates/frr.conf create mode 100644 daemon/core/configservices/frrservices/templates/frrboot.sh create mode 100644 daemon/core/configservices/frrservices/templates/vtysh.conf diff --git a/daemon/core/configservices/frrservices/__init__.py b/daemon/core/configservices/frrservices/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/core/configservices/frrservices/services.py b/daemon/core/configservices/frrservices/services.py new file mode 100644 index 00000000..c4502f86 --- /dev/null +++ b/daemon/core/configservices/frrservices/services.py @@ -0,0 +1,391 @@ +import abc +from typing import Any, Dict + +import netaddr + +from core import constants +from core.configservice.base import ConfigService, ConfigServiceMode +from core.emane.nodes import EmaneNet +from core.nodes.base import CoreNodeBase +from core.nodes.interface import CoreInterface +from core.nodes.network import WlanNode + +GROUP = "FRR" + + +def has_mtu_mismatch(ifc: CoreInterface) -> bool: + """ + Helper to detect MTU mismatch and add the appropriate FRR + mtu-ignore command. This is needed when e.g. a node is linked via a + GreTap device. + """ + if ifc.mtu != 1500: + return True + if not ifc.net: + return False + for i in ifc.net.netifs(): + if i.mtu != ifc.mtu: + return True + return False + + +def get_min_mtu(ifc): + """ + Helper to discover the minimum MTU of interfaces linked with the + given interface. + """ + mtu = ifc.mtu + if not ifc.net: + return mtu + for i in ifc.net.netifs(): + if i.mtu < mtu: + mtu = i.mtu + return mtu + + +def get_router_id(node: CoreNodeBase) -> str: + """ + Helper to return the first IPv4 address of a node as its router ID. + """ + for ifc in node.netifs(): + if getattr(ifc, "control", False): + continue + for a in ifc.addrlist: + a = a.split("/")[0] + if netaddr.valid_ipv4(a): + return a + return "0.0.0.0" + + +class FRRZebra(ConfigService): + name = "FRRzebra" + group = GROUP + directories = ["/usr/local/etc/frr", "/var/run/frr", "/var/log/frr"] + files = [ + "/usr/local/etc/frr/frr.conf", + "frrboot.sh", + "/usr/local/etc/frr/vtysh.conf", + "/usr/local/etc/frr/daemons", + ] + executables = ["zebra"] + dependencies = [] + startup = ["sh frrboot.sh zebra"] + validate = ["pidof zebra"] + shutdown = ["killall zebra"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + + def data(self) -> Dict[str, Any]: + frr_conf = self.files[0] + frr_bin_search = self.node.session.options.get_config( + "frr_bin_search", default="/usr/local/bin /usr/bin /usr/lib/frr" + ).strip('"') + frr_sbin_search = self.node.session.options.get_config( + "frr_sbin_search", default="/usr/local/sbin /usr/sbin /usr/lib/frr" + ).strip('"') + + services = [] + want_ip4 = False + want_ip6 = False + for service in self.node.config_services.values(): + if self.name not in service.dependencies: + continue + if service.ipv4_routing: + want_ip4 = True + if service.ipv6_routing: + want_ip6 = True + services.append(service) + + interfaces = [] + for ifc in self.node.netifs(): + ip4s = [] + ip6s = [] + for x in ifc.addrlist: + addr = x.split("/")[0] + if netaddr.valid_ipv4(addr): + ip4s.append(x) + else: + ip6s.append(x) + is_control = getattr(ifc, "control", False) + interfaces.append((ifc, ip4s, ip6s, is_control)) + + return dict( + frr_conf=frr_conf, + frr_sbin_search=frr_sbin_search, + frr_bin_search=frr_bin_search, + frr_state_dir=constants.FRR_STATE_DIR, + interfaces=interfaces, + want_ip4=want_ip4, + want_ip6=want_ip6, + services=services, + ) + + +class FrrService(abc.ABC): + group = GROUP + directories = [] + files = [] + executables = [] + dependencies = ["FRRzebra"] + startup = [] + validate = [] + shutdown = [] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [] + modes = {} + ipv4_routing = False + ipv6_routing = False + + @abc.abstractmethod + def frr_interface_config(self, ifc: CoreInterface) -> str: + raise NotImplementedError + + @abc.abstractmethod + def frr_config(self) -> str: + raise NotImplementedError + + +class FRROspfv2(FrrService, ConfigService): + """ + The OSPFv2 service provides IPv4 routing for wired networks. It does + not build its own configuration file but has hooks for adding to the + unified frr.conf file. + """ + + name = "FRROSPFv2" + startup = () + shutdown = ["killall ospfd"] + validate = ["pidof ospfd"] + ipv4_routing = True + + def frr_config(self) -> str: + router_id = get_router_id(self.node) + addresses = [] + for ifc in self.node.netifs(): + if getattr(ifc, "control", False): + continue + for a in ifc.addrlist: + addr = a.split("/")[0] + if netaddr.valid_ipv4(addr): + addresses.append(a) + data = dict(router_id=router_id, addresses=addresses) + text = """ + router ospf + router-id ${router_id} + % for addr in addresses: + network ${addr} area 0 + % endfor + ! + """ + return self.render_text(text, data) + + def frr_interface_config(self, ifc: CoreInterface) -> str: + if has_mtu_mismatch(ifc): + return "ip ospf mtu-ignore" + else: + return "" + + +class FRROspfv3(FrrService, ConfigService): + """ + The OSPFv3 service provides IPv6 routing for wired networks. It does + not build its own configuration file but has hooks for adding to the + unified frr.conf file. + """ + + name = "FRROSPFv3" + shutdown = ["killall ospf6d"] + validate = ["pidof ospf6d"] + ipv4_routing = True + ipv6_routing = True + + def frr_config(self) -> str: + router_id = get_router_id(self.node) + ifnames = [] + for ifc in self.node.netifs(): + if getattr(ifc, "control", False): + continue + ifnames.append(ifc.name) + data = dict(router_id=router_id, ifnames=ifnames) + text = """ + router ospf6 + router-id ${router_id} + % for ifname in ifnames: + interface ${ifname} area 0.0.0.0 + % endfor + ! + """ + return self.render_text(text, data) + + def frr_interface_config(self, ifc: CoreInterface) -> str: + mtu = get_min_mtu(ifc) + if mtu < ifc.mtu: + return f"ipv6 ospf6 ifmtu {mtu}" + else: + return "" + + +class FRRBgp(FrrService, ConfigService): + """ + The BGP service provides interdomain routing. + Peers must be manually configured, with a full mesh for those + having the same AS number. + """ + + name = "FRRBGP" + shutdown = ["killall bgpd"] + validate = ["pidof bgpd"] + custom_needed = True + ipv4_routing = True + ipv6_routing = True + + def frr_config(self) -> str: + router_id = get_router_id(self.node) + text = f""" + ! BGP configuration + ! You should configure the AS number below + ! along with this router's peers. + router bgp {self.node.id} + bgp router-id {router_id} + redistribute connected + !neighbor 1.2.3.4 remote-as 555 + ! + """ + return self.clean_text(text) + + def frr_interface_config(self, ifc: CoreInterface) -> str: + return "" + + +class FRRRip(FrrService, ConfigService): + """ + The RIP service provides IPv4 routing for wired networks. + """ + + name = "FRRRIP" + shutdown = ["killall ripd"] + validate = ["pidof ripd"] + ipv4_routing = True + + def frr_config(self) -> str: + text = """ + router rip + redistribute static + redistribute connected + redistribute ospf + network 0.0.0.0/0 + ! + """ + return self.clean_text(text) + + def frr_interface_config(self, ifc: CoreInterface) -> str: + return "" + + +class FRRRipng(FrrService, ConfigService): + """ + The RIP NG service provides IPv6 routing for wired networks. + """ + + name = "FRRRIPNG" + shutdown = ["killall ripngd"] + validate = ["pidof ripngd"] + ipv6_routing = True + + def frr_config(self) -> str: + text = """ + router ripng + redistribute static + redistribute connected + redistribute ospf6 + network ::/0 + ! + """ + return self.clean_text(text) + + def frr_interface_config(self, ifc: CoreInterface) -> str: + return "" + + +class FRRBabel(FrrService, ConfigService): + """ + The Babel service provides a loop-avoiding distance-vector routing + protocol for IPv6 and IPv4 with fast convergence properties. + """ + + name = "FRRBabel" + shutdown = ["killall babeld"] + validate = ["pidof babeld"] + ipv6_routing = True + + def frr_config(self) -> str: + ifnames = [] + for ifc in self.node.netifs(): + if getattr(ifc, "control", False): + continue + ifnames.append(ifc.name) + text = """ + router babel + % for ifname in ifnames: + network ${ifname} + % endfor + redistribute static + redistribute ipv4 connected + ! + """ + data = dict(ifnames=ifnames) + return self.render_text(text, data) + + def frr_interface_config(self, ifc: CoreInterface) -> str: + if isinstance(ifc.net, (WlanNode, EmaneNet)): + text = """ + babel wireless + no babel split-horizon + """ + else: + text = """ + babel wired + babel split-horizon + """ + return self.clean_text(text) + + +class FRRpimd(FrrService, ConfigService): + """ + PIM multicast routing based on XORP. + """ + + name = "FRRpimd" + shutdown = ["killall pimd"] + validate = ["pidof pimd"] + ipv4_routing = True + + def frr_config(self) -> str: + ifname = "eth0" + for ifc in self.node.netifs(): + if ifc.name != "lo": + ifname = ifc.name + break + + text = f""" + router mfea + ! + router igmp + ! + router pim + !ip pim rp-address 10.0.0.1 + ip pim bsr-candidate {ifname} + ip pim rp-candidate {ifname} + !ip pim spt-threshold interval 10 bytes 80000 + ! + """ + return self.clean_text(text) + + def frr_interface_config(self, ifc: CoreInterface) -> str: + text = """ + ip mfea + ip igmp + ip pim + """ + return self.clean_text(text) diff --git a/daemon/core/configservices/frrservices/templates/daemons b/daemon/core/configservices/frrservices/templates/daemons new file mode 100644 index 00000000..0f6bda53 --- /dev/null +++ b/daemon/core/configservices/frrservices/templates/daemons @@ -0,0 +1,59 @@ +# +# When activation a daemon at the first time, a config file, even if it is +# empty, has to be present *and* be owned by the user and group "frr", else +# the daemon will not be started by /etc/init.d/frr. The permissions should +# be u=rw,g=r,o=. +# When using "vtysh" such a config file is also needed. It should be owned by +# group "frrvty" and set to ug=rw,o= though. Check /etc/pam.d/frr, too. +# +# The watchfrr and zebra daemons are always started. +# +bgpd=yes +ospfd=yes +ospf6d=yes +ripd=yes +ripngd=yes +isisd=yes +pimd=yes +ldpd=yes +nhrpd=yes +eigrpd=yes +babeld=yes +sharpd=yes +pbrd=yes +bfdd=yes +fabricd=yes + +# +# If this option is set the /etc/init.d/frr script automatically loads +# the config via "vtysh -b" when the servers are started. +# Check /etc/pam.d/frr if you intend to use "vtysh"! +# +vtysh_enable=yes +zebra_options=" -A 127.0.0.1 -s 90000000" +bgpd_options=" -A 127.0.0.1" +ospfd_options=" -A 127.0.0.1" +ospf6d_options=" -A ::1" +ripd_options=" -A 127.0.0.1" +ripngd_options=" -A ::1" +isisd_options=" -A 127.0.0.1" +pimd_options=" -A 127.0.0.1" +ldpd_options=" -A 127.0.0.1" +nhrpd_options=" -A 127.0.0.1" +eigrpd_options=" -A 127.0.0.1" +babeld_options=" -A 127.0.0.1" +sharpd_options=" -A 127.0.0.1" +pbrd_options=" -A 127.0.0.1" +staticd_options="-A 127.0.0.1" +bfdd_options=" -A 127.0.0.1" +fabricd_options="-A 127.0.0.1" + +# The list of daemons to watch is automatically generated by the init script. +#watchfrr_options="" + +# for debugging purposes, you can specify a "wrap" command to start instead +# of starting the daemon directly, e.g. to use valgrind on ospfd: +# ospfd_wrap="/usr/bin/valgrind" +# or you can use "all_wrap" for all daemons, e.g. to use perf record: +# all_wrap="/usr/bin/perf record --call-graph -" +# the normal daemon command is added to this at the end. diff --git a/daemon/core/configservices/frrservices/templates/frr.conf b/daemon/core/configservices/frrservices/templates/frr.conf new file mode 100644 index 00000000..748c8692 --- /dev/null +++ b/daemon/core/configservices/frrservices/templates/frr.conf @@ -0,0 +1,25 @@ +% for ifc, ip4s, ip6s, is_control in interfaces: +interface ${ifc.name} + % if want_ip4: + % for addr in ip4s: + ip address ${addr} + % endfor + % endif + % if want_ip6: + % for addr in ip6s: + ipv6 address ${addr} + % endfor + % endif + % if not is_control: + % for service in services: + % for line in service.frr_interface_config(ifc).split("\n"): + ${line} + % endfor + % endfor + % endif +! +% endfor + +% for service in services: +${service.frr_config()} +% endfor diff --git a/daemon/core/configservices/frrservices/templates/frrboot.sh b/daemon/core/configservices/frrservices/templates/frrboot.sh new file mode 100644 index 00000000..5a6a0e3d --- /dev/null +++ b/daemon/core/configservices/frrservices/templates/frrboot.sh @@ -0,0 +1,95 @@ +#!/bin/sh +# auto-generated by zebra service (frr.py) +FRR_CONF="${frr_conf}" +FRR_SBIN_SEARCH="${frr_sbin_search}" +FRR_BIN_SEARCH="${frr_bin_search}" +FRR_STATE_DIR="${frr_state_dir}" + +searchforprog() +{ + prog=$1 + searchpath=$@ + ret= + for p in $searchpath; do + if [ -x $p/$prog ]; then + ret=$p + break + fi + done + echo $ret +} + +confcheck() +{ + CONF_DIR=`dirname $FRR_CONF` + # if /etc/frr exists, point /etc/frr/frr.conf -> CONF_DIR + if [ "$CONF_DIR" != "/etc/frr" ] && [ -d /etc/frr ] && [ ! -e /etc/frr/frr.conf ]; then + ln -s $CONF_DIR/frr.conf /etc/frr/frr.conf + fi + # if /etc/frr exists, point /etc/frr/vtysh.conf -> CONF_DIR + if [ "$CONF_DIR" != "/etc/frr" ] && [ -d /etc/frr ] && [ ! -e /etc/frr/vtysh.conf ]; then + ln -s $CONF_DIR/vtysh.conf /etc/frr/vtysh.conf + fi +} + +bootdaemon() +{ + FRR_SBIN_DIR=$(searchforprog $1 $FRR_SBIN_SEARCH) + if [ "z$FRR_SBIN_DIR" = "z" ]; then + echo "ERROR: FRR's '$1' daemon not found in search path:" + echo " $FRR_SBIN_SEARCH" + return 1 + fi + + flags="" + + if [ "$1" = "pimd" ] && \\ + grep -E -q '^[[:space:]]*router[[:space:]]+pim6[[:space:]]*$' $FRR_CONF; then + flags="$flags -6" + fi + + #force FRR to use CORE generated conf file + flags="$flags -d -f $FRR_CONF" + $FRR_SBIN_DIR/$1 $flags + + if [ "$?" != "0" ]; then + echo "ERROR: FRR's '$1' daemon failed to start!:" + return 1 + fi +} + +bootfrr() +{ + FRR_BIN_DIR=$(searchforprog 'vtysh' $FRR_BIN_SEARCH) + if [ "z$FRR_BIN_DIR" = "z" ]; then + echo "ERROR: FRR's 'vtysh' program not found in search path:" + echo " $FRR_BIN_SEARCH" + return 1 + fi + + # fix /var/run/frr permissions + id -u frr 2>/dev/null >/dev/null + if [ "$?" = "0" ]; then + chown frr $FRR_STATE_DIR + fi + + bootdaemon "zebra" + for r in rip ripng ospf6 ospf bgp babel; do + if grep -q "^router \\<$${}{r}\\>" $FRR_CONF; then + bootdaemon "$${}{r}d" + fi + done + + if grep -E -q '^[[:space:]]*router[[:space:]]+pim6?[[:space:]]*$' $FRR_CONF; then + bootdaemon "pimd" + fi + + $FRR_BIN_DIR/vtysh -b +} + +if [ "$1" != "zebra" ]; then + echo "WARNING: '$1': all FRR daemons are launched by the 'zebra' service!" + exit 1 +fi +confcheck +bootfrr diff --git a/daemon/core/configservices/frrservices/templates/vtysh.conf b/daemon/core/configservices/frrservices/templates/vtysh.conf new file mode 100644 index 00000000..e0ab9cb6 --- /dev/null +++ b/daemon/core/configservices/frrservices/templates/vtysh.conf @@ -0,0 +1 @@ +service integrated-vtysh-config diff --git a/daemon/core/configservices/quaggaservices/services.py b/daemon/core/configservices/quaggaservices/services.py index bc937038..32ce99be 100644 --- a/daemon/core/configservices/quaggaservices/services.py +++ b/daemon/core/configservices/quaggaservices/services.py @@ -274,6 +274,9 @@ class Bgp(QuaggaService, ConfigService): ipv4_routing = True ipv6_routing = True + def quagga_config(self) -> str: + return "" + def quagga_interface_config(self, ifc: CoreInterface) -> str: router_id = get_router_id(self.node) text = f""" @@ -310,6 +313,9 @@ class Rip(QuaggaService, ConfigService): """ return self.clean_text(text) + def quagga_interface_config(self, ifc: CoreInterface) -> str: + return "" + class Ripng(QuaggaService, ConfigService): """ @@ -332,6 +338,9 @@ class Ripng(QuaggaService, ConfigService): """ return self.clean_text(text) + def quagga_interface_config(self, ifc: CoreInterface) -> str: + return "" + class Babel(QuaggaService, ConfigService): """ From 6f2a84071041b1c3b9d14827b6cc6ed8284e2bc6 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 10:59:47 -0800 Subject: [PATCH 28/37] update simple service to show all config types --- daemon/core/configservices/simpleservice.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py index 3284471d..e727fe82 100644 --- a/daemon/core/configservices/simpleservice.py +++ b/daemon/core/configservices/simpleservice.py @@ -15,14 +15,19 @@ class SimpleService(ConfigService): shutdown = [] validation_mode = ConfigServiceMode.BLOCKING default_configs = [ - Configuration(_id="value1", _type=ConfigDataTypes.STRING, label="Value 1"), - Configuration(_id="value2", _type=ConfigDataTypes.STRING, label="Value 2"), - Configuration(_id="value3", _type=ConfigDataTypes.STRING, label="Value 3"), + Configuration(_id="value1", _type=ConfigDataTypes.STRING, label="Text"), + Configuration(_id="value2", _type=ConfigDataTypes.BOOL, label="Boolean"), + Configuration( + _id="value3", + _type=ConfigDataTypes.STRING, + label="Multiple Choice", + options=["value1", "value2", "value3"], + ), ] modes = { - "mode1": {"value1": "m1", "value2": "m1", "value3": "m1"}, - "mode2": {"value1": "m2", "value2": "m2", "value3": "m2"}, - "mode3": {"value1": "m3", "value2": "m3", "value3": "m3"}, + "mode1": {"value1": "value1", "value2": "0", "value3": "value2"}, + "mode2": {"value1": "value2", "value2": "1", "value3": "value3"}, + "mode3": {"value1": "value3", "value2": "0", "value3": "value1"}, } def get_text_template(self, name: str) -> str: From 93ad6b588a9a140838484870b0083ca915730bd3 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 12:51:57 -0800 Subject: [PATCH 29/37] updates to coretk to fix displaying boolean configs properly and updating config values for config services when changing mode --- daemon/core/gui/widgets.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/daemon/core/gui/widgets.py b/daemon/core/gui/widgets.py index e0dc4b25..9c07e8c7 100644 --- a/daemon/core/gui/widgets.py +++ b/daemon/core/gui/widgets.py @@ -99,7 +99,7 @@ class ConfigFrame(ttk.Notebook): label.grid(row=index, pady=PADY, padx=PADX, sticky="w") value = tk.StringVar() if option.type == core_pb2.ConfigOptionType.BOOL: - select = tuple(option.select) + select = ("On", "Off") combobox = ttk.Combobox( tab.frame, textvariable=value, values=select, state="readonly" ) @@ -186,7 +186,13 @@ class ConfigFrame(ttk.Notebook): def set_values(self, config: Dict[str, str]) -> None: for name, data in config.items(): + option = self.config[name] value = self.values[name] + if option.type == core_pb2.ConfigOptionType.BOOL: + if data == "1": + data = "On" + else: + data = "Off" value.set(data) From a85cb92196b11e2337689a0446e5e08bc87f3f5a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 13:19:08 -0800 Subject: [PATCH 30/37] removed options not needed for config data for GUI support for both old and new GUIs --- daemon/core/emane/bypass.py | 1 - daemon/core/emulator/sessionconfig.py | 3 --- daemon/core/location/mobility.py | 6 +----- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/daemon/core/emane/bypass.py b/daemon/core/emane/bypass.py index 24f5d45a..8454d8ce 100644 --- a/daemon/core/emane/bypass.py +++ b/daemon/core/emane/bypass.py @@ -21,7 +21,6 @@ class EmaneBypassModel(emanemodel.EmaneModel): _id="none", _type=ConfigDataTypes.BOOL, default="0", - options=["True", "False"], label="There are no parameters for the bypass model.", ) ] diff --git a/daemon/core/emulator/sessionconfig.py b/daemon/core/emulator/sessionconfig.py index 38322efd..5f2d5916 100644 --- a/daemon/core/emulator/sessionconfig.py +++ b/daemon/core/emulator/sessionconfig.py @@ -36,21 +36,18 @@ class SessionConfig(ConfigurableManager, ConfigurableOptions): _id="enablerj45", _type=ConfigDataTypes.BOOL, default="1", - options=["On", "Off"], label="Enable RJ45s", ), Configuration( _id="preservedir", _type=ConfigDataTypes.BOOL, default="0", - options=["On", "Off"], label="Preserve session dir", ), Configuration( _id="enablesdt", _type=ConfigDataTypes.BOOL, default="0", - options=["On", "Off"], label="Enable SDT3D output", ), Configuration( diff --git a/daemon/core/location/mobility.py b/daemon/core/location/mobility.py index 703eaa9f..4689d217 100644 --- a/daemon/core/location/mobility.py +++ b/daemon/core/location/mobility.py @@ -906,11 +906,7 @@ class Ns2ScriptedMobility(WayPointMobility): label="refresh time (ms)", ), Configuration( - _id="loop", - _type=ConfigDataTypes.BOOL, - default="1", - options=["On", "Off"], - label="loop", + _id="loop", _type=ConfigDataTypes.BOOL, default="1", label="loop" ), Configuration( _id="autostart", From 05a60f684e7f6f2d0524e5cd7f33bcced85eee83 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 16:01:44 -0800 Subject: [PATCH 31/37] updated config service logging to debug --- daemon/core/configservice/base.py | 10 +--------- daemon/core/configservice/manager.py | 4 ++-- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 39685768..b0033a0c 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -38,7 +38,6 @@ class ConfigService(abc.ABC): self.node = node class_file = inspect.getfile(self.__class__) templates_path = pathlib.Path(class_file).parent.joinpath(TEMPLATES_DIR) - logging.info(templates_path) self.templates = TemplateLookup(directories=templates_path) self.config = {} self.custom_templates = {} @@ -182,7 +181,7 @@ class ConfigService(abc.ABC): else: text = self.get_text_template(name) rendered = self.render_text(text, data) - logging.info( + logging.debug( "node(%s) service(%s) template(%s): \n%s", self.node.name, self.name, @@ -241,13 +240,6 @@ class ConfigService(abc.ABC): ) def render_template(self, basename: str, data: Dict[str, Any] = None) -> str: - logging.info( - "node(%s) service(%s) rendering template(%s): %s", - self.node.name, - self.name, - basename, - data, - ) try: template = self.templates.get_template(basename) return self._render(template, data) diff --git a/daemon/core/configservice/manager.py b/daemon/core/configservice/manager.py index 2cdbd817..af98e685 100644 --- a/daemon/core/configservice/manager.py +++ b/daemon/core/configservice/manager.py @@ -43,10 +43,10 @@ class ConfigServiceManager: subdirs.append(path) service_errors = [] for subdir in subdirs: - logging.info("loading config services from: %s", subdir) + logging.debug("loading config services from: %s", subdir) services = utils.load_classes(str(subdir), ConfigService) for service in services: - logging.info("found service: %s", service) + logging.debug("found service: %s", service) try: self.add(service) except CoreError as e: From b02f2251cd9053dd8e09f3d5da0ca5daef3510ab Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 16:32:36 -0800 Subject: [PATCH 32/37] added config service template files to MANIFEST file for core daemon --- daemon/MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/MANIFEST.in b/daemon/MANIFEST.in index 40dbefc8..c46dc828 100644 --- a/daemon/MANIFEST.in +++ b/daemon/MANIFEST.in @@ -1 +1,2 @@ graft core/gui/data +graft core/configservices/*/templates From bc2fb628cc21abe698c8a3b5105d2983952b0a46 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 17:12:14 -0800 Subject: [PATCH 33/37] updates to Pipefiles, requirements.txt, and setup.py to account for addition of mako dependency --- daemon/Pipfile | 1 - daemon/Pipfile.lock | 25 ++++++++++++------------- daemon/requirements.txt | 14 ++++++++------ daemon/setup.py.in | 1 + 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/daemon/Pipfile b/daemon/Pipfile index 63a1d240..d55b248f 100644 --- a/daemon/Pipfile +++ b/daemon/Pipfile @@ -21,4 +21,3 @@ mock = "*" [packages] core = {editable = true,path = "."} -mako = "*" diff --git a/daemon/Pipfile.lock b/daemon/Pipfile.lock index aa6bcd51..b3cacedc 100644 --- a/daemon/Pipfile.lock +++ b/daemon/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3d8bcb3cbfddc5803b6f2309330c77e815b44ed81f42644f4d42b625e65b5d24" + "sha256": "d702e6eed5a1362bf261543572bbffd2e8a87140b8d8cb07b99fb0d25220a2b5" }, "pipfile-spec": 6, "requires": {}, @@ -201,10 +201,9 @@ }, "mako": { "hashes": [ - "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b" + "sha256:2984a6733e1d472796ceef37ad48c26f4a984bb18119bb2dbc37a44d8f6e75a4" ], - "index": "pypi", - "version": "==1.1.0" + "version": "==1.1.1" }, "markupsafe": { "hashes": [ @@ -576,10 +575,10 @@ }, "packaging": { "hashes": [ - "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", - "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" + "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73", + "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334" ], - "version": "==20.0" + "version": "==20.1" }, "pluggy": { "hashes": [ @@ -649,11 +648,11 @@ }, "pytest": { "hashes": [ - "sha256:9f8d44f4722b3d06b41afaeb8d177cfbe0700f8351b1fc755dd27eedaa3eb9e0", - "sha256:f5d3d0e07333119fe7d4af4ce122362dc4053cdd34a71d2766290cf5369c64ad" + "sha256:1d122e8be54d1a709e56f82e2d85dcba3018313d64647f38a91aec88c239b600", + "sha256:c13d1943c63e599b98cf118fcb9703e4d7bde7caa9a432567bcdcae4bf512d20" ], "index": "pypi", - "version": "==5.3.3" + "version": "==5.3.4" }, "pyyaml": { "hashes": [ @@ -701,10 +700,10 @@ }, "zipp": { "hashes": [ - "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", - "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" + "sha256:b338014b9bc7102ca69e0fb96ed07215a8954d2989bc5d83658494ab2ba634af", + "sha256:e013e7800f60ec4dde789ebf4e9f7a54236e4bbf5df2a1a4e20ce9e1d9609d67" ], - "version": "==1.0.0" + "version": "==2.0.1" } } } diff --git a/daemon/requirements.txt b/daemon/requirements.txt index eaddd657..d0defc6b 100644 --- a/daemon/requirements.txt +++ b/daemon/requirements.txt @@ -3,13 +3,15 @@ cffi==1.13.2 cryptography==2.8 fabric==2.5.0 grpcio==1.26.0 -invoke==1.3.0 +invoke==1.4.0 lxml==4.4.2 +Mako==1.1.1 +MarkupSafe==1.1.1 netaddr==0.7.19 paramiko==2.7.1 -pillow==6.2.1 -protobuf==3.11.1 +Pillow==7.0.0 +protobuf==3.11.2 pycparser==2.19 -pynacl==1.3.0 -pyyaml==5.2 -six==1.13.0 +PyNaCl==1.3.0 +PyYAML==5.3 +six==1.14.0 diff --git a/daemon/setup.py.in b/daemon/setup.py.in index e0faf01d..bdef71ab 100644 --- a/daemon/setup.py.in +++ b/daemon/setup.py.in @@ -39,6 +39,7 @@ setup( "netaddr", "invoke", "lxml", + "mako", "pillow", "protobuf", "pyyaml", From 22ff48182a7262de0063b6f2d1e9a668f26e20bd Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 17:27:12 -0800 Subject: [PATCH 34/37] added some basic documentation to the config service manager and dependency finder --- daemon/core/configservice/dependencies.py | 30 +++++++++++++++++++++-- daemon/core/configservice/manager.py | 27 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/daemon/core/configservice/dependencies.py b/daemon/core/configservice/dependencies.py index 7f62a563..49c8041c 100644 --- a/daemon/core/configservice/dependencies.py +++ b/daemon/core/configservice/dependencies.py @@ -7,11 +7,15 @@ if TYPE_CHECKING: class ConfigServiceDependencies: """ - Can generate boot paths for services, based on their dependencies. Will validate - that all services will be booted and that all dependencies exist within the services provided. + Generates sets of services to start in order of their dependencies. """ def __init__(self, services: Dict[str, "ConfigService"]) -> None: + """ + Create a ConfigServiceDependencies instance. + + :param services: services for determining dependency sets + """ # helpers to check validity self.dependents = {} self.booted = set() @@ -28,6 +32,11 @@ class ConfigServiceDependencies: self.visiting = set() def boot_paths(self) -> List[List["ConfigService"]]: + """ + Find services sets based on dependencies. + + :return: lists of lists of services that can be started in parallel + """ paths = [] for name in self.node_services: service = self.node_services[name] @@ -50,16 +59,33 @@ class ConfigServiceDependencies: return paths def _reset(self) -> None: + """ + Clear out metadata used for finding service dependency sets. + + :return: nothing + """ self.path = [] self.visited.clear() self.visiting.clear() def _start(self, service: "ConfigService") -> List["ConfigService"]: + """ + Starts a oath for checking dependencies for a given service. + + :param service: service to check dependencies for + :return: list of config services to start in order + """ logging.debug("starting service dependency check: %s", service.name) self._reset() return self._visit(service) def _visit(self, current_service: "ConfigService") -> List["ConfigService"]: + """ + Visits a service when discovering dependency chains for service. + + :param current_service: service being visited + :return: list of dependent services for a visited service + """ logging.debug("visiting service(%s): %s", current_service.name, self.path) self.visited.add(current_service.name) self.visiting.add(current_service.name) diff --git a/daemon/core/configservice/manager.py b/daemon/core/configservice/manager.py index af98e685..1f806f7b 100644 --- a/daemon/core/configservice/manager.py +++ b/daemon/core/configservice/manager.py @@ -8,16 +8,37 @@ from core.errors import CoreError class ConfigServiceManager: + """ + Manager for configurable services. + """ + def __init__(self): + """ + Create a ConfigServiceManager instance. + """ self.services = {} def get_service(self, name: str) -> Type[ConfigService]: + """ + Retrieve a service by name. + + :param name: name of service + :return: service class + :raises CoreError: when service is not found + """ service_class = self.services.get(name) if service_class is None: raise CoreError(f"service does not exit {name}") return service_class def add(self, service: ConfigService) -> None: + """ + Add service to manager, checking service requirements have been met. + + :param service: service to add to manager + :return: nothing + :raises CoreError: when service is a duplicate or has unmet executables + """ name = service.name logging.debug("loading service: class(%s) name(%s)", service.__class__, name) @@ -38,6 +59,12 @@ class ConfigServiceManager: self.services[name] = service def load(self, path: str) -> List[str]: + """ + Search path provided for configurable services and add them for being managed. + + :param path: path to search configurable services + :return: list errors when loading and adding services + """ path = pathlib.Path(path) subdirs = [x for x in path.iterdir() if x.is_dir()] subdirs.append(path) From 80073dd8c88f4b1417c3ca793e4506fea9a5747c Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 24 Jan 2020 22:15:21 -0800 Subject: [PATCH 35/37] adding documentation to config service base class --- daemon/core/configservice/base.py | 121 ++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index b0033a0c..47cd6f62 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -28,6 +28,10 @@ class ConfigServiceBootError(Exception): class ConfigService(abc.ABC): + """ + Base class for creating configurable services. + """ + # validation period in seconds, how frequent validation is attempted validation_period = 0.5 @@ -35,6 +39,11 @@ class ConfigService(abc.ABC): validation_timer = 5 def __init__(self, node: CoreNode) -> None: + """ + Create ConfigService instance. + + :param node: node this service is assigned to + """ self.node = node class_file = inspect.getfile(self.__class__) templates_path = pathlib.Path(class_file).parent.joinpath(TEMPLATES_DIR) @@ -47,6 +56,13 @@ class ConfigService(abc.ABC): @staticmethod def clean_text(text: str) -> str: + """ + Returns space stripped text for string literals, while keeping space + indentations. + + :param text: text to clean + :return: cleaned text + """ return inspect.cleandoc(text) @property @@ -110,6 +126,13 @@ class ConfigService(abc.ABC): raise NotImplementedError def start(self) -> None: + """ + Creates services files/directories, runs startup, and validates based on + validation mode. + + :return: nothing + :raises ConfigServiceBootError: when there is an error starting service + """ logging.info("node(%s) service(%s) starting...", self.node.name, self.name) self.create_dirs() self.create_files() @@ -122,6 +145,11 @@ class ConfigService(abc.ABC): self.run_validation() def stop(self) -> None: + """ + Stop service using shutdown commands. + + :return: nothing + """ for cmd in self.shutdown: try: self.node.cmd(cmd) @@ -132,10 +160,21 @@ class ConfigService(abc.ABC): ) def restart(self) -> None: + """ + Restarts service by running stop and then start. + + :return: nothing + """ self.stop() self.start() def create_dirs(self) -> None: + """ + Creates directories for service. + + :return: nothing + :raises CoreError: when there is a failure creating a directory + """ for directory in self.directories: try: self.node.privatedir(directory) @@ -146,15 +185,39 @@ class ConfigService(abc.ABC): ) def data(self) -> Dict[str, Any]: + """ + Returns key/value data, used when rendering file templates. + + :return: key/value template data + """ return {} def set_template(self, name: str, template: str) -> None: + """ + Store custom template to render for a given file. + + :param name: file to store custom template for + :param template: custom template to render + :return: nothing + """ self.custom_templates[name] = template def get_text_template(self, name: str) -> str: + """ + Retrieves text based template for files that do not have a file based template. + + :param name: name of file to get template for + :return: template to render + """ raise CoreError(f"service({self.name}) unknown template({name})") def get_templates(self) -> Dict[str, str]: + """ + Retrieves mapping of file names to templates for all cases, which + includes custom templates, file templates, and text templates. + + :return: mapping of files to templates + """ templates = {} for name in self.files: basename = pathlib.Path(name).name @@ -170,6 +233,11 @@ class ConfigService(abc.ABC): return templates def create_files(self) -> None: + """ + Creates service files inside associated node. + + :return: nothing + """ data = self.data() for name in self.files: basename = pathlib.Path(name).name @@ -191,6 +259,14 @@ class ConfigService(abc.ABC): self.node.nodefile(name, rendered) def run_startup(self, wait: bool) -> None: + """ + Run startup commands for service on node. + + :param wait: wait successful command exit status when True, ignore status + otherwise + :return: nothing + :raises ConfigServiceBootError: when a command that waits fails + """ for cmd in self.startup: try: self.node.cmd(cmd, wait=wait) @@ -200,6 +276,12 @@ class ConfigService(abc.ABC): ) def run_validation(self) -> None: + """ + Runs validation commands for service on node. + + :return: nothing + :raises ConfigServiceBootError: if there is a validation failure + """ start = time.monotonic() cmds = self.validate[:] index = 0 @@ -222,6 +304,13 @@ class ConfigService(abc.ABC): ) def _render(self, template: Template, data: Dict[str, Any] = None) -> str: + """ + Renders template providing all associated data to template. + + :param template: template to render + :param data: service specific defined data for template + :return: rendered template + """ if data is None: data = {} return template.render_unicode( @@ -229,6 +318,13 @@ class ConfigService(abc.ABC): ) def render_text(self, text: str, data: Dict[str, Any] = None) -> str: + """ + Renders text based template providing all associated data to template. + + :param text: text to render + :param data: service specific defined data for template + :return: rendered template + """ text = self.clean_text(text) try: template = Template(text) @@ -240,6 +336,13 @@ class ConfigService(abc.ABC): ) def render_template(self, basename: str, data: Dict[str, Any] = None) -> str: + """ + Renders file based template providing all associated data to template. + + :param basename: base name for file to render + :param data: service specific defined data for template + :return: rendered template + """ try: template = self.templates.get_template(basename) return self._render(template, data) @@ -250,16 +353,34 @@ class ConfigService(abc.ABC): ) def _define_config(self, configs: List[Configuration]) -> None: + """ + Initializes default configuration data. + + :param configs: configs to initialize + :return: nothing + """ for config in configs: self.config[config.id] = config def render_config(self) -> Dict[str, str]: + """ + Returns configuration data key/value pairs for rendering a template. + + :return: nothing + """ if self.custom_config: return self.custom_config else: return {k: v.default for k, v in self.config.items()} def set_config(self, data: Dict[str, str]) -> None: + """ + Set configuration data from key/value pairs. + + :param data: configuration key/values to set + :return: nothing + :raise CoreError: when an unknown configuration value is given + """ for key, value in data.items(): if key not in self.config: raise CoreError(f"unknown config: {key}") From 09aa882017a99a1590f6fa860f90d7b04a2aa282 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 27 Jan 2020 10:57:16 -0800 Subject: [PATCH 36/37] fixed config service to leverage validation_timer properly, added unit tests for config service base class --- daemon/core/configservice/base.py | 12 +- daemon/tests/test_config_services.py | 300 +++++++++++++++++++++++++++ 2 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 daemon/tests/test_config_services.py diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 47cd6f62..82598988 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -140,7 +140,7 @@ class ConfigService(abc.ABC): self.run_startup(wait) if not wait: if self.validation_mode == ConfigServiceMode.TIMER: - time.sleep(self.validation_timer) + self.wait_validation() else: self.run_validation() @@ -275,6 +275,14 @@ class ConfigService(abc.ABC): f"node({self.node.name}) service({self.name}) failed startup: {e}" ) + def wait_validation(self) -> None: + """ + Waits for a period of time to consider service started successfully. + + :return: nothing + """ + time.sleep(self.validation_timer) + def run_validation(self) -> None: """ Runs validation commands for service on node. @@ -298,7 +306,7 @@ class ConfigService(abc.ABC): ) time.sleep(self.validation_period) - if cmds and time.monotonic() - start > 0: + if cmds and time.monotonic() - start > self.validation_timer: raise ConfigServiceBootError( f"node({self.node.name}) service({self.name}) failed to validate" ) diff --git a/daemon/tests/test_config_services.py b/daemon/tests/test_config_services.py new file mode 100644 index 00000000..eaba4d47 --- /dev/null +++ b/daemon/tests/test_config_services.py @@ -0,0 +1,300 @@ +from unittest import mock + +import pytest + +from core.config import Configuration +from core.configservice.base import ( + ConfigService, + ConfigServiceBootError, + ConfigServiceMode, +) +from core.emulator.enumerations import ConfigDataTypes +from core.errors import CoreCommandError, CoreError + +TEMPLATE_TEXT = "echo hello" + + +class MyService(ConfigService): + name = "MyService" + group = "MyGroup" + directories = ["/usr/local/lib"] + files = ["test.sh"] + executables = [] + dependencies = [] + startup = [f"sh {files[0]}"] + validate = [f"pidof {files[0]}"] + shutdown = [f"pkill {files[0]}"] + validation_mode = ConfigServiceMode.BLOCKING + default_configs = [ + Configuration(_id="value1", _type=ConfigDataTypes.STRING, label="Text"), + Configuration(_id="value2", _type=ConfigDataTypes.BOOL, label="Boolean"), + Configuration( + _id="value3", + _type=ConfigDataTypes.STRING, + label="Multiple Choice", + options=["value1", "value2", "value3"], + ), + ] + modes = { + "mode1": {"value1": "value1", "value2": "0", "value3": "value2"}, + "mode2": {"value1": "value2", "value2": "1", "value3": "value3"}, + "mode3": {"value1": "value3", "value2": "0", "value3": "value1"}, + } + + def get_text_template(self, name: str) -> str: + return TEMPLATE_TEXT + + +class TestConfigServices: + def test_set_template(self): + # given + node = mock.MagicMock() + text = "echo custom" + service = MyService(node) + + # when + service.set_template(MyService.files[0], text) + + # then + assert MyService.files[0] in service.custom_templates + assert service.custom_templates[MyService.files[0]] == text + + def test_create_directories(self): + # given + node = mock.MagicMock() + service = MyService(node) + + # when + service.create_dirs() + + # then + node.privatedir.assert_called_with(MyService.directories[0]) + + def test_create_files_custom(self): + # given + node = mock.MagicMock() + service = MyService(node) + text = "echo custom" + service.set_template(MyService.files[0], text) + + # when + service.create_files() + + # then + node.nodefile.assert_called_with(MyService.files[0], text) + + def test_create_files_text(self): + # given + node = mock.MagicMock() + service = MyService(node) + + # when + service.create_files() + + # then + node.nodefile.assert_called_with(MyService.files[0], TEMPLATE_TEXT) + + def test_run_startup(self): + # given + node = mock.MagicMock() + wait = True + service = MyService(node) + + # when + service.run_startup(wait=wait) + + # then + node.cmd.assert_called_with(MyService.startup[0], wait=wait) + + def test_run_startup_exception(self): + # given + node = mock.MagicMock() + node.cmd.side_effect = CoreCommandError(1, "error") + service = MyService(node) + + # when + with pytest.raises(ConfigServiceBootError): + service.run_startup(wait=True) + + def test_shutdown(self): + # given + node = mock.MagicMock() + service = MyService(node) + + # when + service.stop() + + # then + node.cmd.assert_called_with(MyService.shutdown[0]) + + def test_run_validation(self): + # given + node = mock.MagicMock() + service = MyService(node) + + # when + service.run_validation() + + # then + node.cmd.assert_called_with(MyService.validate[0]) + + def test_run_validation_timer(self): + # given + node = mock.MagicMock() + service = MyService(node) + service.validation_mode = ConfigServiceMode.TIMER + service.validation_timer = 0 + + # when + service.run_validation() + + # then + node.cmd.assert_called_with(MyService.validate[0]) + + def test_run_validation_timer_exception(self): + # given + node = mock.MagicMock() + node.cmd.side_effect = CoreCommandError(1, "error") + service = MyService(node) + service.validation_mode = ConfigServiceMode.TIMER + service.validation_period = 0 + service.validation_timer = 0 + + # when + with pytest.raises(ConfigServiceBootError): + service.run_validation() + + def test_run_validation_non_blocking(self): + # given + node = mock.MagicMock() + service = MyService(node) + service.validation_mode = ConfigServiceMode.NON_BLOCKING + service.validation_period = 0 + service.validation_timer = 0 + + # when + service.run_validation() + + # then + node.cmd.assert_called_with(MyService.validate[0]) + + def test_run_validation_non_blocking_exception(self): + # given + node = mock.MagicMock() + node.cmd.side_effect = CoreCommandError(1, "error") + service = MyService(node) + service.validation_mode = ConfigServiceMode.NON_BLOCKING + service.validation_period = 0 + service.validation_timer = 0 + + # when + with pytest.raises(ConfigServiceBootError): + service.run_validation() + + def test_render_config(self): + # given + node = mock.MagicMock() + service = MyService(node) + + # when + config = service.render_config() + + # then + assert config == {"value1": "", "value2": "", "value3": ""} + + def test_render_config_custom(self): + # given + node = mock.MagicMock() + service = MyService(node) + custom_config = {"value1": "1", "value2": "2", "value3": "3"} + service.set_config(custom_config) + + # when + config = service.render_config() + + # then + assert config == custom_config + + def test_set_config(self): + # given + node = mock.MagicMock() + service = MyService(node) + custom_config = {"value1": "1", "value2": "2", "value3": "3"} + + # when + service.set_config(custom_config) + + # then + assert service.custom_config == custom_config + + def test_set_config_exception(self): + # given + node = mock.MagicMock() + service = MyService(node) + custom_config = {"value4": "1"} + + # when + with pytest.raises(CoreError): + service.set_config(custom_config) + + def test_start_blocking(self): + # given + node = mock.MagicMock() + service = MyService(node) + service.create_dirs = mock.MagicMock() + service.create_files = mock.MagicMock() + service.run_startup = mock.MagicMock() + service.run_validation = mock.MagicMock() + service.wait_validation = mock.MagicMock() + + # when + service.start() + + # then + service.create_files.assert_called_once() + service.create_dirs.assert_called_once() + service.run_startup.assert_called_once() + service.run_validation.assert_not_called() + service.wait_validation.assert_not_called() + + def test_start_timer(self): + # given + node = mock.MagicMock() + service = MyService(node) + service.validation_mode = ConfigServiceMode.TIMER + service.create_dirs = mock.MagicMock() + service.create_files = mock.MagicMock() + service.run_startup = mock.MagicMock() + service.run_validation = mock.MagicMock() + service.wait_validation = mock.MagicMock() + + # when + service.start() + + # then + service.create_files.assert_called_once() + service.create_dirs.assert_called_once() + service.run_startup.assert_called_once() + service.run_validation.assert_not_called() + service.wait_validation.assert_called_once() + + def test_start_non_blocking(self): + # given + node = mock.MagicMock() + service = MyService(node) + service.validation_mode = ConfigServiceMode.NON_BLOCKING + service.create_dirs = mock.MagicMock() + service.create_files = mock.MagicMock() + service.run_startup = mock.MagicMock() + service.run_validation = mock.MagicMock() + service.wait_validation = mock.MagicMock() + + # when + service.start() + + # then + service.create_files.assert_called_once() + service.create_dirs.assert_called_once() + service.run_startup.assert_called_once() + service.run_validation.assert_called_once() + service.wait_validation.assert_not_called() From 45fb32c8349e840e11a19f8334d4177c40667e04 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Mon, 27 Jan 2020 11:44:00 -0800 Subject: [PATCH 37/37] renamed config service dependency finder method to startup_paths, added pydocs to config service related methods --- daemon/core/api/grpc/server.py | 57 ++++++++++++++++++++++- daemon/core/configservice/dependencies.py | 20 ++++---- daemon/core/nodes/base.py | 27 +++++++++-- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index eee6a446..3a207ce4 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -101,6 +101,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): :param context: :return: session object that satisfies, if session not found then raise an exception + :raises Exception: raises grpc exception when session does not exist """ session = self.coreemu.sessions.get(session_id) if not session: @@ -117,6 +118,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): :param node_id: node id :param context: :return: node object that satisfies. If node not found then raise an exception. + :raises Exception: raises grpc exception when node does not exist """ try: return session.get_node(node_id) @@ -126,6 +128,14 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): def validate_service( self, name: str, context: ServicerContext ) -> Type[ConfigService]: + """ + Validates a configuration service is a valid known service. + + :param name: name of service to validate + :param context: grpc context + :return: class for service to validate + :raises Exception: raises grpc exception when service does not exist + """ service = self.coreemu.service_manager.services.get(name) if not service: context.abort(grpc.StatusCode.NOT_FOUND, f"unknown service {name}") @@ -138,7 +148,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): Start a session. :param request: start session request - :param context: grcp context + :param context: grpc context :return: start session response """ logging.debug("start session: %s", request) @@ -235,7 +245,7 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): Stop a running session. :param request: stop session request - :param context: grcp context + :param context: grpc context :return: stop session response """ logging.debug("stop session: %s", request) @@ -1475,6 +1485,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): def GetConfigServices( self, request: GetConfigServicesRequest, context: ServicerContext ) -> GetConfigServicesResponse: + """ + Gets all currently known configuration services. + + :param request: get config services request + :param context: grpc context + :return: get config services response + """ services = [] for service in self.coreemu.service_manager.services.values(): service_proto = ConfigService( @@ -1497,6 +1514,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): def GetNodeConfigService( self, request: GetNodeConfigServiceRequest, context: ServicerContext ) -> GetNodeConfigServiceResponse: + """ + Gets configuration, for a given configuration service, for a given node. + + :param request: get node config service request + :param context: grpc context + :return: get node config service response + """ session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) self.validate_service(request.name, context) @@ -1511,6 +1535,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): def GetConfigServiceDefaults( self, request: GetConfigServiceDefaultsRequest, context: ServicerContext ) -> GetConfigServiceDefaultsResponse: + """ + Get default values for a given configuration service. + + :param request: get config service defaults request + :param context: grpc context + :return: get config service defaults response + """ service_class = self.validate_service(request.name, context) service = service_class(None) templates = service.get_templates() @@ -1536,6 +1567,14 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): def GetNodeConfigServiceConfigs( self, request: GetNodeConfigServiceConfigsRequest, context: ServicerContext ) -> GetNodeConfigServiceConfigsResponse: + """ + Get current custom templates and config for configuration services for a given + node. + + :param request: get node config service configs request + :param context: grpc context + :return: get node config service configs response + """ session = self.get_session(request.session_id, context) configs = [] for node in session.nodes.values(): @@ -1557,6 +1596,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): def GetNodeConfigServices( self, request: GetNodeConfigServicesRequest, context: ServicerContext ) -> GetNodeConfigServicesResponse: + """ + Get configuration services for a given node. + + :param request: get node config services request + :param context: grpc context + :return: get node config services response + """ session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) services = node.config_services.keys() @@ -1565,6 +1611,13 @@ class CoreGrpcServer(core_pb2_grpc.CoreApiServicer): def SetNodeConfigService( self, request: SetNodeConfigServiceRequest, context: ServicerContext ) -> SetNodeConfigServiceResponse: + """ + Set custom config, for a given configuration service, for a given node. + + :param request: set node config service request + :param context: grpc context + :return: set node config service response + """ session = self.get_session(request.session_id, context) node = self.get_node(session, request.node_id, context) self.validate_service(request.name, context) diff --git a/daemon/core/configservice/dependencies.py b/daemon/core/configservice/dependencies.py index 49c8041c..92eede79 100644 --- a/daemon/core/configservice/dependencies.py +++ b/daemon/core/configservice/dependencies.py @@ -18,7 +18,7 @@ class ConfigServiceDependencies: """ # helpers to check validity self.dependents = {} - self.booted = set() + self.started = set() self.node_services = {} for service in services.values(): self.node_services[service.name] = service @@ -31,18 +31,18 @@ class ConfigServiceDependencies: self.visited = set() self.visiting = set() - def boot_paths(self) -> List[List["ConfigService"]]: + def startup_paths(self) -> List[List["ConfigService"]]: """ - Find services sets based on dependencies. + Find startup path sets based on service dependencies. :return: lists of lists of services that can be started in parallel """ paths = [] for name in self.node_services: service = self.node_services[name] - if service.name in self.booted: + if service.name in self.started: logging.debug( - "skipping service that will already be booted: %s", service.name + "skipping service that will already be started: %s", service.name ) continue @@ -50,10 +50,10 @@ class ConfigServiceDependencies: if path: paths.append(path) - if self.booted != set(self.node_services): + if self.started != set(self.node_services): raise ValueError( - "failure to boot all services: %s != %s" - % (self.booted, self.node_services.keys()) + "failure to start all services: %s != %s" + % (self.started, self.node_services.keys()) ) return paths @@ -109,8 +109,8 @@ class ConfigServiceDependencies: self._visit(service) # add service when bottom is found - logging.debug("adding service to boot path: %s", current_service.name) - self.booted.add(current_service.name) + logging.debug("adding service to startup path: %s", current_service.name) + self.started.add(current_service.name) self.path.append(current_service) self.visiting.remove(current_service.name) diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index af2854f7..19951efa 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -285,22 +285,41 @@ class CoreNodeBase(NodeBase): self.nodedir = None self.tmpnodedir = False - def add_config_service(self, service_class: "ConfigServiceType"): + def add_config_service(self, service_class: "ConfigServiceType") -> None: + """ + Adds a configuration service to the node. + + :param service_class: configuration service class to assign to node + :return: nothing + """ name = service_class.name if name in self.config_services: raise CoreError(f"node({self.name}) already has service({name})") self.config_services[name] = service_class(self) def set_service_config(self, name: str, data: Dict[str, str]) -> None: + """ + Sets configuration service custom config data. + + :param name: name of configuration service + :param data: custom config data to set + :return: nothing + """ service = self.config_services.get(name) if service is None: raise CoreError(f"node({self.name}) does not have service({name})") service.set_config(data) def start_config_services(self) -> None: - boot_paths = ConfigServiceDependencies(self.config_services).boot_paths() - for boot_path in boot_paths: - for service in boot_path: + """ + Determins startup paths and starts configuration services, based on their + dependency chains. + + :return: nothing + """ + startup_paths = ConfigServiceDependencies(self.config_services).startup_paths() + for startup_path in startup_paths: + for service in startup_path: service.start() def makenodedir(self) -> None: