From 2a554984832c199d2c57b721fc4fbeb2ac219931 Mon Sep 17 00:00:00 2001 From: Philip Guyton Date: Thu, 1 Feb 2024 18:56:26 +0000 Subject: [PATCH 1/3] Add rockstor-build systemd service #2793 Move build.sh execution from within rpm %posttrans script to it's own dedicated rockstor-build.service. Enabling greater fidelity and control over the environment and timing; and eases development and user feedback on build.sh failures in the future. Partnered with rockstor.spec changes in rockstor-rpmbuild repo. ## Includes - New rockstor-build.service file. - After= & Requires= entries in rockstor-pre on rockstor-build, to extend our service cascade. - Trivial build.sh and pkg_mgmt.py comment updates. - Add the new rockstor-build.service to initrock.py to assist in asserting the service akin to all other rockstor services. Mostly redundant given our rpm service management, but nice-to-have. --- build.sh | 6 +++--- conf/rockstor-build.service | 21 +++++++++++++++++++++ conf/rockstor-pre.service | 2 ++ src/rockstor/scripts/initrock.py | 1 + src/rockstor/system/pkg_mgmt.py | 2 +- 5 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 conf/rockstor-build.service diff --git a/build.sh b/build.sh index af25e3783..a202ec036 100644 --- a/build.sh +++ b/build.sh @@ -4,14 +4,14 @@ set -o errexit # Install Poetry, a dependency management, packaging, and build system. # Uninstall legacy/transitional Poetry version of 1.1.15 -PATH="$HOME/.local/bin:$PATH" # account for more constrained environments. +PATH="$HOME/.local/bin:$PATH" # ensure legacy path. if which poetry && poetry --version | grep -q "1.1.15"; then echo "Poetry version 1.1.15 found - UNINSTALLING" curl -sSL https://install.python-poetry.org | python3 - --uninstall rm --force /root/.local/bin/poetry # remove dangling dead link. fi PATH="${PATH//'/root/.local/bin:'/''}" # null all legacy poetry paths -# We are run, outside of development, only by RPM's %posttrans. +# We are run by rockstor-build.service. # As such our .venv dir has already been removed in %post (update mode). echo "Unset VIRTUAL_ENV" # Redundant when updating from rockstor 5.0.3-0 onwards: src/rockstor/system/pkg_mgmt.py @@ -88,7 +88,7 @@ export Environment="PASSWORD_STORE_DIR=/root/.password-store" # Additional collectstatic options --clear --dry-run export DJANGO_SETTINGS_MODULE=settings # must be run in project root: -/usr/local/bin/poetry run django-admin collectstatic --no-input --verbosity 2 +poetry run django-admin collectstatic --no-input --verbosity 2 echo echo "ROCKSTOR BUILD SCRIPT COMPLETED" diff --git a/conf/rockstor-build.service b/conf/rockstor-build.service new file mode 100644 index 000000000..7beae7822 --- /dev/null +++ b/conf/rockstor-build.service @@ -0,0 +1,21 @@ +[Unit] +Description=Build Rockstor +ConditionPathIsDirectory=!/opt/rockstor/.venv +After=postgresql.service +After=NetworkManager.service +After=NetworkManager-wait-online.service +Requires=postgresql.service +Requires=NetworkManager.service +# https://pypi.org/ to install/update Poetry, & have it build/rebuild .venv. +Requires=NetworkManager-wait-online.service + +[Service] +Environment="DJANGO_SETTINGS_MODULE=settings" +Environment="PASSWORD_STORE_DIR=/root/.password-store" +WorkingDirectory=/opt/rockstor +ExecStart=/opt/rockstor/build.sh +Type=oneshot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/conf/rockstor-pre.service b/conf/rockstor-pre.service index e5320e8a8..c93a5f743 100644 --- a/conf/rockstor-pre.service +++ b/conf/rockstor-pre.service @@ -1,7 +1,9 @@ [Unit] Description=Tasks required prior to starting Rockstor +After=rockstor-build.service After=postgresql.service After=NetworkManager.service +Requires=rockstor-build.service Requires=postgresql.service Requires=NetworkManager.service diff --git a/src/rockstor/scripts/initrock.py b/src/rockstor/scripts/initrock.py index 68980252b..088ada52d 100644 --- a/src/rockstor/scripts/initrock.py +++ b/src/rockstor/scripts/initrock.py @@ -95,6 +95,7 @@ SYSTEMD_OVERRIDE_DIR = "/etc/systemd/system" ROCKSTOR_SYSTEMD_SERVICES = [ + "rockstor-build.service", # Build/Rebuild .venv & jslibs, init `pass`. "rockstor-pre.service", # Loads us (initrock.py). "rockstor.service", "rockstor-bootstrap.service", diff --git a/src/rockstor/system/pkg_mgmt.py b/src/rockstor/system/pkg_mgmt.py index 111436894..66f9300c9 100644 --- a/src/rockstor/system/pkg_mgmt.py +++ b/src/rockstor/system/pkg_mgmt.py @@ -458,7 +458,7 @@ def update_run(subscription=None, update_all_other=False): # Unset inherited VIRTUAL_ENV environmental variable before invoking rpm/zypper atfo.write("unset VIRTUAL_ENV\n") atfo.write(pkg_in_up_rockstor) - # rockstor-bootstrap Requires rockstor which Requires rockstor-pre (initrock) + # rockstor-bootstrap Requires rockstor which Requires rockstor-pre (initrock) which Requires rockstor-build atfo.write("{} start rockstor-bootstrap\n".format(SYSTEMCTL)) else: # update_all_other True so update all bar the rockstor package. logger.info( From 964bd2bbfae692836786e1b6dd49511ff22ef161 Mon Sep 17 00:00:00 2001 From: Philip Guyton Date: Mon, 5 Feb 2024 18:26:12 +0000 Subject: [PATCH 2/3] (t) Samba shares not accessible - 5.0.6-0 & 5.0.7-0 #2794 Introduce 'poetry-plugin-dotenv' to enable Poetry to establish environmental variables from a .env file. Add PASSWORD_STORE_DIR & DJANGO_SETTINGS_MODULE variables to inform OS level 'password-store' app, & our .venv Django of their configuration. ## Includes: - Additional logging to poetry-install.txt to indicate plugins installed. - Include new `.env` file in project.toml for sdist inclusion. - NO SECRETS indicator in new .env file. - Add `source .env` to build.sh to ease development, the .env file is also read by build.sh's dedicated rockstor-build.service. - Pin, in build.sh, poetry-plugin-dotenv to latest 0.6.11 - Adopt new `poetry run mnt-share share-name`, with required `cd`, in new `root preexec` Samba share config creation. - Incidentally fix legacy Poetry removal regression introduced when build.sh execution was moved to a systemd service from rpm %posttrans. - Add smb.conf preexec migration procedure to initrock. - Remove redundant smb & nmb restarts from prior preexec migrations. - Resource .env file in all relevant rockstor*.service files via `EnvironmentFile=` directive. - Normalise on `/usr/local/bin/poetry run` script invocation in all relevant rockstor*.service files. - Modify developer instructions (build.sh) to account for new poetry-plugin-dotenv. --- .env | 14 ++++++++++++++ build.sh | 25 +++++++++++++++---------- conf/rockstor-bootstrap.service | 5 ++--- conf/rockstor-build.service | 3 +-- conf/rockstor-pre.service | 3 +-- conf/rockstor.service | 3 +-- pyproject.toml | 1 + src/rockstor/scripts/initrock.py | 32 +++++++++++++++++++++++++++++++- src/rockstor/system/samba.py | 5 +++-- 9 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 .env mode change 100644 => 100755 build.sh diff --git a/.env b/.env new file mode 100644 index 000000000..f719655c8 --- /dev/null +++ b/.env @@ -0,0 +1,14 @@ +# NO SECRETS - ENVIRONMENTAL VARIABLES ONLY +# All entires should be compatible with all of the following uses: +# - poetry-plugin-dotenv: https://pypi.org/project/poetry-plugin-dotenv/ +# - python-dotenv: https://pypi.org/project/python-dotenv/ +# - systemd: https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#EnvironmentFile= +# - bash `source` command: https://www.gnu.org/software/bash/manual/bash.html#index-source + +# password-store working directory: GNUPG/gpg encrypted. +# https://www.passwordstore.org/ +# https://git.zx2c4.com/password-store/about/ +PASSWORD_STORE_DIR=/root/.password-store + +# Django +DJANGO_SETTINGS_MODULE=settings diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 index a202ec036..fccf3ed9f --- a/build.sh +++ b/build.sh @@ -4,7 +4,7 @@ set -o errexit # Install Poetry, a dependency management, packaging, and build system. # Uninstall legacy/transitional Poetry version of 1.1.15 -PATH="$HOME/.local/bin:$PATH" # ensure legacy path. +PATH="/root/.local/bin:$PATH" # ensure legacy path. if which poetry && poetry --version | grep -q "1.1.15"; then echo "Poetry version 1.1.15 found - UNINSTALLING" curl -sSL https://install.python-poetry.org | python3 - --uninstall @@ -33,6 +33,9 @@ export PIPX_MAN_DIR=/usr/local/share/man # manual page location for pipx-instal # https://python-poetry.org/docs/#installing-with-pipx pipx ensurepath pipx install --python python3.11 poetry==1.7.1 +# https://pypi.org/project/poetry-plugin-dotenv/ +# https://python-poetry.org/docs/master/plugins/#using-plugins +pipx inject --verbose poetry poetry-plugin-dotenv==0.6.11 pipx list # Install project dependencies defined in cwd pyproject.toml using poetry.toml @@ -43,6 +46,7 @@ pipx list # ** --no-ansi avoids special characters ** env > poetry-install.txt poetry --version >> poetry-install.txt +poetry self show plugins >> poetry-install.txt # /usr/local/bin/poetry -> /opt/pipx/venvs/poetry poetry install -vvv --no-interaction --no-ansi >> poetry-install.txt 2>&1 echo @@ -78,15 +82,17 @@ fi # Ensure GNUPG is setup for 'pass' (Idempotent) /usr/bin/gpg --quick-generate-key --batch --passphrase '' rockstor@localhost || true -# Init 'pass' in ~ using above GPG key, and generate Django SECRET_KEY -export Environment="PASSWORD_STORE_DIR=/root/.password-store" +# Init 'pass' in .env defined PASSWORD_STORE_DIR using above GPG key, and generate Django SECRET_KEY +set -o allexport +echo "Sourcing ${pwd}.env" +source .env # also read by rockstor-build.service +set +o allexport /usr/bin/pass init rockstor@localhost /usr/bin/pass generate --no-symbols --force python-keyring/rockstor/SECRET_KEY 100 # Collect all static files in the STATIC_ROOT subdirectory. See settings.py. # /opt/rockstor/static # Additional collectstatic options --clear --dry-run -export DJANGO_SETTINGS_MODULE=settings # must be run in project root: poetry run django-admin collectstatic --no-input --verbosity 2 echo @@ -95,9 +101,8 @@ echo "ROCKSTOR BUILD SCRIPT COMPLETED" echo echo "If installing from source, from scratch, for development; i.e. NOT via RPM:" echo "Note GnuPG & password-store ExecStartPre steps in /opt/rockstor/conf/rockstor-pre.service" -echo "1. Run 'cd /opt/rockstor'." -echo "2. Run 'systemctl start postgresql'." -echo "3. Run 'export DJANGO_SETTINGS_MODULE=settings'." -echo "4. Run 'export PASSWORD_STORE_DIR=/root/.password-store'." -echo "5. Run 'poetry run initrock' as root (equivalent to rockstor-pre.service ExecStart)." -echo "6. Run 'systemctl enable --now rockstor-bootstrap'." \ No newline at end of file +echo "1. Run 'systemctl start postgresql'." +echo "2. Run 'cd /opt/rockstor'." +echo "3. Run './build.sh'." +echo "4. Run 'poetry run initrock' as root (equivalent to rockstor-pre.service ExecStart)." +echo "5. Run 'systemctl enable --now rockstor-bootstrap'." \ No newline at end of file diff --git a/conf/rockstor-bootstrap.service b/conf/rockstor-bootstrap.service index 8d9e8b699..d280b66b7 100644 --- a/conf/rockstor-bootstrap.service +++ b/conf/rockstor-bootstrap.service @@ -4,10 +4,9 @@ After=rockstor.service Requires=rockstor.service [Service] -Environment="DJANGO_SETTINGS_MODULE=settings" -Environment="PASSWORD_STORE_DIR=/root/.password-store" WorkingDirectory=/opt/rockstor -ExecStart=/opt/rockstor/.venv/bin/bootstrap +EnvironmentFile=/opt/rockstor/.env +ExecStart=/usr/local/bin/poetry run bootstrap Type=oneshot RemainAfterExit=yes diff --git a/conf/rockstor-build.service b/conf/rockstor-build.service index 7beae7822..0423992e0 100644 --- a/conf/rockstor-build.service +++ b/conf/rockstor-build.service @@ -10,9 +10,8 @@ Requires=NetworkManager.service Requires=NetworkManager-wait-online.service [Service] -Environment="DJANGO_SETTINGS_MODULE=settings" -Environment="PASSWORD_STORE_DIR=/root/.password-store" WorkingDirectory=/opt/rockstor +EnvironmentFile=/opt/rockstor/.env ExecStart=/opt/rockstor/build.sh Type=oneshot RemainAfterExit=yes diff --git a/conf/rockstor-pre.service b/conf/rockstor-pre.service index c93a5f743..6aeae7670 100644 --- a/conf/rockstor-pre.service +++ b/conf/rockstor-pre.service @@ -8,9 +8,8 @@ Requires=postgresql.service Requires=NetworkManager.service [Service] -Environment="DJANGO_SETTINGS_MODULE=settings" -Environment="PASSWORD_STORE_DIR=/root/.password-store" WorkingDirectory=/opt/rockstor +EnvironmentFile=/opt/rockstor/.env # Avoid `pass` stdout leaking generated passwords (N.B. 2>&1 >/dev/null failed). StandardOutput=null # Idempotent: failure tolerated for pgp as key likely already exists (rc 2). diff --git a/conf/rockstor.service b/conf/rockstor.service index 8f809b782..88e6ff7e1 100644 --- a/conf/rockstor.service +++ b/conf/rockstor.service @@ -4,9 +4,8 @@ After=rockstor-pre.service Requires=rockstor-pre.service [Service] -Environment="DJANGO_SETTINGS_MODULE=settings" -Environment="PASSWORD_STORE_DIR=/root/.password-store" WorkingDirectory=/opt/rockstor +EnvironmentFile=/opt/rockstor/.env ExecStart=/usr/local/bin/poetry run supervisord -c /opt/rockstor/etc/supervisord.conf ExecStop=/usr/local/bin/poetry run supervisorctl shutdown ExecReload=/usr/local/bin/poetry run supervisorctl reload diff --git a/pyproject.toml b/pyproject.toml index c4acee101..d7c3e748d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ include = [ "build.sh", # master build script "poetry.toml", # poetry config "poetry.lock", # current poetry established dependency lock file. + ".env", # poetry-plugin-dotenv default source file. { path = "conf" }, # Configuration directories { path = "etc" }, { path = "var" }, # Some processes depend on this tree existing. diff --git a/src/rockstor/scripts/initrock.py b/src/rockstor/scripts/initrock.py index 088ada52d..eab3d6601 100644 --- a/src/rockstor/scripts/initrock.py +++ b/src/rockstor/scripts/initrock.py @@ -116,9 +116,10 @@ # use None to use the current mask of the target file defined at . # services: Python List of service(s) to restart, if any, after modifying the file. LocalFile = namedtuple("LocalFile", "path mask services") +# samba_config's "root preexec = ..." migrations do not required service restarts. LOCAL_FILES = { "samba_config": LocalFile( - path="/etc/samba/smb.conf", mask=None, services=["nmb", "smb"] + path="/etc/samba/smb.conf", mask=None, services=None ), "rockstor_crontab": LocalFile( path="/etc/cron.d/rockstortab", mask=stat.S_IRUSR | stat.S_IWUSR, services=None @@ -463,6 +464,33 @@ def establish_poetry_paths(): logger.info("### DONE establishing poetry path to binaries in local files.") +def update_smb_conf_preexec(): + """ + 5.0.8-0 onwards adopts a new smb.conf preexec command for all new Samba exports. + Modify existing shares accordingly. Example for test_share01: + root preexec = "/opt/rockstor/.venv/bin/mnt-share test_share01" + root preexec = sh -c "cd /opt/rockstor/ && poetry run mnt-share test_share01" + Avoids premature DB requirement re: + - refresh_smb_config(list(SambaShare.objects.all())) + - refresh_smb_discovery(list(SambaShare.objects.all())) + """ + logger.info("### BEGIN Establishing SMB config preexec update...") + smb_conf = LOCAL_FILES["samba_config"] + pattern = f'"{BASE_DIR}.venv/bin/' + replacement = f'sh -c "cd {BASE_DIR} && poetry run ' + if os.path.isfile(smb_conf.path): + fh, npath = mkstemp() + altered = replace_pattern_inline(smb_conf.path, npath, pattern, replacement) + if altered: # smb_conf.mask assumed None + shutil.copystat(smb_conf.path, npath) + shutil.move(npath, smb_conf.path) + logger.info("smb.conf preexec format updated") + else: + os.remove(npath) + logger.info("smb.conf preexec already updated") + logger.info("### DONE Establishing SMB config preexec update...") + + def set_api_client_secret(): """ Set/reset the API client secret which is used internally by OAUTH_INTERNAL_APP = "cliapp", @@ -649,6 +677,8 @@ def main(): establish_poetry_paths() + update_smb_conf_preexec() + if __name__ == "__main__": main() diff --git a/src/rockstor/system/samba.py b/src/rockstor/system/samba.py index 9f6e8f18f..1c1879de5 100644 --- a/src/rockstor/system/samba.py +++ b/src/rockstor/system/samba.py @@ -49,14 +49,15 @@ def test_parm(config="/etc/samba/smb.conf"): def rockstor_smb_config(fo, exports): - mnt_helper = os.path.join(settings.ROOT_DIR, ".venv/bin/mnt-share") + mnt_helper = "poetry run mnt-share" fo.write("{}\n".format(RS_SHARES_HEADER)) for e in exports: admin_users = "" for au in e.admin_users.all(): admin_users = "{}{} ".format(admin_users, au.username) fo.write("[{}]\n".format(e.share.name)) - fo.write(' root preexec = "{} {}"\n'.format(mnt_helper, e.share.name)) + # Requires `poetry run` in ROOT_DIR to gain .env defined environment. + fo.write(f" root preexec = sh -c \"cd {settings.ROOT_DIR} && {mnt_helper} {e.share.name}\"\n") fo.write(" root preexec close = yes\n") fo.write(" comment = {}\n".format(e.comment.encode("utf-8"))) fo.write(" path = {}\n".format(e.path)) From e0c79b4c08c59c0bd7f4c08c3bfd43183265bd9e Mon Sep 17 00:00:00 2001 From: Philip Guyton Date: Mon, 12 Feb 2024 14:38:47 +0000 Subject: [PATCH 3/3] Bump versions to a 5.0.8 base - testing branch #2800 pyproject.toml build.sh --- build.sh | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index fccf3ed9f..a9d56695e 100755 --- a/build.sh +++ b/build.sh @@ -53,7 +53,7 @@ echo # Add js libs. See: https://github.com/rockstor/rockstor-jslibs # Set jslibs_version of GitHub release: -jslibs_version=5.0.7 +jslibs_version=5.0.8 jslibs_url=https://github.com/rockstor/rockstor-jslibs/archive/refs/tags/"${jslibs_version}".tar.gz # Check for rpm embedded, or previously downloaded jslibs. diff --git a/pyproject.toml b/pyproject.toml index d7c3e748d..3a6c9ad2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "rockstor" -version = "5.0.7" +version = "5.0.8" description = "Btrfs Network Attached Storage (NAS) Appliance." homepage = "https://rockstor.com/" repository = "https://github.com/rockstor/rockstor-core"