From 191393e4a1db111c66ff34e4a556e203255ea77b Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Wed, 17 Dec 2025 00:31:00 +0000 Subject: [PATCH] [experiment] Use hermetic zstd --- MODULE.bazel | 2 +- MODULE.bazel.lock | 4 +++- minidock/container_data.bzl | 9 +++++++- minidock/container_data_tools/BUILD | 1 + minidock/container_data_tools/build_tar.py | 26 ++++++++++++++-------- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3cdba4a..efda61a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -8,7 +8,7 @@ module( # Required dependencies bazel_dep(name = "platforms", version = "0.0.11") - +bazel_dep(name = "zstd", version = "1.5.7") bazel_dep(name = "rules_multitool", version = "1.11.1") multitool = use_extension("@rules_multitool//multitool:extension.bzl", "multitool") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index dfec10c..c4c1bff 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -141,7 +141,9 @@ "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", + "https://bcr.bazel.build/modules/zstd/1.5.7/MODULE.bazel": "f5780cdbd6f4c5bb985a20f839844316fe48fb5e463056f372dbc37cfabdf450", + "https://bcr.bazel.build/modules/zstd/1.5.7/source.json": "f72c48184b6528ffc908a5a2bcbf3070c6684f3db03da2182c8ca999ae5f5cfd" }, "selectedYankedVersions": {}, "moduleExtensions": { diff --git a/minidock/container_data.bzl b/minidock/container_data.bzl index 0b95590..27e6357 100644 --- a/minidock/container_data.bzl +++ b/minidock/container_data.bzl @@ -115,11 +115,12 @@ def __container_data_impl( args.add(ctx.attr.zstd_compression_level, format = "--zstd_compression_level=%s") args.add(compression, format = "--compression=%s") args.add(ctx.attr.mtime, format = "--mtime=%s") + args.add(ctx.executable._zstd_tool.path, format = "--zstd_path=%s") ctx.actions.run( executable = ctx.executable._build_tar, arguments = [args], - inputs = ctx.files.tars + [manifest_file] + files, + inputs = ctx.files.tars + [manifest_file] + files + [ctx.executable._zstd_tool], outputs = [layer], use_default_shell_env = True, mnemonic = "ContainerData", @@ -144,6 +145,12 @@ container_data = rule( cfg = "exec", executable = True, ), + "_zstd_tool": attr.label( + default = Label("@zstd//:zstd_cli"), + cfg = "exec", + executable = True, + allow_single_file = True, + ), "data_path": attr.string( doc = """Root path of the files. diff --git a/minidock/container_data_tools/BUILD b/minidock/container_data_tools/BUILD index 1d0f565..c1d0fa1 100644 --- a/minidock/container_data_tools/BUILD +++ b/minidock/container_data_tools/BUILD @@ -1,5 +1,6 @@ py_binary( name = "build_tar", srcs = ["build_tar.py"], + data = ["@zstd//:zstd_cli"], visibility = ["//visibility:public"], ) diff --git a/minidock/container_data_tools/build_tar.py b/minidock/container_data_tools/build_tar.py index 38ce19b..af114cd 100644 --- a/minidock/container_data_tools/build_tar.py +++ b/minidock/container_data_tools/build_tar.py @@ -47,6 +47,7 @@ def __init__(self, compression='', gzip_compression_level=9, zstd_compression_level=3, + zstd_path='zstd', root_directory='./', default_mtime=None, preserve_tar_mtimes=True): @@ -56,6 +57,7 @@ def __init__(self, compression: compression type: bzip2, bz2, gz, tgz, xz, lzma, zstd. gzip_compression_level: compression level for gzip (1-9). zstd_compression_level: compression level for zstd (1-22). + zstd_path: path to the zstd executable. root_directory: virtual root to prepend to elements in the archive. default_mtime: default mtime to use for elements in the archive. May be an integer or the value 'portable' to use the date @@ -72,6 +74,7 @@ def __init__(self, # Support zstd compression through zstd command line tool self.zstd = compression == 'zstd' self.zstd_compression_level = zstd_compression_level + self.zstd_path = zstd_path self.name = name self.root_directory = root_directory.rstrip('/') self.preserve_mtime = preserve_tar_mtimes @@ -320,10 +323,10 @@ def add_tar(self, # Note that we buffer the file in memory and it can have an important # memory footprint but it's probably fine as we don't use them for really # large files. - if subprocess.call('which zstd', shell=True, stdout=subprocess.PIPE): + if not os.path.exists(self.zstd_path): raise self.Error('Cannot handle .zstd compression: ' - 'zstd command not found.') - p = subprocess.Popen('zstd -dc %s' % tar, + 'zstd command not found at %s.' % self.zstd_path) + p = subprocess.Popen('%s -dc %s' % (self.zstd_path, tar), shell=True, stdout=subprocess.PIPE) f = io.BytesIO(p.stdout.read()) @@ -405,12 +408,12 @@ def close(self): if self.zstd: # Support zstd compression through zstd command line tool # Following same pattern as xz to maintain no-external-dependencies principle - if subprocess.call('which zstd', shell=True, stdout=subprocess.PIPE): + if not os.path.exists(self.zstd_path): raise self.Error('Cannot handle .zstd compression: ' - 'zstd command not found.') + 'zstd command not found at %s.' % self.zstd_path) subprocess.call( - 'mv {0} {0}.d && zstd -z -{1} {0}.d && mv {0}.d.zst {0}'.format( - self.name, self.zstd_compression_level), + 'mv {0} {0}.d && {2} -z -{1} {0}.d && mv {0}.d.zst {0}'.format( + self.name, self.zstd_compression_level, self.zstd_path), shell=True, stdout=subprocess.PIPE) @@ -446,12 +449,13 @@ def parse_pkg_name(metadata, filename): def __init__(self, output, directory, root_directory, default_mtime, enable_mtime_preservation, - force_posixpath, gzip_compression_level, zstd_compression_level=3, compression='gz'): + force_posixpath, gzip_compression_level, zstd_compression_level=3, zstd_path='zstd', compression='gz'): self.directory = directory self.output = output self.compression = compression self.gzip_compression_level = gzip_compression_level self.zstd_compression_level = zstd_compression_level + self.zstd_path = zstd_path self.root_directory = root_directory self.default_mtime = default_mtime self.enable_mtime_preservation = enable_mtime_preservation @@ -463,6 +467,7 @@ def __enter__(self): self.compression, self.gzip_compression_level, self.zstd_compression_level, + self.zstd_path, self.root_directory, self.default_mtime, self.enable_mtime_preservation, @@ -717,7 +722,7 @@ def main(FLAGS): FLAGS.root_directory, FLAGS.mtime, FLAGS.enable_mtime_preservation, FLAGS.force_posixpath, FLAGS.gzip_compression_level, - FLAGS.zstd_compression_level, FLAGS.compression) as output: + FLAGS.zstd_compression_level, FLAGS.zstd_path, FLAGS.compression) as output: def file_attributes(filename): if filename.startswith('/'): filename = filename[1:] @@ -831,6 +836,9 @@ def validate_link(l): parser.add_argument('--zstd_compression_level', type=int, default=3, help='Set the zstd compression level to use (1-22).') + parser.add_argument('--zstd_path', type=str, default='zstd', + help='Path to the zstd executable.') + parser.add_argument('--compression', type=str, default='gz', help='Set the compression type: gz, bz2, xz, lzma, zstd, or empty string for no compression.')