Skip to content

Making bundler respect hermetic requirement #733

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
a-ovchinnikov merged 3 commits into from
Nov 21, 2024
Merged

Making bundler respect hermetic requirement #733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
575660d
Making bundler respect hermetic requirement
a-ovchinnikov Nov 14, 2024
a24bead
bundler: Forbidding more network interactions
a-ovchinnikov Nov 15, 2024
d061c72
Revert "tests: integration: bundler: Temporarily disable E2E tests"
a-ovchinnikov Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 51 additions & 10 deletions cachi2/core/package_managers/bundler/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from cachi2.core.models.sbom import Component
from cachi2.core.package_managers.bundler.parser import (
GemPlatformSpecificDependency,
GitDependency,
ParseResult,
PathDependency,
parse_lockfile,
Expand All @@ -32,17 +33,20 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
_prepare_environment_variables_for_hermetic_build()
)
project_files: list[ProjectFile] = []
git_paths = []

for package in request.bundler_packages:
path_within_root = request.source_dir.join_within_root(package.path)
components.extend(
_resolve_bundler_package(
package_dir=path_within_root,
output_dir=request.output_dir,
allow_binary=package.allow_binary,
)
_comps, _git_paths = _resolve_bundler_package(
package_dir=path_within_root,
output_dir=request.output_dir,
allow_binary=package.allow_binary,
)
project_files.append(_prepare_for_hermetic_build(request.source_dir, request.output_dir))
components.extend(_comps)
git_paths.extend(_git_paths)
project_files.append(
_prepare_for_hermetic_build(request.source_dir, request.output_dir, git_paths)
)

return RequestOutput.from_obj_list(
components=components,
Expand All @@ -51,11 +55,17 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
)


# Aliases for git dependency name and git dependency name as
# it is written to file system:
DepName = str
FSDepName = str


def _resolve_bundler_package(
package_dir: RootedPath,
output_dir: RootedPath,
allow_binary: bool = False,
) -> list[Component]:
) -> tuple[list[Component], list[tuple[DepName, FSDepName]]]:
"""Process a request for a single bundler package."""
deps_dir = output_dir.join_within_root("deps", "bundler")
deps_dir.path.mkdir(parents=True, exist_ok=True)
Expand All @@ -72,17 +82,20 @@ def _resolve_bundler_package(
)

components = [Component(name=name, version=version, purl=main_package_purl.to_string())]
git_paths = []
for dep in dependencies:
dep.download_to(deps_dir)
if isinstance(dep, GemPlatformSpecificDependency):
properties = PropertySet(bundler_package_binary=True).to_properties()
else:
properties = []
if isinstance(dep, GitDependency):
git_paths.append((dep.name, dep.name + "-" + dep.ref[:12]))

c = Component(name=dep.name, version=dep.version, purl=dep.purl, properties=properties)
components.append(c)

return components
return components, git_paths


def _get_main_package_name_and_version(
Expand Down Expand Up @@ -154,17 +167,45 @@ def _prepare_environment_variables_for_hermetic_build() -> list[EnvironmentVaria
]


def _prepare_for_hermetic_build(source_dir: RootedPath, output_dir: RootedPath) -> ProjectFile:
def _prepare_for_hermetic_build(
source_dir: RootedPath, output_dir: RootedPath, git_paths: Optional[list] = None
) -> ProjectFile:
"""Prepare a package for hermetic build by injecting necessary config."""
potential_bundle_config = source_dir.join_within_root(".bundle/config").path
hermetic_config = dedent(
"""
BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler"
BUNDLE_DEPLOYMENT: "true"
BUNDLE_NO_PRUNE: "true"
BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
BUNDLE_DISABLE_VERSION_CHECK: "true"
BUNDLE_VERSION: "system"
"""
)
# Note: if a package depends on a git revision then the following variables
# are necessary for a hermetic build:
# BUNDLE_DISABLE_LOCAL_BRANCH_CHECK
# BUNDLE_DISABLE_LOCAL_REVISION_CHECK
# because otherwise some (potentially all, depending on exact set of
# ecosystem components versions, environment variables and celestial
# alignment) Bundler versions will try to fetch the latest changes of the
# remotes which may be present even when instructed not to with --local
# flag.
# See https://bundler.io/guides/git.html#local-git-repos for details.
# (or https://github.com/rubygems/bundler-site/blob/
# 9ff3b76e9866524ecefe165633ffb547f0004a99/source/guides/git.html.md
# if the link above ceases to exist).
if git_paths is not None:
hermetic_config += 'BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"\n'
hermetic_config += 'BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"\n'
for packname, dirname in git_paths:
# "-" in variable names is deprecated in Bundler and now generates
# a warning and a suggestion to replace all dashes with triple
# underscores. Package names sometimes contain dashes:
varname = "BUNDLE_LOCAL." + packname.upper().replace("-", "___")
location = "${output_dir}/deps/bundler/" + dirname
config_entry = varname + f': "{location}"'
hermetic_config += f"{config_entry}\n"
if potential_bundle_config.is_file():
config_data = potential_bundle_config.read_text()
config_data += hermetic_config
Expand Down
11 changes: 11 additions & 0 deletions docs/bundler.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ works correctly:
- BUNDLE_DEPLOYMENT: "true"
- BUNDLE_NO_PRUNE: "true"
- BUNDLE_VERSION: "system"
- BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
- BUNDLE_DISABLE_VERSION_CHECK: "true"


### BUNDLE_CACHE_PATH

Expand All @@ -89,6 +92,14 @@ Leave outdated gems unpruned.

The version of Bundler to use when running under the Bundler environment.

### BUNDLE_ALLOW_OFFLINE_INSTALL

Allow Bundler to use cached data when installing without network access.

### BUNDLE_DISABLE_VERSION_CHECK

Stop Bundler from checking if a newer Bundler version is available on rubygems.org.

**Note**: _A prefetch could fail when Bundler versions differ between the build
system and lockfile and when the former is outdated. Therefore we do not recommend
using mismatching or outdated versions of Bundler in build systems as this might
Expand Down
2 changes: 0 additions & 2 deletions tests/integration/test_bundler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,6 @@ def test_bundler_packages(
),
],
)
# FIXME: Re-enable the test once we have a proper fix or at least a workaround in place
@pytest.mark.skip(reason="E2E tests currently broken due to bundler refusing to work offline")
def test_e2e_bundler(
test_params: utils.TestParameters,
check_cmd: list[str],
Expand Down
5 changes: 5 additions & 0 deletions tests/integration/test_data/bundler_everything_present/.build-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,9 @@ project_files:
BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler"
BUNDLE_DEPLOYMENT: "true"
BUNDLE_NO_PRUNE: "true"
BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
BUNDLE_DISABLE_VERSION_CHECK: "true"
BUNDLE_VERSION: "system"
BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"
BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"
BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684"
5 changes: 5 additions & 0 deletions tests/integration/test_data/bundler_everything_present_except_gemspec/.build-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,9 @@ project_files:
BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler"
BUNDLE_DEPLOYMENT: "true"
BUNDLE_NO_PRUNE: "true"
BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
BUNDLE_DISABLE_VERSION_CHECK: "true"
BUNDLE_VERSION: "system"
BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"
BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"
BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684"
11 changes: 10 additions & 1 deletion tests/unit/package_managers/bundler/test_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def test_resolve_bundler_package(
mock_parse_lockfile.return_value = deps
mock_get_main_package_name_and_version.return_value = ("name", None)

components = _resolve_bundler_package(package_dir=package_dir, output_dir=output_dir)
components, git_paths = _resolve_bundler_package(package_dir=package_dir, output_dir=output_dir)

mock_parse_lockfile.assert_called_once_with(package_dir, False)
mock_get_main_package_name_and_version.assert_called_once_with(package_dir, deps)
Expand All @@ -71,6 +71,7 @@ def test_resolve_bundler_package(
mock_path_dep_download_to.assert_called_with(deps_dir)

assert len(components) == len(deps) + 1 # + 1 for the "main" package
assert len(git_paths) == 1 # since there is exactly one git dependency
assert deps_dir.path.exists()


Expand Down Expand Up @@ -127,6 +128,8 @@ def test__prepare_for_hermetic_build_injects_necessary_variable_into_empty_confi
BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler"
BUNDLE_DEPLOYMENT: "true"
BUNDLE_NO_PRUNE: "true"
BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
BUNDLE_DISABLE_VERSION_CHECK: "true"
BUNDLE_VERSION: "system"
"""
)
Expand All @@ -147,6 +150,8 @@ def test__prepare_for_hermetic_build_injects_necessary_variable_into_existing_co
BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler"
BUNDLE_DEPLOYMENT: "true"
BUNDLE_NO_PRUNE: "true"
BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
BUNDLE_DISABLE_VERSION_CHECK: "true"
BUNDLE_VERSION: "system"
"""
)
Expand Down Expand Up @@ -177,6 +182,8 @@ def test__prepare_for_hermetic_build_injects_necessary_variable_into_existing_al
BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler"
BUNDLE_DEPLOYMENT: "true"
BUNDLE_NO_PRUNE: "true"
BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
BUNDLE_DISABLE_VERSION_CHECK: "true"
BUNDLE_VERSION: "system"
"""
)
Expand Down Expand Up @@ -212,6 +219,8 @@ def test__prepare_for_hermetic_build_ignores_a_directory_in_place_of_config(
BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler"
BUNDLE_DEPLOYMENT: "true"
BUNDLE_NO_PRUNE: "true"
BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
BUNDLE_DISABLE_VERSION_CHECK: "true"
BUNDLE_VERSION: "system"
"""
)
Expand Down
Loading