Skip to content

Commit

Permalink
chore: Replaced apt with apt-get (#804)
Browse files Browse the repository at this point in the history
Signed-off-by: Griffin <[email protected]>
Co-authored-by: Sertaç Özercan <[email protected]>
  • Loading branch information
prakrit55 and sozercan authored Oct 24, 2024
1 parent ee516e0 commit 6dd5b5e
Show file tree
Hide file tree
Showing 40 changed files with 67 additions and 67 deletions.
44 changes: 22 additions & 22 deletions pkg/pkgmgr/dpkg.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,12 +189,12 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, up
llb.ResolveModeDefault,
)
updated := toolingBase.Run(
llb.Shlex("apt update"),
llb.Shlex("apt-get update"),
llb.WithProxy(utils.GetProxy()),
llb.IgnoreCache,
).Root()

const installBusyBoxCmd = "apt install busybox-static"
const installBusyBoxCmd = "apt-get install busybox-static"
busyBoxInstalled := updated.Run(llb.Shlex(installBusyBoxCmd), llb.WithProxy(utils.GetProxy())).Root()
busyBoxApplied := dm.config.ImageState.File(llb.Copy(busyBoxInstalled, "/bin/busybox", "/bin/busybox"))
mkFolders := busyBoxApplied.File(llb.Mkdir(resultsPath, 0o744, llb.WithParents(true)))
Expand Down Expand Up @@ -298,19 +298,19 @@ func GetPackageInfo(file string) (string, string, error) {
}

// Patch a regular debian image with:
// - sh and apt installed on the image
// - sh and apt-get installed on the image
// - valid dpkg status on the image
//
// Images with neither (i.e. Google Debian Distroless) should be patched with unpackAndMergeUpdates.
func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages, ignoreErrors bool) (*llb.State, []byte, error) {
aptUpdated := dm.config.ImageState.Run(
llb.Shlex("apt update"),
aptGetUpdated := dm.config.ImageState.Run(
llb.Shlex("apt-get update"),
llb.WithProxy(utils.GetProxy()),
llb.IgnoreCache,
).Root()

checkUpgradable := `sh -c "apt list --upgradable 2>/dev/null | grep -q "upgradable" || exit 1"`
aptUpdated = aptUpdated.Run(llb.Shlex(checkUpgradable)).Root()
checkUpgradable := `sh -c "apt-get -s upgrade 2>/dev/null | grep -q "^Inst" || exit 1"`
aptGetUpdated = aptGetUpdated.Run(llb.Shlex(checkUpgradable)).Root()

// Install all requested update packages without specifying the version. This works around:
// - Reports being slightly out of date, where a newer security revision has displaced the one specified leading to not found errors.
Expand All @@ -319,29 +319,29 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U

var installCmd string
if updates != nil {
aptInstallTemplate := `sh -c "apt install --no-install-recommends -y %s && apt clean -y"`
aptGetInstallTemplate := `sh -c "apt-get install --no-install-recommends -y %s && apt-get clean -y"`
pkgStrings := []string{}
for _, u := range updates {
pkgStrings = append(pkgStrings, u.Name)
}
installCmd = fmt.Sprintf(aptInstallTemplate, strings.Join(pkgStrings, " "))
installCmd = fmt.Sprintf(aptGetInstallTemplate, strings.Join(pkgStrings, " "))
} else {
// if updates is not specified, update all packages
installCmd = `sh -c "output=$(apt upgrade -y && apt clean -y && apt autoremove 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi"`
installCmd = `sh -c "output=$(apt-get upgrade -y && apt-get clean -y && apt-get autoremove 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi"`
}

aptInstalled := aptUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root()
aptGetInstalled := aptGetUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root()

// Validate no errors were encountered if updating all
if updates == nil && !ignoreErrors {
aptInstalled = aptInstalled.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root()
aptGetInstalled = aptGetInstalled.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root()
}

// Write results.manifest to host for post-patch validation
const outputResultsTemplate = `sh -c 'grep "^Package:\|^Version:" "%s" >> "%s"'`
outputResultsCmd := fmt.Sprintf(outputResultsTemplate, dpkgStatusPath, resultManifest)
resultsWritten := aptInstalled.Dir(resultsPath).Run(llb.Shlex(outputResultsCmd)).Root()
resultsDiff := llb.Diff(aptInstalled, resultsWritten)
resultsWritten := aptGetInstalled.Dir(resultsPath).Run(llb.Shlex(outputResultsCmd)).Root()
resultsDiff := llb.Diff(aptGetInstalled, resultsWritten)

resultsBytes, err := buildkit.ExtractFileFromState(ctx, dm.config.Client, &resultsDiff, filepath.Join(resultsPath, resultManifest))
if err != nil {
Expand All @@ -354,7 +354,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U
prevPatchDiff := llb.Diff(dm.config.ImageState, dm.config.PatchedImageState)

// Diff the base image and new patches
newPatchDiff := llb.Diff(aptUpdated, aptInstalled)
newPatchDiff := llb.Diff(aptGetUpdated, aptGetInstalled)

// Merging these two diffs will discard everything in the filesystem that hasn't changed
// Doing llb.Scratch ensures we can keep everything in the filesystem that has not changed
Expand All @@ -368,7 +368,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U
}

// Diff the installed updates and merge that into the target image
patchDiff := llb.Diff(aptUpdated, aptInstalled)
patchDiff := llb.Diff(aptGetUpdated, aptGetInstalled)
patchMerge := llb.Merge([]llb.State{dm.config.ImageState, patchDiff})

return &patchMerge, resultsBytes, nil
Expand All @@ -387,9 +387,9 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers
llb.ResolveModeDefault,
)

// Run apt update && apt download list of updates to target folder
// Run apt-get update && apt-get download list of updates to target folder
updated := toolingBase.Run(
llb.Shlex("apt update"),
llb.Shlex("apt-get update"),
llb.WithProxy(utils.GetProxy()),
llb.IgnoreCache,
).Root()
Expand All @@ -411,7 +411,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers
while IFS=':' read -r package version; do
pkg_name=$(echo "$package" | sed 's/^"\(.*\)"$/\1/')
pkg_version=$(echo "$version" | sed 's/^"\(.*\)"$/\1/')
latest_version=$(apt show $pkg_name 2>/dev/null | awk -F ': ' '/Version:/{print $2}')
latest_version=$(apt-cache show $pkg_name 2>/dev/null | awk -F ': ' '/Version:/{print $2}')
if [ "$latest_version" != "$pkg_version" ]; then
update_packages="$update_packages $pkg_name"
Expand All @@ -436,17 +436,17 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers
var downloadCmd string
pkgStrings := []string{}
if updates != nil {
aptDownloadTemplate := "apt download --no-install-recommends %s"
aptGetDownloadTemplate := "apt-get download --no-install-recommends %s"
for _, u := range updates {
pkgStrings = append(pkgStrings, u.Name)
}
downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " "))
downloadCmd = fmt.Sprintf(aptGetDownloadTemplate, strings.Join(pkgStrings, " "))
} else {
// only update the outdated pacakges from packages.txt
downloadCmd = `
packages=$(<packages.txt)
for package in $packages; do
output=$(apt download --no-install-recommends "$package" 2>&1)
output=$(apt-get download --no-install-recommends "$package" 2>&1)
if [ $? -ne 0 ]; then
echo "$output" >>error_log.txt
fi
Expand Down
14 changes: 7 additions & 7 deletions pkg/pkgmgr/dpkg_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ var (
func TestDpkgParseResultsManifest(t *testing.T) {
t.Run("valid manifest", func(t *testing.T) {
expectedMap := map[string]string{
"apt": "1.8.2.3",
"apt-get": "1.8.2.3",
"base-files": "10.3+deb10u13",
}
actualMap, err := dpkgParseResultsManifest(validDPKGManifest)
Expand Down Expand Up @@ -286,19 +286,19 @@ func TestValidateDebianPackageVersions(t *testing.T) {
{
name: "version lower than requested",
updates: unversioned.UpdatePackages{
{Name: "apt", FixedVersion: "2.0"},
{Name: "apt-get", FixedVersion: "2.0"},
},
cmp: dpkgComparer,
resultsBytes: validDPKGManifest,
ignoreErrors: false,
expectedError: `1 error occurred:
* downloaded package apt version 1.8.2.3 lower than required 2.0 for update`,
expectedErrPkgs: []string{"apt"},
* downloaded package apt-get version 1.8.2.3 lower than required 2.0 for update`,
expectedErrPkgs: []string{"apt-get"},
},
{
name: "version lower than requested with ignore errors",
updates: unversioned.UpdatePackages{
{Name: "apt", FixedVersion: "2.0"},
{Name: "apt-get", FixedVersion: "2.0"},
},
cmp: dpkgComparer,
resultsBytes: validDPKGManifest,
Expand All @@ -307,7 +307,7 @@ func TestValidateDebianPackageVersions(t *testing.T) {
{
name: "version equal to requested",
updates: unversioned.UpdatePackages{
{Name: "apt", FixedVersion: "1.8.2.3"},
{Name: "apt-get", FixedVersion: "1.8.2.3"},
},
cmp: dpkgComparer,
resultsBytes: validDPKGManifest,
Expand All @@ -316,7 +316,7 @@ func TestValidateDebianPackageVersions(t *testing.T) {
{
name: "version greater than requested",
updates: unversioned.UpdatePackages{
{Name: "apt", FixedVersion: "0.9"},
{Name: "apt-get", FixedVersion: "0.9"},
},
cmp: dpkgComparer,
resultsBytes: validDPKGManifest,
Expand Down
2 changes: 1 addition & 1 deletion pkg/pkgmgr/testdata/dpkg_invalid.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Package: apt
Package: apt-get
Version: a.b.c.d
Package: base-files
Version: x.y.z
2 changes: 1 addition & 1 deletion pkg/pkgmgr/testdata/dpkg_valid.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Package: apt
Package: apt-get
Version: 1.8.2.3
Package: base-files
Version: 10.3+deb10u13
2 changes: 1 addition & 1 deletion website/docs/design.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ title: Design

The design of copa arises from the application of those tenets to the observed issues in previous efforts directly update container images via rebasing, for example, the experimental [`crane rebase`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/rebase.md):

- Rebasing requires that all actors involved in creation of the image are coordinated so that some layers can be switched out without breaking the image. Attempting to switch out layers in the container overlay structure is brittle because most existing containers are created by writing over shared configuration files and data stores in base images. For example, an `apt install` during image creation will overwrite the dpkg `status` file in the base image, which will mask any package updates in a rebased layer. Since many existing container scanners rely on the reported package status to find vulnerable package versions, this can cause new vulnerabilities to not be reported or for patched binaries not to be recognized by the scanners.
- Rebasing requires that all actors involved in creation of the image are coordinated so that some layers can be switched out without breaking the image. Attempting to switch out layers in the container overlay structure is brittle because most existing containers are created by writing over shared configuration files and data stores in base images. For example, an `apt-get install` during image creation will overwrite the dpkg `status` file in the base image, which will mask any package updates in a rebased layer. Since many existing container scanners rely on the reported package status to find vulnerable package versions, this can cause new vulnerabilities to not be reported or for patched binaries not to be recognized by the scanners.

To avoid breaking integration with the existing container ecosystem, copa patches the filesystem bundle as a whole instead of as a collection of layers so that the resulting image state is consistent. This strategy also allows copa to patch vulnerabilties introduced at any layer in the image, including OS packages added in the app layers that is not addressed by a simple rebase. It also supports the core tenet of supporting patching without requiring coordination with all the publishers of the base images that a given image transitively depends on.

Expand Down
2 changes: 1 addition & 1 deletion website/docs/faq.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ title: FAQ

## What kind of vulnerabilities can Copa patch?

Copa is capable of patching "OS level" vulnerabilities. This includes packages (like `openssl`) in the image that are managed by a package manager such as `apt` or `yum`. Copa is not currently capable of patching vulnerabilities at the "application level" such as Python packages or Go modules (see [below](#what-kind-of-vulnerabilities-can-copa-not-patch) for more details).
Copa is capable of patching "OS level" vulnerabilities. This includes packages (like `openssl`) in the image that are managed by a package manager such as `apt-get` or `yum`. Copa is not currently capable of patching vulnerabilities at the "application level" such as Python packages or Go modules (see [below](#what-kind-of-vulnerabilities-can-copa-not-patch) for more details).


## What kind of vulnerabilities can Copa not patch?
Expand Down
2 changes: 1 addition & 1 deletion website/docs/introduction.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ In addition to filling the operational gap not met by left-shift security practi
The `copa` tool is an extensible engine that:

1. Parses the needed update packages from the container image’s vulnerability report produced by a scanner like Trivy. New adapters can be written to accommodate more report formats.
2. Obtains and processes the needed update packages using the appropriate package manager tools such as apt, apk, etc. New adapters can be written to support more package managers.
2. Obtains and processes the needed update packages using the appropriate package manager tools such as apt-get, apk, etc. New adapters can be written to support more package managers.
3. Applies the resulting update binaries to the container image using buildkit.

<img title="report-driven vulnerability patching" src="/copacetic/website/img/vulnerability-patch.png" />
Expand Down
2 changes: 1 addition & 1 deletion website/docs/quick-start.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ This guide illustrates how to patch outdated containers with `copa`.
```bash
$ docker history $IMAGE-patched
IMAGE CREATED CREATED BY SIZE COMMENT
262dacfeb193 About a minute ago mount / from exec sh -c apt install --no-ins… 41.1MB buildkit.exporter.image.v0
262dacfeb193 About a minute ago mount / from exec sh -c apt-get install --no-ins… 41.1MB buildkit.exporter.image.v0
<missing> 20 months ago /bin/sh -c #(nop) CMD ["nginx" "-g" "daemon… 0B
<missing> 20 months ago /bin/sh -c #(nop) STOPSIGNAL SIGQUIT 0B
<missing> 20 months ago /bin/sh -c #(nop) EXPOSE 80 0B
Expand Down
2 changes: 1 addition & 1 deletion website/versioned_docs/version-v0.1.x/design.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ title: Design

The design of copa arises from the application of those tenets to the observed issues in previous efforts directly update container images via rebasing, for example, the experimental [`crane rebase`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/rebase.md):

- Rebasing requires that all actors involved in creation of the image are coordinated so that some layers can be switched out without breaking the image. Attempting to switch out layers in the container overlay structure is brittle because most existing containers are created by writing over shared configuration files and data stores in base images. For example, an `apt install` during image creation will overwrite the dpkg `status` file in the base image, which will mask any package updates in a rebased layer. Since many existing container scanners rely on the reported package status to find vulnerable package versions, this can cause new vulnerabilities to not be reported or for patched binaries not to be recognized by the scanners.
- Rebasing requires that all actors involved in creation of the image are coordinated so that some layers can be switched out without breaking the image. Attempting to switch out layers in the container overlay structure is brittle because most existing containers are created by writing over shared configuration files and data stores in base images. For example, an `apt-get install` during image creation will overwrite the dpkg `status` file in the base image, which will mask any package updates in a rebased layer. Since many existing container scanners rely on the reported package status to find vulnerable package versions, this can cause new vulnerabilities to not be reported or for patched binaries not to be recognized by the scanners.

To avoid breaking integration with the existing container ecosystem, copa patches the filesystem bundle as a whole instead of as a collection of layers so that the resulting image state is consistent. This strategy also allows copa to patch vulnerabilties introduced at any layer in the image, including OS packages added in the app layers that is not addressed by a simple rebase. It also supports the core tenet of supporting patching without requiring coordination with all the publishers of the base images that a given image transitively depends on.

Expand Down
2 changes: 1 addition & 1 deletion website/versioned_docs/version-v0.1.x/faq.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ title: FAQ

## What kind of vulnerabilities can Copa patch?

Copa is capable of patching "OS level" vulnerabilities. This includes packages (like `openssl`) in the image that are managed by a package manager such as `apt` or `yum`. Copa is not currently capable of patching vulnerabilities at the "application level" such as Python packages or Go modules (see [below](#what-kind-of-vulnerabilities-can-copa-not-patch) for more details).
Copa is capable of patching "OS level" vulnerabilities. This includes packages (like `openssl`) in the image that are managed by a package manager such as `apt-get` or `yum`. Copa is not currently capable of patching vulnerabilities at the "application level" such as Python packages or Go modules (see [below](#what-kind-of-vulnerabilities-can-copa-not-patch) for more details).


## What kind of vulnerabilities can Copa not patch?
Expand Down
2 changes: 1 addition & 1 deletion website/versioned_docs/version-v0.1.x/introduction.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ In addition to filling the operational gap not met by left-shift security practi
The `copa` tool is an extensible engine that:

1. Parses the needed update packages from the container image’s vulnerability report produced by a scanner like Trivy. New adapters can be written to accommodate more report formats.
2. Obtains and processes the needed update packages using the appropriate package manager tools such as apt, apk, etc. New adapters can be written to support more package managers.
2. Obtains and processes the needed update packages using the appropriate package manager tools such as apt-get, apk, etc. New adapters can be written to support more package managers.
3. Applies the resulting update binaries to the container image using buildkit.

<img title="report-driven vulnerability patching" src="/copacetic/website/img/vulnerability-patch.png" />
Expand Down
2 changes: 1 addition & 1 deletion website/versioned_docs/version-v0.1.x/quick-start.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ This sample illustrates how to patch containers using vulnerability reports with
```bash
$ docker history docker.io/library/nginx:1.21.6-patched
IMAGE CREATED CREATED BY SIZE COMMENT
a372df41e06d 1 minute ago mount / from exec sh -c apt install --no-ins… 26.1MB buildkit.exporter.image.v0
a372df41e06d 1 minute ago mount / from exec sh -c apt-get install --no-ins… 26.1MB buildkit.exporter.image.v0
<missing> 3 months ago CMD ["nginx" "-g" "daemon off;"] 0B buildkit.dockerfile.v0
<missing> 3 months ago STOPSIGNAL SIGQUIT 0B buildkit.dockerfile.v0
<missing> 3 months ago EXPOSE map[80/tcp:{}] 0B buildkit.dockerfile.v0
Expand Down
Loading

0 comments on commit 6dd5b5e

Please sign in to comment.