Skip to content

Add optional checksums #400

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 3, 2025
Merged

Add optional checksums #400

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,10 @@ It is possible by forcing md5 checksums on data by using the `--md5` option.

To test [POST Object](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html) operations use `-post` parameter.

To add a [checksum](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html) to the uploaded objects, use `--checksum` parameter.
The following checksums are supported: `CRC32` (composite), `CRC32-FO` (full object), `CRC32C`, `CRC32-FO`, `CRC32C`, `SHA1`, `SHA256` and `CRC64NVME`.
Adding a checksum will always disable MD5 checksums.

## DELETE

Benchmarking delete operations will attempt to delete as many objects it can within `--duration`.
Expand Down
13 changes: 7 additions & 6 deletions cli/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,12 +175,13 @@ func getClient(ctx *cli.Context, host string) (*minio.Client, error) {
lookup = minio.BucketLookupPath
}
cl, err := minio.New(host, &minio.Options{
Creds: creds,
Secure: ctx.Bool("tls"),
Region: ctx.String("region"),
BucketLookup: lookup,
CustomMD5: md5simd.NewServer().NewHash,
Transport: transport,
Creds: creds,
Secure: ctx.Bool("tls"),
Region: ctx.String("region"),
BucketLookup: lookup,
CustomMD5: md5simd.NewServer().NewHash,
Transport: transport,
TrailingHeaders: useTrailingHeaders.Load(),
})
if err != nil {
return nil, err
Expand Down
5 changes: 3 additions & 2 deletions cli/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,14 +331,15 @@ func getCommon(ctx *cli.Context, src func() generator.Source) bench.Common {
// set burst to 1 as limiter will always be called to wait for 1 token
rpsLimiter = rate.NewLimiter(rate.Limit(rpsLimit), 1)
}

// Create put options now, so ensure that trailing headers are set.
putOpts := putOpts(ctx)
return bench.Common{
Client: newClient(ctx),
Concurrency: ctx.Int("concurrent"),
Source: src,
Bucket: ctx.String("bucket"),
Location: ctx.String("region"),
PutOpts: putOpts(ctx),
PutOpts: putOpts,
DiscardOutput: noOps,
ExtraOut: extra,
RpsLimiter: rpsLimiter,
Expand Down
49 changes: 48 additions & 1 deletion cli/put.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,14 @@
package cli

import (
"errors"
"fmt"
"math/rand"
"strings"
"sync/atomic"

"github.com/minio/cli"
"github.com/minio/mc/pkg/probe"
"github.com/minio/minio-go/v7"
"github.com/minio/pkg/v3/console"
"github.com/minio/warp/pkg/bench"
Expand All @@ -44,6 +47,11 @@ var putFlags = []cli.Flag{
Name: "post",
Usage: "Use PostObject for upload. Will force single part upload",
},
cli.StringFlag{
Name: "checksum",
Usage: "Add checksum to uploaded object. Values: CRC64NVME, CRC32[-FO], CRC32C[-FO], SHA1 or SHA256. Requires server trailing headers (AWS, MinIO)",
Value: "",
},
}

var PutCombinedFlags = combineFlags(globalFlags, ioFlags, putFlags, genFlags, benchFlags, analyzeFlags)
Expand Down Expand Up @@ -82,11 +90,13 @@ const metadataChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ12345
// putOpts retrieves put options from the context.
func putOpts(ctx *cli.Context) minio.PutObjectOptions {
pSize, _ := toSize(ctx.String("part.size"))
useMD5, checksumType := parseChecksum(ctx)
options := minio.PutObjectOptions{
ServerSideEncryption: newSSE(ctx),
DisableMultipart: ctx.Bool("disable-multipart"),
DisableContentSha256: ctx.Bool("disable-sha256-payload"),
SendContentMd5: ctx.Bool("md5"),
SendContentMd5: useMD5,
Checksum: checksumType,
StorageClass: ctx.String("storage-class"),
PartSize: pSize,
}
Expand Down Expand Up @@ -134,3 +144,40 @@ func checkPutSyntax(ctx *cli.Context) {
checkAnalyze(ctx)
checkBenchmark(ctx)
}

var useTrailingHeaders atomic.Bool

func parseChecksum(ctx *cli.Context) (useMD5 bool, ct minio.ChecksumType) {
useMD5 = ctx.Bool("md5")
if cs := ctx.String("checksum"); cs != "" {
switch strings.ToUpper(cs) {
case "CRC32":
ct = minio.ChecksumCRC32
case "CRC32C":
ct = minio.ChecksumCRC32C
case "CRC32-FO":
ct = minio.ChecksumFullObjectCRC32
case "CRC32C-FO":
ct = minio.ChecksumFullObjectCRC32C
case "SHA1":
ct = minio.ChecksumSHA1
case "SHA256":
ct = minio.ChecksumSHA256
case "CRC64N", "CRC64NVME":
ct = minio.ChecksumCRC64NVME
case "MD5":
useMD5 = true
default:
err := fmt.Errorf("unknown checksum type: %s. Should be one of CRC64NVME, MD5, CRC32, CRC32C, CRC32-FO, CRC32C-FO, SHA1 or SHA256", cs)
fatalIf(probe.NewError(err), "")
}
if ct.IsSet() {
useTrailingHeaders.Store(true)
if useMD5 {
err := errors.New("cannot combine MD5 with checksum")
fatalIf(probe.NewError(err), "")
}
}
}
return
}
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ toolchain go1.24.3
require (
github.com/bygui86/multi-profile/v2 v2.1.0
github.com/charmbracelet/bubbles v0.21.0
github.com/charmbracelet/bubbletea v1.3.4
github.com/charmbracelet/bubbletea v1.3.5
github.com/charmbracelet/lipgloss v1.1.0
github.com/cheggaaa/pb v1.0.29
github.com/dustin/go-humanize v1.0.1
Expand All @@ -19,7 +19,7 @@ require (
github.com/minio/madmin-go/v4 v4.0.13
github.com/minio/mc v0.0.0-20250506164133-19d87ba47505
github.com/minio/md5-simd v1.1.2
github.com/minio/minio-go/v7 v7.0.92-0.20250515110726-4f25bfc12706
github.com/minio/minio-go/v7 v7.0.92
github.com/minio/pkg/v3 v3.1.8
github.com/minio/websocket v1.6.0
github.com/muesli/termenv v0.16.0
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs=
github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg=
github.com/charmbracelet/bubbletea v1.3.4 h1:kCg7B+jSCFPLYRA52SDZjr51kG/fMUEoPoZrkaDHyoI=
github.com/charmbracelet/bubbletea v1.3.4/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo=
github.com/charmbracelet/bubbletea v1.3.5 h1:JAMNLTbqMOhSwoELIr0qyP4VidFq72/6E9j7HHmRKQc=
github.com/charmbracelet/bubbletea v1.3.5/go.mod h1:TkCnmH+aBd4LrXhXcqrKiYwRs7qyQx5rBgH5fVY3v54=
github.com/charmbracelet/colorprofile v0.3.1 h1:k8dTHMd7fgw4bnFd7jXTLZrSU/CQrKnL3m+AxCzDz40=
github.com/charmbracelet/colorprofile v0.3.1/go.mod h1:/GkGusxNs8VB/RSOh3fu0TJmQ4ICMMPApIIVn0KszZ0=
github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
Expand Down Expand Up @@ -119,8 +119,8 @@ github.com/minio/mc v0.0.0-20250506164133-19d87ba47505 h1:zPiINQs+HwJUyiSNynfyAo
github.com/minio/mc v0.0.0-20250506164133-19d87ba47505/go.mod h1:u2sfW0ODOs2EGYdRlh9YjVdJ6VjB4GwIYA82Bl/bmAU=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.92-0.20250515110726-4f25bfc12706 h1:Tj9KPjo+7+s20m/tJYlwV/7qQdU8of2ROQ7bEdeNgIo=
github.com/minio/minio-go/v7 v7.0.92-0.20250515110726-4f25bfc12706/go.mod h1:vTIc8DNcnAZIhyFsk8EB90AbPjj3j68aWIEQCiPj7d0=
github.com/minio/minio-go/v7 v7.0.92 h1:jpBFWyRS3p8P/9tsRc+NuvqoFi7qAmTCFPoRFmobbVw=
github.com/minio/minio-go/v7 v7.0.92/go.mod h1:vTIc8DNcnAZIhyFsk8EB90AbPjj3j68aWIEQCiPj7d0=
github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA=
github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ=
github.com/minio/pkg/v3 v3.1.8 h1:mcnJod5IvEUtk9KjincKB2e1IJJXcSHnZ+LxhFMeqlk=
Expand Down
Loading