Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/crio.conf.5.md
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,9 @@ conmon-rs (`runtime_type = "pod"`) supports this configuration for exec and atta
Path to the seccomp.json profile which is used as the default seccomp profile for the runtime. If not specified, then the `crio.runtime` seccomp profile will be used.
If that is also not specified, then the internal default seccomp profile will be used.

**container_create_timeout**=240
The timeout for container creation operations in seconds. If not set, defaults to 240 seconds. If set to a value less than 30 seconds, it will be automatically adjusted to 30 seconds (the minimum allowed value). This allows different runtime handlers to have different container creation timeouts, which is useful for VM-based runtimes that may need longer timeouts than OCI runtimes.

### CRIO.RUNTIME.WORKLOADS TABLE

The "crio.runtime.workloads" table defines a list of workloads - a way to customize the behavior of a pod and container.
Expand Down
2 changes: 0 additions & 2 deletions internal/oci/oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ const (
ContainerStateRunning = "running"
// ContainerStateStopped represents the stopped state of a container.
ContainerStateStopped = "stopped"
// ContainerCreateTimeout represents the value of container creating timeout.
ContainerCreateTimeout = 240 * time.Second

// killContainerTimeout is the timeout that we wait for the container to
// be SIGKILLed.
Expand Down
7 changes: 5 additions & 2 deletions internal/oci/runtime_oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,9 @@ func (r *runtimeOCI) CreateContainer(ctx context.Context, c *Container, cgroupPa
return nil
}

// Get the container create timeout for this runtime handler
timeout := time.Duration(r.handler.ContainerCreateTimeout) * time.Second

var stderrBuf bytes.Buffer

parentPipe, childPipe, err := newPipe()
Expand Down Expand Up @@ -330,8 +333,8 @@ func (r *runtimeOCI) CreateContainer(ctx context.Context, c *Container, cgroupPa

return errors.New("container create failed")
}
case <-time.After(ContainerCreateTimeout):
log.Errorf(ctx, "Container creation timeout (%v)", ContainerCreateTimeout)
case <-time.After(timeout):
log.Errorf(ctx, "Container creation timeout (%v)", timeout)

return errors.New("create container timeout")
}
Expand Down
53 changes: 27 additions & 26 deletions internal/oci/runtime_vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,11 @@ import (
// runtimeVM is the Runtime interface implementation that is more appropriate
// for VM based container runtimes.
type runtimeVM struct {
path string
fifoDir string
configPath string
exitsPath string
pullImage bool
ctx context.Context
client *ttrpc.Client
task task.TaskService
exitsPath string
ctx context.Context
client *ttrpc.Client
task task.TaskService
handler *config.RuntimeHandler

sync.Mutex
ctrs map[string]containerInfo
Expand Down Expand Up @@ -97,13 +94,10 @@ func newRuntimeVM(handler *config.RuntimeHandler, exitsPath string) RuntimeImpl
typeurl.Register(&rspec.WindowsResources{}, prefix, "opencontainers/runtime-spec", major, "WindowsResources")

return &runtimeVM{
path: handler.RuntimePath,
configPath: handler.RuntimeConfigPath,
exitsPath: exitsPath,
pullImage: handler.RuntimePullImage,
fifoDir: filepath.Join(handler.RuntimeRoot, "crio", "fifo"),
ctx: context.Background(),
ctrs: make(map[string]containerInfo),
exitsPath: exitsPath,
ctx: context.Background(),
handler: handler,
ctrs: make(map[string]containerInfo),
}
}

Expand Down Expand Up @@ -151,15 +145,18 @@ func (r *runtimeVM) CreateContainer(ctx context.Context, c *Container, cgroupPar
c.opLock.Lock()
defer c.opLock.Unlock()

// Get the container create timeout for this runtime handler
timeout := time.Duration(r.handler.ContainerCreateTimeout) * time.Second

// Lets ensure we're able to properly get construct the Options
// that we'll pass to the ContainerCreateTask, as admins can set
// the runtime_config_path to an arbitrary location. Also, lets
// fail early if something goes wrong.
var opts *anypb.Any = nil

if r.configPath != "" {
if r.handler.RuntimeConfigPath != "" {
runtimeOptions := &runtimeoptions.Options{
ConfigPath: r.configPath,
ConfigPath: r.handler.RuntimeConfigPath,
}

marshaledOtps, err := typeurl.MarshalAny(runtimeOptions)
Expand All @@ -175,7 +172,7 @@ func (r *runtimeVM) CreateContainer(ctx context.Context, c *Container, cgroupPar
return err
}

containerIO, err := r.createContainerIO(ctx, c, cio.WithNewFIFOs(r.fifoDir, c.terminal, c.stdin))
containerIO, err := r.createContainerIO(ctx, c, cio.WithNewFIFOs(filepath.Join(r.handler.RuntimeRoot, "crio", "fifo"), c.terminal, c.stdin))
if err != nil {
return err
}
Expand Down Expand Up @@ -205,7 +202,7 @@ func (r *runtimeVM) CreateContainer(ctx context.Context, c *Container, cgroupPar
Options: opts,
}

if r.pullImage {
if r.handler.RuntimePullImage {
err := addVolumeMountsToCreateRequest(ctx, request, c)
if err != nil {
log.Warnf(ctx, "Failed to add KataVirtualVolume information to CreateContainer: %v", err)
Expand All @@ -214,9 +211,13 @@ func (r *runtimeVM) CreateContainer(ctx context.Context, c *Container, cgroupPar

createdCh := make(chan error)

// Create a context with timeout for the task creation
taskCtx, taskCancel := context.WithTimeout(ctx, timeout)
defer taskCancel()

go func() {
// Create the container
if resp, err := r.task.Create(r.ctx, request); err != nil {
if resp, err := r.task.Create(taskCtx, request); err != nil {
createdCh <- errdefs.FromGRPC(err)
} else if err := c.state.SetInitPid(int(resp.GetPid())); err != nil {
createdCh <- err
Expand All @@ -230,14 +231,14 @@ func (r *runtimeVM) CreateContainer(ctx context.Context, c *Container, cgroupPar
if err != nil {
return fmt.Errorf("CreateContainer failed: %w", err)
}
case <-time.After(ContainerCreateTimeout):
case <-time.After(timeout):
if err := r.remove(c.ID(), ""); err != nil {
return err
return fmt.Errorf("failed to cleanup container after creation timeout has reached: %w", err)
}

<-createdCh

return fmt.Errorf("CreateContainer timeout (%v)", ContainerCreateTimeout)
return fmt.Errorf("CreateContainer timeout (%v)", timeout)
}

return nil
Expand All @@ -263,7 +264,7 @@ func (r *runtimeVM) startRuntimeDaemon(ctx context.Context, c *Container) error
cmd, err := client.Command(
r.ctx,
&client.CommandConfig{
Runtime: r.path,
Runtime: r.handler.RuntimePath,
Path: c.BundlePath(),
Args: args,
},
Expand Down Expand Up @@ -468,7 +469,7 @@ func (r *runtimeVM) execContainerCommon(ctx context.Context, c *Container, cmd [
}

// Create IO fifos
execIO, err := cio.NewExecIO(c.ID(), r.fifoDir, tty, stdin != nil)
execIO, err := cio.NewExecIO(c.ID(), filepath.Join(r.handler.RuntimeRoot, "crio", "fifo"), tty, stdin != nil)
if err != nil {
return execError, errdefs.FromGRPC(err)
}
Expand Down Expand Up @@ -902,7 +903,7 @@ func (r *runtimeVM) restoreContainerIO(ctx context.Context, c *Container, state
Stderr: state.GetStderr(),
}
// The existing fifos is created by NewFIFOSetInDir. stdin, stdout, stderr should exist
// in a same temporary directory under r.fifoDir. crio is responsible for removing these
// in a same temporary directory under the fifo directory. crio is responsible for removing these
// files after container io is closed.
var iofiles []string
if cioCfg.Stdin != "" {
Expand Down
33 changes: 31 additions & 2 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ const (
defaultGRPCMaxMsgSize = 80 * 1024 * 1024
// default minimum memory for all other runtimes.
defaultContainerMinMemory = 12 * 1024 * 1024 // 12 MiB
// defaultContainerCreateTimeout is the default timeout for container creation operations in seconds.
defaultContainerCreateTimeout = 240
// minimumContainerCreateTimeout is the minimum allowed timeout for container creation operations in seconds.
minimumContainerCreateTimeout = 30
// minimum memory for crun, the default runtime.
defaultContainerMinMemoryCrun = 500 * 1024 // 500 KiB
OCIBufSize = 8192
Expand Down Expand Up @@ -296,6 +300,10 @@ type RuntimeHandler struct {
// If that is also set to "", the internal default seccomp profile will be applied.
SeccompProfile string `toml:"seccomp_profile,omitempty"`

// ContainerCreateTimeout is the timeout for container creation operations in seconds.
// If not set, defaults to 240 seconds.
ContainerCreateTimeout int64 `toml:"container_create_timeout,omitempty"`

// seccompConfig is the seccomp configuration for the handler.
seccompConfig *seccomp.Config
}
Expand Down Expand Up @@ -1431,8 +1439,9 @@ func getDefaultMonitorGroup(isSystemd bool) string {

func defaultRuntimeHandler(isSystemd bool) *RuntimeHandler {
return &RuntimeHandler{
RuntimeType: DefaultRuntimeType,
RuntimeRoot: DefaultRuntimeRoot,
RuntimeType: DefaultRuntimeType,
RuntimeRoot: DefaultRuntimeRoot,
ContainerCreateTimeout: defaultContainerCreateTimeout,
AllowedAnnotations: []string{
annotations.OCISeccompBPFHookAnnotation,
annotations.DevicesAnnotation,
Expand Down Expand Up @@ -1814,6 +1823,10 @@ func (r *RuntimeHandler) Validate(name string) error {
logrus.Errorf("Unable to set minimum container memory for runtime handler %q: %v", name, err)
}

if err := r.ValidateContainerCreateTimeout(name); err != nil {
logrus.Errorf("Unable to set container create timeout for runtime handler %q: %v", name, err)
}

if err := r.ValidateNoSyncLog(); err != nil {
return fmt.Errorf("no sync log: %w", err)
}
Expand Down Expand Up @@ -1956,6 +1969,22 @@ func (r *RuntimeHandler) ValidateContainerMinMemory(name string) error {
return nil
}

// ValidateContainerCreateTimeout sets the default container create timeout if not configured.
func (r *RuntimeHandler) ValidateContainerCreateTimeout(name string) error {
switch {
case r.ContainerCreateTimeout == 0:
r.ContainerCreateTimeout = defaultContainerCreateTimeout
logrus.Infof("Runtime handler %q container create timeout not set, using default: %d seconds", name, r.ContainerCreateTimeout)
case r.ContainerCreateTimeout < minimumContainerCreateTimeout:
logrus.Warnf("Runtime handler %q container create timeout (%d seconds) is less than minimum (%d seconds), setting to minimum: %d seconds", name, r.ContainerCreateTimeout, minimumContainerCreateTimeout, minimumContainerCreateTimeout)
r.ContainerCreateTimeout = minimumContainerCreateTimeout
default:
logrus.Infof("Runtime handler %q container create timeout set to: %d seconds", name, r.ContainerCreateTimeout)
}

return nil
}

// ValidateWebsocketStreaming can be used to verify if the runtime supports WebSocket streaming.
func (r *RuntimeHandler) ValidateWebsocketStreaming(name string) error {
if r.RuntimeType != RuntimeTypePod {
Expand Down
104 changes: 104 additions & 0 deletions pkg/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1780,4 +1780,108 @@ var _ = t.Describe("Config", func() {
Expect(ok).To(BeTrue())
})
})

t.Describe("ValidateContainerCreateTimeout", func() {
It("should set default timeout when not configured", func() {
// Given
handler := &config.RuntimeHandler{}

// When
err := handler.ValidateContainerCreateTimeout("test-runtime")

// Then
Expect(err).ToNot(HaveOccurred())
Expect(handler.ContainerCreateTimeout).To(Equal(int64(240)))
})

It("should use configured timeout when valid", func() {
// Given
handler := &config.RuntimeHandler{
ContainerCreateTimeout: 600, // 10 minutes
}

// When
err := handler.ValidateContainerCreateTimeout("test-runtime")

// Then
Expect(err).ToNot(HaveOccurred())
Expect(handler.ContainerCreateTimeout).To(Equal(int64(600)))
})

It("should set minimum timeout when below minimum", func() {
// Given
handler := &config.RuntimeHandler{
ContainerCreateTimeout: 15, // Below minimum of 30
}

// When
err := handler.ValidateContainerCreateTimeout("test-runtime")

// Then
Expect(err).ToNot(HaveOccurred())
Expect(handler.ContainerCreateTimeout).To(Equal(int64(30)))
})

It("should allow minimum timeout", func() {
// Given
handler := &config.RuntimeHandler{
ContainerCreateTimeout: 30, // Exactly minimum
}

// When
err := handler.ValidateContainerCreateTimeout("test-runtime")

// Then
Expect(err).ToNot(HaveOccurred())
Expect(handler.ContainerCreateTimeout).To(Equal(int64(30)))
})

It("should handle zero timeout by setting default", func() {
// Given
handler := &config.RuntimeHandler{
ContainerCreateTimeout: 0,
}

// When
err := handler.ValidateContainerCreateTimeout("test-runtime")

// Then
Expect(err).ToNot(HaveOccurred())
Expect(handler.ContainerCreateTimeout).To(Equal(int64(240)))
})

It("should handle negative timeout by setting minimum", func() {
// Given
handler := &config.RuntimeHandler{
ContainerCreateTimeout: -10,
}

// When
err := handler.ValidateContainerCreateTimeout("test-runtime")

// Then
Expect(err).ToNot(HaveOccurred())
Expect(handler.ContainerCreateTimeout).To(Equal(int64(30)))
})

It("should set different timeouts for different runtime handlers", func() {
// Given
sut.Runtimes[config.DefaultRuntime] = &config.RuntimeHandler{
RuntimePath: validFilePath,
ContainerCreateTimeout: 300, // 5 minutes for OCI runtime
}
sut.Runtimes["kata"] = &config.RuntimeHandler{
RuntimePath: validFilePath,
ContainerCreateTimeout: 600, // 10 minutes for VM runtime
}

// When
err := sut.ValidateRuntimes()

// Then
Expect(err).ToNot(HaveOccurred())
Expect(sut.Runtimes[config.DefaultRuntime].ContainerCreateTimeout).To(Equal(int64(300)))
Expect(sut.Runtimes["kata"].ContainerCreateTimeout).To(Equal(int64(600)))
})
})
})
6 changes: 6 additions & 0 deletions pkg/config/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -1257,6 +1257,7 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run
# default_annotations = {}
# stream_websockets = false
# seccomp_profile = ""
# container_create_timeout = 240
# Where:
# - runtime-handler: Name used to identify the runtime.
# - runtime_path (optional, string): Absolute path to the runtime executable in
Expand Down Expand Up @@ -1320,6 +1321,11 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run
# seccomp profile for the runtime.
# If not specified or set to "", the runtime seccomp_profile will be used.
# If that is also not specified or set to "", the internal default seccomp profile will be applied.
# - container_create_timeout (optional, int64): The timeout for container creation operations in seconds.
# If not set, defaults to 240 seconds. If set to a value less than 30 seconds, it will be automatically
# adjusted to 30 seconds (the minimum allowed value). This allows different runtime handlers to have
# different container creation timeouts, which is useful for VM-based runtimes that may need longer
# timeouts than OCI runtimes.
#
# Using the seccomp notifier feature:
#
Expand Down
Loading