diff --git a/internal/cli/ci.go b/internal/cli/ci.go index e915dca2d2..36af540676 100644 --- a/internal/cli/ci.go +++ b/internal/cli/ci.go @@ -53,6 +53,8 @@ See blog post for additional information: https://blog.tilt.dev/2020/04/16/how-t cmd.Flags().Lookup("logactions").Hidden = true cmd.Flags().StringVar(&c.outputSnapshotOnExit, "output-snapshot-on-exit", "", "If specified, Tilt will dump a snapshot of its state to the specified path when it exits") + cmd.Flags().DurationVar(&ciTimeout, "timeout", model.CITimeoutDefault, + "Timeout to wait for CI to pass. Set to 0 for no timeout.") return cmd } @@ -101,3 +103,5 @@ func (c *ciCmd) run(ctx context.Context, args []string) error { } return err } + +var ciTimeout time.Duration diff --git a/internal/cli/wire.go b/internal/cli/wire.go index 16f535dfd6..ca173812e0 100644 --- a/internal/cli/wire.go +++ b/internal/cli/wire.go @@ -129,6 +129,7 @@ var BaseWireSet = wire.NewSet( controllers.WireSet, + provideCITimeoutFlag, provideWebVersion, provideWebMode, provideWebURL, @@ -354,3 +355,7 @@ func wireLsp(ctx context.Context, l logger.Logger, subcommand model.TiltSubcomma wire.Build(UpWireSet, newLspDeps, newAnalytics) return cmdLspDeps{}, nil } + +func provideCITimeoutFlag() model.CITimeoutFlag { + return model.CITimeoutFlag(ciTimeout) +} diff --git a/internal/cli/wire_gen.go b/internal/cli/wire_gen.go index e6ee250cc6..9d41893e95 100644 --- a/internal/cli/wire_gen.go +++ b/internal/cli/wire_gen.go @@ -74,6 +74,7 @@ import ( "github.com/tilt-dev/tilt/internal/store" "github.com/tilt-dev/tilt/internal/store/liveupdates" "github.com/tilt-dev/tilt/internal/tiltfile" + "github.com/tilt-dev/tilt/internal/tiltfile/cisettings" "github.com/tilt-dev/tilt/internal/tiltfile/config" "github.com/tilt-dev/tilt/internal/tiltfile/k8scontext" "github.com/tilt-dev/tilt/internal/tiltfile/tiltextension" @@ -115,6 +116,8 @@ func wireTiltfileResult(ctx context.Context, analytics2 *analytics.TiltAnalytics scheme := v1alpha1.NewScheme() extensionReconciler := extension.NewReconciler(deferredClient, scheme, analytics2) tiltextensionPlugin := tiltextension.NewPlugin(reconciler, extensionReconciler) + ciTimeoutFlag := provideCITimeoutFlag() + cisettingsPlugin := cisettings.NewPlugin(ciTimeoutFlag) realClientCreator := _wireRealClientCreatorValue restConfigOrError := k8s.ProvideRESTConfig(clientConfig) clientsetOrError := k8s.ProvideClientset(restConfigOrError) @@ -131,7 +134,7 @@ func wireTiltfileResult(ctx context.Context, analytics2 *analytics.TiltAnalytics env := localexec.DefaultEnv(webPort, webHost) processExecer := localexec.NewProcessExecer(env) defaults := _wireDefaultsValue - tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics2, plugin, versionPlugin, configPlugin, tiltextensionPlugin, dockerComposeClient, webHost, processExecer, defaults, product) + tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics2, plugin, versionPlugin, configPlugin, tiltextensionPlugin, cisettingsPlugin, dockerComposeClient, webHost, processExecer, defaults, product) cliCmdTiltfileResultDeps := newTiltfileResultDeps(tiltfileLoader) return cliCmdTiltfileResultDeps, nil } @@ -182,13 +185,15 @@ func wireDockerPrune(ctx context.Context, analytics2 *analytics.TiltAnalytics, s scheme := v1alpha1.NewScheme() extensionReconciler := extension.NewReconciler(deferredClient, scheme, analytics2) tiltextensionPlugin := tiltextension.NewPlugin(reconciler, extensionReconciler) + ciTimeoutFlag := provideCITimeoutFlag() + cisettingsPlugin := cisettings.NewPlugin(ciTimeoutFlag) dockerComposeClient := dockercompose.NewDockerComposeClient(localEnv) webHost := provideWebHost() webPort := provideWebPort() env := localexec.DefaultEnv(webPort, webHost) processExecer := localexec.NewProcessExecer(env) defaults := _wireDefaultsValue - tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics2, plugin, versionPlugin, configPlugin, tiltextensionPlugin, dockerComposeClient, webHost, processExecer, defaults, product) + tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics2, plugin, versionPlugin, configPlugin, tiltextensionPlugin, cisettingsPlugin, dockerComposeClient, webHost, processExecer, defaults, product) cliDpDeps := newDPDeps(compositeClient, client, tiltfileLoader) return cliDpDeps, nil } @@ -307,11 +312,13 @@ func wireCmdUp(ctx context.Context, analytics3 *analytics.TiltAnalytics, cmdTags } extensionReconciler := extension.NewReconciler(deferredClient, scheme, analytics3) tiltextensionPlugin := tiltextension.NewPlugin(extensionrepoReconciler, extensionReconciler) + ciTimeoutFlag := provideCITimeoutFlag() + cisettingsPlugin := cisettings.NewPlugin(ciTimeoutFlag) dockerComposeClient := dockercompose.NewDockerComposeClient(localEnv) defaults := _wireDefaultsValue - tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics3, plugin, versionPlugin, configPlugin, tiltextensionPlugin, dockerComposeClient, webHost, processExecer, defaults, product) + tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics3, plugin, versionPlugin, configPlugin, tiltextensionPlugin, cisettingsPlugin, dockerComposeClient, webHost, processExecer, defaults, product) engineMode := _wireEngineModeValue - tiltfileReconciler := tiltfile2.NewReconciler(storeStore, tiltfileLoader, compositeClient, deferredClient, scheme, engineMode, k8sKubeContextOverride, k8sNamespaceOverride) + tiltfileReconciler := tiltfile2.NewReconciler(storeStore, tiltfileLoader, compositeClient, deferredClient, scheme, engineMode, k8sKubeContextOverride, k8sNamespaceOverride, ciTimeoutFlag) togglebuttonReconciler := togglebutton.NewReconciler(deferredClient, scheme) dockerUpdater := containerupdate.NewDockerUpdater(compositeClient) execUpdater := containerupdate.NewExecUpdater(client) @@ -515,11 +522,13 @@ func wireCmdCI(ctx context.Context, analytics3 *analytics.TiltAnalytics, subcomm } extensionReconciler := extension.NewReconciler(deferredClient, scheme, analytics3) tiltextensionPlugin := tiltextension.NewPlugin(extensionrepoReconciler, extensionReconciler) + ciTimeoutFlag := provideCITimeoutFlag() + cisettingsPlugin := cisettings.NewPlugin(ciTimeoutFlag) dockerComposeClient := dockercompose.NewDockerComposeClient(localEnv) defaults := _wireDefaultsValue - tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics3, plugin, versionPlugin, configPlugin, tiltextensionPlugin, dockerComposeClient, webHost, processExecer, defaults, product) + tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics3, plugin, versionPlugin, configPlugin, tiltextensionPlugin, cisettingsPlugin, dockerComposeClient, webHost, processExecer, defaults, product) engineMode := _wireStoreEngineModeValue - tiltfileReconciler := tiltfile2.NewReconciler(storeStore, tiltfileLoader, compositeClient, deferredClient, scheme, engineMode, k8sKubeContextOverride, k8sNamespaceOverride) + tiltfileReconciler := tiltfile2.NewReconciler(storeStore, tiltfileLoader, compositeClient, deferredClient, scheme, engineMode, k8sKubeContextOverride, k8sNamespaceOverride, ciTimeoutFlag) togglebuttonReconciler := togglebutton.NewReconciler(deferredClient, scheme) dockerUpdater := containerupdate.NewDockerUpdater(compositeClient) execUpdater := containerupdate.NewExecUpdater(client) @@ -719,11 +728,13 @@ func wireCmdUpdog(ctx context.Context, analytics3 *analytics.TiltAnalytics, cmdT } extensionReconciler := extension.NewReconciler(deferredClient, scheme, analytics3) tiltextensionPlugin := tiltextension.NewPlugin(extensionrepoReconciler, extensionReconciler) + ciTimeoutFlag := provideCITimeoutFlag() + cisettingsPlugin := cisettings.NewPlugin(ciTimeoutFlag) dockerComposeClient := dockercompose.NewDockerComposeClient(localEnv) defaults := _wireDefaultsValue - tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics3, plugin, versionPlugin, configPlugin, tiltextensionPlugin, dockerComposeClient, webHost, processExecer, defaults, product) + tiltfileLoader := tiltfile.ProvideTiltfileLoader(analytics3, plugin, versionPlugin, configPlugin, tiltextensionPlugin, cisettingsPlugin, dockerComposeClient, webHost, processExecer, defaults, product) engineMode := _wireEngineModeValue2 - tiltfileReconciler := tiltfile2.NewReconciler(storeStore, tiltfileLoader, compositeClient, deferredClient, scheme, engineMode, k8sKubeContextOverride, k8sNamespaceOverride) + tiltfileReconciler := tiltfile2.NewReconciler(storeStore, tiltfileLoader, compositeClient, deferredClient, scheme, engineMode, k8sKubeContextOverride, k8sNamespaceOverride, ciTimeoutFlag) togglebuttonReconciler := togglebutton.NewReconciler(deferredClient, scheme) dockerUpdater := containerupdate.NewDockerUpdater(compositeClient) execUpdater := containerupdate.NewExecUpdater(k8sClient) @@ -960,6 +971,8 @@ func wireDownDeps(ctx context.Context, tiltAnalytics *analytics.TiltAnalytics, s scheme := v1alpha1.NewScheme() extensionReconciler := extension.NewReconciler(deferredClient, scheme, tiltAnalytics) tiltextensionPlugin := tiltextension.NewPlugin(reconciler, extensionReconciler) + ciTimeoutFlag := provideCITimeoutFlag() + cisettingsPlugin := cisettings.NewPlugin(ciTimeoutFlag) realClientCreator := _wireRealClientCreatorValue restConfigOrError := k8s.ProvideRESTConfig(clientConfig) clientsetOrError := k8s.ProvideClientset(restConfigOrError) @@ -976,7 +989,7 @@ func wireDownDeps(ctx context.Context, tiltAnalytics *analytics.TiltAnalytics, s env := localexec.DefaultEnv(webPort, webHost) processExecer := localexec.NewProcessExecer(env) defaults := _wireDefaultsValue - tiltfileLoader := tiltfile.ProvideTiltfileLoader(tiltAnalytics, plugin, versionPlugin, configPlugin, tiltextensionPlugin, dockerComposeClient, webHost, processExecer, defaults, product) + tiltfileLoader := tiltfile.ProvideTiltfileLoader(tiltAnalytics, plugin, versionPlugin, configPlugin, tiltextensionPlugin, cisettingsPlugin, dockerComposeClient, webHost, processExecer, defaults, product) downDeps := ProvideDownDeps(tiltfileLoader, dockerComposeClient, k8sClient, processExecer) return downDeps, nil } @@ -1081,7 +1094,8 @@ var K8sWireSet = wire.NewSet(k8s.ProvideClusterProduct, k8s.ProvideClusterName, ProvideNamespaceOverride) var BaseWireSet = wire.NewSet( - K8sWireSet, tiltfile.WireSet, git.ProvideGitRemote, localexec.DefaultEnv, localexec.NewProcessExecer, wire.Bind(new(localexec.Execer), new(*localexec.ProcessExecer)), docker.SwitchWireSet, dockercompose.NewDockerComposeClient, clockwork.NewRealClock, engine.DeployerWireSet, engine.NewBuildController, local.NewServerController, kubernetesdiscovery.NewContainerRestartDetector, k8swatch.NewServiceWatcher, k8swatch.NewEventWatchManager, uisession2.NewSubscriber, uiresource2.NewSubscriber, configs.NewConfigsController, configs.NewTriggerQueueSubscriber, telemetry.NewController, cloud.WireSet, cloudurl.ProvideAddress, k8srollout.NewPodMonitor, telemetry.NewStartTracker, session2.NewController, build.ProvideClock, provideClock, hud.WireSet, prompt.WireSet, wire.Value(openurl.OpenURL(openurl.BrowserOpen)), provideLogActions, store.NewStore, wire.Bind(new(store.RStore), new(*store.Store)), wire.Bind(new(store.Dispatcher), new(*store.Store)), dockerprune.NewDockerPruner, provideTiltInfo, engine.NewUpper, analytics2.NewAnalyticsUpdater, analytics2.ProvideAnalyticsReporter, provideUpdateModeFlag, fsevent.ProvideWatcherMaker, fsevent.ProvideTimerMaker, controllers.WireSet, provideWebVersion, + K8sWireSet, tiltfile.WireSet, git.ProvideGitRemote, localexec.DefaultEnv, localexec.NewProcessExecer, wire.Bind(new(localexec.Execer), new(*localexec.ProcessExecer)), docker.SwitchWireSet, dockercompose.NewDockerComposeClient, clockwork.NewRealClock, engine.DeployerWireSet, engine.NewBuildController, local.NewServerController, kubernetesdiscovery.NewContainerRestartDetector, k8swatch.NewServiceWatcher, k8swatch.NewEventWatchManager, uisession2.NewSubscriber, uiresource2.NewSubscriber, configs.NewConfigsController, configs.NewTriggerQueueSubscriber, telemetry.NewController, cloud.WireSet, cloudurl.ProvideAddress, k8srollout.NewPodMonitor, telemetry.NewStartTracker, session2.NewController, build.ProvideClock, provideClock, hud.WireSet, prompt.WireSet, wire.Value(openurl.OpenURL(openurl.BrowserOpen)), provideLogActions, store.NewStore, wire.Bind(new(store.RStore), new(*store.Store)), wire.Bind(new(store.Dispatcher), new(*store.Store)), dockerprune.NewDockerPruner, provideTiltInfo, engine.NewUpper, analytics2.NewAnalyticsUpdater, analytics2.ProvideAnalyticsReporter, provideUpdateModeFlag, fsevent.ProvideWatcherMaker, fsevent.ProvideTimerMaker, controllers.WireSet, provideCITimeoutFlag, + provideWebVersion, provideWebMode, provideWebURL, provideWebPort, @@ -1161,3 +1175,7 @@ type DumpImageDeployRefDeps struct { DockerBuilder *build.DockerBuilder DockerClient docker.Client } + +func provideCITimeoutFlag() model.CITimeoutFlag { + return model.CITimeoutFlag(ciTimeout) +} diff --git a/internal/controllers/core/session/reconciler_test.go b/internal/controllers/core/session/reconciler_test.go index 5f36deaafb..b1699d1625 100644 --- a/internal/controllers/core/session/reconciler_test.go +++ b/internal/controllers/core/session/reconciler_test.go @@ -178,6 +178,30 @@ func TestExitControlCI_GracePeriod(t *testing.T) { f.requireDoneWithError("exceeded grace period: Pod pod-a in error state due to container c1: ErrImagePull") } +func TestExitControlCI_Timeout(t *testing.T) { + f := newFixture(t, store.EngineModeCI) + + var session v1alpha1.Session + f.MustGet(types.NamespacedName{Name: "Tiltfile"}, &session) + session.Spec.CI = &v1alpha1.SessionCISpec{Timeout: &metav1.Duration{Duration: time.Minute}} + f.Update(&session) + + m := manifestbuilder.New(f, "fe").WithK8sYAML(testyaml.SanchoYAML).Build() + f.upsertManifest(m) + + f.MustReconcile(sessionKey) + f.requireNotDone() + + f.clock.Advance(50 * time.Second) + + f.MustReconcile(sessionKey) + f.requireNotDone() + + f.clock.Advance(20 * time.Second) + f.MustReconcile(sessionKey) + f.requireDoneWithError("Timeout after 1m0s") +} + func TestExitControlCI_PodRunningContainerError(t *testing.T) { f := newFixture(t, store.EngineModeCI) @@ -619,7 +643,11 @@ func newFixture(t testing.TB, engineMode store.EngineMode) *fixture { clock := clockwork.NewFakeClock() r := NewReconciler(cfb.Client, st, clock) cf := cfb.Build(r) - cf.Create(sessions.FromTiltfile(tf, nil, engineMode)) + + session := sessions.FromTiltfile(tf, nil, model.CITimeoutFlag(model.CITimeoutDefault), engineMode) + session.Status.StartTime = apis.NewMicroTime(clock.Now()) + cf.Create(session) + return &fixture{ ControllerFixture: cf, tf: tdf, diff --git a/internal/controllers/core/session/status.go b/internal/controllers/core/session/status.go index 1a190c19a8..454aef5b38 100644 --- a/internal/controllers/core/session/status.go +++ b/internal/controllers/core/session/status.go @@ -42,6 +42,17 @@ func (r *Reconciler) makeLatestStatus(session *v1alpha1.Session, result *ctrl.Re }) r.processExitCondition(session.Spec, &state, &status) + + // If there's a global timeout, schedule a requeue. + ci := session.Spec.CI + if ci != nil && ci.Timeout != nil && ci.Timeout.Duration > 0 { + timeout := ci.Timeout.Duration + requeueAfter := timeout - r.clock.Since(session.Status.StartTime.Time) + if result.RequeueAfter == 0 || result.RequeueAfter < requeueAfter { + result.RequeueAfter = requeueAfter + } + } + return status } @@ -90,6 +101,14 @@ func (r *Reconciler) processExitCondition(spec v1alpha1.SessionSpec, state *stor if allResourcesOK && len(status.Targets) > 1 { status.Done = true } + + // Enforce a global timeout. + ci := spec.CI + if status.Error == "" && ci != nil && ci.Timeout != nil && ci.Timeout.Duration > 0 && + r.clock.Since(status.StartTime.Time) > ci.Timeout.Duration { + status.Done = true + status.Error = fmt.Sprintf("Timeout after %s", ci.Timeout.Duration) + } } // errToString returns a stringified version of an error or an empty string if the error is nil. diff --git a/internal/controllers/core/tiltfile/api.go b/internal/controllers/core/tiltfile/api.go index a42f34f609..c6786c2743 100644 --- a/internal/controllers/core/tiltfile/api.go +++ b/internal/controllers/core/tiltfile/api.go @@ -55,6 +55,7 @@ func updateOwnedObjects( tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult, changeEnabledResources bool, + ciTimeoutFlag model.CITimeoutFlag, mode store.EngineMode, defaultK8sConnection *v1alpha1.KubernetesClusterConnection, ) error { @@ -78,7 +79,7 @@ func updateOwnedObjects( } } - apiObjects := toAPIObjects(nn, tf, tlr, mode, defaultK8sConnection, disableSources) + apiObjects := toAPIObjects(nn, tf, tlr, ciTimeoutFlag, mode, defaultK8sConnection, disableSources) // Propagate labels and owner references from the parent tiltfile. for _, objMap := range apiObjects { @@ -223,6 +224,7 @@ func toAPIObjects( nn types.NamespacedName, tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult, + ciTimeoutFlag model.CITimeoutFlag, mode store.EngineMode, defaultK8sConnection *v1alpha1.KubernetesClusterConnection, disableSources disableSourceMap, @@ -248,7 +250,7 @@ func toAPIObjects( result.AddSetForType(&v1alpha1.UIButton{}, toCancelButtons(tlr)) } - result.AddSetForType(&v1alpha1.Session{}, toSessionObjects(nn, tf, tlr, mode)) + result.AddSetForType(&v1alpha1.Session{}, toSessionObjects(nn, tf, tlr, ciTimeoutFlag, mode)) result.AddSetForType(&v1alpha1.UIResource{}, toUIResourceObjects(tf, tlr, disableSources)) watchInputs := WatchInputs{ @@ -589,12 +591,12 @@ func toImageMapObjects(tlr *tiltfile.TiltfileLoadResult, disableSources disableS } // Creates an object representing the tilt session and exit conditions. -func toSessionObjects(nn types.NamespacedName, tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult, mode store.EngineMode) apiset.TypedObjectSet { +func toSessionObjects(nn types.NamespacedName, tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult, ciTimeoutFlag model.CITimeoutFlag, mode store.EngineMode) apiset.TypedObjectSet { result := apiset.TypedObjectSet{} if nn.Name != model.MainTiltfileManifestName.String() { return result } - result[sessions.DefaultSessionName] = sessions.FromTiltfile(tf, tlr, mode) + result[sessions.DefaultSessionName] = sessions.FromTiltfile(tf, tlr, ciTimeoutFlag, mode) return result } diff --git a/internal/controllers/core/tiltfile/api_test.go b/internal/controllers/core/tiltfile/api_test.go index e55914e087..066c9a3a4f 100644 --- a/internal/controllers/core/tiltfile/api_test.go +++ b/internal/controllers/core/tiltfile/api_test.go @@ -376,7 +376,7 @@ func TestReconciledTypesCompleteness(t *testing.T) { fe := manifestbuilder.New(f, "fe").WithK8sYAML(testyaml.SanchoYAML).Build() tlr := &tiltfile.TiltfileLoadResult{Manifests: []model.Manifest{fe}} ds := toDisableSources(tlr) - objs := toAPIObjects(nn, tf, tlr, store.EngineModeCI, &v1alpha1.KubernetesClusterConnection{}, ds) + objs := toAPIObjects(nn, tf, tlr, 0, store.EngineModeCI, &v1alpha1.KubernetesClusterConnection{}, ds) reconciledTypes := make(map[schema.GroupVersionResource]bool) for _, t := range typesToReconcile { @@ -414,7 +414,7 @@ func newAPIFixture(t testing.TB) *apiFixture { } func (f *apiFixture) updateOwnedObjects(nn types.NamespacedName, tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult) error { - return updateOwnedObjects(f.ctx, f.c, nn, tf, tlr, false, store.EngineModeUp, + return updateOwnedObjects(f.ctx, f.c, nn, tf, tlr, false, 0, store.EngineModeUp, &v1alpha1.KubernetesClusterConnection{}) } diff --git a/internal/controllers/core/tiltfile/reconciler.go b/internal/controllers/core/tiltfile/reconciler.go index ef3eac44f8..8857ead36c 100644 --- a/internal/controllers/core/tiltfile/reconciler.go +++ b/internal/controllers/core/tiltfile/reconciler.go @@ -49,6 +49,7 @@ type Reconciler struct { requeuer *indexer.Requeuer engineMode store.EngineMode loadCount int // used to differentiate spans + ciTimeoutFlag model.CITimeoutFlag runs map[types.NamespacedName]*runStatus @@ -87,6 +88,7 @@ func NewReconciler( engineMode store.EngineMode, k8sContextOverride k8s.KubeContextOverride, k8sNamespaceOverride k8s.NamespaceOverride, + ciTimeoutFlag model.CITimeoutFlag, ) *Reconciler { return &Reconciler{ st: st, @@ -99,6 +101,7 @@ func NewReconciler( engineMode: engineMode, k8sContextOverride: k8sContextOverride, k8sNamespaceOverride: k8sNamespaceOverride, + ciTimeoutFlag: ciTimeoutFlag, } } @@ -119,7 +122,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( r.deleteExistingRun(nn) // Delete owned objects - err := updateOwnedObjects(ctx, r.ctrlClient, nn, nil, nil, false, r.engineMode, r.defaultK8sConnection()) + err := updateOwnedObjects(ctx, r.ctrlClient, nn, nil, nil, false, r.ciTimeoutFlag, r.engineMode, r.defaultK8sConnection()) if err != nil { return ctrl.Result{}, err } @@ -134,7 +137,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( run := r.runs[nn] if run == nil { // Initialize the UISession and filewatch if this has never been initialized before. - err := updateOwnedObjects(ctx, r.ctrlClient, nn, &tf, nil, false, r.engineMode, r.defaultK8sConnection()) + err := updateOwnedObjects(ctx, r.ctrlClient, nn, &tf, nil, false, r.ciTimeoutFlag, r.engineMode, r.defaultK8sConnection()) if err != nil { return ctrl.Result{}, err } @@ -357,7 +360,7 @@ func (r *Reconciler) handleLoaded( tlr *tiltfile.TiltfileLoadResult) error { // TODO(nick): Rewrite to handle multiple tiltfiles. changeEnabledResources := entry.ArgsChanged && tlr != nil && tlr.Error == nil - err := updateOwnedObjects(ctx, r.ctrlClient, nn, tf, tlr, changeEnabledResources, r.engineMode, + err := updateOwnedObjects(ctx, r.ctrlClient, nn, tf, tlr, changeEnabledResources, r.ciTimeoutFlag, r.engineMode, r.defaultK8sConnection()) if err != nil { // If updating the API server fails, just return the error, so that the diff --git a/internal/controllers/core/tiltfile/reconciler_test.go b/internal/controllers/core/tiltfile/reconciler_test.go index 93f6193118..b2ab14df7a 100644 --- a/internal/controllers/core/tiltfile/reconciler_test.go +++ b/internal/controllers/core/tiltfile/reconciler_test.go @@ -496,7 +496,7 @@ func newFixture(t *testing.T) *fixture { st := NewTestingStore() tfl := tiltfile.NewFakeTiltfileLoader() d := docker.NewFakeClient() - r := NewReconciler(st, tfl, d, cfb.Client, v1alpha1.NewScheme(), store.EngineModeUp, "", "") + r := NewReconciler(st, tfl, d, cfb.Client, v1alpha1.NewScheme(), store.EngineModeUp, "", "", 0) q := workqueue.NewRateLimitingQueue( workqueue.NewItemExponentialFailureRateLimiter(time.Millisecond, time.Millisecond)) _ = r.requeuer.Start(context.Background(), handler.Funcs{}, q) diff --git a/internal/engine/upper_test.go b/internal/engine/upper_test.go index 427aa2e387..9a35c42e79 100644 --- a/internal/engine/upper_test.go +++ b/internal/engine/upper_test.go @@ -94,6 +94,7 @@ import ( "github.com/tilt-dev/tilt/internal/testutils/servicebuilder" "github.com/tilt-dev/tilt/internal/testutils/tempdir" "github.com/tilt-dev/tilt/internal/tiltfile" + "github.com/tilt-dev/tilt/internal/tiltfile/cisettings" "github.com/tilt-dev/tilt/internal/tiltfile/config" "github.com/tilt-dev/tilt/internal/tiltfile/k8scontext" "github.com/tilt-dev/tilt/internal/tiltfile/tiltextension" @@ -3216,7 +3217,9 @@ func newTestFixture(t *testing.T, options ...fixtureOptions) *testFixture { extPlugin := tiltextension.NewFakePlugin( tiltextension.NewFakeExtRepoReconciler(f.Path()), tiltextension.NewFakeExtReconciler(f.Path())) - realTFL := tiltfile.ProvideTiltfileLoader(ta, k8sContextPlugin, versionPlugin, configPlugin, extPlugin, + ciSettingsPlugin := cisettings.NewPlugin(0) + realTFL := tiltfile.ProvideTiltfileLoader(ta, + k8sContextPlugin, versionPlugin, configPlugin, extPlugin, ciSettingsPlugin, fakeDcc, "localhost", execer, feature.MainDefaults, env) tfl := tiltfile.NewFakeTiltfileLoader() cc := configs.NewConfigsController(cdc) @@ -3265,7 +3268,7 @@ func newTestFixture(t *testing.T, options ...fixtureOptions) *testFixture { dcds := dockercomposeservice.NewDisableSubscriber(ctx, fakeDcc, clock) dcr := dockercomposeservice.NewReconciler(cdc, fakeDcc, dockerClient, st, sch, dcds) - tfr := ctrltiltfile.NewReconciler(st, tfl, dockerClient, cdc, sch, engineMode, "", "") + tfr := ctrltiltfile.NewReconciler(st, tfl, dockerClient, cdc, sch, engineMode, "", "", 0) tbr := togglebutton.NewReconciler(cdc, sch) extr := extension.NewReconciler(cdc, sch, ta) extrr, err := extensionrepo.NewReconciler(cdc, st, base) diff --git a/internal/store/sessions/object.go b/internal/store/sessions/object.go index 37774263f7..1bce1df2a0 100644 --- a/internal/store/sessions/object.go +++ b/internal/store/sessions/object.go @@ -10,6 +10,7 @@ import ( "github.com/tilt-dev/tilt/internal/tiltfile" "github.com/tilt-dev/tilt/pkg/apis" "github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1" + "github.com/tilt-dev/tilt/pkg/model" ) const DefaultSessionName = "Tiltfile" @@ -17,7 +18,7 @@ const DefaultSessionName = "Tiltfile" var processStartTime = time.Now() var processPID = int64(os.Getpid()) -func FromTiltfile(tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult, mode store.EngineMode) *v1alpha1.Session { +func FromTiltfile(tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult, ciTimeoutFlag model.CITimeoutFlag, mode store.EngineMode) *v1alpha1.Session { s := &v1alpha1.Session{ ObjectMeta: metav1.ObjectMeta{ Name: DefaultSessionName, @@ -31,6 +32,10 @@ func FromTiltfile(tf *v1alpha1.Tiltfile, tlr *tiltfile.TiltfileLoadResult, mode }, } + if mode == store.EngineModeCI { + s.Spec.CI = &v1alpha1.SessionCISpec{Timeout: &metav1.Duration{Duration: time.Duration(ciTimeoutFlag)}} + } + // TLR may be nil if the tiltfile hasn't finished loading yet. if tlr != nil { s.Spec.CI = tlr.CISettings diff --git a/internal/tiltfile/api/__init__.py b/internal/tiltfile/api/__init__.py index 379fbcc5e1..30b7f32792 100644 --- a/internal/tiltfile/api/__init__.py +++ b/internal/tiltfile/api/__init__.py @@ -1272,11 +1272,13 @@ def update_settings( """ def ci_settings( - k8s_grace_period: str='') -> None: + k8s_grace_period: str='', + timeout: str='') -> None: """Configures 'tilt ci' mode. Args: - k8s_grace_period: Grace period given for Kubernetes resources to recover after they start failing. + k8s_grace_period: Grace period given for Kubernetes resources to recover after they start failing. A duration string. + timeout: Timeout for the whole CI pipeline. A duration string. Defaults to '30m'. """ def watch_settings(ignore: Union[str, List[str]]) -> None: diff --git a/internal/tiltfile/cisettings/ci_settings.go b/internal/tiltfile/cisettings/ci_settings.go index 52ced20126..82bb9c0b41 100644 --- a/internal/tiltfile/cisettings/ci_settings.go +++ b/internal/tiltfile/cisettings/ci_settings.go @@ -10,17 +10,24 @@ import ( "github.com/tilt-dev/tilt/internal/tiltfile/starkit" "github.com/tilt-dev/tilt/internal/tiltfile/value" "github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1" + "github.com/tilt-dev/tilt/pkg/model" ) // Implements functions for dealing with ci settings. -type Plugin struct{} +type Plugin struct { + ciTimeoutFlag model.CITimeoutFlag +} -func NewPlugin() Plugin { - return Plugin{} +func NewPlugin(ciTimeoutFlag model.CITimeoutFlag) Plugin { + return Plugin{ + ciTimeoutFlag: ciTimeoutFlag, + } } func (e Plugin) NewState() interface{} { - return &v1alpha1.SessionCISpec{} + return &v1alpha1.SessionCISpec{ + Timeout: &metav1.Duration{Duration: time.Duration(e.ciTimeoutFlag)}, + } } func (e Plugin) OnStart(env *starkit.Environment) error { @@ -29,8 +36,10 @@ func (e Plugin) OnStart(env *starkit.Environment) error { func (e *Plugin) ciSettings(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var k8sGracePeriod value.Duration = -1 + var timeout value.Duration = -1 if err := starkit.UnpackArgs(thread, fn.Name(), args, kwargs, - "k8s_grace_period?", &k8sGracePeriod); err != nil { + "k8s_grace_period?", &k8sGracePeriod, + "timeout?", &timeout); err != nil { return nil, err } @@ -39,6 +48,10 @@ func (e *Plugin) ciSettings(thread *starlark.Thread, fn *starlark.Builtin, args settings = settings.DeepCopy() settings.K8sGracePeriod = &metav1.Duration{Duration: time.Duration(k8sGracePeriod)} } + if timeout != -1 { + settings = settings.DeepCopy() + settings.Timeout = &metav1.Duration{Duration: time.Duration(timeout)} + } return settings }) diff --git a/internal/tiltfile/cisettings/ci_settings_test.go b/internal/tiltfile/cisettings/ci_settings_test.go index 3513287dff..c355404b83 100644 --- a/internal/tiltfile/cisettings/ci_settings_test.go +++ b/internal/tiltfile/cisettings/ci_settings_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tilt-dev/tilt/internal/tiltfile/starkit" + "github.com/tilt-dev/tilt/pkg/model" ) func TestK8sGracePeriod(t *testing.T) { @@ -21,6 +22,7 @@ ci_settings(k8s_grace_period='3m') ci, err := GetState(result) require.NoError(t, err) require.Equal(t, 3*time.Minute, ci.K8sGracePeriod.Duration) + require.Equal(t, model.CITimeoutDefault, ci.Timeout.Duration) } func TestK8sGracePeriodOverride(t *testing.T) { @@ -53,6 +55,21 @@ ci_settings() require.Equal(t, 3*time.Minute, ci.K8sGracePeriod.Duration) } +func TestTimeout(t *testing.T) { + f := newFixture(t) + f.File("Tiltfile", ` +ci_settings(timeout='3m') +ci_settings() +`) + + result, err := f.ExecFile("Tiltfile") + require.NoError(t, err) + + ci, err := GetState(result) + require.NoError(t, err) + require.Equal(t, 3*time.Minute, ci.Timeout.Duration) +} + func newFixture(t testing.TB) *starkit.Fixture { - return starkit.NewFixture(t, NewPlugin()) + return starkit.NewFixture(t, NewPlugin(model.CITimeoutFlag(model.CITimeoutDefault))) } diff --git a/internal/tiltfile/tiltfile.go b/internal/tiltfile/tiltfile.go index 0c4e3f0cd0..40b979d46f 100644 --- a/internal/tiltfile/tiltfile.go +++ b/internal/tiltfile/tiltfile.go @@ -101,6 +101,7 @@ func ProvideTiltfileLoader( versionPlugin version.Plugin, configPlugin *config.Plugin, extensionPlugin *tiltextension.Plugin, + ciSettingsPlugin cisettings.Plugin, dcCli dockercompose.DockerComposeClient, webHost model.WebHost, execer localexec.Execer, @@ -112,6 +113,7 @@ func ProvideTiltfileLoader( versionPlugin: versionPlugin, configPlugin: configPlugin, extensionPlugin: extensionPlugin, + ciSettingsPlugin: ciSettingsPlugin, dcCli: dcCli, webHost: webHost, execer: execer, @@ -130,6 +132,7 @@ type tiltfileLoader struct { versionPlugin version.Plugin configPlugin *config.Plugin extensionPlugin *tiltextension.Plugin + ciSettingsPlugin cisettings.Plugin fDefaults feature.Defaults env clusterid.Product } @@ -171,7 +174,7 @@ func (tfl tiltfileLoader) Load(ctx context.Context, tf *corev1alpha1.Tiltfile, p tlr.Tiltignore = tiltignore s := newTiltfileState(ctx, tfl.dcCli, tfl.webHost, tfl.execer, tfl.k8sContextPlugin, tfl.versionPlugin, - tfl.configPlugin, tfl.extensionPlugin, feature.FromDefaults(tfl.fDefaults)) + tfl.configPlugin, tfl.extensionPlugin, tfl.ciSettingsPlugin, feature.FromDefaults(tfl.fDefaults)) manifests, result, err := s.loadManifests(tf) diff --git a/internal/tiltfile/tiltfile_state.go b/internal/tiltfile/tiltfile_state.go index 5969e73be0..94d2060b3c 100644 --- a/internal/tiltfile/tiltfile_state.go +++ b/internal/tiltfile/tiltfile_state.go @@ -19,6 +19,7 @@ import ( "github.com/tilt-dev/tilt/internal/controllers/apis/liveupdate" "github.com/tilt-dev/tilt/internal/controllers/apiset" "github.com/tilt-dev/tilt/internal/localexec" + "github.com/tilt-dev/tilt/internal/tiltfile/cisettings" "github.com/tilt-dev/tilt/internal/tiltfile/hasher" "github.com/tilt-dev/tilt/internal/tiltfile/links" "github.com/tilt-dev/tilt/internal/tiltfile/print" @@ -34,7 +35,6 @@ import ( "github.com/tilt-dev/tilt/internal/ospath" "github.com/tilt-dev/tilt/internal/sliceutils" "github.com/tilt-dev/tilt/internal/tiltfile/analytics" - "github.com/tilt-dev/tilt/internal/tiltfile/cisettings" "github.com/tilt-dev/tilt/internal/tiltfile/config" "github.com/tilt-dev/tilt/internal/tiltfile/dockerprune" "github.com/tilt-dev/tilt/internal/tiltfile/encoding" @@ -85,6 +85,7 @@ type tiltfileState struct { versionPlugin version.Plugin configPlugin *config.Plugin extensionPlugin *tiltextension.Plugin + ciSettingsPlugin cisettings.Plugin features feature.FeatureSet // added to during execution @@ -163,6 +164,7 @@ func newTiltfileState( versionPlugin version.Plugin, configPlugin *config.Plugin, extensionPlugin *tiltextension.Plugin, + ciSettingsPlugin cisettings.Plugin, features feature.FeatureSet) *tiltfileState { return &tiltfileState{ ctx: ctx, @@ -173,6 +175,7 @@ func newTiltfileState( versionPlugin: versionPlugin, configPlugin: configPlugin, extensionPlugin: extensionPlugin, + ciSettingsPlugin: ciSettingsPlugin, buildIndex: newBuildIndex(), k8sObjectIndex: tiltfile_k8s.NewState(), k8sByName: make(map[string]*k8sResource), @@ -223,7 +226,7 @@ func (s *tiltfileState) loadManifests(tf *v1alpha1.Tiltfile) ([]model.Manifest, telemetry.NewPlugin(), metrics.NewPlugin(), updatesettings.NewPlugin(), - cisettings.NewPlugin(), + s.ciSettingsPlugin, secretsettings.NewPlugin(), encoding.NewPlugin(), shlex.NewPlugin(), diff --git a/internal/tiltfile/tiltfile_test.go b/internal/tiltfile/tiltfile_test.go index 6ccc04fd9f..3f520bcdc8 100644 --- a/internal/tiltfile/tiltfile_test.go +++ b/internal/tiltfile/tiltfile_test.go @@ -36,6 +36,7 @@ import ( "github.com/tilt-dev/tilt/internal/sliceutils" "github.com/tilt-dev/tilt/internal/testutils" "github.com/tilt-dev/tilt/internal/testutils/tempdir" + "github.com/tilt-dev/tilt/internal/tiltfile/cisettings" "github.com/tilt-dev/tilt/internal/tiltfile/config" "github.com/tilt-dev/tilt/internal/tiltfile/hasher" tiltfile_k8s "github.com/tilt-dev/tilt/internal/tiltfile/k8s" @@ -5796,8 +5797,9 @@ func (f *fixture) newTiltfileLoader() TiltfileLoader { extr := tiltextension.NewFakeExtReconciler(f.Path()) extrr := tiltextension.NewFakeExtRepoReconciler(f.Path()) extPlugin := tiltextension.NewFakePlugin(extrr, extr) + ciSettingsPlugin := cisettings.NewPlugin(0) return ProvideTiltfileLoader(f.ta, k8sContextPlugin, versionPlugin, configPlugin, - extPlugin, dcc, f.webHost, execer, f.features, f.k8sEnv) + extPlugin, ciSettingsPlugin, dcc, f.webHost, execer, f.features, f.k8sEnv) } func newFixture(t *testing.T) *fixture { diff --git a/internal/tiltfile/wire.go b/internal/tiltfile/wire.go index 4c56290078..3f5d4dac3a 100644 --- a/internal/tiltfile/wire.go +++ b/internal/tiltfile/wire.go @@ -3,6 +3,7 @@ package tiltfile import ( "github.com/google/wire" + "github.com/tilt-dev/tilt/internal/tiltfile/cisettings" "github.com/tilt-dev/tilt/internal/tiltfile/config" "github.com/tilt-dev/tilt/internal/tiltfile/k8scontext" "github.com/tilt-dev/tilt/internal/tiltfile/tiltextension" @@ -15,4 +16,5 @@ var WireSet = wire.NewSet( version.NewPlugin, config.NewPlugin, tiltextension.NewPlugin, + cisettings.NewPlugin, ) diff --git a/pkg/model/ci.go b/pkg/model/ci.go new file mode 100644 index 0000000000..8b48b52dfb --- /dev/null +++ b/pkg/model/ci.go @@ -0,0 +1,8 @@ +package model + +import "time" + +// Inject the flag-specified CI timeout. +type CITimeoutFlag time.Duration + +const CITimeoutDefault = 30 * time.Minute diff --git a/pkg/webview/view.pb.go b/pkg/webview/view.pb.go index 192751592e..065ea9b653 100644 --- a/pkg/webview/view.pb.go +++ b/pkg/webview/view.pb.go @@ -741,12 +741,12 @@ func (x *VersionSettings) GetCheckUpdates() bool { // Our websocket service has two kinds of View messages: // -// 1) On initialization, we send down the complete view state -// (TiltStartTime, UISession, UIResources, and LogList) +// 1. On initialization, we send down the complete view state +// (TiltStartTime, UISession, UIResources, and LogList) // -// 2) On every change, we send down the resources that have -// changed since the last send(). -// (new logs and any updated UISession/UIResource objects). +// 2. On every change, we send down the resources that have +// changed since the last send(). +// (new logs and any updated UISession/UIResource objects). // // All other fields are obsolete, but are needed for deserializing // old snapshots. @@ -1268,9 +1268,9 @@ func (x *UploadSnapshotResponse) GetUrl() string { // NOTE(nick): This is obsolete. // // Our websocket service has two kinds of messages: -// 1) On initialization, we send down the complete view state -// 2) On every change, we send down the resources that have -// changed since the last send(). +// 1. On initialization, we send down the complete view state +// 2. On every change, we send down the resources that have +// changed since the last send(). type AckWebsocketRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache