Send logs for pods in bad status #4572
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -20,8 +20,6 @@ import ( | |||||
| "context" | ||||||
| "fmt" | ||||||
|
|
||||||
| kpav1alpha1 "github.com/knative/serving/pkg/apis/autoscaling/v1alpha1" | ||||||
| "github.com/knative/serving/pkg/apis/serving/v1alpha1" | ||||||
| "github.com/knative/serving/pkg/reconciler/revision/resources" | ||||||
|
|
@@ -32,12 +30,20 @@ import ( | |||||
| "k8s.io/apimachinery/pkg/api/equality" | ||||||
| apierrs "k8s.io/apimachinery/pkg/api/errors" | ||||||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||||||
| "knative.dev/pkg/logging" | ||||||
| "knative.dev/pkg/logging/logkey" | ||||||
| ) | ||||||
|
|
||||||
| // userFacing is the key used to represent whether the logs should be surfaced | ||||||
| // to user | ||||||
| // TODO: Move this to knative/pkg/logging/logkey | ||||||
| const userFacing = "userfacing" | ||||||
|
|
||||||
| func (c *Reconciler) reconcileDeployment(ctx context.Context, rev *v1alpha1.Revision) error { | ||||||
| ns := rev.Namespace | ||||||
| deploymentName := resourcenames.Deployment(rev) | ||||||
| logger := logging.FromContext(ctx).With(zap.String(logkey.Deployment, deploymentName)) | ||||||
| userFacingLogger := logger.With(zap.Bool(userFacing, true)) | ||||||
|
|
||||||
| deployment, err := c.deploymentLister.Deployments(ns).Get(deploymentName) | ||||||
| if apierrs.IsNotFound(err) { | ||||||
|
|
@@ -66,32 +72,52 @@ func (c *Reconciler) reconcileDeployment(ctx context.Context, rev *v1alpha1.Revi | |||||
| } | ||||||
|
|
||||||
| // If a container keeps crashing (no active pods in the deployment although we want some) | ||||||
| if *deployment.Spec.Replicas > deployment.Status.AvailableReplicas { | ||||||
|
Contributor
There was a problem hiding this comment. This will always be true if we're scaling up, right? |
||||||
| pods, err := c.KubeClientSet.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(deployment.Spec.Selector)}) | ||||||
| if err != nil { | ||||||
| logger.Errorf("Error getting pods: %v", err) | ||||||
| } else if len(pods.Items) > 0 { | ||||||
| // Should change revision status if all pods are crashing | ||||||
| shouldMarkRev := deployment.Status.AvailableReplicas == 0 | ||||||
| for _, pod := range pods.Items { | ||||||
| // Update the revision status if pod cannot be scheduled(possibly resource constraints) | ||||||
|
Contributor
There was a problem hiding this comment.
Suggested change
|
||||||
| // If pod cannot be scheduled then we expect the container status to be empty. | ||||||
| for _, cond := range pod.Status.Conditions { | ||||||
| if cond.Type == corev1.PodScheduled && cond.Status == corev1.ConditionFalse { | ||||||
|
Contributor
There was a problem hiding this comment. #4136 tries to grab this status from the deployment rather than from individual pods. Is that possible here as well? |
||||||
| userFacingLogger.Warnf("Pod %s of revision %s cannot be scheduled: %s/%s", pod.Name, rev.Name, cond.Reason, cond.Message) | ||||||
| if shouldMarkRev { | ||||||
| logger.Infof("%s marking resources unavailable with: %s: %s", rev.Name, cond.Reason, cond.Message) | ||||||
| rev.Status.MarkResourcesUnavailable(cond.Reason, cond.Message) | ||||||
| } | ||||||
| break | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| for _, status := range pod.Status.ContainerStatuses { | ||||||
| // This is based on the fact that rev.Spec.GetContainer() returns | ||||||
| // the user container. | ||||||
| if status.Name == rev.Spec.GetContainer().Name { | ||||||
| if t := status.LastTerminationState.Terminated; t != nil { | ||||||
| userFacingLogger.Warnf("Container %s in pod %s of revision %s is in terminated status: %s/%s", | ||||||
|
Contributor
There was a problem hiding this comment. Should we produce events instead of logs? They'd be actually userfacing. On that note: Are these events already produced for the individual pods? |
||||||
| status.Name, pod.Name, rev.Name, t.Reason, t.Message) | ||||||
| if shouldMarkRev { | ||||||
| logger.Infof("%s marking exiting with: %d/%s: %s", rev.Name, t.ExitCode, t.Reason, t.Message) | ||||||
| rev.Status.MarkContainerExiting(t.ExitCode, t.Reason, t.Message) | ||||||
|
Contributor
There was a problem hiding this comment. So if I have 3 pods one is ok, 2 are crashing but for different reasons, you'll set the status twice to different things? |
||||||
| } | ||||||
| } else if w := status.State.Waiting; w != nil && hasDeploymentTimedOut(deployment) { | ||||||
| userFacingLogger.Warnf("Container %s in pod %s of revision %s is timeout in waiting status: %s/%s", | ||||||
| status.Name, pod.Name, rev.Name, w.Reason, w.Message) | ||||||
| if shouldMarkRev { | ||||||
| logger.Infof("%s marking resources unavailable with: %s: %s", rev.Name, w.Reason, w.Message) | ||||||
| rev.Status.MarkResourcesUnavailable(w.Reason, w.Message) | ||||||
| } | ||||||
| } | ||||||
| break | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| if shouldMarkRev { | ||||||
| // Arbitrarily check the very first pod, as they all should be crashing | ||||||
|
Contributor
There was a problem hiding this comment. This comment seems wrong now. |
||||||
| break | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Golint comments: exported method RevisionStatus.MarkContainerExiting should have comment or be unexported. More info.