| c3ddbeec-0d36-4457-84c2-f3cbce063d9a |
repro |
|
💥 |
WARNING in br_nf_local_in (2) |
2026/03/06 05:59 |
2026/03/06 05:59 |
2026/03/06 06:16 |
31e9c887f7dc24e04b3ca70d0d54fc34141844b0 |
Error 504, Message: Deadline expired before operation could complete., Status: DEADLINE_EXCEEDED, Details: [map[@type:type.googleapis.com/google.rpc.DebugInfo detail:[ORIGINAL ERROR] generic::deadline_exceeded: [original: beyond::dependency::DEADLINE_EXCEEDED] Deadline exceeded during decode.; Failed while executing Op 'Decode'; Failed to close the streaming context; status = DEADLINE_EXCEEDED: Deadline exceeded during decode.; Failed while executing Op 'Decode' [type.googleapis.com/stairways.orch.ErrorLoc='learning/serving/servables/wiz/orch_wiz_servable.cc:1027']
=== Source Location Trace: ===
learning/brain/research/stairways/orchestrator/leaf/chain_stub.cc:125
learning/serving/servables/wiz/orch_wiz_servable.cc:1037
; Failed to run inference for model: go/debugstr
name: "prod-common-global__/aistudio/gemini-v4m-rev22-riftrunner-sc__main__/aistudio/gemini-v4m-rev22-riftrunner-sc__2026020501__prefill__variant-sha__f96c8406-5f3a-4706-8967-e126111084bc"
version {
value: 1
}
signature_name: "serving_default"
; [model=/aistudio/gemini-v4m-rev22-riftrunner-sc,target=PredictionService]; for endpoint: [google.rpc.error_details_ext] { message: "Deadline expired before operation could complete." details { type_url: "type.googleapis.com/language_labs.genai.debug.GeminiApiDebugInfo" value: "\222\001\375\006\n\335\006Deadline exceeded during decode.; Failed while executing Op \'Decode\'; Failed to close the streaming context; status = DEADLINE_EXCEEDED: Deadline exceeded during decode.; Failed while executing Op \'Decode\' [type.googleapis.com/stairways.orch.ErrorLoc=\'learning/serving/servables/wiz/orch_wiz_servable.cc:1027\']\n=== Source Location Trace: === \nlearning/brain/research/stairways/orchestrator/leaf/chain_stub.cc:125\nlearning/serving/servables/wiz/orch_wiz_servable.cc:1037\n; Failed to run inference for model: go/debugstr \nname: \"prod-common-global__/aistudio/gemini-v4m-rev22-riftrunner-sc__main__/aistudio/gemini-v4m-rev22-riftrunner-sc__2026020501__prefill__variant-sha__f96c8406-5f3a-4706-8967-e126111084bc\"\nversion {\n value: 1\n}\nsignature_name: \"serving_default\"\n; [model=/aistudio/gemini-v4m-rev22-riftrunner-sc,target=PredictionService]; for endpoint: \022\033net/rpc/rpc-status.cc:562:0" } } 525002922 { 3 { 1: "/aistudio/gemini-v4m-rev22-riftrunner-sc" } 6 { 5 { 1 { 1: "/aistudio/gemini-v4m-rev22-riftrunner-sc" 3: "serving_default" } 2: 131072 3: 1 } 6 { 2: 1 3: 0 4: "Non-retryable error: Deadline exceeded during decode.; Failed while executing Op \'Decode\'; Failed to close the streaming context; status = DEADLINE_EXCEEDED: Deadline exceeded during decode.; Failed while executing Op \'Decode\' [type.googleapis.com/stairways.orch.ErrorLoc=\'learning/serving/servables/wiz/orch_wiz_servable.cc:1027\']\n=== Source Location Trace: === \nlearning/brain/research/stairways/orchestrator/leaf/chain_stub.cc:125\nlearning/serving/servables/wiz/orch_wiz_servable.cc:1037\n; Failed to run inference for model: go/debugstr \nname: \"prod-common-global__/aistudio/gemini-v4m-rev22-riftrunner-sc__main__/aistudio/gemini-v4m-rev22-riftrunner-sc__2026020501__prefill__variant-sha__f96c8406-5f3a-4706-8967-e126111084bc\"\nversion {\n value: 1\n}\nsignature_name: \"serving_default\"\n; [model=/aistudio/gemini-v4m-rev22-riftrunner-sc,target=PredictionService]; for endpoint: " 5 { 1 { 1: 1772777186 2: 274691543 } 2 { 1: 597 2: 620730513 } } 6: "\200\200\004" } } }]] |