-
Notifications
You must be signed in to change notification settings - Fork 47
fix(push): prevent hang and tag race in apify push (#1131) #1134
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
07e801a
cfe0e26
d9bf8ed
b1d8b3e
9e26c3e
d8e957b
1cac10f
1867aca
5a40540
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,7 +6,12 @@ import type { Actor, ActorCollectionCreateOptions, ActorDefaultRunOptions } from | |
| import open from 'open'; | ||
|
|
||
| import { fetchManifest } from '@apify/actor-templates'; | ||
| import { ACTOR_JOB_STATUSES, ACTOR_SOURCE_TYPES, MAX_MULTIFILE_BYTES } from '@apify/consts'; | ||
| import { | ||
| ACTOR_JOB_STATUSES, | ||
| ACTOR_JOB_TERMINAL_STATUSES, | ||
| ACTOR_SOURCE_TYPES, | ||
| MAX_MULTIFILE_BYTES, | ||
| } from '@apify/consts'; | ||
| import { createHmacSignature } from '@apify/utilities'; | ||
|
|
||
| import { ApifyCommand } from '../../lib/command-framework/apify-command.js'; | ||
|
|
@@ -25,6 +30,7 @@ import { | |
| getLocalUserInfo, | ||
| getLoggedClientOrThrow, | ||
| outputJobLog, | ||
| parseWaitForFinishMillis, | ||
| printJsonToStdout, | ||
| } from '../../lib/utils.js'; | ||
|
|
||
|
|
@@ -190,9 +196,7 @@ export class ActorsPushCommand extends ApifyCommand<typeof ActorsPushCommand> { | |
| buildTag = DEFAULT_BUILD_TAG; | ||
| } | ||
|
|
||
| const waitForFinishMillis = Number.isNaN(this.flags.waitForFinish) | ||
| ? undefined | ||
| : Number.parseInt(this.flags.waitForFinish!, 10) * 1000; | ||
| const waitForFinishMillis = parseWaitForFinishMillis(this.flags.waitForFinish); | ||
|
|
||
| // User can override actorId of pushing Actor. | ||
| // It causes that we push Actor to this id but attributes in localConfig will remain same. | ||
|
|
@@ -359,18 +363,16 @@ Skipping push. Use --force to override.`, | |
| waitForFinish: 2, // NOTE: We need to wait some time to Apify open stream and we can create connection | ||
| }); | ||
|
|
||
| try { | ||
| // While the log is streaming, forward interrupt signals to a | ||
| // platform-side abort so the build doesn't keep running after the | ||
| // user gives up waiting (Ctrl+C, SIGTERM from a parent process, | ||
| // SIGHUP from a closing terminal). The `using` binding guarantees | ||
| // the listener is removed before we poll for final status. | ||
| using _signalHandler = useAbortJobOnSignal({ | ||
| apifyClient, | ||
| kind: 'build', | ||
| jobId: build.id, | ||
| }); | ||
| // Forward interrupt signals (Ctrl+C, SIGTERM, SIGHUP) to a platform-side | ||
| // abort for the lifetime of log streaming AND status polling, so the | ||
| // build doesn't keep running after the user gives up waiting. | ||
| using _signalHandler = useAbortJobOnSignal({ | ||
| apifyClient, | ||
| kind: 'build', | ||
| jobId: build.id, | ||
| }); | ||
|
|
||
| try { | ||
| await outputJobLog({ job: build, timeoutMillis: waitForFinishMillis, apifyClient }); | ||
| } catch (err) { | ||
| warning({ message: 'Can not get log:' }); | ||
|
|
@@ -379,6 +381,42 @@ Skipping push. Use --force to override.`, | |
|
|
||
| build = (await apifyClient.build(build.id).get())!; | ||
|
|
||
| // `outputJobLog` can return before the build is actually terminal (stream | ||
| // ended early, timeout hit). Poll so the status branches below see the | ||
| // real outcome. With no --wait-for-finish, the flag documents "waits | ||
| // forever", so poll without a deadline. | ||
| const deadline = waitForFinishMillis === undefined ? Infinity : Date.now() + waitForFinishMillis; | ||
| while (!ACTOR_JOB_TERMINAL_STATUSES.includes(build.status as never) && Date.now() < deadline) { | ||
| await new Promise((resolve) => setTimeout(resolve, 1000)); | ||
| build = (await apifyClient.build(build.id).get())!; | ||
| } | ||
|
|
||
| // Platform updates `taggedBuilds[buildTag]` asynchronously after the | ||
| // build finishes. Wait until the tag points at this build so callers | ||
| // (including --json automation) that immediately | ||
| // `actor.start({ build: buildTag })` don't race it. Capped at 30s so an | ||
| // unknown platform delay can't stall push forever. | ||
| if (build.status === ACTOR_JOB_STATUSES.SUCCEEDED && buildTag) { | ||
| // 30s budget is independent of --wait-for-finish: the build is already | ||
| // done, we're only waiting on the platform to update the tag pointer. | ||
| const tagDeadline = Date.now() + 30_000; | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 30s is wayy too much, i'd bail out after 5 max. Also lets print out a message that we are "applying the build tag" (even tho the console does it) just so users know why their terminal is hanging |
||
| let tagApplied = false; | ||
| while (Date.now() < tagDeadline) { | ||
| const a = await actorClient.get(); | ||
| if (a?.taggedBuilds?.[buildTag]?.buildId === build.id) { | ||
| tagApplied = true; | ||
| break; | ||
| } | ||
| await new Promise((resolve) => setTimeout(resolve, 1000)); | ||
| } | ||
| if (!tagApplied) { | ||
| warning({ | ||
| message: `Build succeeded but tag "${buildTag}" did not update within 30s; subsequent calls referencing this tag may race.`, | ||
| }); | ||
| process.exitCode = CommandExitCodes.BuildTimedOut; | ||
| } | ||
| } | ||
|
|
||
| if (this.flags.json) { | ||
| printJsonToStdout(build); | ||
| return; | ||
|
|
@@ -403,9 +441,11 @@ Skipping push. Use --force to override.`, | |
| // @ts-expect-error FIX THESE TYPES 😢 | ||
| } else if (build.status === ACTOR_JOB_STATUSES.READY) { | ||
| warning({ message: 'Build is waiting for allocation.' }); | ||
| process.exitCode = CommandExitCodes.BuildTimedOut; | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is technically wrong, if I pass in |
||
| // @ts-expect-error FIX THESE TYPES 😢 | ||
| } else if (build.status === ACTOR_JOB_STATUSES.RUNNING) { | ||
| warning({ message: 'Build is still running.' }); | ||
| process.exitCode = CommandExitCodes.BuildTimedOut; | ||
| // @ts-expect-error FIX THESE TYPES 😢 | ||
| } else if (build.status === ACTOR_JOB_STATUSES.ABORTED || build.status === ACTOR_JOB_STATUSES.ABORTING) { | ||
| warning({ message: 'Build was aborted!' }); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -607,7 +607,7 @@ export const outputJobLog = async ({ | |
| } | ||
| }); | ||
|
|
||
| if (timeoutMillis) { | ||
| if (timeoutMillis !== undefined) { | ||
| nodeTimeout = setTimeout(() => { | ||
| stream.destroy(); | ||
| resolve('timeouts'); | ||
|
|
@@ -840,3 +840,10 @@ export function shellConfigFile(userHomeDirectory: string, shell: ReturnType<typ | |
| } | ||
| } | ||
| } | ||
|
|
||
| export function parseWaitForFinishMillis(flag: string | undefined): number | undefined { | ||
| if (flag === undefined) return undefined; | ||
| const parsed = Number.parseInt(flag, 10); | ||
| if (!Number.isFinite(parsed) || parsed < 0) return undefined; | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. mmmm, can you compare what happens right now with |
||
| return parsed * 1000; | ||
| } | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok this test is kinda useless :D |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| import { parseWaitForFinishMillis } from '../../../src/lib/utils.js'; | ||
|
|
||
| describe('parseWaitForFinishMillis()', () => { | ||
| it('returns undefined when flag is omitted', () => { | ||
| expect(parseWaitForFinishMillis(undefined)).toBeUndefined(); | ||
| }); | ||
|
|
||
| it('returns undefined for non-numeric input', () => { | ||
| expect(parseWaitForFinishMillis('abc')).toBeUndefined(); | ||
| }); | ||
|
|
||
| it('returns 0 for zero (no wait)', () => { | ||
| expect(parseWaitForFinishMillis('0')).toBe(0); | ||
| }); | ||
|
|
||
| it('returns undefined for negative values', () => { | ||
| expect(parseWaitForFinishMillis('-5')).toBeUndefined(); | ||
| }); | ||
|
|
||
| it('converts positive seconds to milliseconds', () => { | ||
| expect(parseWaitForFinishMillis('30')).toBe(30_000); | ||
| }); | ||
| }); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I feel like the deadline should be computed right before starting the build, not after waiting for the outputLog to finalizr/crash.
also, won't this cause a double-wait? Imagine an actor that takes 4minutes to build, if --waitForFinish is set to 30s, we print for 29s then connection dies for the log stream, we wait another 30s after? Ideally we calculate a delta between build start, and log end, and wait out the remaining time, if any, right?