fix: graceful instance restart on docker exit

This commit is contained in:
Ben Allfree 2023-10-20 03:50:57 -07:00
parent 23bf51ebe5
commit fb30978d8f
3 changed files with 28 additions and 45 deletions

View File

@ -61,9 +61,6 @@ global.EventSource = require('eventsource')
version: DAEMON_PB_SEMVER,
name: PUBLIC_MOTHERSHIP_NAME,
slug: PUBLIC_MOTHERSHIP_NAME,
onUnexpectedStop: () => {
error(`migrate had an unexpected stop. Check it out`)
},
},
{ logger },
)
@ -71,20 +68,14 @@ global.EventSource = require('eventsource')
info(`Migrating done`)
}
info(`Serving`)
const { url } = await pbService.spawn(
{
const { url } = await pbService.spawn({
command: 'serve',
isMothership: true,
version: DAEMON_PB_SEMVER,
name: PUBLIC_MOTHERSHIP_NAME,
slug: PUBLIC_MOTHERSHIP_NAME,
port: MOTHERSHIP_PORT,
onUnexpectedStop: () => {
error(`serve had an unexpected stop. Check it out`)
},
},
{ logger },
)
})
/**
* Launch services

View File

@ -257,26 +257,6 @@ export const instanceService = mkSingleton(
port: newPort,
env: instance.secrets || {},
version,
onUnexpectedStop: (code) => {
warn(
`PocketBase processes exited unexpectedly with ${code}. Putting in maintenance mode.`,
)
shutdownManager.add(async () => {
await updateInstance(instance.id, {
maintenance: true,
})
userInstanceLogger.error(
`Putting instance in maintenance mode because it shut down with return code ${code}. `,
)
})
setImmediate(() => {
_safeShutdown(
new Error(
`PocketBase processes exited unexpectedly with ${code}. Putting in maintenance mode.`,
),
).catch(error)
})
},
})
return cp
} catch (e) {
@ -289,10 +269,27 @@ export const instanceService = mkSingleton(
)
}
})()
const { pid: _pid } = childProcess
const { pid: _pid, exited } = childProcess
const pid = _pid()
exited.then((code) => {
dbg(`PocketBase processes exited with ${code}.`)
if (code !== 0) {
shutdownManager.add(async () => {
userInstanceLogger.error(
`Putting instance in maintenance mode because it shut down with return code ${code}. `,
)
await updateInstance(instance.id, {
maintenance: true,
})
})
}
setImmediate(() => {
_safeShutdown().catch(error)
})
})
assertTruthy(pid, `Expected PID here but got ${pid}`)
dbg(`PocketBase instance PID: ${pid}`)
systemInstanceLogger.breadcrumb(`pid:${pid}`)
shutdownManager.add(async () => {
dbg(`killing ${id}`)

View File

@ -39,7 +39,6 @@ export type SpawnConfig = {
env?: Env
stdout?: MemoryStream
stderr?: MemoryStream
onUnexpectedStop: (code: number | null) => void
}
export type PocketbaseServiceApi = AsyncReturnType<
typeof createPocketbaseService
@ -91,7 +90,6 @@ export const createPocketbaseService = async (
name,
slug,
port,
onUnexpectedStop,
isMothership,
env,
stderr,
@ -208,7 +206,6 @@ export const createPocketbaseService = async (
`Unexpected stop with code ${StatusCode} and error ${err}`,
)
error(`${slug} stopped unexpectedly with code ${err}`, data)
onUnexpectedStop?.(StatusCode)
resolveExit(StatusCode || 999)
} else {
resolveExit(0)
@ -223,7 +220,6 @@ export const createPocketbaseService = async (
if (!container) {
iLogger.error(`Could not start container`)
error(`${slug} could not start container`)
onUnexpectedStop?.(999)
resolveExit(999)
}
})
@ -249,9 +245,8 @@ export const createPocketbaseService = async (
kill: async () => {
unsub()
if (!container) {
throw new Error(
`Attempt to kill a PocketBase process that was never running.`,
)
dbg(`Already exited`)
return
}
iLogger.info(`Stopping instance`)
await container.stop()