diff --git a/.changeset/1719707021337.md b/.changeset/1719707021337.md new file mode 100644 index 00000000..f5cc91d1 --- /dev/null +++ b/.changeset/1719707021337.md @@ -0,0 +1,5 @@ +--- +'@pockethost/plugin-launcher-spawn': minor +--- + +Relocated instance database, added support for starting/stopping/listing instances via CLI, added remote internal API for starting/stopping/listing instances. \ No newline at end of file diff --git a/packages/plugin-launcher-spawn/package.json b/packages/plugin-launcher-spawn/package.json index 3a9fa6f5..ad5bb5c6 100644 --- a/packages/plugin-launcher-spawn/package.json +++ b/packages/plugin-launcher-spawn/package.json @@ -22,19 +22,26 @@ }, "license": "MIT", "dependencies": { + "@s-libs/micro-dash": "^16.1.0", "@types/node": "^20.8.10", "@types/semver": "^7.5.4", "async-mutex": "^0.5.0", + "boolean": "^3.2.0", "bottleneck": "^2.19.5", + "commander": "^11.1.0", + "express": "^4.18.2", "get-port": "^6.1.2", "glob": "^10.4.2", "gobot": "1.0.0-alpha.40", + "lowdb": "^7.0.1", "semver": "^7.6.2" }, "peerDependencies": { "pockethost": "workspace:^1.5.0" }, "devDependencies": { + "@types/express": "^4.17.21", + "@types/lowdb": "^1.0.15", "typescript": "^5.4.5" } } diff --git a/packages/plugin-launcher-spawn/src/InternalApp.ts b/packages/plugin-launcher-spawn/src/InternalApp.ts new file mode 100644 index 00000000..debc2a35 --- /dev/null +++ b/packages/plugin-launcher-spawn/src/InternalApp.ts @@ -0,0 +1,83 @@ +import { boolean } from 'boolean' +import express, { RequestHandler } from 'express' +import { + doGetOneInstanceByExactCriteriaFilter, + doGetOrProvisionInstanceUrlFilter, + doIsInstanceRunningFilter, + doKillInstanceAction, +} from 'pockethost' +import { PLUGIN_NAME } from './constants' +import { DbService } from './db' +import { dbg, info } from './log' + +const getInstanceBySubdomainOrId = async (subdomainOrId: string) => { + { + const instance = await doGetOneInstanceByExactCriteriaFilter(undefined, { + id: subdomainOrId, + }) + if (instance) return instance + } + { + const instance = await doGetOneInstanceByExactCriteriaFilter(undefined, { + subdomain: subdomainOrId, + }) + if (instance) return instance + } +} + +export const InternalApp = (): RequestHandler => { + const app = express() + app.get('/start/:subdomainOrInstanceId', async (req, res) => { + const { subdomainOrInstanceId } = req.params + dbg(`Got start request for ${subdomainOrInstanceId}`) + const instance = await getInstanceBySubdomainOrId(subdomainOrInstanceId) + + dbg(`Instance:`, instance) + if (!instance) { + res.status(404).send(`Instance not found`) + return + } + const url = await doGetOrProvisionInstanceUrlFilter(undefined, { instance }) + res.send(`Started instance ${subdomainOrInstanceId} at ${url}`) + }) + + app.get('/stop/:subdomainOrInstanceId', async (req, res) => { + const { subdomainOrInstanceId } = req.params + dbg(`Got stop request for ${subdomainOrInstanceId}`) + const instance = await getInstanceBySubdomainOrId(subdomainOrInstanceId) + dbg(`Instance:`, instance) + if (!instance) { + res.status(404).send(`Instance not found`) + return + } + await doKillInstanceAction({ instance }) + res.send(`Stopped instance ${instance.subdomain} (${instance.id})`) + }) + + app.get('/list', async (req, res) => { + const shouldListAll = boolean(req.query.all) + dbg(`Got list request. Should list all: ${shouldListAll}`) + const { getInstancesByExactCriteria } = await DbService() + const instances = getInstancesByExactCriteria({}) + info(`Listing instances`) + const final = [] + for (const instance of instances) { + const { subdomain, id, version } = instance + const isRunning = await doIsInstanceRunningFilter(false, { + instance, + }) + if (!shouldListAll && !isRunning) continue + final.push({ instance, isRunning }) + } + res.json(final) + }) + + app.use('*', (req, res) => { + dbg(`Got request to internal app ${req.url}`) + res.send(`Got request to internal app ${req.url}`) + }) + + const root = express() + root.use(`/${PLUGIN_NAME}`, app) + return root +} diff --git a/packages/plugin-launcher-spawn/src/LauncherCommand.ts b/packages/plugin-launcher-spawn/src/LauncherCommand.ts new file mode 100644 index 00000000..008cc7dd --- /dev/null +++ b/packages/plugin-launcher-spawn/src/LauncherCommand.ts @@ -0,0 +1,107 @@ +import { Command } from 'commander' +import { + INTERNAL_APP_AUTH_HEADER, + INTERNAL_APP_SECRET, + INTERNAL_APP_URL, + PUBLIC_INSTANCE_URL, + tryFetch, +} from 'pockethost/core' +import { PLUGIN_NAME } from './constants' +import { dbg, error, info } from './log' + +export const LauncherCommand = () => + new Command(`spawn`) + .description(`Spawn launcher commands`) + .addCommand( + new Command(`start`) + .argument(``) + .description(`Start an instance`) + .action(async (subdomainOrId: string) => { + const internalRequestUrl = INTERNAL_APP_URL( + PLUGIN_NAME, + `start/${subdomainOrId}`, + ) + dbg( + `Starting instance ${subdomainOrId} by sending request to ${internalRequestUrl}`, + ) + const res = await fetch(internalRequestUrl, { + headers: { [INTERNAL_APP_AUTH_HEADER]: INTERNAL_APP_SECRET() }, + }) + if (!res.ok) { + error( + `Failed to start instance ${subdomainOrId}: ${res.status} ${ + res.statusText + } - ${await res.text()}`, + ) + } else { + info(`Started instance ${subdomainOrId}: ${await res.text()}`) + } + }), + ) + .addCommand( + new Command(`stop`) + .argument(``) + .description(`Stop an instance`) + .action(async (subdomainOrId) => { + const internalRequestUrl = INTERNAL_APP_URL( + PLUGIN_NAME, + `stop/${subdomainOrId}`, + ) + dbg( + `Stopping instance ${subdomainOrId} by sending request to ${internalRequestUrl}`, + ) + const res = await tryFetch(internalRequestUrl, { + headers: { [INTERNAL_APP_AUTH_HEADER]: INTERNAL_APP_SECRET() }, + }) + const text = await res.text() + if (!res.ok) { + error( + `Failed to start instance ${subdomainOrId}: ${res.status} ${res.statusText} - ${text}`, + ) + } else { + info(`Stopped instance ${subdomainOrId}: ${text}`) + } + }), + ) + .addCommand( + new Command(`list`) + .alias(`ls`) + .description(`List instances`) + .option(`-a, --all`, `List all instances (not just running instances)`) + .action(async (options) => { + const internalRequestUrl = INTERNAL_APP_URL( + PLUGIN_NAME, + `list?all=${!!options.all}`, + ) + dbg(`Listing instances by sending request to ${internalRequestUrl}`) + const res = await tryFetch(internalRequestUrl, { + headers: { [INTERNAL_APP_AUTH_HEADER]: INTERNAL_APP_SECRET() }, + }) + const text = await res.text() + if (!res.ok) { + error( + `Failed to list instances: ${res.status} ${res.statusText} - ${text}`, + ) + } else { + const instances = JSON.parse(text) as { + instance: any + isRunning: boolean + }[] + if (!instances.length) { + info(`No instances found. Try --all`) + return + } + for (const { + instance, + instance: { subdomain, id, version }, + isRunning, + } of instances) { + info( + `${PUBLIC_INSTANCE_URL(instance)} [${id}|${version}|${ + isRunning ? 'running' : 'stopped' + }]`, + ) + } + } + }), + ) diff --git a/packages/plugin-launcher-spawn/src/constants.ts b/packages/plugin-launcher-spawn/src/constants.ts index b49b14af..badde9eb 100644 --- a/packages/plugin-launcher-spawn/src/constants.ts +++ b/packages/plugin-launcher-spawn/src/constants.ts @@ -1,12 +1,13 @@ import { join } from 'path' -import { DEBUG } from 'pockethost' -import { PH_HOME, Settings, logSettings, mkPath } from 'pockethost/core' +import { PH_HOME, Settings, mkPath } from 'pockethost/core' export const PLUGIN_NAME = `plugin-launcher-spawn` export const HOME_DIR = process.env.PH_LAUNCHER_SPAWN_HOME || join(PH_HOME(), PLUGIN_NAME) +export const PLUGIN_DATA = (...paths: string[]) => join(HOME_DIR, ...paths) + export const settings = Settings({ PH_LAUNCHER_SPAWN_HOME: mkPath(HOME_DIR, { create: true }), }) diff --git a/packages/plugin-launcher-spawn/src/db.ts b/packages/plugin-launcher-spawn/src/db.ts new file mode 100644 index 00000000..d6a98f59 --- /dev/null +++ b/packages/plugin-launcher-spawn/src/db.ts @@ -0,0 +1,52 @@ +import { find } from '@s-libs/micro-dash' +import { JSONFileSyncPreset } from 'lowdb/node' +import { InstanceFields, InstanceId, mkSingleton, pocketNow } from 'pockethost' +import { PLUGIN_DATA } from './constants' +import { info } from './log' + +export const DbService = mkSingleton(async () => { + const db = JSONFileSyncPreset<{ + instances: { [key: InstanceId]: InstanceFields } + }>(PLUGIN_DATA('db.json'), { instances: {} }) + + const deleteInstance = (id: InstanceId) => { + db.update((data) => { + delete data.instances[id] + }) + info(`Deleted instance ${id}`) + } + + const createOrUpdateInstance = (instance: InstanceFields) => { + if (!instance.id) { + throw new Error(`Instance id is required`) + } + db.update((data) => { + data.instances[instance.id] = { + ...instance, + updated: instance.id in data.instances ? pocketNow() : instance.updated, + } + }) + info(`Added/updated instance ${instance.id}`) + } + + const getInstanceById = (id: InstanceId) => db.data.instances[id] + const getInstanceBySubdomain = (subdomain: string) => + find(db.data.instances, (v, k) => v.subdomain === subdomain) + + const getInstancesByExactCriteria = ( + criteria: Partial, + ) => + Object.values(db.data.instances).filter((instance) => + Object.entries(criteria).every( + ([k, v]) => instance[k as unknown as keyof InstanceFields] === v, + ), + ) + + return { + createOrUpdateInstance, + deleteInstance, + getInstanceById, + getInstanceBySubdomain, + getInstancesByExactCriteria, + } +}) diff --git a/packages/plugin-launcher-spawn/src/launcher.ts b/packages/plugin-launcher-spawn/src/launcher.ts new file mode 100644 index 00000000..c83b8a92 --- /dev/null +++ b/packages/plugin-launcher-spawn/src/launcher.ts @@ -0,0 +1,192 @@ +import { Mutex } from 'async-mutex' +import fs, { mkdirSync } from 'fs' +import getPort from 'get-port' +import { globSync } from 'glob' +import { gobot } from 'gobot' +import path from 'path' +import { + Bind, + InstanceFields, + doAfterInstanceStartedAction, + doAfterInstanceStoppedAction, + doInstanceConfigFilter, + doInstanceLogAction, + onKillInstanceAction, +} from 'pockethost' +import { INSTANCE_DATA_DIR, exitHook, tryFetch } from 'pockethost/core' +import { gte } from 'semver' +import { dbg } from './log' + +const deleteFiles = (globPattern: string) => { + const files = globSync(globPattern) + files.forEach((file) => { + dbg(`Deleting ${file}`) + fs.unlinkSync(file) + }) +} + +const copyFiles = (binds: Bind[], destination: string) => { + binds.forEach((bind) => { + const srcFiles = globSync(bind.src) + srcFiles.forEach((srcFile) => { + const relativePath = path.relative(bind.base, srcFile) + const destFile = path.join(destination, relativePath) + const destDir = path.dirname(destFile) + dbg(`Copying ${srcFile} to ${destFile}`, { relativePath, destDir, bind }) + + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }) + } + + fs.copyFileSync(srcFile, destFile) + }) + }) +} + +const escape = (path: string) => `"${path}"` + +const launchMutex = new Mutex() + +export const mkLauncher = (instance: InstanceFields) => { + const { dev, subdomain, version, secrets } = instance + return new Promise(async (resolve, reject) => { + const bot = await gobot(`pocketbase`, { version }) + const realVersion = await bot.maxSatisfyingVersion(version) + if (!realVersion) { + throw new Error(`No PocketBase version satisfying ${version}`) + } + + const instanceConfig = await doInstanceConfigFilter({ + env: {}, + binds: { + data: [], + hooks: [], + migrations: [], + public: [], + }, + }) + + dbg(`instanceConfig`, { instanceConfig }) + + const dataDir = INSTANCE_DATA_DIR(subdomain, `pb_data`) + const hooksDir = INSTANCE_DATA_DIR(subdomain, `pb_hooks`) + const migrationsDir = INSTANCE_DATA_DIR(subdomain, `pb_migrations`) + const publicDir = INSTANCE_DATA_DIR(subdomain, `pb_public`) + ;[dataDir, hooksDir, migrationsDir, publicDir].forEach((dir) => { + return mkdirSync(dir, { recursive: true }) + }) + const { binds, env } = instanceConfig + copyFiles(binds.data, dataDir) + copyFiles(binds.hooks, hooksDir) + copyFiles(binds.migrations, migrationsDir) + copyFiles(binds.public, publicDir) + + return launchMutex.runExclusive(async () => { + dbg(`got lock`) + const port = await getPort() + const args = [ + `serve`, + `--dir`, + escape(dataDir), + `--hooksDir`, + escape(hooksDir), + `--migrationsDir`, + escape(migrationsDir), + `--publicDir`, + escape(publicDir), + `--http`, + `0.0.0.0:${port}`, + ] + if (dev && gte(realVersion, `0.20.1`)) args.push(`--dev`) + doInstanceLogAction({ + instance, + type: 'stdout', + data: `Launching: ${await bot.getBinaryFilePath()} ${args.join(' ')}`, + }) + bot.run(args, { env: { ...secrets, ...env } }, (proc) => { + proc.stdout.on('data', (data) => { + data + .toString() + .trim() + .split(`\n`) + .forEach((line: string) => { + doInstanceLogAction({ + instance, + type: 'stdout', + data: line, + }) + }) + }) + proc.stderr.on('data', (data) => { + data + .toString() + .trim() + .split(`\n`) + .forEach((line: string) => { + doInstanceLogAction({ + instance, + type: 'stderr', + data: line, + }) + }) + }) + + const unsubKillAction = onKillInstanceAction(async (context) => { + dbg(`kill action`, { context, instance }) + if (context.instance.id !== instance.id) return + kill() + }) + + const kill = () => { + dbg(`killing ${subdomain}`) + doInstanceLogAction({ + instance, + type: 'stdout', + data: `Forcibly killing PocketBase process`, + }) + proc.kill() + } + + const unsubExitHook = exitHook(kill) + const unsub = () => { + unsubExitHook() + unsubKillAction() + } + proc.on('exit', (code) => { + unsub() + doInstanceLogAction({ + instance, + type: 'stdout', + data: `PocketBase process exited with code ${code}`, + }) + doAfterInstanceStoppedAction({ instance, url }) + dbg(`${subdomain} process exited with code ${code}`) + }) + const url = `http://localhost:${port}` + doInstanceLogAction({ + instance, + type: 'stdout', + data: `Waiting for PocketBase to start on ${url}`, + }) + tryFetch(url) + .then(() => { + doInstanceLogAction({ + instance, + type: 'stdout', + data: `PocketBase started on ${url}`, + }) + doAfterInstanceStartedAction({ instance, url }) + return resolve(url) + }) + .catch((e) => { + doInstanceLogAction({ + instance, + type: 'stderr', + data: `PocketBase failed to start on ${url}: ${e}`, + }) + reject(e) + }) + }) + }) + }) +} diff --git a/packages/plugin-launcher-spawn/src/log.ts b/packages/plugin-launcher-spawn/src/log.ts index 54fb6ab9..0865d2e8 100644 --- a/packages/plugin-launcher-spawn/src/log.ts +++ b/packages/plugin-launcher-spawn/src/log.ts @@ -2,4 +2,4 @@ import { LoggerService } from 'pockethost' import { PLUGIN_NAME } from './constants' const logger = LoggerService().create(PLUGIN_NAME) -export const { dbg, info } = logger +export const { dbg, info, error } = logger diff --git a/packages/plugin-launcher-spawn/src/plugin.ts b/packages/plugin-launcher-spawn/src/plugin.ts index 5f64be63..9b718c8b 100644 --- a/packages/plugin-launcher-spawn/src/plugin.ts +++ b/packages/plugin-launcher-spawn/src/plugin.ts @@ -1,23 +1,18 @@ -import { Mutex } from 'async-mutex' -import fs, { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs' -import getPort from 'get-port' -import { globSync } from 'glob' -import { gobot } from 'gobot' -import path from 'path' import { - Bind, - InstanceFields, PocketHostPlugin, - doAfterInstanceStartedAction, - doAfterInstanceStoppedAction, - doInstanceConfigFilter, - doInstanceLogAction, mkInstance, onAfterInstanceFoundAction, + onAfterInstanceStoppedAction, onAfterServerStartAction, + onAppMountedAction, + onCliCommandsFilter, + onGetAllInstancesByExactCriteriaFilter, onGetInstanceByRequestInfoFilter, + onGetOneInstanceByExactCriteriaFilter, onGetOrProvisionInstanceUrlFilter, + onIsInstanceRunningFilter, onNewInstanceRecordFilter, + onSettingsFilter, } from 'pockethost' import { APEX_DOMAIN, @@ -25,70 +20,31 @@ import { PORT, PUBLIC_INSTANCE_URL, } from 'pockethost/core' +import { InternalApp } from './InternalApp' +import { LauncherCommand } from './LauncherCommand' import { PLUGIN_NAME, settings } from './constants' import { DbService } from './db' +import { mkLauncher } from './launcher' import { dbg, info } from './log' -const deleteFiles = (globPattern: string) => { - const files = globSync(globPattern) - files.forEach((file) => { - dbg(`Deleting ${file}`) - fs.unlinkSync(file) - }) -} - -const copyFiles = (binds: Bind[], destination: string) => { - binds.forEach((bind) => { - const srcFiles = globSync(bind.src) - srcFiles.forEach((srcFile) => { - const relativePath = path.relative(bind.base, srcFile) - const destFile = path.join(destination, relativePath) - const destDir = path.dirname(destFile) - dbg(`Copying ${srcFile} to ${destFile}`, { relativePath, destDir, bind }) - - if (!fs.existsSync(destDir)) { - fs.mkdirSync(destDir, { recursive: true }) - } - - fs.copyFileSync(srcFile, destFile) - }) - }) -} - -const escape = (path: string) => `"${path}"` - -const metaPath = (name: string) => { - const instanceRootDir = INSTANCE_DATA_DIR(name) - mkdirSync(instanceRootDir, { recursive: true }) - const instanceMetaFile = INSTANCE_DATA_DIR(name, `meta.json`) - return instanceMetaFile -} - -const readMeta = (path: string) => - JSON.parse(readFileSync(path, 'utf-8').toString()) - -const getOrCreateMeta = async (name: string) => { - const instanceMetaFile = metaPath(name) - const meta = await (async () => { - if (!existsSync(instanceMetaFile)) { - const meta = await mkInstance(name) - writeFileSync(instanceMetaFile, JSON.stringify(meta, null, 2)) - } - const meta = readMeta(instanceMetaFile) - return meta - })() - return meta -} - -const writeMeta = async (instance: InstanceFields) => { - const { subdomain } = instance - const instanceMetaFile = metaPath(subdomain) - writeFileSync(instanceMetaFile, JSON.stringify(instance, null, 2)) -} - export const plugin: PocketHostPlugin = async ({}) => { dbg(`initializing ${PLUGIN_NAME}`) + const { + getInstanceBySubdomain, + createOrUpdateInstance, + getInstancesByExactCriteria, + } = await DbService({}) + + onCliCommandsFilter(async (commands) => { + return [...commands, LauncherCommand()] + }) + + onAppMountedAction(async ({ internalApp }) => { + dbg(`Mounting internal app`) + internalApp.use(InternalApp()) + }) + /** Display some informational alerts to help the user get started. */ onAfterServerStartAction(async () => { const protocol = HTTP_PROTOCOL() @@ -106,8 +62,10 @@ export const plugin: PocketHostPlugin = async ({}) => { /** When a request comes in, return an instance based on subdomain */ onGetInstanceByRequestInfoFilter(async (instance, context) => { const { subdomain } = context - const meta = await getOrCreateMeta(subdomain) - return { ...instance, ...meta } + return { + ...(instance || (await mkInstance(subdomain))), + ...getInstanceBySubdomain(subdomain), + } }) /** @@ -118,27 +76,23 @@ export const plugin: PocketHostPlugin = async ({}) => { */ onNewInstanceRecordFilter(async (instance) => { const { subdomain } = instance - const path = metaPath(subdomain) - if (!existsSync(path)) return instance - const meta = await readMeta(path) - return { ...instance, ...meta } + return { ...instance, ...getInstanceBySubdomain(subdomain), id: subdomain } }) /** After an instance has been found, store it to the db */ onAfterInstanceFoundAction(async (context) => { const { instance } = context - await writeMeta(instance) + dbg({ instance }) + createOrUpdateInstance(instance) }) const instances: { [_: string]: Promise } = {} - const launchMutex = new Mutex() - /** * The workhorse. This filter is responsible for launching PocketBase and * returning an endpoint URL. */ - onGetOrProvisionInstanceUrlFilter(async (url: string, { instance }) => { + onGetOrProvisionInstanceUrlFilter(async (url, { instance }) => { const { dev, subdomain, version, secrets } = instance if (subdomain in instances) return instances[subdomain]! @@ -147,7 +101,22 @@ export const plugin: PocketHostPlugin = async ({}) => { return (instances[subdomain] = mkLauncher(instance)) }) + onAfterInstanceStoppedAction(async ({ instance }) => { + const { subdomain } = instance + delete instances[subdomain] + }) + onSettingsFilter(async (allSettings) => ({ ...allSettings, ...settings })) + onIsInstanceRunningFilter(async (isRunning, { instance }) => { + return isRunning || !!instances[instance.subdomain] + }) + + onGetOneInstanceByExactCriteriaFilter(async (instance, criteria) => { + return instance || getInstancesByExactCriteria(criteria)[0] + }) + + onGetAllInstancesByExactCriteriaFilter(async (instances, criteria) => { + return [...instances, ...getInstancesByExactCriteria(criteria)] }) }