feat(launcher-spawn): Relocated instance database, added support for starting/stopping/listing instances via CLI, added remote internal API for starting/stopping/listing instances.

.changeset/1719707021337.md

@@ -0,0 +1,5 @@
+---
+'@pockethost/plugin-launcher-spawn': minor
+---
+
+Relocated instance database, added support for starting/stopping/listing instances via CLI, added remote internal API for starting/stopping/listing instances.

packages/plugin-launcher-spawn/package.json

@@ -22,19 +22,26 @@
   },
   "license": "MIT",
   "dependencies": {
+    "@s-libs/micro-dash": "^16.1.0",
     "@types/node": "^20.8.10",
     "@types/semver": "^7.5.4",
     "async-mutex": "^0.5.0",
+    "boolean": "^3.2.0",
     "bottleneck": "^2.19.5",
+    "commander": "^11.1.0",
+    "express": "^4.18.2",
     "get-port": "^6.1.2",
     "glob": "^10.4.2",
     "gobot": "1.0.0-alpha.40",
+    "lowdb": "^7.0.1",
     "semver": "^7.6.2"
   },
   "peerDependencies": {
     "pockethost": "workspace:^1.5.0"
   },
   "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/lowdb": "^1.0.15",
     "typescript": "^5.4.5"
   }
 }

packages/plugin-launcher-spawn/src/InternalApp.ts

@@ -0,0 +1,83 @@
+import { boolean } from 'boolean'
+import express, { RequestHandler } from 'express'
+import {
+  doGetOneInstanceByExactCriteriaFilter,
+  doGetOrProvisionInstanceUrlFilter,
+  doIsInstanceRunningFilter,
+  doKillInstanceAction,
+} from 'pockethost'
+import { PLUGIN_NAME } from './constants'
+import { DbService } from './db'
+import { dbg, info } from './log'
+
+const getInstanceBySubdomainOrId = async (subdomainOrId: string) => {
+  {
+    const instance = await doGetOneInstanceByExactCriteriaFilter(undefined, {
+      id: subdomainOrId,
+    })
+    if (instance) return instance
+  }
+  {
+    const instance = await doGetOneInstanceByExactCriteriaFilter(undefined, {
+      subdomain: subdomainOrId,
+    })
+    if (instance) return instance
+  }
+}
+
+export const InternalApp = (): RequestHandler => {
+  const app = express()
+  app.get('/start/:subdomainOrInstanceId', async (req, res) => {
+    const { subdomainOrInstanceId } = req.params
+    dbg(`Got start request for ${subdomainOrInstanceId}`)
+    const instance = await getInstanceBySubdomainOrId(subdomainOrInstanceId)
+
+    dbg(`Instance:`, instance)
+    if (!instance) {
+      res.status(404).send(`Instance not found`)
+      return
+    }
+    const url = await doGetOrProvisionInstanceUrlFilter(undefined, { instance })
+    res.send(`Started instance ${subdomainOrInstanceId} at ${url}`)
+  })
+
+  app.get('/stop/:subdomainOrInstanceId', async (req, res) => {
+    const { subdomainOrInstanceId } = req.params
+    dbg(`Got stop request for ${subdomainOrInstanceId}`)
+    const instance = await getInstanceBySubdomainOrId(subdomainOrInstanceId)
+    dbg(`Instance:`, instance)
+    if (!instance) {
+      res.status(404).send(`Instance not found`)
+      return
+    }
+    await doKillInstanceAction({ instance })
+    res.send(`Stopped instance ${instance.subdomain} (${instance.id})`)
+  })
+
+  app.get('/list', async (req, res) => {
+    const shouldListAll = boolean(req.query.all)
+    dbg(`Got list request. Should list all: ${shouldListAll}`)
+    const { getInstancesByExactCriteria } = await DbService()
+    const instances = getInstancesByExactCriteria({})
+    info(`Listing instances`)
+    const final = []
+    for (const instance of instances) {
+      const { subdomain, id, version } = instance
+      const isRunning = await doIsInstanceRunningFilter(false, {
+        instance,
+      })
+      if (!shouldListAll && !isRunning) continue
+      final.push({ instance, isRunning })
+    }
+    res.json(final)
+  })
+
+  app.use('*', (req, res) => {
+    dbg(`Got request to internal app ${req.url}`)
+    res.send(`Got request to internal app ${req.url}`)
+  })
+
+  const root = express()
+  root.use(`/${PLUGIN_NAME}`, app)
+  return root
+}

packages/plugin-launcher-spawn/src/LauncherCommand.ts

@@ -0,0 +1,107 @@
+import { Command } from 'commander'
+import {
+  INTERNAL_APP_AUTH_HEADER,
+  INTERNAL_APP_SECRET,
+  INTERNAL_APP_URL,
+  PUBLIC_INSTANCE_URL,
+  tryFetch,
+} from 'pockethost/core'
+import { PLUGIN_NAME } from './constants'
+import { dbg, error, info } from './log'
+
+export const LauncherCommand = () =>
+  new Command(`spawn`)
+    .description(`Spawn launcher commands`)
+    .addCommand(
+      new Command(`start`)
+        .argument(`<subdomain|id>`)
+        .description(`Start an instance`)
+        .action(async (subdomainOrId: string) => {
+          const internalRequestUrl = INTERNAL_APP_URL(
+            PLUGIN_NAME,
+            `start/${subdomainOrId}`,
+          )
+          dbg(
+            `Starting instance ${subdomainOrId} by sending request to ${internalRequestUrl}`,
+          )
+          const res = await fetch(internalRequestUrl, {
+            headers: { [INTERNAL_APP_AUTH_HEADER]: INTERNAL_APP_SECRET() },
+          })
+          if (!res.ok) {
+            error(
+              `Failed to start instance ${subdomainOrId}: ${res.status} ${
+                res.statusText
+              } - ${await res.text()}`,
+            )
+          } else {
+            info(`Started instance ${subdomainOrId}: ${await res.text()}`)
+          }
+        }),
+    )
+    .addCommand(
+      new Command(`stop`)
+        .argument(`<instance>`)
+        .description(`Stop an instance`)
+        .action(async (subdomainOrId) => {
+          const internalRequestUrl = INTERNAL_APP_URL(
+            PLUGIN_NAME,
+            `stop/${subdomainOrId}`,
+          )
+          dbg(
+            `Stopping instance ${subdomainOrId} by sending request to ${internalRequestUrl}`,
+          )
+          const res = await tryFetch(internalRequestUrl, {
+            headers: { [INTERNAL_APP_AUTH_HEADER]: INTERNAL_APP_SECRET() },
+          })
+          const text = await res.text()
+          if (!res.ok) {
+            error(
+              `Failed to start instance ${subdomainOrId}: ${res.status} ${res.statusText} - ${text}`,
+            )
+          } else {
+            info(`Stopped instance ${subdomainOrId}: ${text}`)
+          }
+        }),
+    )
+    .addCommand(
+      new Command(`list`)
+        .alias(`ls`)
+        .description(`List instances`)
+        .option(`-a, --all`, `List all instances (not just running instances)`)
+        .action(async (options) => {
+          const internalRequestUrl = INTERNAL_APP_URL(
+            PLUGIN_NAME,
+            `list?all=${!!options.all}`,
+          )
+          dbg(`Listing instances by sending request to ${internalRequestUrl}`)
+          const res = await tryFetch(internalRequestUrl, {
+            headers: { [INTERNAL_APP_AUTH_HEADER]: INTERNAL_APP_SECRET() },
+          })
+          const text = await res.text()
+          if (!res.ok) {
+            error(
+              `Failed to list instances: ${res.status} ${res.statusText} - ${text}`,
+            )
+          } else {
+            const instances = JSON.parse(text) as {
+              instance: any
+              isRunning: boolean
+            }[]
+            if (!instances.length) {
+              info(`No instances found. Try --all`)
+              return
+            }
+            for (const {
+              instance,
+              instance: { subdomain, id, version },
+              isRunning,
+            } of instances) {
+              info(
+                `${PUBLIC_INSTANCE_URL(instance)} [${id}|${version}|${
+                  isRunning ? 'running' : 'stopped'
+                }]`,
+              )
+            }
+          }
+        }),
+    )

packages/plugin-launcher-spawn/src/constants.ts

@@ -1,12 +1,13 @@
 import { join } from 'path'
-import { DEBUG } from 'pockethost'
-import { PH_HOME, Settings, logSettings, mkPath } from 'pockethost/core'
+import { PH_HOME, Settings, mkPath } from 'pockethost/core'
 
 export const PLUGIN_NAME = `plugin-launcher-spawn`
 
 export const HOME_DIR =
   process.env.PH_LAUNCHER_SPAWN_HOME || join(PH_HOME(), PLUGIN_NAME)
 
+export const PLUGIN_DATA = (...paths: string[]) => join(HOME_DIR, ...paths)
+
 export const settings = Settings({
   PH_LAUNCHER_SPAWN_HOME: mkPath(HOME_DIR, { create: true }),
 })

packages/plugin-launcher-spawn/src/db.ts

@@ -0,0 +1,52 @@
+import { find } from '@s-libs/micro-dash'
+import { JSONFileSyncPreset } from 'lowdb/node'
+import { InstanceFields, InstanceId, mkSingleton, pocketNow } from 'pockethost'
+import { PLUGIN_DATA } from './constants'
+import { info } from './log'
+
+export const DbService = mkSingleton(async () => {
+  const db = JSONFileSyncPreset<{
+    instances: { [key: InstanceId]: InstanceFields }
+  }>(PLUGIN_DATA('db.json'), { instances: {} })
+
+  const deleteInstance = (id: InstanceId) => {
+    db.update((data) => {
+      delete data.instances[id]
+    })
+    info(`Deleted instance ${id}`)
+  }
+
+  const createOrUpdateInstance = (instance: InstanceFields) => {
+    if (!instance.id) {
+      throw new Error(`Instance id is required`)
+    }
+    db.update((data) => {
+      data.instances[instance.id] = {
+        ...instance,
+        updated: instance.id in data.instances ? pocketNow() : instance.updated,
+      }
+    })
+    info(`Added/updated instance ${instance.id}`)
+  }
+
+  const getInstanceById = (id: InstanceId) => db.data.instances[id]
+  const getInstanceBySubdomain = (subdomain: string) =>
+    find(db.data.instances, (v, k) => v.subdomain === subdomain)
+
+  const getInstancesByExactCriteria = (
+    criteria: Partial<InstanceFields | { [_: string]: number | string }>,
+  ) =>
+    Object.values(db.data.instances).filter((instance) =>
+      Object.entries(criteria).every(
+        ([k, v]) => instance[k as unknown as keyof InstanceFields] === v,
+      ),
+    )
+
+  return {
+    createOrUpdateInstance,
+    deleteInstance,
+    getInstanceById,
+    getInstanceBySubdomain,
+    getInstancesByExactCriteria,
+  }
+})

packages/plugin-launcher-spawn/src/launcher.ts

@@ -0,0 +1,192 @@
+import { Mutex } from 'async-mutex'
+import fs, { mkdirSync } from 'fs'
+import getPort from 'get-port'
+import { globSync } from 'glob'
+import { gobot } from 'gobot'
+import path from 'path'
+import {
+  Bind,
+  InstanceFields,
+  doAfterInstanceStartedAction,
+  doAfterInstanceStoppedAction,
+  doInstanceConfigFilter,
+  doInstanceLogAction,
+  onKillInstanceAction,
+} from 'pockethost'
+import { INSTANCE_DATA_DIR, exitHook, tryFetch } from 'pockethost/core'
+import { gte } from 'semver'
+import { dbg } from './log'
+
+const deleteFiles = (globPattern: string) => {
+  const files = globSync(globPattern)
+  files.forEach((file) => {
+    dbg(`Deleting ${file}`)
+    fs.unlinkSync(file)
+  })
+}
+
+const copyFiles = (binds: Bind[], destination: string) => {
+  binds.forEach((bind) => {
+    const srcFiles = globSync(bind.src)
+    srcFiles.forEach((srcFile) => {
+      const relativePath = path.relative(bind.base, srcFile)
+      const destFile = path.join(destination, relativePath)
+      const destDir = path.dirname(destFile)
+      dbg(`Copying ${srcFile} to ${destFile}`, { relativePath, destDir, bind })
+
+      if (!fs.existsSync(destDir)) {
+        fs.mkdirSync(destDir, { recursive: true })
+      }
+
+      fs.copyFileSync(srcFile, destFile)
+    })
+  })
+}
+
+const escape = (path: string) => `"${path}"`
+
+const launchMutex = new Mutex()
+
+export const mkLauncher = (instance: InstanceFields) => {
+  const { dev, subdomain, version, secrets } = instance
+  return new Promise<string>(async (resolve, reject) => {
+    const bot = await gobot(`pocketbase`, { version })
+    const realVersion = await bot.maxSatisfyingVersion(version)
+    if (!realVersion) {
+      throw new Error(`No PocketBase version satisfying ${version}`)
+    }
+
+    const instanceConfig = await doInstanceConfigFilter({
+      env: {},
+      binds: {
+        data: [],
+        hooks: [],
+        migrations: [],
+        public: [],
+      },
+    })
+
+    dbg(`instanceConfig`, { instanceConfig })
+
+    const dataDir = INSTANCE_DATA_DIR(subdomain, `pb_data`)
+    const hooksDir = INSTANCE_DATA_DIR(subdomain, `pb_hooks`)
+    const migrationsDir = INSTANCE_DATA_DIR(subdomain, `pb_migrations`)
+    const publicDir = INSTANCE_DATA_DIR(subdomain, `pb_public`)
+    ;[dataDir, hooksDir, migrationsDir, publicDir].forEach((dir) => {
+      return mkdirSync(dir, { recursive: true })
+    })
+    const { binds, env } = instanceConfig
+    copyFiles(binds.data, dataDir)
+    copyFiles(binds.hooks, hooksDir)
+    copyFiles(binds.migrations, migrationsDir)
+    copyFiles(binds.public, publicDir)
+
+    return launchMutex.runExclusive(async () => {
+      dbg(`got lock`)
+      const port = await getPort()
+      const args = [
+        `serve`,
+        `--dir`,
+        escape(dataDir),
+        `--hooksDir`,
+        escape(hooksDir),
+        `--migrationsDir`,
+        escape(migrationsDir),
+        `--publicDir`,
+        escape(publicDir),
+        `--http`,
+        `0.0.0.0:${port}`,
+      ]
+      if (dev && gte(realVersion, `0.20.1`)) args.push(`--dev`)
+      doInstanceLogAction({
+        instance,
+        type: 'stdout',
+        data: `Launching: ${await bot.getBinaryFilePath()} ${args.join(' ')}`,
+      })
+      bot.run(args, { env: { ...secrets, ...env } }, (proc) => {
+        proc.stdout.on('data', (data) => {
+          data
+            .toString()
+            .trim()
+            .split(`\n`)
+            .forEach((line: string) => {
+              doInstanceLogAction({
+                instance,
+                type: 'stdout',
+                data: line,
+              })
+            })
+        })
+        proc.stderr.on('data', (data) => {
+          data
+            .toString()
+            .trim()
+            .split(`\n`)
+            .forEach((line: string) => {
+              doInstanceLogAction({
+                instance,
+                type: 'stderr',
+                data: line,
+              })
+            })
+        })
+
+        const unsubKillAction = onKillInstanceAction(async (context) => {
+          dbg(`kill action`, { context, instance })
+          if (context.instance.id !== instance.id) return
+          kill()
+        })
+
+        const kill = () => {
+          dbg(`killing ${subdomain}`)
+          doInstanceLogAction({
+            instance,
+            type: 'stdout',
+            data: `Forcibly killing PocketBase process`,
+          })
+          proc.kill()
+        }
+
+        const unsubExitHook = exitHook(kill)
+        const unsub = () => {
+          unsubExitHook()
+          unsubKillAction()
+        }
+        proc.on('exit', (code) => {
+          unsub()
+          doInstanceLogAction({
+            instance,
+            type: 'stdout',
+            data: `PocketBase process exited with code ${code}`,
+          })
+          doAfterInstanceStoppedAction({ instance, url })
+          dbg(`${subdomain} process exited with code ${code}`)
+        })
+        const url = `http://localhost:${port}`
+        doInstanceLogAction({
+          instance,
+          type: 'stdout',
+          data: `Waiting for PocketBase to start on ${url}`,
+        })
+        tryFetch(url)
+          .then(() => {
+            doInstanceLogAction({
+              instance,
+              type: 'stdout',
+              data: `PocketBase started on ${url}`,
+            })
+            doAfterInstanceStartedAction({ instance, url })
+            return resolve(url)
+          })
+          .catch((e) => {
+            doInstanceLogAction({
+              instance,
+              type: 'stderr',
+              data: `PocketBase failed to start on ${url}: ${e}`,
+            })
+            reject(e)
+          })
+      })
+    })
+  })
+}

packages/plugin-launcher-spawn/src/log.ts

@@ -2,4 +2,4 @@ import { LoggerService } from 'pockethost'
 import { PLUGIN_NAME } from './constants'
 
 const logger = LoggerService().create(PLUGIN_NAME)
-export const { dbg, info } = logger
+export const { dbg, info, error } = logger

packages/plugin-launcher-spawn/src/plugin.ts

@@ -1,23 +1,18 @@
-import { Mutex } from 'async-mutex'
-import fs, { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
-import getPort from 'get-port'
-import { globSync } from 'glob'
-import { gobot } from 'gobot'
-import path from 'path'
 import {
-  Bind,
-  InstanceFields,
   PocketHostPlugin,
-  doAfterInstanceStartedAction,
-  doAfterInstanceStoppedAction,
-  doInstanceConfigFilter,
-  doInstanceLogAction,
   mkInstance,
   onAfterInstanceFoundAction,
+  onAfterInstanceStoppedAction,
   onAfterServerStartAction,
+  onAppMountedAction,
+  onCliCommandsFilter,
+  onGetAllInstancesByExactCriteriaFilter,
   onGetInstanceByRequestInfoFilter,
+  onGetOneInstanceByExactCriteriaFilter,
   onGetOrProvisionInstanceUrlFilter,
+  onIsInstanceRunningFilter,
   onNewInstanceRecordFilter,
+  onSettingsFilter,
 } from 'pockethost'
 import {
   APEX_DOMAIN,
@@ -25,70 +20,31 @@ import {
   PORT,
   PUBLIC_INSTANCE_URL,
 } from 'pockethost/core'
+import { InternalApp } from './InternalApp'
+import { LauncherCommand } from './LauncherCommand'
 import { PLUGIN_NAME, settings } from './constants'
 import { DbService } from './db'
+import { mkLauncher } from './launcher'
 import { dbg, info } from './log'
 
-const deleteFiles = (globPattern: string) => {
-  const files = globSync(globPattern)
-  files.forEach((file) => {
-    dbg(`Deleting ${file}`)
-    fs.unlinkSync(file)
-  })
-}
-
-const copyFiles = (binds: Bind[], destination: string) => {
-  binds.forEach((bind) => {
-    const srcFiles = globSync(bind.src)
-    srcFiles.forEach((srcFile) => {
-      const relativePath = path.relative(bind.base, srcFile)
-      const destFile = path.join(destination, relativePath)
-      const destDir = path.dirname(destFile)
-      dbg(`Copying ${srcFile} to ${destFile}`, { relativePath, destDir, bind })
-
-      if (!fs.existsSync(destDir)) {
-        fs.mkdirSync(destDir, { recursive: true })
-      }
-
-      fs.copyFileSync(srcFile, destFile)
-    })
-  })
-}
-
-const escape = (path: string) => `"${path}"`
-
-const metaPath = (name: string) => {
-  const instanceRootDir = INSTANCE_DATA_DIR(name)
-  mkdirSync(instanceRootDir, { recursive: true })
-  const instanceMetaFile = INSTANCE_DATA_DIR(name, `meta.json`)
-  return instanceMetaFile
-}
-
-const readMeta = (path: string) =>
-  JSON.parse(readFileSync(path, 'utf-8').toString())
-
-const getOrCreateMeta = async (name: string) => {
-  const instanceMetaFile = metaPath(name)
-  const meta = await (async () => {
-    if (!existsSync(instanceMetaFile)) {
-      const meta = await mkInstance(name)
-      writeFileSync(instanceMetaFile, JSON.stringify(meta, null, 2))
-    }
-    const meta = readMeta(instanceMetaFile)
-    return meta
-  })()
-  return meta
-}
-
-const writeMeta = async (instance: InstanceFields) => {
-  const { subdomain } = instance
-  const instanceMetaFile = metaPath(subdomain)
-  writeFileSync(instanceMetaFile, JSON.stringify(instance, null, 2))
-}
-
 export const plugin: PocketHostPlugin = async ({}) => {
   dbg(`initializing ${PLUGIN_NAME}`)
 
+  const {
+    getInstanceBySubdomain,
+    createOrUpdateInstance,
+    getInstancesByExactCriteria,
+  } = await DbService({})
+
+  onCliCommandsFilter(async (commands) => {
+    return [...commands, LauncherCommand()]
+  })
+
+  onAppMountedAction(async ({ internalApp }) => {
+    dbg(`Mounting internal app`)
+    internalApp.use(InternalApp())
+  })
+
   /** Display some informational alerts to help the user get started. */
   onAfterServerStartAction(async () => {
     const protocol = HTTP_PROTOCOL()
@@ -106,8 +62,10 @@ export const plugin: PocketHostPlugin = async ({}) => {
   /** When a request comes in, return an instance based on subdomain */
   onGetInstanceByRequestInfoFilter(async (instance, context) => {
     const { subdomain } = context
-    const meta = await getOrCreateMeta(subdomain)
-    return { ...instance, ...meta }
+    return {
+      ...(instance || (await mkInstance(subdomain))),
+      ...getInstanceBySubdomain(subdomain),
+    }
   })
 
   /**
@@ -118,27 +76,23 @@ export const plugin: PocketHostPlugin = async ({}) => {
    */
   onNewInstanceRecordFilter(async (instance) => {
     const { subdomain } = instance
-    const path = metaPath(subdomain)
-    if (!existsSync(path)) return instance
-    const meta = await readMeta(path)
-    return { ...instance, ...meta }
+    return { ...instance, ...getInstanceBySubdomain(subdomain), id: subdomain }
   })
 
   /** After an instance has been found, store it to the db */
   onAfterInstanceFoundAction(async (context) => {
     const { instance } = context
-    await writeMeta(instance)
+    dbg({ instance })
+    createOrUpdateInstance(instance)
   })
 
   const instances: { [_: string]: Promise<string> } = {}
 
-  const launchMutex = new Mutex()
-
   /**
    * The workhorse. This filter is responsible for launching PocketBase and
    * returning an endpoint URL.
    */
-  onGetOrProvisionInstanceUrlFilter(async (url: string, { instance }) => {
+  onGetOrProvisionInstanceUrlFilter(async (url, { instance }) => {
     const { dev, subdomain, version, secrets } = instance
 
     if (subdomain in instances) return instances[subdomain]!
@@ -147,7 +101,22 @@ export const plugin: PocketHostPlugin = async ({}) => {
     return (instances[subdomain] = mkLauncher(instance))
   })
 
+  onAfterInstanceStoppedAction(async ({ instance }) => {
+    const { subdomain } = instance
+    delete instances[subdomain]
+  })
+
   onSettingsFilter(async (allSettings) => ({ ...allSettings, ...settings }))
 
+  onIsInstanceRunningFilter(async (isRunning, { instance }) => {
+    return isRunning || !!instances[instance.subdomain]
+  })
+
+  onGetOneInstanceByExactCriteriaFilter(async (instance, criteria) => {
+    return instance || getInstancesByExactCriteria(criteria)[0]
+  })
+
+  onGetAllInstancesByExactCriteriaFilter(async (instances, criteria) => {
+    return [...instances, ...getInstancesByExactCriteria(criteria)]
   })
 }