From 26d1070c20dcd797258ca12f9e86ffbcdf26e915 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Sat, 23 Nov 2024 21:13:20 -0800 Subject: [PATCH] enh: instance volumes --- .changeset/mean-news-sing.md | 5 ++ .../EdgeCommand/FtpCommand/FtpService/PhFs.ts | 10 +++- .../pockethost/src/common/schema/Instance.ts | 1 + packages/pockethost/src/constants.ts | 7 ++- packages/pockethost/src/core/dir.ts | 3 +- .../1732403125_updated_instances.js | 52 +++++++++++++++++++ .../services/InstanceLoggerService/index.ts | 30 ++++++++--- .../src/services/InstanceService/index.ts | 7 ++- .../src/services/PocketBaseService/index.ts | 7 ++- .../pockethost/src/services/RealtimeLog.ts | 6 ++- 10 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 .changeset/mean-news-sing.md create mode 100644 packages/pockethost/src/mothership-app/pb_migrations/1732403125_updated_instances.js diff --git a/.changeset/mean-news-sing.md b/.changeset/mean-news-sing.md new file mode 100644 index 00000000..3e05dac5 --- /dev/null +++ b/.changeset/mean-news-sing.md @@ -0,0 +1,5 @@ +--- +'pockethost': minor +--- + +Added support for instances hosted on alternate volumes diff --git a/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts index 263860f5..f30964b3 100644 --- a/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts +++ b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts @@ -51,7 +51,7 @@ const checkBun = ( [`bun.lockb`, `package.json`].includes(maybeImportant || '')) if (isImportant) { - const logger = InstanceLogWriter(instance.id, `exec`) + const logger = InstanceLogWriter(instance.id, instance.volume, `exec`) logger.info(`${maybeImportant} changed, running bun install`) launchBunInstall(instance, virtualPath, cwd).catch(console.error) } @@ -103,7 +103,7 @@ const launchBunInstall = (() => { runCache[cwd] = { runAgain: true } while (runCache[cwd]!.runAgain) { runCache[cwd]!.runAgain = false - const logger = InstanceLogWriter(instance.id, `exec`) + const logger = InstanceLogWriter(instance.id, instance.volume, `exec`) logger.info(`Launching 'bun install' in ${virtualPath}`) await runBun(cwd, logger) } @@ -174,6 +174,9 @@ export class PhFs implements FileSystem { `Instance must be in maintenance mode to access ${instanceRootDir}`, ) } + if (instance.volume) { + fsPathParts.push(instance.volume) + } fsPathParts.push(instance.id) dbg({ fsPathParts, @@ -222,6 +225,9 @@ export class PhFs implements FileSystem { this.cwd = clientPath return this.currentDirectory() }) + .catch((e) => { + throw new Error(`no such file or directory: ${path}`) + }) } async list(path = '.') { diff --git a/packages/pockethost/src/common/schema/Instance.ts b/packages/pockethost/src/common/schema/Instance.ts index 0a6c064d..762541d8 100644 --- a/packages/pockethost/src/common/schema/Instance.ts +++ b/packages/pockethost/src/common/schema/Instance.ts @@ -33,6 +33,7 @@ export type InstanceFields = BaseFields & { dev: boolean cname_active: boolean notifyMaintenanceMode: boolean + volume: string idleTtl: number } & TExtra diff --git a/packages/pockethost/src/constants.ts b/packages/pockethost/src/constants.ts index 61e24741..f9f5caa5 100644 --- a/packages/pockethost/src/constants.ts +++ b/packages/pockethost/src/constants.ts @@ -271,8 +271,11 @@ export const mkInstanceUrl = (instance: InstanceFields, ...paths: string[]) => [`${HTTP_PROTOCOL()}//${mkInstanceHostname(instance)}`, paths.join(`/`)] .filter(Boolean) .join('/') -export const mkInstanceDataPath = (instanceId: string, ...path: string[]) => - join(settings().DATA_ROOT, instanceId, ...path) +export const mkInstanceDataPath = ( + volume: string, + instanceId: string, + ...path: string[] +) => DATA_ROOT(volume, instanceId, ...path) export const logConstants = () => { const vars = { diff --git a/packages/pockethost/src/core/dir.ts b/packages/pockethost/src/core/dir.ts index 6f570f08..e7da66a4 100644 --- a/packages/pockethost/src/core/dir.ts +++ b/packages/pockethost/src/core/dir.ts @@ -4,12 +4,13 @@ import { mkInstanceDataPath } from '../constants' export function ensureInstanceDirectoryStructure( instanceId: string, + volume: string, logger: Logger, ) { const { dbg } = logger ;['pb_data', 'pb_migrations', 'pb_public', 'logs', 'pb_hooks'].forEach( (dir) => { - const path = mkInstanceDataPath(instanceId, dir) + const path = mkInstanceDataPath(volume, instanceId, dir) if (!existsSync(path)) { dbg(`Creating ${path}`) mkdirSync(path, { recursive: true }) diff --git a/packages/pockethost/src/mothership-app/pb_migrations/1732403125_updated_instances.js b/packages/pockethost/src/mothership-app/pb_migrations/1732403125_updated_instances.js new file mode 100644 index 00000000..880371d0 --- /dev/null +++ b/packages/pockethost/src/mothership-app/pb_migrations/1732403125_updated_instances.js @@ -0,0 +1,52 @@ +/// +migrate((db) => { + const dao = new Dao(db) + const collection = dao.findCollectionByNameOrId("etae8tuiaxl6xfv") + + // remove + collection.schema.removeField("1vrc1wfd") + + // add + collection.schema.addField(new SchemaField({ + "system": false, + "id": "kd017nrg", + "name": "volume", + "type": "text", + "required": false, + "presentable": false, + "unique": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } + })) + + return dao.saveCollection(collection) +}, (db) => { + const dao = new Dao(db) + const collection = dao.findCollectionByNameOrId("etae8tuiaxl6xfv") + + // add + collection.schema.addField(new SchemaField({ + "system": false, + "id": "1vrc1wfd", + "name": "volume", + "type": "select", + "required": false, + "presentable": false, + "unique": false, + "options": { + "maxSelect": 1, + "values": [ + ".", + "sfo-2-data" + ] + } + })) + + // remove + collection.schema.removeField("kd017nrg") + + return dao.saveCollection(collection) +}) diff --git a/packages/pockethost/src/services/InstanceLoggerService/index.ts b/packages/pockethost/src/services/InstanceLoggerService/index.ts index fa028486..d7c6c9ab 100644 --- a/packages/pockethost/src/services/InstanceLoggerService/index.ts +++ b/packages/pockethost/src/services/InstanceLoggerService/index.ts @@ -25,13 +25,22 @@ export type LogEntry = { time: string } -export function InstanceLogWriter(instanceId: string, target: string) { +export function InstanceLogWriter( + instanceId: string, + volume: string, + target: string, +) { const logger = LoggerService().create(instanceId).breadcrumb({ target }) const { dbg, info, error, warn } = logger - ensureInstanceDirectoryStructure(instanceId, logger) + ensureInstanceDirectoryStructure(instanceId, volume, logger) - const logFile = mkInstanceDataPath(instanceId, `logs`, `${target}.log`) + const logFile = mkInstanceDataPath( + volume, + instanceId, + `logs`, + `${target}.log`, + ) const appendLogEntry = (msg: string, stream: 'stdout' | 'stderr') => { appendFile( @@ -58,15 +67,24 @@ export function InstanceLogWriter(instanceId: string, target: string) { return api } -export function InstanceLogReader(instanceId: string, target: string) { +export function InstanceLogReader( + instanceId: string, + volume: string, + target: string, +) { const logger = LoggerService().create(instanceId).breadcrumb({ target }) const { dbg, info, error, warn } = logger - ensureInstanceDirectoryStructure(instanceId, logger) + ensureInstanceDirectoryStructure(instanceId, volume, logger) const api = { tail: (linesBack: number, data: (line: LogEntry) => void): UnsubFunc => { - const logFile = mkInstanceDataPath(instanceId, `logs`, `${target}.log`) + const logFile = mkInstanceDataPath( + volume, + instanceId, + `logs`, + `${target}.log`, + ) let tid: any let unsub: any diff --git a/packages/pockethost/src/services/InstanceService/index.ts b/packages/pockethost/src/services/InstanceService/index.ts index bd347204..65ba9497 100644 --- a/packages/pockethost/src/services/InstanceService/index.ts +++ b/packages/pockethost/src/services/InstanceService/index.ts @@ -71,7 +71,11 @@ export const instanceService = mkSingleton( `${subdomain}:${id}:${version}`, ) const { dbg, warn, error, info, trace } = systemInstanceLogger - const userInstanceLogger = InstanceLogWriter(instance.id, `exec`) + const userInstanceLogger = InstanceLogWriter( + instance.id, + instance.volume, + `exec`, + ) shutdownManager.push(() => { dbg(`Shutting down`) @@ -123,6 +127,7 @@ export const instanceService = mkSingleton( const spawnArgs: SpawnConfig = { subdomain: instance.subdomain, instanceId: instance.id, + volume: instance.volume, dev: instance.dev, extraBinds: flatten([ globSync(join(INSTANCE_APP_MIGRATIONS_DIR(), '*.js')).map( diff --git a/packages/pockethost/src/services/PocketBaseService/index.ts b/packages/pockethost/src/services/PocketBaseService/index.ts index a62965db..081b7ae7 100644 --- a/packages/pockethost/src/services/PocketBaseService/index.ts +++ b/packages/pockethost/src/services/PocketBaseService/index.ts @@ -23,6 +23,7 @@ export type Env = { [_: string]: string } export type SpawnConfig = { subdomain: string instanceId: string + volume: string version?: string extraBinds?: string[] env?: Env @@ -77,6 +78,7 @@ export const createPocketbaseService = async ( version, subdomain, instanceId, + volume, extraBinds, env, stderr, @@ -85,7 +87,7 @@ export const createPocketbaseService = async ( } = _cfg logger.breadcrumb({ subdomain, instanceId }) - const iLogger = InstanceLogWriter(instanceId, 'exec') + const iLogger = InstanceLogWriter(instanceId, volume, 'exec') const _version = version || maxVersion // If _version is blank, we use the max version available const realVersion = await bot.maxSatisfyingVersion(_version) @@ -123,7 +125,7 @@ export const createPocketbaseService = async ( dbg(data.toString()) }) const Binds = [ - `${mkInstanceDataPath(instanceId)}:${mkContainerHomePath()}`, + `${mkInstanceDataPath(volume, instanceId)}:${mkContainerHomePath()}`, `${binPath}:${mkContainerHomePath(`pocketbase`)}:ro`, ] @@ -155,6 +157,7 @@ export const createPocketbaseService = async ( Hard: 4096, }, ], + ExtraHosts: [`minio:host-gateway`], }, Tty: false, ExposedPorts: { diff --git a/packages/pockethost/src/services/RealtimeLog.ts b/packages/pockethost/src/services/RealtimeLog.ts index 80778375..882ad0c2 100644 --- a/packages/pockethost/src/services/RealtimeLog.ts +++ b/packages/pockethost/src/services/RealtimeLog.ts @@ -69,7 +69,11 @@ export const realtimeLog = mkSingleton(async (config: RealtimeLogConfig) => { dbg(`Instance is `, instance) /** Get a database connection */ - const instanceLogger = InstanceLogReader(instance.id, `exec`) + const instanceLogger = InstanceLogReader( + instance.id, + instance.volume, + `exec`, + ) /** Start the stream */ res.writeHead(200, {