From bdbcc479cf6ebffbd2afd0daf6f75942f00518de Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Mon, 16 Sep 2024 19:06:21 +0000 Subject: [PATCH] wip --- .../EdgeCommand/FtpCommand/FtpService/PhFs.ts | 324 +++++++----------- .../FtpCommand/FtpService/guards.ts | 47 +++ .../FtpCommand/FtpService/index.ts | 47 --- 3 files changed, 174 insertions(+), 244 deletions(-) create mode 100644 packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/guards.ts diff --git a/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts index 7482a17b..3f5a780e 100644 --- a/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts +++ b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/PhFs.ts @@ -1,21 +1,9 @@ import { compact, map } from '@s-libs/micro-dash' -import { - Mode, - constants, - createReadStream, - createWriteStream, - existsSync, - mkdirSync, -} from 'fs' +import Bottleneck from 'bottleneck' +import { spawn } from 'child_process' +import { Mode, constants, createReadStream, createWriteStream } from 'fs' import { FileStat, FileSystem, FtpConnection } from 'ftp-srv' -import { isAbsolute, join, normalize, resolve, sep } from 'path' -import { - FolderNamesMap, - INSTANCE_ROOT_VIRTUAL_FOLDER_NAMES, - MAINTENANCE_ONLY_FOLDER_NAMES, - VirtualFolderNames, - virtualFolderGuard, -} from '.' +import { dirname, isAbsolute, join, normalize, resolve, sep } from 'path' import { InstanceFields, Logger, @@ -24,7 +12,9 @@ import { newId, } from '../../../../../common' import { DATA_ROOT } from '../../../../../core' +import { InstanceLogger, InstanceLoggerApi } from '../../../../../services' import * as fsAsync from './fs-async' +import { MAINTENANCE_ONLY_INSTANCE_ROOTS } from './guards' export type PathError = { cause: { @@ -43,6 +33,73 @@ export type PathError = { const UNIX_SEP_REGEX = /\//g const WIN_SEP_REGEX = /\\/g +const checkBun = (virtualPath: string, cwd: string) => { + const [subdomain, maybeImportant, ...rest] = virtualPath + .split('/') + .filter((p) => !!p) + if (!subdomain) { + throw new Error(`Subdomain not found in path: ${virtualPath}`) + } + + const isImportant = + rest.length === 0 && + [`bun.lockb`, `package.json`].includes(maybeImportant || '') + + if (isImportant) { + const logger = InstanceLogger(subdomain, `exec`, { ttl: 5000 }) + logger.info(`${maybeImportant} changed, running bun install`) + launchBunInstall(subdomain, virtualPath, cwd).catch(console.error) + } +} + +const runBun = (() => { + const bunLimiter = new Bottleneck({ maxConcurrent: 1 }) + return (cwd: string, logger: InstanceLoggerApi) => + bunLimiter.schedule( + () => + new Promise((resolve) => { + const proc = spawn( + '/root/.bun/bin/bun', + ['install', `--no-save`, `--production`, `--ignore-scripts`], + { cwd }, + ) + const tid = setTimeout(() => { + logger.error(`bun timeout after 10s`) + proc.kill() + }, 10000) + proc.stdout.on('data', (data) => { + logger.info(`${data}`) + }) + proc.stderr.on('data', (data) => { + logger.error(`${data}`) + }) + proc.on('close', (code) => { + logger.info(`bun exited with: ${code}`) + clearTimeout(tid) + resolve(code) + }) + }), + ) +})() + +const launchBunInstall = (() => { + const runCache: { [key: string]: { runAgain: boolean } } = {} + return async (subdomain: string, virtualPath: string, cwd: string) => { + if (cwd in runCache) { + runCache[cwd]!.runAgain = true + return + } + runCache[cwd] = { runAgain: true } + while (runCache[cwd]!.runAgain) { + runCache[cwd]!.runAgain = false + const logger = InstanceLogger(subdomain, `exec`) + logger.info(`Launching 'bun install' in ${virtualPath}`) + await runBun(cwd, logger) + } + delete runCache[cwd] + } +})() + export class PhFs implements FileSystem { private log: Logger connection: FtpConnection @@ -64,106 +121,77 @@ export class PhFs implements FileSystem { return this._root } - async _resolvePath(path = '.') { + async _resolvePath(virtualPath = '.') { const { dbg } = this.log.create(`_resolvePath`) // Unix separators normalize nicer on both unix and win platforms - const resolvedPath = path.replace(WIN_SEP_REGEX, '/') + const resolvedVirtualPath = virtualPath.replace(WIN_SEP_REGEX, '/') // Join cwd with new path - const joinedPath = isAbsolute(resolvedPath) - ? normalize(resolvedPath) - : join('/', this.cwd, resolvedPath) + const finalVirtualPath = isAbsolute(resolvedVirtualPath) + ? normalize(resolvedVirtualPath) + : join('/', this.cwd, resolvedVirtualPath) - console.log(`***joinedPath`, { joinedPath }) + console.log(`***finalVirtualPath`, { finalVirtualPath }) // Create local filesystem path using the platform separator - const [empty, subdomain, virtualRootFolderName, ...pathFromRootFolder] = - joinedPath.split('/') + const [empty, subdomain, ...restOfVirtualPath] = finalVirtualPath.split('/') dbg({ - joinedPath, + finalVirtualPath, subdomain, - virtualRootFolderName, - restOfPath: pathFromRootFolder, + restOfVirtualPath, }) - // Check if the root folder name is valid - if (virtualRootFolderName) { - virtualFolderGuard(virtualRootFolderName) - } - // Begin building the physical path const fsPathParts: string[] = [this._root] // Check if the instance is valid const instance = await (async () => { console.log(`***checking validity`, { subdomain }) - if (subdomain) { - const instance = await this.client - .collection(`instances`) - .getFirstListItem(`subdomain='${subdomain}'`) - if (!instance) { - throw new Error(`${subdomain} not found.`) - } - fsPathParts.push(instance.id) - if (virtualRootFolderName) { - virtualFolderGuard(virtualRootFolderName) - const physicalFolderName = FolderNamesMap[virtualRootFolderName] - dbg({ - fsPathParts, - virtualRootFolderName, - physicalFolderName, - instance, - }) - if ( - MAINTENANCE_ONLY_FOLDER_NAMES.includes(virtualRootFolderName) && - !instance.maintenance - ) { - throw new Error( - `Instance must be in maintenance mode to access ${virtualRootFolderName}`, - ) - } - fsPathParts.push(physicalFolderName) - // Ensure folder exists - const rootFolderFsPath = resolve( - join(...fsPathParts) - .replace(UNIX_SEP_REGEX, sep) - .replace(WIN_SEP_REGEX, sep), - ) - dbg({ rootFolderFsPath }) - if (!existsSync(rootFolderFsPath)) { - mkdirSync(rootFolderFsPath, { recursive: true }) - } - } - if (resolvedPath.length > 0) fsPathParts.push(...pathFromRootFolder) - return instance + if (!subdomain) return + const instance = await this.client + .collection(`instances`) + .getFirstListItem(`subdomain='${subdomain}'`) + if (!instance) { + throw new Error(`${subdomain} not found.`) } + const instanceRootDir = restOfVirtualPath[0] + if ( + instanceRootDir && + MAINTENANCE_ONLY_INSTANCE_ROOTS.includes(instanceRootDir) && + !instance.maintenance + ) { + throw new Error( + `Instance must be in maintenance mode to access ${instanceRootDir}`, + ) + } + fsPathParts.push(instance.id) + dbg({ + fsPathParts, + instance, + }) + return instance })() // Finalize the fs path const fsPath = resolve( - join(...fsPathParts) + join(...fsPathParts, ...restOfVirtualPath) .replace(UNIX_SEP_REGEX, sep) .replace(WIN_SEP_REGEX, sep), ) // Create FTP client path using unix separator - const clientPath = joinedPath.replace(WIN_SEP_REGEX, '/') + const clientPath = finalVirtualPath.replace(WIN_SEP_REGEX, '/') dbg({ clientPath, fsPath, - subdomain, - virtualRootFolderName, - restOfPath: pathFromRootFolder, + restOfVirtualPath, instance, }) return { clientPath, fsPath, - subdomain, - rootFolderName: virtualRootFolderName as VirtualFolderNames | undefined, - pathFromRootFolder, instance, } } @@ -192,13 +220,12 @@ export class PhFs implements FileSystem { .breadcrumb(`cwd:${this.cwd}`) .breadcrumb(path) - const { fsPath, subdomain, rootFolderName, instance } = - await this._resolvePath(path) + const { fsPath, instance } = await this._resolvePath(path) /* If a subdomain is not specified, we are in the user's root. List all subdomains. */ - if (subdomain === '') { + if (!instance) { const instances = await this.client.collection(`instances`).getFullList() return instances.map((i) => { return { @@ -212,30 +239,8 @@ export class PhFs implements FileSystem { } /* - If we have a subdomain, then it should have resolved to an instance owned by that user - */ - if (!instance) { - throw new Error( - `Something as gone wrong. An instance without a subdomain is not possible.`, - ) - } - - /* - If there is no root folder name, then we are in the instance root. In this case, list - our allowed folder names. - */ - if (!rootFolderName) { - return INSTANCE_ROOT_VIRTUAL_FOLDER_NAMES.map((name) => ({ - isDirectory: () => true, - mode: 0o755, - size: 0, - mtime: Date.parse(instance.updated), - name: name, - })) - } - - /* - If we do have a root folder name, then we list its contents + If we do have a root folder name, it will include teh instance ID at this point + List it's contents */ return fsAsync .readdir(fsPath) @@ -260,8 +265,7 @@ export class PhFs implements FileSystem { } async get(fileName: string): Promise { - const { fsPath, subdomain, instance, rootFolderName, pathFromRootFolder } = - await this._resolvePath(fileName) + const { fsPath, instance, clientPath } = await this._resolvePath(fileName) const { dbg, error } = this.log .create(`get`) @@ -271,9 +275,9 @@ export class PhFs implements FileSystem { dbg(`get`) /* - If the subdomain is not specified, we are in the root + If the instance subdomain is not specified, we are in the root */ - if (!subdomain) { + if (!instance) { return { isDirectory: () => true, mode: 0o755, @@ -283,36 +287,17 @@ export class PhFs implements FileSystem { } } - /* - We are in a subdomain, we must have an instance - */ - if (!instance) { - throw new Error(`Expected instance here`) - } - + const isInstanceRoot = clientPath.split('/').filter((p) => !!p).length === 0 /* If we don't have a root folder, we're at the instance subdomain level */ - if (!rootFolderName) { + if (!isInstanceRoot) { return { isDirectory: () => true, mode: 0o755, size: 0, mtime: Date.parse(instance.updated), - name: subdomain, - } - } - - /* - If we don't have a path beneath the root folder, we're at a root folder level - */ - if (!pathFromRootFolder) { - return { - isDirectory: () => true, - mode: 0o755, - size: 0, - mtime: Date.parse(instance.updated), - name: rootFolderName, + name: instance.subdomain, } } @@ -336,19 +321,11 @@ export class PhFs implements FileSystem { .breadcrumb(fileName) dbg(`write`) - const { fsPath, clientPath, rootFolderName, pathFromRootFolder, instance } = - await this._resolvePath(fileName) + const { fsPath, clientPath, instance } = await this._resolvePath(fileName) assert(instance, `Instance expected here`) const { append, start } = options || {} - /* - If we are not in a root folder, disallow writing - */ - if (pathFromRootFolder.length === 0) { - throw new Error(`write not allowed at this level`) - } - const stream = createWriteStream(fsPath, { flags: !append ? 'w+' : 'a+', start, @@ -360,8 +337,10 @@ export class PhFs implements FileSystem { fsAsync.unlink(fsPath) }) stream.once('close', () => { - dbg(`write(${fileName}) closing`) + const virtualPath = join(this.cwd, fileName) + dbg(`write(${virtualPath}) closing`) stream.end() + checkBun(virtualPath, dirname(fsPath)) }) return { stream, @@ -379,18 +358,10 @@ export class PhFs implements FileSystem { .breadcrumb(fileName) dbg(`read`) - const { fsPath, clientPath, pathFromRootFolder } = - await this._resolvePath(fileName) + const { fsPath, clientPath } = await this._resolvePath(fileName) const { start } = options || {} - /* - If we are not in a root folder, disallow reading - */ - if (pathFromRootFolder.length === 0) { - throw new Error(`read not allowed at this level`) - } - return fsAsync .stat(fsPath) .then((stat) => { @@ -412,17 +383,9 @@ export class PhFs implements FileSystem { .breadcrumb(path) dbg(`delete`) - const { fsPath, clientPath, pathFromRootFolder, rootFolderName, instance } = - await this._resolvePath(path) + const { fsPath, instance } = await this._resolvePath(path) assert(instance, `Instance expected here`) - /* - Disallow deleting if not inside root folder - */ - if (pathFromRootFolder.length === 0) { - throw new Error(`delete not allowed at this level`) - } - return fsAsync.stat(fsPath).then((stat) => { if (stat.isDirectory()) { return fsAsync.rmdir(fsPath) @@ -438,15 +401,7 @@ export class PhFs implements FileSystem { .breadcrumb(path) dbg(`mkdir`) - const { fsPath, clientPath, pathFromRootFolder } = - await this._resolvePath(path) - - /* - Disallow making directories if not inside root folder - */ - if (pathFromRootFolder.length === 0) { - throw new Error(`mkdir not allowed at this level`) - } + const { fsPath } = await this._resolvePath(path) return fsAsync.mkdir(fsPath, { recursive: true }).then(() => fsPath) } @@ -459,29 +414,11 @@ export class PhFs implements FileSystem { .breadcrumb(to) dbg(`rename`) - const { - fsPath: fromPath, - pathFromRootFolder: fromPathFromRootFolder, - rootFolderName: fromRootFolderName, - instance, - } = await this._resolvePath(from) + const { fsPath: fromPath, instance } = await this._resolvePath(from) - const { - fsPath: toPath, - pathFromRootFolder: toPathFromRootFolder, - rootFolderName: toRootFolderName, - } = await this._resolvePath(to) + const { fsPath: toPath } = await this._resolvePath(to) assert(instance, `Instance expected here`) - /* - Disallow making directories if not inside root folder - */ - if (fromPathFromRootFolder.length === 0) { - throw new Error(`rename not allowed at this level`) - } - if (toPathFromRootFolder.length === 0) { - throw new Error(`rename not allowed at this level`) - } return fsAsync.rename(fromPath, toPath) } @@ -494,15 +431,8 @@ export class PhFs implements FileSystem { .breadcrumb(mode.toString()) dbg(`chmod`) - const { fsPath, clientPath, pathFromRootFolder } = - await this._resolvePath(path) + const { fsPath } = await this._resolvePath(path) - /* - Disallow making directories if not inside root folder - */ - if (pathFromRootFolder.length === 0) { - throw new Error(`chmod not allowed at this level`) - } return // noop return fsAsync.chmod(fsPath, mode) } diff --git a/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/guards.ts b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/guards.ts new file mode 100644 index 00000000..3b3faf4c --- /dev/null +++ b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/guards.ts @@ -0,0 +1,47 @@ +import { keys, values } from '@s-libs/micro-dash' + +export enum VirtualFolderNames { + Cache = `.cache`, + Data = 'pb_data', + Public = 'pb_public', + Migrations = 'pb_migrations', + Hooks = 'pb_hooks', +} + +export enum PhysicalFolderNames { + Cache = `.cache`, + Data = 'pb_data', + Public = 'pb_public', + Migrations = 'pb_migrations', + Hooks = 'pb_hooks', +} +export const MAINTENANCE_ONLY_INSTANCE_ROOTS: string[] = [ + VirtualFolderNames.Data, +] + +export const FolderNamesMap: { + [_ in VirtualFolderNames]: PhysicalFolderNames +} = { + [VirtualFolderNames.Cache]: PhysicalFolderNames.Cache, + [VirtualFolderNames.Data]: PhysicalFolderNames.Data, + [VirtualFolderNames.Public]: PhysicalFolderNames.Public, + [VirtualFolderNames.Migrations]: PhysicalFolderNames.Migrations, + [VirtualFolderNames.Hooks]: PhysicalFolderNames.Hooks, +} as const + +export const INSTANCE_ROOT_VIRTUAL_FOLDER_NAMES = keys(FolderNamesMap) +export const INSTANCE_ROOT_PHYSICAL_FOLDER_NAMES = values(FolderNamesMap) + +export function isInstanceRootVirtualFolder( + name: string, +): name is VirtualFolderNames { + return INSTANCE_ROOT_VIRTUAL_FOLDER_NAMES.includes(name as VirtualFolderNames) +} + +export function virtualFolderGuard( + name: string, +): asserts name is VirtualFolderNames { + if (!isInstanceRootVirtualFolder(name)) { + // throw new Error(`Accessing ${name} is not allowed.`) + } +} diff --git a/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/index.ts b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/index.ts index b36c5b7f..035bd0a8 100644 --- a/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/index.ts +++ b/packages/pockethost/src/cli/commands/EdgeCommand/FtpCommand/FtpService/index.ts @@ -1,4 +1,3 @@ -import { keys, values } from '@s-libs/micro-dash' import { readFileSync } from 'fs' import { FtpSrv } from 'ftp-srv' import { @@ -21,52 +20,6 @@ import { PhFs } from './PhFs' export type FtpConfig = { mothershipUrl: string } -export enum VirtualFolderNames { - Cache = `.cache`, - Data = 'pb_data', - Public = 'pb_public', - Migrations = 'pb_migrations', - Hooks = 'pb_hooks', -} - -export enum PhysicalFolderNames { - Cache = `.cache`, - Data = 'pb_data', - Public = 'pb_public', - Migrations = 'pb_migrations', - Hooks = 'pb_hooks', -} -export const MAINTENANCE_ONLY_FOLDER_NAMES: VirtualFolderNames[] = [ - VirtualFolderNames.Data, -] - -export const FolderNamesMap: { - [_ in VirtualFolderNames]: PhysicalFolderNames -} = { - [VirtualFolderNames.Cache]: PhysicalFolderNames.Cache, - [VirtualFolderNames.Data]: PhysicalFolderNames.Data, - [VirtualFolderNames.Public]: PhysicalFolderNames.Public, - [VirtualFolderNames.Migrations]: PhysicalFolderNames.Migrations, - [VirtualFolderNames.Hooks]: PhysicalFolderNames.Hooks, -} as const - -export const INSTANCE_ROOT_VIRTUAL_FOLDER_NAMES = keys(FolderNamesMap) -export const INSTANCE_ROOT_PHYSICAL_FOLDER_NAMES = values(FolderNamesMap) - -export function isInstanceRootVirtualFolder( - name: string, -): name is VirtualFolderNames { - return INSTANCE_ROOT_VIRTUAL_FOLDER_NAMES.includes(name as VirtualFolderNames) -} - -export function virtualFolderGuard( - name: string, -): asserts name is VirtualFolderNames { - if (!isInstanceRootVirtualFolder(name)) { - throw new Error(`Accessing ${name} is not allowed.`) - } -} - export const ftpService = mkSingleton((config: Partial = {}) => { const { mothershipUrl } = mergeConfig( {