log refactor

This commit is contained in:
Ben Allfree 2022-12-27 16:29:28 -08:00
parent f3650cbdad
commit ea432684d7
27 changed files with 175 additions and 109 deletions

View File

@ -9,7 +9,7 @@
"lint:fix": "prettier -w \"./**/*.ts\"",
"build": "concurrently 'yarn:build:*'",
"build:www": "cd packages/pockethost.io && yarn build",
"dev": "concurrently 'yarn:dev:*'",
"dev": "NODE_ENV=development concurrently 'yarn:dev:*'",
"dev:proxy": "cd packages/proxy && yarn dev",
"dev:www": "cd packages/pockethost.io && yarn dev",
"dev:daemon": "cd packages/daemon && yarn dev",
@ -50,4 +50,4 @@
"dependencies": {
"postinstall-postinstall": "^2.1.0"
}
}
}

View File

@ -1,38 +1,68 @@
import { mkSingleton } from './mkSingleton'
export type Config = {
raw?: boolean
debug: boolean
pfx: string[]
pfx?: string[]
}
export type Logger = ReturnType<typeof createLogger>
const MAX_BUF = 1000
let _curIdx = 0
const _buf: any = []
export const createLogger = (config: Partial<Config>) => {
const _config: Config = {
const _config: Required<Config> = {
raw: false,
debug: true,
pfx: [''],
...config,
}
const { debug, pfx } = _config
const { pfx } = _config
const _pfx = (s: string) =>
[s, ...pfx]
.filter((v) => !!v)
.map((p) => `[${p}]`)
.join(' ')
const _log = (shouldDisplay: boolean, ...args: any[]) => {
if (_buf.length < MAX_BUF) {
_buf.push(args)
} else {
_buf[_curIdx] = args
_curIdx++
if (_curIdx === MAX_BUF) _curIdx = 0
}
if (shouldDisplay) console.log(...args)
}
const raw = (...args: any[]) => {
_log(_config.raw, _pfx('RAW'), ...args)
}
const dbg = (...args: any[]) => {
if (!debug) return
console.log(_pfx('DBG'), ...args)
_log(_config.debug, _pfx('DBG'), ...args)
}
const warn = (...args: any[]) => {
console.log(_pfx('WARN'), ...args)
_log(true, _pfx('WARN'), ...args)
}
const info = (...args: any[]) => {
console.log(_pfx(`INFO`), ...args)
_log(true, _pfx(`INFO`), ...args)
}
const error = (...args: any[]) => {
console.error(_pfx(`ERROR`), ...args)
_log(true, _pfx(`ERROR`), ...args)
console.error(`========================`)
;[..._buf.slice(_curIdx, MAX_BUF), ..._buf.slice(0, _curIdx)].forEach(
(args) => {
console.error(...args)
}
)
console.error(`========================`)
}
const create = (s: string) =>
@ -41,5 +71,7 @@ export const createLogger = (config: Partial<Config>) => {
pfx: [..._config.pfx, s],
})
return { dbg, warn, info, error, create }
return { raw, dbg, warn, info, error, create }
}
export const logger = mkSingleton((config: Config) => createLogger(config))

View File

@ -1,17 +1,11 @@
import { ClientResponseError } from 'pocketbase'
import { Logger } from './Logger'
import { logger } from './Logger'
export type PromiseHelperConfig = {
logger: Logger
}
export type PromiseHelperConfig = {}
export type PromiseHelper = ReturnType<typeof createPromiseHelper>
export const createPromiseHelper = (config: PromiseHelperConfig) => {
const { logger } = config
const { dbg, error, warn } = logger
let inside = ''
let c = 0
const safeCatch = <TIn extends any[], TOut>(
name: string,
@ -20,17 +14,16 @@ export const createPromiseHelper = (config: PromiseHelperConfig) => {
return (...args: TIn) => {
const _c = c++
const uuid = `${name}:${_c}`
const pfx = `[safeCatch:${uuid}]`
// dbg(uuid, ...args)
const pfx = `safeCatch:${uuid}`
const { raw, error, warn } = logger().create(pfx)
raw(`args`, args)
const tid = setTimeout(() => {
warn(pfx, `timeout waiting for ${pfx}`)
warn(`timeout waiting for ${pfx}`)
}, 100)
inside = pfx
return cb(...args)
.then((res) => {
// dbg(uuid, `finished`)
inside = ''
raw(`finished`)
clearTimeout(tid)
return res
})
@ -38,16 +31,15 @@ export const createPromiseHelper = (config: PromiseHelperConfig) => {
if (e instanceof ClientResponseError) {
if (e.status === 400) {
error(
pfx,
`PocketBase API error: It looks like you don't have permission to make this request.`
)
} else if (e.status === 0) {
dbg(pfx, `Client request aborted (duplicate)`)
warn(`Client request aborted (duplicate)`)
} else {
error(pfx, `Unknown PocketBase API error`, JSON.stringify(e))
error(`Unknown PocketBase API error`, JSON.stringify(e))
}
} else {
error(pfx, JSON.stringify(e, null, 2))
error(JSON.stringify(e, null, 2))
}
throw e
})

View File

@ -1,6 +1,7 @@
export * from './assert'
export * from './CleanupManager'
export * from './Logger'
export * from './mkSingleton'
export * from './pocketbase-client-helpers'
export * from './PromiseHelper'
export * from './schema'

View File

@ -0,0 +1,17 @@
export const mkSingleton = <TConfig, TApi>(
factory: (config: TConfig) => TApi
) => {
let _service: TApi | undefined = undefined
return (config?: TConfig) => {
if (_service && config) {
throw new Error(`Attempted to initialize service twice`)
}
if (!_service) {
if (!config) {
throw new Error(`Attempted to use service before initialization`)
}
_service = factory(config)
}
return _service
}
}

View File

@ -38,7 +38,8 @@ export const DAEMON_PB_DATA_DIR = (() => {
return v
})()
export const DEBUG = envb('DEBUG', false)
export const NODE_ENV = env('NODE_ENV', '')
export const DEBUG = envb('DEBUG', NODE_ENV === 'development')
export const DAEMON_PB_BACKUP_SLEEP = envi(`DAEMON_PB_BACKUP_SLEEP`, 100)
export const DAEMON_PB_BACKUP_PAGE_COUNT = envi(

View File

@ -6,6 +6,7 @@ import {
BackupStatus,
InstanceFields,
InstanceId,
logger,
} from '@pockethost/common'
import { safeCatch } from '../util/promiseHelper'
import { MixinContext } from './PbClient'
@ -13,6 +14,8 @@ import { MixinContext } from './PbClient'
export type BackupApi = ReturnType<typeof createBackupMixin>
export const createBackupMixin = (context: MixinContext) => {
const { dbg } = logger().create('BackupMixin')
const { client, rawDb } = context
const createBackup = safeCatch(

View File

@ -4,18 +4,20 @@ import {
InstanceFields_Create,
InstanceId,
InstanceStatus,
logger,
UserFields,
} from '@pockethost/common'
import { reduce } from '@s-libs/micro-dash'
import Bottleneck from 'bottleneck'
import { endOfMonth, startOfMonth } from 'date-fns'
import { dbg } from '../util/logger'
import { safeCatch } from '../util/promiseHelper'
import { MixinContext } from './PbClient'
export type InstanceApi = ReturnType<typeof createInstanceMixin>
export const createInstanceMixin = (context: MixinContext) => {
const { dbg, raw } = logger().create('InstanceMixin')
const { client, rawDb } = context
const createInstance = safeCatch(
@ -98,7 +100,7 @@ export const createInstanceMixin = (context: MixinContext) => {
.where('instanceId', instanceId)
.where('startedAt', '>=', startIso)
.where('startedAt', '<=', endIso)
dbg(query.toString())
raw(query.toString())
const res = await query
const [row] = res
assertExists(row, `Expected row here`)

View File

@ -1,5 +1,9 @@
import { InstanceFields, InvocationFields, pocketNow } from '@pockethost/common'
import { dbg } from '../util/logger'
import {
InstanceFields,
InvocationFields,
logger,
pocketNow,
} from '@pockethost/common'
import { safeCatch } from '../util/promiseHelper'
import { InstanceApi } from './InstanceMIxin'
import { MixinContext } from './PbClient'
@ -8,6 +12,8 @@ export const createInvocationMixin = (
context: MixinContext,
instanceApi: InstanceApi
) => {
const { dbg } = logger().create('InvocationMixin')
const { client } = context
const createInvocation = safeCatch(

View File

@ -1,3 +1,4 @@
import { logger } from '@pockethost/common'
import { Knex } from 'knex'
import {
Collection,
@ -6,7 +7,6 @@ import {
} from 'pocketbase'
import { DAEMON_PB_DATA_DIR, PUBLIC_PB_SUBDOMAIN } from '../constants'
import { Collection_Serialized } from '../migrate/schema'
import { info } from '../util/logger'
import { safeCatch } from '../util/promiseHelper'
import { createBackupMixin } from './BackupMixin'
import { createInstanceMixin } from './InstanceMIxin'
@ -19,6 +19,8 @@ export type PocketbaseClientApi = ReturnType<typeof createPbClient>
export type MixinContext = { client: pocketbaseEs; rawDb: Knex }
export const createPbClient = (url: string) => {
const { info } = logger().create('PbClient')
info(`Initializing client: ${url}`)
const rawDb = createRawPbClient(
`${DAEMON_PB_DATA_DIR}/${PUBLIC_PB_SUBDOMAIN}/pb_data/data.db`

View File

@ -1,11 +1,12 @@
import { InstanceStatus } from '@pockethost/common'
import { InstanceStatus, logger } from '@pockethost/common'
import { PH_BIN_CACHE, PUBLIC_PB_SUBDOMAIN } from '../constants'
import { pocketbase } from '../services/PocketBaseService'
import { info } from '../util/logger'
import { safeCatch } from '../util/promiseHelper'
import { schema } from './schema'
import { withInstance } from './withInstance'
;(async () => {
const { info } = logger().create('migrate')
const pbService = await pocketbase({
cachePath: PH_BIN_CACHE,
checkIntervalMs: 5 * 60 * 1000,

View File

@ -1,8 +1,9 @@
import { logger } from '@pockethost/common'
import { exec } from 'child_process'
import { dbg, error } from '../util/logger'
import { safeCatch } from '../util/promiseHelper'
export const pexec = safeCatch(`pexec`, (cmd: string) => {
const { dbg, error } = logger().create('pexec')
return new Promise<void>((resolve, reject) => {
dbg(cmd)
exec(cmd, (err, stdout, stderr) => {

View File

@ -1,3 +1,4 @@
import { logger } from '@pockethost/common'
import {
DAEMON_PB_PASSWORD,
DAEMON_PB_USERNAME,
@ -7,12 +8,13 @@ import {
} from '../constants'
import { createPbClient, PocketbaseClientApi } from '../db/PbClient'
import { pocketbase } from '../services/PocketBaseService'
import { error, info } from '../util/logger'
import { safeCatch } from '../util/promiseHelper'
export const withInstance = safeCatch(
`withInstance`,
async (cb: (client: PocketbaseClientApi) => Promise<void>) => {
const { info, error } = logger().create('withInstance')
// Add `platform` and `bin` required columns (migrate db json)
try {
const mainProcess = await (

View File

@ -1,6 +1,8 @@
import { logger } from '@pockethost/common'
import {
DAEMON_PB_PASSWORD,
DAEMON_PB_USERNAME,
DEBUG,
PH_BIN_CACHE,
PUBLIC_PB_DOMAIN,
PUBLIC_PB_PROTOCOL,
@ -12,10 +14,13 @@ import { createInstanceService } from './services/InstanceService'
import { pocketbase } from './services/PocketBaseService'
import { createProxyService } from './services/ProxyService'
import { createRpcService } from './services/RpcService'
import { dbg, error, info } from './util/logger'
logger({ debug: DEBUG })
// npm install eventsource --save
global.EventSource = require('eventsource')
;(async () => {
const { dbg, error, info } = logger().create(`server.ts`)
const pbService = await pocketbase({
cachePath: PH_BIN_CACHE,
checkIntervalMs: 1000 * 5 * 60,

View File

@ -6,6 +6,7 @@ import {
BackupInstanceResult,
BackupStatus,
createTimerManager,
logger,
RestoreInstancePayload,
RestoreInstancePayloadSchema,
RestoreInstanceResult,
@ -14,13 +15,14 @@ import {
import Bottleneck from 'bottleneck'
import { PocketbaseClientApi } from '../db/PbClient'
import { backupInstance } from '../util/backupInstance'
import { dbg } from '../util/logger'
import { RpcServiceApi } from './RpcService'
export const createBackupService = async (
client: PocketbaseClientApi,
jobService: RpcServiceApi
) => {
const { dbg } = logger().create('BackupService')
jobService.registerCommand<BackupInstancePayload, BackupInstanceResult>(
RpcCommands.BackupInstance,
BackupInstancePayloadSchema,

View File

@ -6,6 +6,7 @@ import {
createTimerManager,
InstanceId,
InstanceStatus,
logger,
RpcCommands,
} from '@pockethost/common'
import { forEachRight, map } from '@s-libs/micro-dash'
@ -20,7 +21,6 @@ import {
} from '../constants'
import { PocketbaseClientApi } from '../db/PbClient'
import { mkInternalUrl } from '../util/internal'
import { dbg, error, warn } from '../util/logger'
import { now } from '../util/now'
import { safeCatch } from '../util/promiseHelper'
import { pocketbase, PocketbaseProcess } from './PocketBaseService'
@ -41,6 +41,8 @@ export type InstanceServiceConfig = {
export type InstanceServiceApi = AsyncReturnType<typeof createInstanceService>
export const createInstanceService = async (config: InstanceServiceConfig) => {
const { dbg, raw, error, warn } = logger().create('InstanceService')
const { client, rpcService } = config
const { registerCommand } = rpcService
const pbService = await pocketbase()
@ -180,7 +182,7 @@ export const createInstanceService = async (config: InstanceServiceConfig) => {
{
tm.repeat(
safeCatch(`idleCheck`, async () => {
dbg(`${subdomain} idle check: ${openRequestCount} open requests`)
raw(`${subdomain} idle check: ${openRequestCount} open requests`)
if (
openRequestCount === 0 &&
lastRequest + DAEMON_PB_IDLE_TTL < now()
@ -191,7 +193,7 @@ export const createInstanceService = async (config: InstanceServiceConfig) => {
await _api.shutdown()
return false
} else {
dbg(`${openRequestCount} requests remain open on ${subdomain}`)
raw(`${openRequestCount} requests remain open on ${subdomain}`)
}
return true
}),
@ -201,7 +203,7 @@ export const createInstanceService = async (config: InstanceServiceConfig) => {
{
const uptime = safeCatch(`uptime`, async () => {
dbg(`${subdomain} uptime`)
raw(`${subdomain} uptime`)
await clientLimiter.schedule(() =>
client.pingInvocation(invocation)
)
@ -210,7 +212,7 @@ export const createInstanceService = async (config: InstanceServiceConfig) => {
tm.repeat(
() =>
uptime().catch((e) => {
dbg(`Ignoring error`)
warn(`Ignoring error`)
return true
}),
1000

View File

@ -1,4 +1,5 @@
import { createTimerManager } from '@pockethost/common'
import { createTimerManager, logger } from '@pockethost/common'
import { mkSingleton } from '@pockethost/common/src/mkSingleton'
import { keys } from '@s-libs/micro-dash'
import { spawn } from 'child_process'
import { chmodSync, existsSync } from 'fs'
@ -10,11 +11,9 @@ import { AsyncReturnType } from 'type-fest'
import { DAEMON_PB_DATA_DIR } from '../constants'
import { downloadAndExtract } from '../util/downloadAndExtract'
import { mkInternalAddress, mkInternalUrl } from '../util/internal'
import { dbg, error } from '../util/logger'
import { safeCatch } from '../util/promiseHelper'
import { smartFetch } from '../util/smartFetch'
import { tryFetch } from '../util/tryFetch'
import { mkSingleton } from './mkSingleton'
export type PocketbaseCommand = 'serve' | 'upgrade'
export type SpawnConfig = {
@ -53,6 +52,8 @@ export type Releases = Release[]
export const createPocketbaseService = async (
config: PocketbaseServiceConfig
) => {
const { dbg, error } = logger().create('PocketbaseService')
const { cachePath, checkIntervalMs } = config
const tm = createTimerManager({})

View File

@ -1,3 +1,4 @@
import { logger } from '@pockethost/common'
import { createServer } from 'http'
import httpProxy from 'http-proxy'
import { AsyncReturnType } from 'type-fest'
@ -6,7 +7,6 @@ import {
PUBLIC_APP_PROTOCOL,
PUBLIC_PB_SUBDOMAIN,
} from '../constants'
import { dbg, error, info } from '../util/logger'
import { InstanceServiceApi } from './InstanceService'
export type ProxyServiceApi = AsyncReturnType<typeof createProxyService>
@ -16,6 +16,8 @@ export type ProxyServiceConfig = {
instanceManager: InstanceServiceApi
}
export const createProxyService = async (config: ProxyServiceConfig) => {
const { dbg, error, info } = logger().create('ProxyService')
const { instanceManager, coreInternalUrl } = config
const proxy = httpProxy.createProxyServer({})

View File

@ -1,5 +1,6 @@
import {
assertTruthy,
logger,
RpcCommands,
RpcFields,
RpcStatus,
@ -12,7 +13,6 @@ import { default as knexFactory } from 'knex'
import pocketbaseEs from 'pocketbase'
import { AsyncReturnType, JsonObject } from 'type-fest'
import { PocketbaseClientApi } from '../db/PbClient'
import { dbg, error } from '../util/logger'
export type RpcServiceApi = AsyncReturnType<typeof createRpcService>
@ -31,6 +31,8 @@ export type RpcRunner<
export type RpcServiceConfig = { client: PocketbaseClientApi }
export const createRpcService = async (config: RpcServiceConfig) => {
const { dbg, error } = logger().create('RpcService')
const { client } = config
const limiter = new Bottleneck({ maxConcurrent: 1 })

View File

@ -1,18 +0,0 @@
export const mkSingleton = <TConfig, TApi>(
factory: (config: TConfig) => Promise<TApi>
) => {
let _service: TApi | undefined = undefined
return async (config?: TConfig) => {
return new Promise<TApi>(async (resolve) => {
if (_service) {
resolve(_service)
return
}
if (!config) {
throw new Error(`Attempt to get service before initialization.`)
}
_service = await factory(config)
resolve(_service)
})
}
}

View File

@ -1,4 +1,4 @@
import { BackupRecordId, InstanceId } from '@pockethost/common'
import { BackupRecordId, InstanceId, logger } from '@pockethost/common'
import { statSync } from 'fs'
import { basename, resolve } from 'path'
import { chdir, cwd } from 'process'
@ -7,7 +7,6 @@ import tmp from 'tmp'
import { DAEMON_PB_DATA_DIR } from '../constants'
import { pexec } from '../migrate/pexec'
import { ensureDirExists } from './ensureDirExists'
import { dbg, error } from './logger'
import { safeCatch } from './promiseHelper'
export type BackupProgress = {
@ -25,6 +24,8 @@ export const PB_DATA_DIR = `pb_data`
export const execBackup = safeCatch(
`execBackup`,
(src: string, dst: string, progress?: ProgressCallback) => {
const { dbg, error } = logger().create('execBackup')
const db = new Database(src)
const backup = db.backup(dst)
return new Promise<void>((resolve, reject) => {
@ -68,6 +69,8 @@ export const backupInstance = safeCatch(
backupId: BackupRecordId,
progress?: ProgressCallback
) => {
const { dbg, error } = logger().create('backupInstance')
const dataRoot = resolve(DAEMON_PB_DATA_DIR, instanceId)
const backupTgzRoot = resolve(dataRoot, 'backup')
const backupTgzFile = resolve(backupTgzRoot, `${backupId}.tgz`)

View File

@ -1,10 +1,12 @@
import { logger } from '@pockethost/common'
import { chmodSync } from 'fs'
import fetch from 'node-fetch'
import { dirname } from 'path'
import { Extract } from 'unzipper'
import { dbg, error } from './logger'
export const downloadAndExtract = async (url: string, binPath: string) => {
const { dbg, error } = logger().create('downloadAndExtract')
await new Promise<void>(async (resolve, reject) => {
dbg(`Fetching ${url}`)
const res = await fetch(url)

View File

@ -1,7 +1,9 @@
import { logger } from '@pockethost/common'
import { mkdirSync } from 'fs'
import { dbg } from './logger'
export const ensureDirExists = (path: string) => {
const { dbg } = logger().create(`ensureDirExists`)
try {
mkdirSync(path)
} catch (e) {

View File

@ -1,5 +0,0 @@
import { createLogger } from '@pockethost/common'
import { DEBUG } from '../constants'
export const logger = createLogger({ debug: DEBUG })
export const { dbg, info, warn, error } = logger

View File

@ -1,5 +1,4 @@
import { createPromiseHelper } from '@pockethost/common'
import { logger } from './logger'
export const promiseHelper = createPromiseHelper({ logger })
export const promiseHelper = createPromiseHelper({})
export const { safeCatch } = promiseHelper

View File

@ -1,21 +1,31 @@
import { logger } from '@pockethost/common'
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import fetch from 'node-fetch'
import { dirname } from 'path'
import { dbg } from './logger'
export const smartFetch = async <TRet>(
url: string,
path: string
): Promise<TRet> => {
const res = await fetch(url)
if (res.status !== 200) {
if (!existsSync(path)) {
throw new Error(`API down and no cache`)
const { dbg } = logger().create(`smartFetch`)
const data = await (async () => {
try {
const res = await fetch(url)
if (res.status !== 200) {
throw new Error(`API appears to be down`)
}
const data = (await res.json()) as TRet
return data
} catch (e) {
dbg(`Caught error on fetch: ${e}`)
if (!existsSync(path)) {
throw new Error(`API down and no cache`)
}
dbg(`Using data from cache`)
return JSON.parse(readFileSync(path).toString()) as TRet
}
dbg(`Using data from cache`)
return JSON.parse(readFileSync(path).toString()) as TRet
}
const data = await res.json()
})()
mkdirSync(dirname(path), { recursive: true })
writeFileSync(path, JSON.stringify(data))
return data as TRet

View File

@ -1,22 +1,21 @@
import { dbg } from './logger'
import { logger } from '@pockethost/common'
import { safeCatch } from './promiseHelper'
export const tryFetch = safeCatch(
`tryFetch`,
(url: string) =>
new Promise<void>((resolve, reject) => {
const tryFetch = () => {
dbg(`Trying to connect to instance ${url} `)
fetch(url)
.then(() => {
dbg(`Connection to ${url} successful`)
resolve()
})
.catch((e) => {
dbg(`Could not connect to ${url}`)
setTimeout(tryFetch, 1000)
})
}
tryFetch()
})
)
export const tryFetch = safeCatch(`tryFetch`, (url: string) => {
const { dbg } = logger().create('tryFetch')
return new Promise<void>((resolve, reject) => {
const tryFetch = () => {
dbg(`Trying to connect to instance ${url} `)
fetch(url)
.then(() => {
dbg(`Connection to ${url} successful`)
resolve()
})
.catch((e) => {
dbg(`Could not connect to ${url}`)
setTimeout(tryFetch, 1000)
})
}
tryFetch()
})
})