feat: health monitoring to Discord

ecosystem.config.cjs

@@ -25,5 +25,10 @@ module.exports = {
       restart_delay: 60 * 60 * 1000, // 1 hour
       script: 'pnpm download-versions',
     },
+    {
+      name: `edge-health`,
+      restart_delay: 60 * 1000, // 1 minute
+      script: 'pnpm prod:edge:health',
+    },
   ],
 }

package.json

@@ -31,6 +31,7 @@
     "prod:proxy": "dotenv tsx ./src/cli/proxy/server.ts",
     "prod:edge:daemon": "tsx src/cli/edge-daemon.ts",
     "prod:edge:ftp": "tsx src/cli/edge-ftp.ts",
+    "prod:edge:health": "tsx src/cli/edge-health.ts",
     "prod:edge:syslog": "tsx src/cli/edge-syslogd.ts",
     "prod:mothership": "tsx src/cli/mothership.ts",
     "plop": "plop",

src/cli/edge-health.ts

@@ -0,0 +1,131 @@
+import {
+  DAEMON_PORT,
+  DEBUG,
+  DefaultSettingsService,
+  DISCORD_HEALTH_CHANNEL_URL,
+  MOTHERSHIP_NAME,
+  MOTHERSHIP_PORT,
+  SETTINGS,
+} from '$constants'
+import { LoggerService, LogLevelName } from '$shared'
+import { execSync } from 'child_process'
+import fetch from 'node-fetch'
+import { freemem } from 'os'
+
+DefaultSettingsService(SETTINGS)
+
+const DISCORD_URL = DISCORD_HEALTH_CHANNEL_URL()
+
+const { dbg, error, info, warn } = LoggerService({
+  level: DEBUG() ? LogLevelName.Debug : LogLevelName.Info,
+}).create('edge-health.ts')
+
+info(`Starting`)
+
+const _exec = (cmd: string) =>
+  execSync(cmd, { shell: '/bin/bash', maxBuffer: 1024 * 1024 * 10 })
+    .toString()
+    .split(`\n`)
+
+const openFiles = _exec(`lsof -n | awk '$4 ~ /^[0-9]/ {print}'`)
+
+const [freeSpace] = _exec(`df -h / | awk 'NR==2{print $4}'`)
+
+const containers = _exec(
+  `docker ps --format '{{.Names}} {{.Ports}}' | awk '{print $1, $2}' | sed 's/-[0-9]* / /' |  awk -F':' '{print $1, $2}' | awk '{print $1, $3}' | awk -F'->' '{print $1}'`,
+)
+  .map((line) => line.split(/\s+/))
+  .filter((split): split is [string, string] => !!split[0])
+  .filter(([name]) => name !== MOTHERSHIP_NAME())
+  .map(([name, port]) => {
+    return {
+      name,
+      priority: 0,
+      emoji: ':guitar:',
+      port: parseInt(port || '0', 10),
+      isHealthy: false,
+      url: `http://localhost:${port}/api/health`,
+    }
+  })
+
+const checks: {
+  name: string
+  priority: number
+  emoji?: string
+  isHealthy: boolean
+  url: string
+  port?: number
+}[] = [
+  {
+    name: `edge proxy`,
+    priority: 10,
+    emoji: `:park:`,
+    isHealthy: false,
+    url: `https://proxy.pockethost.io/api/health`,
+  },
+  {
+    name: `edge daemon`,
+    priority: 8,
+    emoji: `:imp:`,
+    isHealthy: false,
+    url: `http://localhost:${DAEMON_PORT()}/api/health`,
+  },
+  {
+    name: `mothership`,
+    priority: 9,
+    emoji: `:flying_saucer:`,
+    isHealthy: false,
+    url: `http://localhost:${MOTHERSHIP_PORT()}/api/health`,
+  },
+  ...containers,
+]
+
+await Promise.all(
+  checks.map(async (check) => {
+    const { url } = check
+    dbg({ container: check })
+    try {
+      const res = await fetch(url)
+      dbg({ url, status: res.status })
+      check.isHealthy = res.status === 200
+      return true
+    } catch (e) {
+      dbg(`${url}: ${e}`)
+      check.isHealthy = false
+    }
+  }),
+)
+dbg({ checks })
+
+function getFreeMemoryInGB(): string {
+  const freeMemoryBytes: number = freemem()
+  const freeMemoryGB: number = freeMemoryBytes / Math.pow(1024, 3)
+  return freeMemoryGB.toFixed(2) // Rounds to 2 decimal places
+}
+
+await fetch(DISCORD_URL, {
+  method: 'POST',
+  body: JSON.stringify({
+    content: [
+      `===================`,
+      `Server: SFO-1`,
+      `${new Date()}`,
+      `Free RAM: ${getFreeMemoryInGB()}`,
+      `Free disk: ${freeSpace}`,
+      `${checks.length} containers running and ${openFiles.length} open files.`,
+      ...checks
+        .sort((a, b) => {
+          if (a.priority > b.priority) return -1
+          if (a.priority < b.priority) return 1
+          return a.name.localeCompare(b.name)
+        })
+        .map(
+          ({ name, isHealthy, emoji }) =>
+            `${
+              isHealthy ? ':white_check_mark:' : ':face_vomiting: '
+            } ${emoji} ${name}`,
+        ),
+    ].join(`\n`),
+  }),
+  headers: { 'content-type': 'application/json' },
+})

src/cli/proxy/server.ts

@@ -44,6 +44,11 @@ forEach(hostnameRoutes, (target, host) => {
   app.use(createVhostProxyMiddleware(host, target, IS_DEV()))
 })
 
+app.get(`/api/health`, (req, res, next) => {
+  res.json({ status: 'ok' })
+  res.end()
+})
+
 // Fall-through
 const handler = createProxyMiddleware({
   target: `http://localhost:${DAEMON_PORT()}`,

src/constants.ts

@@ -100,6 +100,8 @@ export const SETTINGS = {
   LS_WEBHOOK_SECRET: mkString(''),
 
   SYSLOGD_PORT: mkNumber(6514),
+
+  DISCORD_HEALTH_CHANNEL_URL: mkString(''),
 }
 ;(() => {
   let passed = true
@@ -225,6 +227,9 @@ export const LS_WEBHOOK_SECRET = () => settings().LS_WEBHOOK_SECRET
 
 export const SYSLOGD_PORT = () => settings().SYSLOGD_PORT
 
+export const DISCORD_HEALTH_CHANNEL_URL = () =>
+  settings().DISCORD_HEALTH_CHANNEL_URL
+
 /** Helpers */
 
 export const MOTHERSHIP_DATA_ROOT = () => INSTANCE_DATA_ROOT(MOTHERSHIP_NAME())

src/services/ProxyService.ts

@@ -2,14 +2,14 @@ import { DAEMON_PORT } from '$constants'
 import {
   Logger,
   LoggerService,
-  SingletonBaseConfig,
   mkSingleton,
+  SingletonBaseConfig,
 } from '$shared'
 import { asyncExitHook } from '$util'
 import cors from 'cors'
 import express, { Request, Response } from 'express'
 import 'express-async-errors'
-import { default as Server, default as httpProxy } from 'http-proxy'
+import { default as httpProxy, default as Server } from 'http-proxy'
 import { AsyncReturnType } from 'type-fest'
 
 export type ProxyServiceApi = AsyncReturnType<typeof proxyService>
@@ -44,6 +44,11 @@ export const proxyService = mkSingleton(async (config: ProxyServiceConfig) => {
 
   server.use(cors())
 
+  server.get('/api/health', (req, res, next) => {
+    res.json({ status: 'ok' })
+    res.end
+  })
+
   server.use((req, res, next) => {
     const host = req.headers.host
     if (!host) {