remove unneeded discordAlert from health check

This commit is contained in:
Ben Allfree 2024-09-25 04:34:23 +00:00
parent 54837acc60
commit 57f5535910

View File

@ -8,7 +8,6 @@ import {
DISCORD_HEALTH_CHANNEL_URL, DISCORD_HEALTH_CHANNEL_URL,
LoggerService, LoggerService,
MOTHERSHIP_PORT, MOTHERSHIP_PORT,
discordAlert,
stringify, stringify,
} from '../../../../core' } from '../../../../core'
@ -24,217 +23,209 @@ export const checkHealth = async () => {
info(`Starting`) info(`Starting`)
try { const _exec = (cmd: string) =>
const _exec = (cmd: string) => execSync(cmd, { shell: '/bin/bash', maxBuffer: 1024 * 1024 * 10 })
execSync(cmd, { shell: '/bin/bash', maxBuffer: 1024 * 1024 * 10 }) .toString()
.toString() .split(`\n`)
.split(`\n`)
type DockerPs = { type DockerPs = {
Command: string Command: string
CreatedAt: string CreatedAt: string
ID: string ID: string
Image: string Image: string
Labels: string Labels: string
LocalVolumes: string LocalVolumes: string
Mounts: string Mounts: string
Names: string Names: string
Networks: string Networks: string
Ports: string Ports: string
RunningFor: string RunningFor: string
Size: string Size: string
State: string State: string
Status: string Status: string
} }
const SAMPLE: DockerPs = { const SAMPLE: DockerPs = {
Command: '"docker-entrypoint.s…"', Command: '"docker-entrypoint.s…"',
CreatedAt: '2024-01-23 04:36:09 +0000 UTC', CreatedAt: '2024-01-23 04:36:09 +0000 UTC',
ID: '6e0921e84391', ID: '6e0921e84391',
Image: 'pockethost-instance', Image: 'pockethost-instance',
Labels: '', Labels: '',
LocalVolumes: '0', LocalVolumes: '0',
Mounts: Mounts:
'/home/pocketho…,/home/pocketho…,/home/pocketho…,/home/pocketho…,/home/pocketho…', '/home/pocketho…,/home/pocketho…,/home/pocketho…,/home/pocketho…,/home/pocketho…',
Names: 'kekbase-1705984569777', Names: 'kekbase-1705984569777',
Networks: 'bridge', Networks: 'bridge',
Ports: '0.0.0.0:44447-\u003e8090/tcp, :::44447-\u003e8090/tcp', Ports: '0.0.0.0:44447-\u003e8090/tcp, :::44447-\u003e8090/tcp',
RunningFor: '7 hours ago', RunningFor: '7 hours ago',
Size: '0B (virtual 146MB)', Size: '0B (virtual 146MB)',
State: 'running', State: 'running',
Status: 'Up 7 hours', Status: 'Up 7 hours',
} }
type Check = { type Check = {
name: string name: string
priority: number priority: number
emoji: string emoji: string
isHealthy: boolean isHealthy: boolean
url: string url: string
// Instance // Instance
port?: number port?: number
ago?: string ago?: string
mem?: string mem?: string
created?: Date created?: Date
} }
const containers = _exec(`docker ps --format '{{json .}}'`) const containers = _exec(`docker ps --format '{{json .}}'`)
.filter((line) => line.trim()) .filter((line) => line.trim())
.map((line) => { .map((line) => {
dbg(line) dbg(line)
return line return line
}) })
.map((line) => JSON.parse(line) as DockerPs) .map((line) => JSON.parse(line) as DockerPs)
.map<Check>((rec) => { .map<Check>((rec) => {
const name = rec.Names.replace(/-\d+/, '') const name = rec.Names.replace(/-\d+/, '')
const port = parseInt(rec.Ports.match(/:(\d+)/)?.[1] || '0', 10) const port = parseInt(rec.Ports.match(/:(\d+)/)?.[1] || '0', 10)
const mem = rec.Size.match(/(\d+MB)/)?.[1] || '0MB' const mem = rec.Size.match(/(\d+MB)/)?.[1] || '0MB'
const created = new Date(rec.CreatedAt) const created = new Date(rec.CreatedAt)
return { return {
name, name,
priority: 0, priority: 0,
emoji: ':octopus:', emoji: ':octopus:',
port, port,
isHealthy: false, isHealthy: false,
url: `http://localhost:${port}/api/health`, url: `http://localhost:${port}/api/health`,
ago: rec.RunningFor, ago: rec.RunningFor,
mem, mem,
created, created,
}
})
function getFreeMemoryInGB(): string {
const freeMemoryBytes: number = freemem()
const freeMemoryGB: number = freeMemoryBytes / Math.pow(1024, 3)
return freeMemoryGB.toFixed(2) // Rounds to 2 decimal places
}
function splitIntoChunks(
lines: string[],
maxChars: number = 2000,
): string[] {
const chunks: string[] = []
let currentChunk: string = ''
lines.forEach((line) => {
// Check if adding the next line exceeds the maxChars limit
if (currentChunk.length + line.length + 1 > maxChars) {
chunks.push(currentChunk)
currentChunk = ''
}
currentChunk += line + '\n' // Add the line and a newline character
})
// Add the last chunk if it's not empty
if (currentChunk) {
chunks.push(currentChunk)
} }
})
return chunks function getFreeMemoryInGB(): string {
const freeMemoryBytes: number = freemem()
const freeMemoryGB: number = freeMemoryBytes / Math.pow(1024, 3)
return freeMemoryGB.toFixed(2) // Rounds to 2 decimal places
}
function splitIntoChunks(lines: string[], maxChars: number = 2000): string[] {
const chunks: string[] = []
let currentChunk: string = ''
lines.forEach((line) => {
// Check if adding the next line exceeds the maxChars limit
if (currentChunk.length + line.length + 1 > maxChars) {
chunks.push(currentChunk)
currentChunk = ''
}
currentChunk += line + '\n' // Add the line and a newline character
})
// Add the last chunk if it's not empty
if (currentChunk) {
chunks.push(currentChunk)
} }
const limiter = new Bottleneck({ maxConcurrent: 1 }) return chunks
}
const send = (lines: string[]) => const limiter = new Bottleneck({ maxConcurrent: 1 })
Promise.all(
splitIntoChunks(lines).map((content) => const send = (lines: string[]) =>
limiter.schedule(() => Promise.all(
fetch(DISCORD_URL, { splitIntoChunks(lines).map((content) =>
method: 'POST', limiter.schedule(() =>
body: stringify({ fetch(DISCORD_URL, {
content, method: 'POST',
}), body: stringify({
headers: { 'content-type': 'application/json' }, content,
}), }),
), headers: { 'content-type': 'application/json' },
}),
), ),
) ),
)
const openFiles = _exec(`cat /proc/sys/fs/file-nr`)[0]?.trim() || `Unknown` const openFiles = _exec(`cat /proc/sys/fs/file-nr`)[0]?.trim() || `Unknown`
await send([ await send([
`===================`, `===================`,
`${new Date()}`, `${new Date()}`,
`CPUs: ${cpu.count()}`, `CPUs: ${cpu.count()}`,
`CPU Usage: ${await cpu.usage()}%`, `CPU Usage: ${await cpu.usage()}%`,
`Free RAM: ${getFreeMemoryInGB()}GB`, `Free RAM: ${getFreeMemoryInGB()}GB`,
`Free Storage /: ${(await drive.info(`/`)).freeGb}GB`, `Free Storage /: ${(await drive.info(`/`)).freeGb}GB`,
`Free Storage /mnt/sfo_data: ${ `Free Storage /mnt/sfo_data: ${
(await drive.info(`/mnt/sfo_data`)).freeGb (await drive.info(`/mnt/sfo_data`)).freeGb
}GB`, }GB`,
`Open files: ${openFiles}`, `Open files: ${openFiles}`,
`Containers: ${containers.length}`, `Containers: ${containers.length}`,
]) ])
const checks: Check[] = [ const checks: Check[] = [
{ {
name: `edge proxy`, name: `edge proxy`,
priority: 10, priority: 10,
emoji: `:park:`, emoji: `:park:`,
isHealthy: false, isHealthy: false,
url: `https://proxy.pockethost.io/_api/health`, url: `https://proxy.pockethost.io/_api/health`,
}, },
{ {
name: `edge daemon`, name: `edge daemon`,
priority: 8, priority: 8,
emoji: `:imp:`, emoji: `:imp:`,
isHealthy: false, isHealthy: false,
url: `http://localhost:${DAEMON_PORT()}/_api/health`, url: `http://localhost:${DAEMON_PORT()}/_api/health`,
}, },
{ {
name: `mothership`, name: `mothership`,
priority: 9, priority: 9,
emoji: `:flying_saucer:`, emoji: `:flying_saucer:`,
isHealthy: false, isHealthy: false,
url: `http://localhost:${MOTHERSHIP_PORT()}/api/health`, url: `http://localhost:${MOTHERSHIP_PORT()}/api/health`,
}, },
...containers, ]
]
await Promise.all( await Promise.all(
checks.map(async (check) => { checks.map(async (check) => {
const { url } = check const { url } = check
dbg({ container: check }) dbg({ container: check })
try { try {
const res = await fetch(url) const res = await fetch(url)
dbg({ url, status: res.status }) dbg({ url, status: res.status })
check.isHealthy = res.status === 200 check.isHealthy = res.status === 200
return true return true
} catch (e) { } catch (e) {
dbg(`${url}: ${e}`) dbg(`${url}: ${e}`)
check.isHealthy = false check.isHealthy = false
}
}),
)
dbg({ checks })
await send([
`---health checks---`,
...checks
.sort((a, b) => {
if (a.priority > b.priority) return -1
if (a.priority < b.priority) return 1
const now = new Date()
const res = +(b.created || now) - +(a.created || now)
if (res) return res
return a.name.localeCompare(b.name)
})
.map(({ name, isHealthy, emoji, mem, ago }) => {
const isInstance = !!mem
if (isInstance) {
return `${
isHealthy ? ':white_check_mark:' : ':face_vomiting: '
} \`${name.padStart(30)} ${(mem || '').padStart(10)} ${(
ago || ''
).padStart(20)}\``
} else {
return `${
isHealthy ? ':white_check_mark:' : ':face_vomiting: '
} ${name}`
} }
}), }),
) ])
dbg({ checks })
await send([
`---health checks---`,
...checks
.sort((a, b) => {
if (a.priority > b.priority) return -1
if (a.priority < b.priority) return 1
const now = new Date()
const res = +(b.created || now) - +(a.created || now)
if (res) return res
return a.name.localeCompare(b.name)
})
.map(({ name, isHealthy, emoji, mem, ago }) => {
const isInstance = !!mem
if (isInstance) {
return `${
isHealthy ? ':white_check_mark:' : ':face_vomiting: '
} \`${name.padStart(30)} ${(mem || '').padStart(10)} ${(
ago || ''
).padStart(20)}\``
} else {
return `${
isHealthy ? ':white_check_mark:' : ':face_vomiting: '
} ${name}`
}
}),
])
} catch (e) {
discordAlert(`${e}`)
}
} }