update: health check

This commit is contained in:
Ben Allfree 2024-01-23 20:28:22 +00:00
parent bd737acaa2
commit 8e94924347
3 changed files with 80 additions and 53 deletions

View File

@ -53,6 +53,7 @@
"type": "module",
"dependencies": {
"@s-libs/micro-dash": "^16.1.0",
"@types/node-os-utils": "^1.3.4",
"@types/winston-syslog": "^2.4.3",
"ajv": "^8.12.0",
"boolean": "^3.2.0",
@ -77,6 +78,7 @@
"memorystream": "^0.3.1",
"nanoid": "^5.0.2",
"node-fetch": "^3.3.2",
"node-os-utils": "^1.3.7",
"ora": "^7.0.1",
"pocketbase": "^0.20.1",
"semver": "^7.5.4",

14
pnpm-lock.yaml generated
View File

@ -11,6 +11,9 @@ importers:
'@s-libs/micro-dash':
specifier: ^16.1.0
version: 16.1.0
'@types/node-os-utils':
specifier: ^1.3.4
version: 1.3.4
'@types/winston-syslog':
specifier: ^2.4.3
version: 2.4.3
@ -83,6 +86,9 @@ importers:
node-fetch:
specifier: ^3.3.2
version: 3.3.2
node-os-utils:
specifier: ^1.3.7
version: 1.3.7
ora:
specifier: ^7.0.1
version: 7.0.1
@ -1532,6 +1538,10 @@ packages:
form-data: 4.0.0
dev: true
/@types/node-os-utils@1.3.4:
resolution: {integrity: sha512-BCUYrbdoO4FUbx6MB9atLNFnkxdliFaxdiTJMIPPiecXIApc5zf4NIqV5G1jWv/ReZvtYyHLs40RkBjHX+vykA==}
dev: false
/@types/node@18.18.8:
resolution: {integrity: sha512-OLGBaaK5V3VRBS1bAkMVP2/W9B+H8meUfl866OrMNQqt7wDgdpWPp5o6gmIc9pB+lIQHSq4ZL8ypeH1vPxcPaQ==}
dependencies:
@ -5559,6 +5569,10 @@ packages:
dev: false
optional: true
/node-os-utils@1.3.7:
resolution: {integrity: sha512-fvnX9tZbR7WfCG5BAy3yO/nCLyjVWD6MghEq0z5FDfN+ZXpLWNITBdbifxQkQ25ebr16G0N7eRWJisOcMEHG3Q==}
dev: false
/node-plop@0.32.0:
resolution: {integrity: sha512-lKFSRSRuDHhwDKMUobdsvaWCbbDRbV3jMUSMiajQSQux1aNUevAZVxUHc2JERI//W8ABPRbi3ebYuSuIzkNIpQ==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}

View File

@ -10,8 +10,11 @@ import { LoggerService, LogLevelName } from '$shared'
import Bottleneck from 'bottleneck'
import { execSync } from 'child_process'
import fetch from 'node-fetch'
import { default as osu } from 'node-os-utils'
import { freemem } from 'os'
const { cpu, drive, openfiles, proc } = osu
DefaultSettingsService(SETTINGS)
const DISCORD_URL = DISCORD_HEALTH_CHANNEL_URL()
@ -105,6 +108,64 @@ const containers = _exec(`docker ps --format '{{json .}}'`)
}
})
function getFreeMemoryInGB(): string {
const freeMemoryBytes: number = freemem()
const freeMemoryGB: number = freeMemoryBytes / Math.pow(1024, 3)
return freeMemoryGB.toFixed(2) // Rounds to 2 decimal places
}
function splitIntoChunks(lines: string[], maxChars: number = 2000): string[] {
const chunks: string[] = []
let currentChunk: string = ''
lines.forEach((line) => {
// Check if adding the next line exceeds the maxChars limit
if (currentChunk.length + line.length + 1 > maxChars) {
chunks.push(currentChunk)
currentChunk = ''
}
currentChunk += line + '\n' // Add the line and a newline character
})
// Add the last chunk if it's not empty
if (currentChunk) {
chunks.push(currentChunk)
}
return chunks
}
const limiter = new Bottleneck({ maxConcurrent: 1 })
const send = (lines: string[]) =>
Promise.all(
splitIntoChunks(lines).map((content) =>
limiter.schedule(() =>
fetch(DISCORD_URL, {
method: 'POST',
body: JSON.stringify({
content,
}),
headers: { 'content-type': 'application/json' },
}),
),
),
)
const driveInfo = await drive.info(`/`)
await send([
`===================`,
`Server: SFO-1`,
`${new Date()}`,
`CPUs: ${cpu.count()}`,
`CPU Usage: ${await cpu.usage()}%`,
`Free RAM: ${getFreeMemoryInGB()}GB`,
`Free disk: ${driveInfo.freeGb}GB`,
`Open files: ${openFiles.length}`,
`Containers: ${containers.length}`,
])
const checks: Check[] = [
{
name: `edge proxy`,
@ -146,20 +207,8 @@ await Promise.all(
}),
)
dbg({ checks })
function getFreeMemoryInGB(): string {
const freeMemoryBytes: number = freemem()
const freeMemoryGB: number = freeMemoryBytes / Math.pow(1024, 3)
return freeMemoryGB.toFixed(2) // Rounds to 2 decimal places
}
const content = [
`===================`,
`Server: SFO-1`,
`${new Date()}`,
`Free RAM: ${getFreeMemoryInGB()}`,
`Free disk: ${freeSpace}`,
`${checks.length} containers running and ${openFiles.length} open files.`,
await send([
`---health checks---`,
...checks
.sort((a, b) => {
if (a.priority > b.priority) return -1
@ -181,42 +230,4 @@ const content = [
isHealthy ? ':white_check_mark:' : ':face_vomiting: '
} ${emoji} ${name}`
}),
]
function splitIntoChunks(lines: string[], maxChars: number = 2000): string[] {
const chunks: string[] = []
let currentChunk: string = ''
lines.forEach((line) => {
// Check if adding the next line exceeds the maxChars limit
if (currentChunk.length + line.length + 1 > maxChars) {
chunks.push(currentChunk)
currentChunk = ''
}
currentChunk += line + '\n' // Add the line and a newline character
})
// Add the last chunk if it's not empty
if (currentChunk) {
chunks.push(currentChunk)
}
return chunks
}
dbg(content)
const limiter = new Bottleneck({ maxConcurrent: 1 })
await Promise.all(
splitIntoChunks(content).map((content) =>
limiter.schedule(() =>
fetch(DISCORD_URL, {
method: 'POST',
body: JSON.stringify({
content,
}),
headers: { 'content-type': 'application/json' },
}),
),
),
)
])