From 8e9492434764ba5b49c4b00d60c5cf460f671b14 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Tue, 23 Jan 2024 20:28:22 +0000 Subject: [PATCH] update: health check --- package.json | 2 + pnpm-lock.yaml | 14 +++++ src/cli/edge-health.ts | 117 ++++++++++++++++++++++------------------- 3 files changed, 80 insertions(+), 53 deletions(-) diff --git a/package.json b/package.json index 6e10cc3f..04dfd289 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "type": "module", "dependencies": { "@s-libs/micro-dash": "^16.1.0", + "@types/node-os-utils": "^1.3.4", "@types/winston-syslog": "^2.4.3", "ajv": "^8.12.0", "boolean": "^3.2.0", @@ -77,6 +78,7 @@ "memorystream": "^0.3.1", "nanoid": "^5.0.2", "node-fetch": "^3.3.2", + "node-os-utils": "^1.3.7", "ora": "^7.0.1", "pocketbase": "^0.20.1", "semver": "^7.5.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2893c52..c19ecf49 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@s-libs/micro-dash': specifier: ^16.1.0 version: 16.1.0 + '@types/node-os-utils': + specifier: ^1.3.4 + version: 1.3.4 '@types/winston-syslog': specifier: ^2.4.3 version: 2.4.3 @@ -83,6 +86,9 @@ importers: node-fetch: specifier: ^3.3.2 version: 3.3.2 + node-os-utils: + specifier: ^1.3.7 + version: 1.3.7 ora: specifier: ^7.0.1 version: 7.0.1 @@ -1532,6 +1538,10 @@ packages: form-data: 4.0.0 dev: true + /@types/node-os-utils@1.3.4: + resolution: {integrity: sha512-BCUYrbdoO4FUbx6MB9atLNFnkxdliFaxdiTJMIPPiecXIApc5zf4NIqV5G1jWv/ReZvtYyHLs40RkBjHX+vykA==} + dev: false + /@types/node@18.18.8: resolution: {integrity: sha512-OLGBaaK5V3VRBS1bAkMVP2/W9B+H8meUfl866OrMNQqt7wDgdpWPp5o6gmIc9pB+lIQHSq4ZL8ypeH1vPxcPaQ==} dependencies: @@ -5559,6 +5569,10 @@ packages: dev: false optional: true + /node-os-utils@1.3.7: + resolution: {integrity: sha512-fvnX9tZbR7WfCG5BAy3yO/nCLyjVWD6MghEq0z5FDfN+ZXpLWNITBdbifxQkQ25ebr16G0N7eRWJisOcMEHG3Q==} + dev: false + /node-plop@0.32.0: resolution: {integrity: sha512-lKFSRSRuDHhwDKMUobdsvaWCbbDRbV3jMUSMiajQSQux1aNUevAZVxUHc2JERI//W8ABPRbi3ebYuSuIzkNIpQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} diff --git a/src/cli/edge-health.ts b/src/cli/edge-health.ts index b4df7f8f..7e4f7183 100644 --- a/src/cli/edge-health.ts +++ b/src/cli/edge-health.ts @@ -10,8 +10,11 @@ import { LoggerService, LogLevelName } from '$shared' import Bottleneck from 'bottleneck' import { execSync } from 'child_process' import fetch from 'node-fetch' +import { default as osu } from 'node-os-utils' import { freemem } from 'os' +const { cpu, drive, openfiles, proc } = osu + DefaultSettingsService(SETTINGS) const DISCORD_URL = DISCORD_HEALTH_CHANNEL_URL() @@ -105,6 +108,64 @@ const containers = _exec(`docker ps --format '{{json .}}'`) } }) +function getFreeMemoryInGB(): string { + const freeMemoryBytes: number = freemem() + const freeMemoryGB: number = freeMemoryBytes / Math.pow(1024, 3) + return freeMemoryGB.toFixed(2) // Rounds to 2 decimal places +} + +function splitIntoChunks(lines: string[], maxChars: number = 2000): string[] { + const chunks: string[] = [] + let currentChunk: string = '' + + lines.forEach((line) => { + // Check if adding the next line exceeds the maxChars limit + if (currentChunk.length + line.length + 1 > maxChars) { + chunks.push(currentChunk) + currentChunk = '' + } + currentChunk += line + '\n' // Add the line and a newline character + }) + + // Add the last chunk if it's not empty + if (currentChunk) { + chunks.push(currentChunk) + } + + return chunks +} + +const limiter = new Bottleneck({ maxConcurrent: 1 }) + +const send = (lines: string[]) => + Promise.all( + splitIntoChunks(lines).map((content) => + limiter.schedule(() => + fetch(DISCORD_URL, { + method: 'POST', + body: JSON.stringify({ + content, + }), + headers: { 'content-type': 'application/json' }, + }), + ), + ), + ) + +const driveInfo = await drive.info(`/`) + +await send([ + `===================`, + `Server: SFO-1`, + `${new Date()}`, + `CPUs: ${cpu.count()}`, + `CPU Usage: ${await cpu.usage()}%`, + `Free RAM: ${getFreeMemoryInGB()}GB`, + `Free disk: ${driveInfo.freeGb}GB`, + `Open files: ${openFiles.length}`, + `Containers: ${containers.length}`, +]) + const checks: Check[] = [ { name: `edge proxy`, @@ -146,20 +207,8 @@ await Promise.all( }), ) dbg({ checks }) - -function getFreeMemoryInGB(): string { - const freeMemoryBytes: number = freemem() - const freeMemoryGB: number = freeMemoryBytes / Math.pow(1024, 3) - return freeMemoryGB.toFixed(2) // Rounds to 2 decimal places -} - -const content = [ - `===================`, - `Server: SFO-1`, - `${new Date()}`, - `Free RAM: ${getFreeMemoryInGB()}`, - `Free disk: ${freeSpace}`, - `${checks.length} containers running and ${openFiles.length} open files.`, +await send([ + `---health checks---`, ...checks .sort((a, b) => { if (a.priority > b.priority) return -1 @@ -181,42 +230,4 @@ const content = [ isHealthy ? ':white_check_mark:' : ':face_vomiting: ' } ${emoji} ${name}` }), -] - -function splitIntoChunks(lines: string[], maxChars: number = 2000): string[] { - const chunks: string[] = [] - let currentChunk: string = '' - - lines.forEach((line) => { - // Check if adding the next line exceeds the maxChars limit - if (currentChunk.length + line.length + 1 > maxChars) { - chunks.push(currentChunk) - currentChunk = '' - } - currentChunk += line + '\n' // Add the line and a newline character - }) - - // Add the last chunk if it's not empty - if (currentChunk) { - chunks.push(currentChunk) - } - - return chunks -} - -dbg(content) - -const limiter = new Bottleneck({ maxConcurrent: 1 }) -await Promise.all( - splitIntoChunks(content).map((content) => - limiter.schedule(() => - fetch(DISCORD_URL, { - method: 'POST', - body: JSON.stringify({ - content, - }), - headers: { 'content-type': 'application/json' }, - }), - ), - ), -) +])