Pushover and API Change truenas

backend/src/alerts/evaluate.js

@@ -0,0 +1,236 @@
+import { sendPushoverMessage } from './pushover.js';
+
+const LEVELS = {
+  none: 0,
+  warn: 1,
+  crit: 2
+};
+
+const alertState = new Map();
+
+const toNumber = (value, fallback) => {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+};
+
+const clampPercent = (value, fallback) => {
+  const parsed = toNumber(value, fallback);
+  if (!Number.isFinite(parsed)) {
+    return fallback;
+  }
+  return Math.max(1, Math.min(100, parsed));
+};
+
+const formatBytes = (bytes) => {
+  const value = Number(bytes);
+  if (!Number.isFinite(value) || value <= 0) {
+    return '0 B';
+  }
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
+  let current = value;
+  let unitIndex = 0;
+  while (current >= 1024 && unitIndex < units.length - 1) {
+    current /= 1024;
+    unitIndex += 1;
+  }
+  return `${current.toFixed(current >= 100 ? 0 : 2)} ${units[unitIndex]}`;
+};
+
+const percentUsed = (usedBytes, totalBytes) => {
+  const total = toNumber(totalBytes, 0);
+  const used = toNumber(usedBytes, 0);
+  if (total <= 0 || used <= 0) {
+    return 0;
+  }
+  return Math.max(0, Math.min(100, (used / total) * 100));
+};
+
+const normalizeSettings = (settings) => {
+  const warnPercent = clampPercent(settings.warnPercent, 80);
+  const critPercent = clampPercent(settings.critPercent, 90);
+  const normalizedCrit = Math.max(critPercent, warnPercent + 1);
+  const cooldownMinutes = Math.max(0, Math.round(toNumber(settings.cooldownMinutes, 60)));
+  return {
+    enabled: settings.enabled !== false,
+    warnPercent,
+    critPercent: normalizedCrit,
+    cooldownMinutes,
+    cooldownMs: cooldownMinutes * 60 * 1000,
+    once: Boolean(settings.once),
+    includePools: settings.includePools !== false
+  };
+};
+
+const mergeSettings = (base, override) => {
+  if (!override || typeof override !== 'object') {
+    return base;
+  }
+  return normalizeSettings({
+    ...base,
+    ...override
+  });
+};
+
+const resolveSettings = (alertsConfig, serverConfig, poolName) => {
+  const base = normalizeSettings(alertsConfig?.defaults || {});
+  const serverAlerts = serverConfig?.alerts || {};
+  const poolOverrides = poolName
+    ? serverAlerts?.pools?.[poolName] || serverAlerts?.pools?.[String(poolName)]
+    : null;
+
+  const serverMerged = mergeSettings(base, serverAlerts);
+  const poolMerged = poolOverrides ? mergeSettings(serverMerged, poolOverrides) : serverMerged;
+
+  return {
+    ...poolMerged,
+    enabled: Boolean(alertsConfig?.enabled) && poolMerged.enabled
+  };
+};
+
+const levelForPercent = (percent, settings) => {
+  if (percent >= settings.critPercent) {
+    return 'crit';
+  }
+  if (percent >= settings.warnPercent) {
+    return 'warn';
+  }
+  return 'none';
+};
+
+const shouldSend = (key, level, settings) => {
+  if (level === 'none') {
+    alertState.delete(key);
+    return false;
+  }
+
+  const now = Date.now();
+  const currentRank = LEVELS[level];
+  const previous = alertState.get(key);
+
+  if (!previous) {
+    alertState.set(key, { lastSentAt: 0, levelRank: 0, latched: false });
+  }
+
+  const state = alertState.get(key);
+
+  if (currentRank > state.levelRank) {
+    return true;
+  }
+
+  if (settings.once && state.latched) {
+    return false;
+  }
+
+  if (!settings.cooldownMs) {
+    return true;
+  }
+
+  return now - state.lastSentAt >= settings.cooldownMs;
+};
+
+const markSent = (key, level, settings) => {
+  const now = Date.now();
+  const levelRank = LEVELS[level] || 0;
+  alertState.set(key, {
+    lastSentAt: now,
+    levelRank,
+    latched: settings.once ? true : false
+  });
+};
+
+const buildServerMessage = (server, percent, level) => {
+  const used = formatBytes(server.usedBytes);
+  const total = formatBytes(server.totalBytes);
+  const rounded = Math.round(percent);
+  const levelLabel = level === 'crit' ? 'CRITICAL' : 'Warning';
+  return {
+    title: `Vault: ${server.name} ${levelLabel}`,
+    message: `Server total at ${rounded}% (${used} / ${total}).`,
+    priority: level === 'crit' ? 1 : 0
+  };
+};
+
+const buildPoolMessage = (server, pool, percent, level) => {
+  const used = formatBytes(pool.usedBytes);
+  const total = formatBytes(pool.totalBytes);
+  const rounded = Math.round(percent);
+  const levelLabel = level === 'crit' ? 'CRITICAL' : 'Warning';
+  return {
+    title: `Vault: ${server.name} / ${pool.name} ${levelLabel}`,
+    message: `${pool.name} at ${rounded}% (${used} / ${total}).`,
+    priority: level === 'crit' ? 1 : 0
+  };
+};
+
+const poolKey = (serverId, poolName) => `${serverId}::pool::${poolName}`;
+const serverKey = (serverId) => `${serverId}::server`;
+
+export async function evaluateAndSendAlerts(payload, storedServers, alertsConfig) {
+  if (!payload || !Array.isArray(payload.servers)) {
+    return { sent: 0, skipped: 0 };
+  }
+
+  const serverMap = new Map((storedServers || []).map((server) => [server.id, server]));
+  let sent = 0;
+  let skipped = 0;
+
+  for (const server of payload.servers) {
+    const stored = serverMap.get(server.id) || {};
+
+    const serverSettings = resolveSettings(alertsConfig, stored, null);
+    if (serverSettings.enabled) {
+      const percent = percentUsed(server.usedBytes, server.totalBytes);
+      const level = levelForPercent(percent, serverSettings);
+      const key = serverKey(server.id);
+      if (shouldSend(key, level, serverSettings)) {
+        try {
+          await sendPushoverMessage(alertsConfig, buildServerMessage(server, percent, level));
+          markSent(key, level, serverSettings);
+          sent += 1;
+        } catch (error) {
+          // eslint-disable-next-line no-console
+          console.error('Pushover server alert failed:', error.message || error);
+        }
+      } else {
+        skipped += 1;
+      }
+    }
+
+    if (!serverSettings.includePools || !Array.isArray(server.pools)) {
+      continue;
+    }
+
+    for (const pool of server.pools) {
+      const poolSettings = resolveSettings(alertsConfig, stored, pool.name);
+      if (!poolSettings.enabled) {
+        continue;
+      }
+      const percent = percentUsed(pool.usedBytes, pool.totalBytes);
+      const level = levelForPercent(percent, poolSettings);
+      const key = poolKey(server.id, pool.name);
+      if (shouldSend(key, level, poolSettings)) {
+        try {
+          await sendPushoverMessage(alertsConfig, buildPoolMessage(server, pool, percent, level));
+          markSent(key, level, poolSettings);
+          sent += 1;
+        } catch (error) {
+          // eslint-disable-next-line no-console
+          console.error('Pushover pool alert failed:', error.message || error);
+        }
+      } else {
+        skipped += 1;
+      }
+    }
+  }
+
+  return { sent, skipped };
+}
+
+export async function sendTestAlert(alertsConfig) {
+  return sendPushoverMessage(alertsConfig, {
+    title: 'Vault: Test Alert',
+    message: 'Pushover integration is working.',
+    priority: 0
+  });
+}
+

backend/src/alerts/pushover.js

@@ -0,0 +1,49 @@
+const PUSHOVER_URL = 'https://api.pushover.net/1/messages.json';
+
+const toNumber = (value, fallback) => {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+};
+
+export async function sendPushoverMessage(config, message) {
+  const userKey = String(config?.pushover?.userKey || '').trim();
+  const apiToken = String(config?.pushover?.apiToken || '').trim();
+
+  if (!userKey || !apiToken) {
+    throw new Error('Pushover requires userKey and apiToken.');
+  }
+
+  const body = new URLSearchParams({
+    token: apiToken,
+    user: userKey,
+    title: String(message?.title || 'Vault Alert'),
+    message: String(message?.message || ''),
+    priority: String(toNumber(message?.priority, 0))
+  });
+
+  if (message?.url) {
+    body.set('url', String(message.url));
+  }
+  if (message?.urlTitle) {
+    body.set('url_title', String(message.urlTitle));
+  }
+  if (message?.sound) {
+    body.set('sound', String(message.sound));
+  }
+
+  const response = await fetch(PUSHOVER_URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded'
+    },
+    body
+  });
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => '');
+    throw new Error(`Pushover error (${response.status}): ${text || 'request failed'}`);
+  }
+
+  return response.json().catch(() => ({ ok: true }));
+}
+

backend/src/providers/truenas.js

@@ -1,4 +1,4 @@
-import { Agent } from 'undici';
+import WebSocket from 'ws';
 
 const bytesToTb = (bytes) => bytes / (1024 ** 4);
 
@@ -16,6 +16,15 @@ const buildBaseUrl = (server) => {
   return url;
 };
 
+const toWebSocketUrl = (baseUrl, wsPath) => {
+  const wsUrl = new URL(baseUrl);
+  wsUrl.protocol = wsUrl.protocol === 'https:' ? 'wss:' : 'ws:';
+  wsUrl.pathname = wsPath;
+  wsUrl.search = '';
+  wsUrl.hash = '';
+  return wsUrl;
+};
+
 const getStatus = (percent) => {
   if (percent >= 85) {
     return { status: 'critical', alerts: 2 };
@@ -31,20 +40,95 @@ const buildPoolEntry = (poolName, totalBytes, usedBytes) => ({
   usedBytes
 });
 
-const getDispatcher = (server) => {
-  if (server?.config?.allowInsecure === false) {
-    return undefined;
+const resolveWsPath = (server) => {
+  const rawPath = server?.config?.wsPath || '/api/current';
+  const normalized = String(rawPath).trim();
+  if (!normalized) {
+    return '/api/current';
   }
-  return new Agent({ connect: { rejectUnauthorized: false } });
+  return normalized.startsWith('/') ? normalized : `/${normalized}`;
 };
 
-const fetchWithAuth = async (url, apiKey, scheme, dispatcher) => fetch(url, {
-  headers: {
-    Accept: 'application/json',
-    Authorization: `${scheme} ${apiKey}`
-  },
-  dispatcher
-});
+const createRpcClient = (server, baseUrl) => {
+  const wsPath = resolveWsPath(server);
+  const wsUrl = toWebSocketUrl(baseUrl, wsPath);
+  const allowInsecure = server?.config?.allowInsecure !== false;
+  const socketOptions = allowInsecure ? { rejectUnauthorized: false } : undefined;
+  const wsProtocol = String(server?.config?.wsProtocol || '').trim();
+  const socket = wsProtocol
+    ? new WebSocket(wsUrl, wsProtocol, socketOptions)
+    : new WebSocket(wsUrl, socketOptions);
+
+  let nextId = 1;
+  const pending = new Map();
+
+  const closeWithError = (error) => {
+    pending.forEach(({ reject }) => reject(error));
+    pending.clear();
+    try {
+      socket.close();
+    } catch {
+      // ignore close errors
+    }
+  };
+
+  socket.on('message', (data) => {
+    try {
+      const payload = JSON.parse(String(data));
+      const entry = pending.get(payload.id);
+      if (!entry) {
+        return;
+      }
+      pending.delete(payload.id);
+      if (payload.error) {
+        entry.reject(new Error(payload.error.message || 'TrueNAS JSON-RPC error.'));
+        return;
+      }
+      entry.resolve(payload.result);
+    } catch (error) {
+      closeWithError(error);
+    }
+  });
+
+  socket.on('error', (error) => {
+    closeWithError(error);
+  });
+
+  const waitForOpen = () => new Promise((resolve, reject) => {
+    const handleOpen = () => {
+      socket.off('error', handleError);
+      resolve();
+    };
+    const handleError = (error) => {
+      socket.off('open', handleOpen);
+      reject(error);
+    };
+    socket.once('open', handleOpen);
+    socket.once('error', handleError);
+  });
+
+  const call = (method, params = []) => new Promise((resolve, reject) => {
+    const id = nextId++;
+    pending.set(id, { resolve, reject });
+    socket.send(JSON.stringify({
+      jsonrpc: '2.0',
+      id,
+      method,
+      params
+    }), (error) => {
+      if (error) {
+        pending.delete(id);
+        reject(error);
+      }
+    });
+  });
+
+  return {
+    waitForOpen,
+    call,
+    close: () => socket.close()
+  };
+};
 
 export async function fetchTrueNasStorage(server) {
   const apiKey = server?.config?.apiKey;
@@ -53,54 +137,54 @@ export async function fetchTrueNasStorage(server) {
   }
 
   const baseUrl = buildBaseUrl(server);
-  const endpoint = new URL('/api/v2.0/pool', baseUrl);
+  const client = createRpcClient(server, baseUrl);
 
-  const dispatcher = getDispatcher(server);
-  let response = await fetchWithAuth(endpoint, apiKey, 'Bearer', dispatcher);
-  if (response.status === 401) {
-    response = await fetchWithAuth(endpoint, apiKey, 'Token', dispatcher);
+  try {
+    await client.waitForOpen();
+    const authResult = await client.call('auth.login_with_api_key', [apiKey]);
+    if (authResult !== true) {
+      throw new Error('TrueNAS authentication failed.');
+    }
+    const pools = await client.call('pool.query', []);
+
+    if (!Array.isArray(pools) || pools.length === 0) {
+      throw new Error('No TrueNAS pool data returned.');
+    }
+
+    let totalBytes = 0;
+    let usedBytes = 0;
+
+    const perPool = pools.map((pool) => {
+      const total = Number(pool.size || 0);
+      const used = Number(pool.allocated || pool.used || 0);
+      totalBytes += total;
+      usedBytes += used;
+      return buildPoolEntry(pool.name || 'zfs-pool', total, used);
+    });
+
+    const totalTb = bytesToTb(totalBytes);
+    const usedTb = bytesToTb(usedBytes);
+    const usedPercent = totalBytes ? Math.round((usedBytes / totalBytes) * 100) : 0;
+    const statusMeta = getStatus(usedPercent);
+
+    return {
+      id: server.id,
+      name: server.name || server.host,
+      platform: 'TrueNAS SCALE',
+      cluster: server.config?.cluster || 'Vault TrueNAS',
+      location: server.config?.location || server.host,
+      pool: 'All pools',
+      totalTb,
+      usedTb,
+      totalBytes,
+      usedBytes,
+      pools: perPool,
+      iops: 0,
+      latencyMs: 0,
+      status: statusMeta.status,
+      alerts: statusMeta.alerts
+    };
+  } finally {
+    client.close();
   }
-
-  if (!response.ok) {
-    throw new Error(`TrueNAS API error (${response.status}).`);
-  }
-
-  const pools = await response.json();
-  if (!Array.isArray(pools) || pools.length === 0) {
-    throw new Error('No TrueNAS pool data returned.');
-  }
-
-  let totalBytes = 0;
-  let usedBytes = 0;
-
-  const perPool = pools.map((pool) => {
-    const total = Number(pool.size || 0);
-    const used = Number(pool.allocated || pool.used || 0);
-    totalBytes += total;
-    usedBytes += used;
-    return buildPoolEntry(pool.name || 'zfs-pool', total, used);
-  });
-
-  const totalTb = bytesToTb(totalBytes);
-  const usedTb = bytesToTb(usedBytes);
-  const usedPercent = totalBytes ? Math.round((usedBytes / totalBytes) * 100) : 0;
-  const statusMeta = getStatus(usedPercent);
-
-  return {
-    id: server.id,
-    name: server.name || server.host,
-    platform: 'TrueNAS SCALE',
-    cluster: server.config?.cluster || 'Vault TrueNAS',
-    location: server.config?.location || server.host,
-    pool: 'All pools',
-    totalTb,
-    usedTb,
-    totalBytes,
-    usedBytes,
-    pools: perPool,
-    iops: 0,
-    latencyMs: 0,
-    status: statusMeta.status,
-    alerts: statusMeta.alerts
-  };
 }

backend/src/server.js

@@ -5,6 +5,8 @@ import { fetchTrueNasStorage } from './providers/truenas.js';
 import { fetchPbsStorage } from './providers/pbs.js';
 import { addServer, applyServerLayout, deleteServer, readServers, setPoolsCollapsed, updateServer } from './store/servers.js';
 import { detectServerType } from './detect.js';
+import { readAlertsConfig, writeAlertsConfig } from './store/alerts.js';
+import { evaluateAndSendAlerts, sendTestAlert } from './alerts/evaluate.js';
 
 const app = express();
 const port = process.env.PORT || 5001;
@@ -144,7 +146,7 @@ app.post('/api/servers/:id/config', async (req, res) => {
 
 app.patch('/api/servers/:id', async (req, res) => {
   const { id } = req.params;
-  const { name } = req.body ?? {};
+  const { name, alerts } = req.body ?? {};
   let nextName = undefined;
   if (name !== undefined) {
     const trimmed = String(name).trim();
@@ -156,8 +158,18 @@ app.patch('/api/servers/:id', async (req, res) => {
     return;
   }
 
+  let nextAlerts = undefined;
+  if (alerts !== undefined) {
+    if (!alerts || typeof alerts !== 'object' || Array.isArray(alerts)) {
+      res.status(400).json({ error: 'alerts must be an object.' });
+      return;
+    }
+    nextAlerts = alerts;
+  }
+
   const updated = await updateServer(id, {
-    ...(name !== undefined ? { name: nextName } : {})
+    ...(name !== undefined ? { name: nextName } : {}),
+    ...(alerts !== undefined ? { alerts: nextAlerts } : {})
   });
 
   if (!updated) {
@@ -207,6 +219,70 @@ app.post('/api/servers/:id/pools-collapsed', async (req, res) => {
   res.json({ server: updated });
 });
 
+app.get('/api/alerts', async (_req, res) => {
+  const config = await readAlertsConfig();
+  res.json({ alerts: config });
+});
+
+app.post('/api/alerts', async (req, res) => {
+  const body = req.body ?? {};
+  const patch = {};
+
+  if (Object.prototype.hasOwnProperty.call(body, 'enabled')) {
+    patch.enabled = Boolean(body.enabled);
+  }
+
+  if (body.pushover && typeof body.pushover === 'object') {
+    patch.pushover = {};
+    if (Object.prototype.hasOwnProperty.call(body.pushover, 'userKey')) {
+      patch.pushover.userKey = String(body.pushover.userKey || '').trim();
+    }
+    if (Object.prototype.hasOwnProperty.call(body.pushover, 'apiToken')) {
+      patch.pushover.apiToken = String(body.pushover.apiToken || '').trim();
+    }
+  }
+
+  if (body.defaults && typeof body.defaults === 'object') {
+    patch.defaults = {};
+    const defaults = body.defaults;
+    if (Object.prototype.hasOwnProperty.call(defaults, 'enabled')) {
+      patch.defaults.enabled = Boolean(defaults.enabled);
+    }
+    if (Object.prototype.hasOwnProperty.call(defaults, 'warnPercent')) {
+      patch.defaults.warnPercent = Number(defaults.warnPercent);
+    }
+    if (Object.prototype.hasOwnProperty.call(defaults, 'critPercent')) {
+      patch.defaults.critPercent = Number(defaults.critPercent);
+    }
+    if (Object.prototype.hasOwnProperty.call(defaults, 'cooldownMinutes')) {
+      patch.defaults.cooldownMinutes = Number(defaults.cooldownMinutes);
+    }
+    if (Object.prototype.hasOwnProperty.call(defaults, 'once')) {
+      patch.defaults.once = Boolean(defaults.once);
+    }
+    if (Object.prototype.hasOwnProperty.call(defaults, 'includePools')) {
+      patch.defaults.includePools = Boolean(defaults.includePools);
+    }
+  }
+
+  const updated = await writeAlertsConfig(patch);
+  res.json({ alerts: updated });
+});
+
+app.post('/api/alerts/test', async (_req, res) => {
+  const config = await readAlertsConfig();
+  if (!config.enabled) {
+    res.status(400).json({ error: 'Alerts are disabled.' });
+    return;
+  }
+  try {
+    await sendTestAlert(config);
+    res.json({ ok: true });
+  } catch (error) {
+    res.status(400).json({ error: error.message || 'Failed to send test alert.' });
+  }
+});
+
 const buildStoragePayload = async () => {
   const stored = await readServers();
   const configured = stored.filter((server) => server.state === 'configured');
@@ -230,6 +306,7 @@ const buildStoragePayload = async () => {
           position: Number.isFinite(server.position) ? server.position : index,
           span: Number.isFinite(server.span) ? server.span : 1,
           poolsCollapsed: Boolean(server.poolsCollapsed),
+          alerts: server.alerts || null,
           name: payload.name || server.name,
           host: server.host
         });
@@ -256,6 +333,21 @@ const refreshStorageCache = async () => {
   try {
     const payload = await buildStoragePayload();
     storageCache = { timestamp: Date.now(), payload };
+    try {
+      const [storedServers, alertsConfig] = await Promise.all([
+        readServers(),
+        readAlertsConfig()
+      ]);
+      queueMicrotask(() => {
+        evaluateAndSendAlerts(payload, storedServers, alertsConfig).catch((error) => {
+          // eslint-disable-next-line no-console
+          console.error('Alert evaluation failed:', error.message || error);
+        });
+      });
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error('Alert setup failed:', error.message || error);
+    }
     return payload;
   } finally {
     storageRefresh = null;

backend/src/store/alerts.js

@@ -0,0 +1,66 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+
+const dataDir = path.resolve(process.cwd(), 'data');
+const dataFile = path.join(dataDir, 'alerts.json');
+
+const DEFAULT_ALERTS = {
+  enabled: false,
+  pushover: {
+    userKey: '',
+    apiToken: ''
+  },
+  defaults: {
+    enabled: true,
+    warnPercent: 80,
+    critPercent: 90,
+    cooldownMinutes: 60,
+    once: false,
+    includePools: true
+  }
+};
+
+async function ensureStore() {
+  await fs.mkdir(dataDir, { recursive: true });
+  try {
+    await fs.access(dataFile);
+  } catch {
+    await fs.writeFile(dataFile, JSON.stringify(DEFAULT_ALERTS, null, 2), 'utf-8');
+  }
+}
+
+const deepMerge = (base, patch) => {
+  if (!patch || typeof patch !== 'object') {
+    return base;
+  }
+  const next = { ...base };
+  Object.entries(patch).forEach(([key, value]) => {
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+      next[key] = deepMerge(base[key] ?? {}, value);
+      return;
+    }
+    next[key] = value;
+  });
+  return next;
+};
+
+export async function readAlertsConfig() {
+  await ensureStore();
+  const raw = await fs.readFile(dataFile, 'utf-8');
+  try {
+    const parsed = JSON.parse(raw);
+    return deepMerge(DEFAULT_ALERTS, parsed);
+  } catch {
+    return { ...DEFAULT_ALERTS };
+  }
+}
+
+export async function writeAlertsConfig(patch) {
+  const current = await readAlertsConfig();
+  const merged = deepMerge(current, patch);
+  await fs.writeFile(dataFile, JSON.stringify(merged, null, 2), 'utf-8');
+  return merged;
+}
+
+export { DEFAULT_ALERTS };
+