From 6ef407f441942a1a16c3b364d54126916239ff48 Mon Sep 17 00:00:00 2001
From: Aaron Bieber <aaron@bolddaemon.com>
Date: Wed, 29 Mar 2023 08:01:58 -0600
Subject: [PATCH] all: enable monitoring with monit

---
 default.nix            |  2 ++
 monitoring/default.nix | 17 +++++++++++++++++
 monitoring/monitrc     | 21 +++++++++++++++++++++
 3 files changed, 40 insertions(+)
 create mode 100644 monitoring/default.nix
 create mode 100644 monitoring/monitrc

diff --git a/default.nix b/default.nix
index 6e01813..b9c77b9 100644
--- a/default.nix
+++ b/default.nix
@@ -49,6 +49,8 @@ in {
     ./system/update.nix
     ./users
 
+    ./monitoring
+
     ./bins
   ];
 
diff --git a/monitoring/default.nix b/monitoring/default.nix
new file mode 100644
index 0000000..f0d48c3
--- /dev/null
+++ b/monitoring/default.nix
@@ -0,0 +1,17 @@
+{ config, ... }:
+
+{
+  config = {
+    sops.secrets = {
+      monit_cfg = {
+        sopsFile = config.xin-secrets.deploy;
+        owner = "root";
+        mode = "400";
+      };
+    };
+    services.monit = {
+      enable = true;
+      config = builtins.readFile ./monitrc;
+    };
+  };
+}
diff --git a/monitoring/monitrc b/monitoring/monitrc
new file mode 100644
index 0000000..6815c6b
--- /dev/null
+++ b/monitoring/monitrc
@@ -0,0 +1,21 @@
+set daemon 30
+set alert alert@bolddaemon.com
+set logfile /var/log/monit.log
+
+include /run/secrets/monit_cfg
+
+ set mail-format {
+ from: alerts@bolddaemon.com
+ subject: $SERVICE $EVENT on $DATE
+ message: $DESCRIPTION
+}
+
+check system $HOST
+    if loadavg (5min) > 3 then alert
+    if loadavg (15min) > 2 then alert
+    if memory usage > 80% for 4 cycles then alert
+    if swap usage > 20% for 4 cycles then alert
+    if cpu usage (user) > 80% for 2 cycles then alert
+    if cpu usage (system) > 20% for 2 cycles then alert
+    if cpu usage (wait) > 80% for 2 cycles then alert
+    if cpu usage > 200% for 4 cycles then alert