all: enable monitoring with monit

default.nix

@@ -49,6 +49,8 @@ in {
     ./system/update.nix
     ./users
 
+    ./monitoring
+
     ./bins
   ];
 

monitoring/default.nix

@@ -0,0 +1,17 @@
+{ config, ... }:
+
+{
+  config = {
+    sops.secrets = {
+      monit_cfg = {
+        sopsFile = config.xin-secrets.deploy;
+        owner = "root";
+        mode = "400";
+      };
+    };
+    services.monit = {
+      enable = true;
+      config = builtins.readFile ./monitrc;
+    };
+  };
+}

monitoring/monitrc

@@ -0,0 +1,21 @@
+set daemon 30
+set alert alert@bolddaemon.com
+set logfile /var/log/monit.log
+
+include /run/secrets/monit_cfg
+
+ set mail-format {
+ from: alerts@bolddaemon.com
+ subject: $SERVICE $EVENT on $DATE
+ message: $DESCRIPTION
+}
+
+check system $HOST
+    if loadavg (5min) > 3 then alert
+    if loadavg (15min) > 2 then alert
+    if memory usage > 80% for 4 cycles then alert
+    if swap usage > 20% for 4 cycles then alert
+    if cpu usage (user) > 80% for 2 cycles then alert
+    if cpu usage (system) > 20% for 2 cycles then alert
+    if cpu usage (wait) > 80% for 2 cycles then alert
+    if cpu usage > 200% for 4 cycles then alert