init: initial commit

README.md

@@ -0,0 +1,3 @@
+Neb or Nebuchadnezzar
+
+This is the main repo of the IXVD infrastructure and it's management tools.

ansible/README.md

@@ -0,0 +1,2 @@
+This is our ansible setup.
+This will only 

ansible/inventory.yml

@@ -0,0 +1,25 @@
+### HOSTS ###
+
+core:
+  hosts:
+    keymaker:
+    apoc:
+
+entertainment:
+  hosts:
+    kid:
+
+experimental:
+  hosts:
+    mouse:
+
+### GROUPS ###
+
+production:
+  children:
+    core:
+    entertainment:
+
+testing:
+  children:
+    experimental:

common/ntfy.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+ntfy_username=""
+ntfy_password=""
+ntfy_token=""
+
+help()
+{
+   echo "Options:"
+   echo "-m     Your message."
+   echo "-p     Notification priority, 1-5, 5 is the highest.  (Optional)"
+   echo "-e     Choose emoji. (https://ntfy.sh/docs/emojis/?h=emo)"
+   echo "-h     Print this help."
+   echo
+   echo "If you want to show if the last command was successful or not, you can do something like this:"
+   echo "yourcommand ; export le=$? ; /path/to/ntfy.sh"
+   echo
+}
+
+
+while getopts "t:m:p:e:h" option; do
+  case $option in
+    t) ntfy_topic=${OPTARG};;
+    m) ntfy_message=${OPTARG};;
+    p) ntfy_prio=${OPTARG};;
+    e) ntfy_emoji=${OPTARG};;
+    h) help
+       exit;;
+    \?)
+       echo "Error: Invalid option"
+       exit;;
+  esac
+done
+shift $((OPTIND-1))
+
+if [ -z "$ntfy_message" ]; then
+  ntfy_message="Done"
+fi
+
+if [ "$ntfy_prio" == "1" ]; then
+  ntfy_prio="min"
+  ntfy_tag="white_small_square"
+elif [ "$ntfy_prio" == "2" ]; then
+  ntfy_prio="low"
+  ntfy_tag="computer"
+elif [ "$ntfy_prio" == "3" ]; then
+  ntfy_prio="default"
+  ntfy_tag="computer"
+elif [ "$ntfy_prio" == "4" ]; then
+  ntfy_prio="high"
+  ntfy_tag="warning"
+elif [ "$ntfy_prio" == "5" ]; then
+  ntfy_prio="max"
+  ntfy_tag="rotating_light"
+else
+  ntfy_prio="default"
+  ntfy_tag="computer"
+fi
+
+if [ -n "$ntfy_emoji" ]; then
+  ntfy_tag="$ntfy_emoji"
+fi
+
+if [ -n "$le" ]; then
+  if [ "$le" == "0" ]; then
+    ntfy_tag="heavy_check_mark"
+  else
+    ntfy_tag="x"
+  fi
+fi
+
+if [ -z "$ntfy_topic" ]; then
+  ntfy_topic="$HOSTNAME"
+fi
+
+if [[ -n $ntfy_password && -n $ntfy_token ]]; then
+  echo "Use ntfy_username and ntfy_password OR ntfy_token"
+  exit 1
+elif [ -n "$ntfy_password" ]; then
+  ntfy_base64=$( echo -n "$ntfy_username:$ntfy_password" | base64 )
+  ntfy_auth="Authorization: Basic $ntfy_base64"
+elif [ -n "$ntfy_token" ]; then
+  ntfy_auth="Authorization: Bearer $ntfy_token"
+else
+  ntfy_auth=""
+fi
+
+curl -s \
+    -H "$ntfy_auth" \
+    -H "Title: $ntfy_topic" \
+    -H "Tags: $ntfy_tag" \
+    -H "Priority: $ntfy_prio" \
+    -d "$ntfy_message" \
+    "$1" > /dev/null

common/restic/backup.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+/neb/common/ntfy.sh -m "performing backup..." -e "warning" https://push.ixvd.net/infra-backups
+if restic \
+    -r "sftp://ixvd_backup_storage//backups/restic-$(hostname)" \
+    -p "/neb/secrets/restic/$(hostname).secret" \
+    backup \
+    --tag auto \
+    /srv /home /etc; then
+    /neb/common/ntfy.sh -m "backup succeeded" -e "warning" https://push.ixvd.net/infra-backups
+else
+    /neb/common/ntfy.sh -p "urgent" -m "backup failed" -e "warning,skull" https://push.ixvd.net/infra-backups
+fi

common/restic/forget.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+notify() {
+    curl \
+        -X POST \
+        -H "Authorization: $(cat ${NTFY_SECRET_FILE:-/etc/ixvd/secrets/ntfy/infra-backups})" \
+        -d "$(hostname): $@" \
+        https://push.ixvd.net/infra-backups
+}
+
+notify "forgetting old backups..."
+if restic \
+    -r "sftp://ixvd_backup_storage//backups/restic-$(hostname)" \
+    -p "/etc/ixvd/secrets/restic/$(hostname).secret" \
+    forget \
+    --keep-last 10 \
+    --keep-tag keep \
+    --keep-daily 20 \
+    --keep-weekly 20 \
+    --keep-monthly 6 \
+    --keep-yearly 10; then
+    notify "forget succeeded"
+else
+    notify "forget failed"
+fi

common/restic/prune.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+notify() {
+    curl \
+        -X POST \
+        -H "Title: $0" \
+        -H "Priority: urgent" \
+        -H "Tags: warning" \
+        -d "$(hostname): $@" \
+        https://push.ixvd.net/infra-backups
+}
+
+notify "pruning old backups..."

common/restic/restic.crontab

@@ -0,0 +1,10 @@
+# /etc/cron.d/restic.crontab
+
+# every 4 hours create a backup
+0 */4 * * * root /neb/common/restic/backup.sh
+
+# every day at 2 in the morning, forget old backups
+0 2 * * * root /neb/common/restic/forget.sh
+
+# every 1st of the month at 2 in the morning, prune
+0 2 1 * * root /neb/common/restic/prune.sh

common/ssh_config

@@ -0,0 +1,21 @@
+#####################
+## IXVD ssh config ##
+#####################
+
+# This config describes what the bastion hose is and how to jump
+
+
+Host link
+    HostName link.ixvd.net
+
+Host keymaker
+    ProxyJump link
+
+Host apoc
+    ProxyJump link
+
+Host kid
+    ProxyJump link
+
+Host mouse
+    ProxyJump link

setup.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# setup new server
+
+## PREFLIGHT
+
+if ! [ "$UID" = "0" ]; then
+    echo "please perform as root, attempting to escalate privileges..."
+    exec sudo $0
+    exit 1
+fi
+
+if ! ssh git@git.ixvd.net; then
+    echo "can't connect to git, please setup ssh"
+    exit 1
+fi
+
+## MAIN
+
+if ! [ -d "/neb" ]; then
+    echo "setting up ixvd/neb repo"
+    git clone \
+        --recurse-submodules \
+        git@git.ixvd.net:ixvd/neb \
+        /neb
+fi
+
+if ! [ -d "/etc/cron.d/restic.crontab" ]; then
+    echo "setting up