migrate: kid; matrix, firefish, search engine

servers/keymaker/docker-compose.d/custom/nginx/conf.d/ixvd-web.conf

@@ -5,4 +5,14 @@ server {
     location / {
         proxy_pass http://ixvd-web$request_uri;
     }
+
+    # matrix
+    location /.well-known/matrix/server {
+        return 200 '{ "m.server": "matrix.ixvd.net:443" }';
+    }
+
+    location /.well-known/matrix/client {
+        add_header Access-Control-Allow-Origin '*';
+        return 200 '{ "m.homeserver": { "base_url": "https://matrix.ixvd.net" } }';
+    }
 }

servers/kid/README.md

@@ -0,0 +1 @@
+server configuration for `kid`.

servers/kid/docker-compose.d/README.md

@@ -0,0 +1 @@
+This folder houses all docker-compose files

servers/kid/docker-compose.d/custom/cinny/Dockerfile

@@ -0,0 +1,3 @@
+FROM ghcr.io/cinnyapp/cinny:latest
+
+COPY ./config.json /app/config.json

servers/kid/docker-compose.d/custom/cinny/config.json

@@ -0,0 +1,9 @@
+{
+  "defaultHomeserver": 0,
+  "homeserverList": [
+    "matrix.ixvd.net",
+    "matrix.org",
+    "mozilla.org"
+  ],
+  "allowCustomHomeservers": true
+}

servers/kid/docker-compose.d/custom/firefish/Dockerfile

@@ -0,0 +1,12 @@
+FROM registry.joinfirefish.org/firefish/firefish:latest
+
+RUN apk add \
+    curl \
+    jq
+
+COPY config/default.yml /firefish/.config/default.yml
+COPY docker-entrypoint.d/ /docker-entrypoint.d/
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
+
+ENTRYPOINT ["sh", "/docker-entrypoint.sh"]

servers/kid/docker-compose.d/custom/firefish/config/default.yml

@@ -0,0 +1,253 @@
+#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Firefish configuration
+#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+# After starting your server, please don't change the URL! Doing so will break federation.
+
+#   ┌─────┐
+#───┘ URL └─────────────────────────────────────────────────────
+
+# Final accessible URL seen by a user.
+url: https://fedi.ixvd.net
+
+#   ┌───────────────────────┐
+#───┘ Port and TLS settings └───────────────────────────────────
+
+#
+# Firefish requires a reverse proxy to support HTTPS connections.
+#
+#                 +----- https://example.com/ ------------+
+#   +------+      |+-------------+      +----------------+|
+#   | User | ---> || Proxy (443) | ---> | Firefish (3000) ||
+#   +------+      |+-------------+      +----------------+|
+#                 +---------------------------------------+
+#
+#   You need to set up a reverse proxy. (e.g. nginx, caddy)
+#   An encrypted connection with HTTPS is highly recommended
+#   because tokens may be transferred in GET requests.
+
+# The port that your Firefish server should listen on.
+port: 3000
+
+#   ┌──────────────────────────┐
+#───┘ PostgreSQL configuration └────────────────────────────────
+
+db:
+  host: firefish-db
+  port: 5432
+  #ssl: false
+  # Database name
+  db: [POSTGRES_DB]
+
+  # Auth
+  user: [POSTGRES_USER]
+  pass: [POSTGRES_PASSWORD]
+
+  # Whether disable Caching queries
+  #disableCache: true
+
+  # Extra Connection options
+  #extra:
+  #  ssl:
+  #   host: localhost
+  #   rejectUnauthorized: false
+
+#   ┌─────────────────────┐
+#───┘ Redis configuration └─────────────────────────────────────
+
+redis:
+  host: firefish-redis
+  port: 6379
+  #tls:
+  #  host: localhost
+  #  rejectUnauthorized: false
+  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+  #pass: example-pass
+  #prefix: example-prefix
+  #db: 1
+  #user: default
+
+  #   ┌─────────────────────────────┐
+  #───┘ Cache server configuration └─────────────────────────────────────
+
+  # A Redis-compatible server (DragonflyDB, Keydb, Redis) for caching
+  # If left blank, it will use the Redis server from above
+
+  #cacheServer:
+  #host: localhost
+  #port: 6379
+  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+  #pass: example-pass
+  #prefix: example-prefix
+  #db: 1
+
+# Please configure either MeiliSearch *or* Sonic.
+# If both MeiliSearch and Sonic configurations are present, MeiliSearch will take precedence.
+
+#   ┌───────────────────────────┐
+#───┘ MeiliSearch configuration └─────────────────────────────────────
+# meilisearch:
+  # host: meilisearch
+  # port: 7700
+  # ssl: false
+  # apiKey: [MEILISEARCH_API_KEY]
+
+#   ┌─────────────────────┐
+#───┘ Sonic configuration └─────────────────────────────────────
+
+sonic:
+ host: [SONIC_HOST]
+ port: [SONIC_PORT]
+ auth: [SONIC_SECRET]
+ collection: notes
+ bucket: default
+
+
+#   ┌───────────────┐
+#───┘ ID generation └───────────────────────────────────────────
+
+# No need to uncomment in most cases, but you may want to change
+# these settings if you plan to run a large and/or distributed server.
+
+# cuid:
+#   # Min 16, Max 24
+#   length: 16
+#
+#   # Set this to a unique string across workers (e.g., machine's hostname)
+#   # if your workers are running in multiple hosts.
+#   fingerprint: my-fingerprint
+
+
+#   ┌─────────────────────┐
+#───┘ Other configuration └─────────────────────────────────────
+
+# Maximum length of a post (default 3000, max 100000)
+#maxNoteLength: 3000
+
+# Maximum length of an image caption (default 1500, max 8192)
+#maxCaptionLength: 1500
+
+# Reserved usernames that only the administrator can register with
+reservedUsernames: [
+  'root',
+  'admin',
+  'administrator',
+  'me',
+  'system'
+]
+
+# Whether disable HSTS
+#disableHsts: true
+
+# Number of worker processes
+#clusterLimit: 1
+
+# Worker only mode
+#onlyQueueProcessor: 1
+
+# Job concurrency per worker
+# deliverJobConcurrency: 128
+# inboxJobConcurrency: 16
+
+# Job rate limiter
+# deliverJobPerSec: 128
+# inboxJobPerSec: 16
+
+# Job attempts
+# deliverJobMaxAttempts: 12
+# inboxJobMaxAttempts: 8
+
+# IP address family used for outgoing request (ipv4, ipv6 or dual)
+#outgoingAddressFamily: ipv4
+
+# Syslog option
+#syslog:
+#  host: localhost
+#  port: 514
+
+# Proxy for HTTP/HTTPS
+#proxy: http://127.0.0.1:3128
+
+#proxyBypassHosts: [
+#  'web.kaiteki.app',
+#  'example.com',
+#  '192.0.2.8'
+#]
+
+# Proxy for SMTP/SMTPS
+#proxySmtp: http://127.0.0.1:3128   # use HTTP/1.1 CONNECT
+#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4
+#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
+
+# Media Proxy
+#mediaProxy: https://example.com/proxy
+
+# Proxy remote files (default: false)
+#proxyRemoteFiles: true
+
+#allowedPrivateNetworks: [
+#  '127.0.0.1/32'
+#]
+
+# TWA
+#twa:
+#  nameSpace: android_app
+#  packageName: tld.domain.twa
+#  sha256CertFingerprints: ['AB:CD:EF']
+
+# Upload or download file size limits (bytes)
+#maxFileSize: 262144000
+
+#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Congrats, you've reached the end of the config file needed for most deployments!
+# Enjoy your Firefish server!
+#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+
+
+
+#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Managed hosting settings
+# >>> NORMAL SELF-HOSTERS, STAY AWAY! <<<
+# >>> YOU DON'T NEED THIS! <<<
+# Each category is optional, but if each item in each category is mandatory!
+# If you mess this up, that's on you, you've been warned...
+#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+#maxUserSignups: 100
+#isManagedHosting: true
+#deepl:
+#  managed: true
+#  authKey: ''
+#  isPro: false
+#
+#email:
+#  managed: true
+#  address: 'example@email.com'
+#  host: 'email.com'
+#  port: 587
+#  user: 'example@email.com'
+#  pass: ''
+#  useImplicitSslTls: false
+#
+#objectStorage:
+#  managed: true
+#  baseUrl: ''
+#  bucket: ''
+#  prefix: ''
+#  endpoint: ''
+#  region: ''
+#  accessKey: ''
+#  secretKey: ''
+#  useSsl: true
+#  connnectOverProxy: false
+#  setPublicReadOnUpload: true
+#  s3ForcePathStyle: true
+
+# !!!!!!!!!!
+# >>>>>> AGAIN, NORMAL SELF-HOSTERS, STAY AWAY! <<<<<<
+# >>>>>> YOU DON'T NEED THIS, ABOVE SETTINGS ARE FOR MANAGED HOSTING ONLY! <<<<<<
+# !!!!!!!!!!
+
+# Seriously. Do NOT fill out the above settings if you're self-hosting.
+# They're much better off being set from the control panel.

servers/kid/docker-compose.d/custom/firefish/docker-entrypoint.d/50-replace-config-with-env.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+POSTGRES_USER=${POSTGRES_USER:-}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
+POSTGRES_DB=${POSTGRES_DB:-firefish}
+SONIC_HOST=${SONIC_HOST:-}
+SONIC_PORT=${SONIC_PORT:-1491}
+SONIC_SECRET=${SONIC_SECRET:-}
+
+if [ -f /firefish/.config/default.yml ]; then
+  sed -i "s/\[POSTGRES_USER\]/$POSTGRES_USER/g" /firefish/.config/default.yml
+  sed -i "s/\[POSTGRES_PASSWORD\]/$POSTGRES_PASSWORD/g" /firefish/.config/default.yml
+  sed -i "s/\[POSTGRES_DB\]/$POSTGRES_DB/g" /firefish/.config/default.yml
+  sed -i "s/\[SONIC_HOST\]/$SONIC_HOST/g" /firefish/.config/default.yml
+  sed -i "s/\[SONIC_PORT\]/$SONIC_PORT/g" /firefish/.config/default.yml
+  sed -i "s/\[SONIC_SECRET\]/$SONIC_SECRET/g" /firefish/.config/default.yml
+fi

servers/kid/docker-compose.d/custom/firefish/docker-entrypoint.d/90-start-firefish.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo "Starting firefish..."
+/sbin/tini -- pnpm run migrateandstart

servers/kid/docker-compose.d/custom/firefish/docker-entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+for file in /docker-entrypoint.d/*; do
+  if [ -x "$file" ]; then
+    echo "Running $file"
+    "$file"
+  else
+    echo "Sourcing $file"
+    . "$file"
+  fi
+done

servers/kid/docker-compose.d/custom/nginx/Dockerfile

@@ -0,0 +1,13 @@
+FROM nginx:alpine
+
+RUN apk add \
+    certbot \
+    certbot-nginx
+
+COPY content /usr/share/nginx/html
+COPY conf.d/ /etc/nginx/conf.d/
+COPY nginx.conf /etc/nginx/nginx.conf
+
+COPY entrypoint.sh /entrypoint
+ENTRYPOINT [ "sh", "/entrypoint" ]
+CMD [ "nginx", "-g", "daemon off;" ]

servers/kid/docker-compose.d/custom/nginx/conf.d/default.conf

@@ -0,0 +1,21 @@
+server {
+    listen 80;
+    server_name kid.ixvd.net;
+
+    # SSL is managed by certbot, no need for a ssl listen; it will be generated automagically!
+
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+    # default html page
+    location / {
+        root /usr/share/nginx/html;
+        index index.html index.htm;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root /usr/share/nginx/html;
+    }
+}

servers/kid/docker-compose.d/custom/nginx/conf.d/firefish.conf

@@ -0,0 +1,12 @@
+server {
+    listen 80;
+    server_name fedi.ixvd.net;
+
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+    location / {
+        proxy_pass http://firefish:3000$request_uri;
+    }
+}

servers/kid/docker-compose.d/custom/nginx/conf.d/matrix.conf

@@ -0,0 +1,27 @@
+server {
+    listen 80;
+    server_name matrix.ixvd.net;
+    
+    proxy_set_header Host      $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_read_timeout         600;
+
+    location = / {
+        return 302 https://cinny.ixvd.net;
+    }
+
+    # matrix
+    location /.well-known/matrix/server {
+        return 200 '{ "m.server": "matrix.ixvd.net:443" }';
+    }
+
+    location /.well-known/matrix/client {
+        add_header Access-Control-Allow-Origin '*';
+        return 200 '{ "m.homeserver": { "base_url": "https://matrix.ixvd.net" } }';
+    }
+
+    location /_matrix {
+        proxy_pass http://conduit:6167; 
+        client_max_body_size 0;
+    }
+}

servers/kid/docker-compose.d/custom/nginx/conf.d/search.conf

@@ -0,0 +1,16 @@
+server {
+    listen 80;
+    server_name search.localhost;
+	access_log /dev/null;
+	error_log /dev/null;
+
+    location / {
+	    proxy_set_header X-Real-IP $remote_addr;
+	    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	    proxy_set_header X-Forwarded-Proto $scheme;
+	    proxy_set_header Host $host;
+	    proxy_set_header X-NginX-Proxy true;
+        proxy_pass http://whoogle:5000;
+    }
+
+}

servers/kid/docker-compose.d/custom/nginx/content/index.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <title>Welcome to nginx!</title>
+    <style>
+        html {
+            color-scheme: light dark;
+        }
+
+        body {
+            width: 35em;
+            margin: 0 auto;
+            font-family: Tahoma, Verdana, Arial, sans-serif;
+        }
+    </style>
+</head>
+
+<body>
+    <h1>Welcome to nginx!</h1>
+    <hr/>
+    <span>If you're seeing this, it means the admin was too lazy to remove this page.</span><br/>
+    <span>Expected something here? contact the admin: webmaster@ixvd.net</span>
+</body>
+
+</html>

servers/kid/docker-compose.d/custom/nginx/entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+trap exit TERM
+
+if [ -n "${CERTBOT_DOMAINS}" ]; then
+    echo "registering..."
+    if ! certbot show_account; then
+        certbot register -n \
+            --agree-tos \
+            -m "${CERTBOT_EMAIL}"
+    fi
+
+    for d in $(echo "${CERTBOT_DOMAINS}" | sed 's/,/ /g'); do
+        echo "requesting for $d..."
+        certbot --nginx -n --keep -d "$d"
+    done
+
+    while :; do
+        echo "renewing domains..."
+        certbot --nginx --keep -n renew
+        sleep 12h &
+        wait $!
+    done &
+else
+    echo "skipping certbot due to no domains!"
+fi &
+
+exec "$@"

servers/kid/docker-compose.d/custom/nginx/nginx.conf

@@ -0,0 +1,35 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    # docker resolver and quad9;
+    resolver 127.0.0.11 9.9.9.9 ipv6=off;
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}

servers/kid/docker-compose.d/custom/sonic/Dockerfile

@@ -0,0 +1,2 @@
+FROM valeriansaliou/sonic:v1.4.3
+COPY sonic.cfg /etc/sonic.cfg

servers/kid/docker-compose.d/custom/sonic/sonic.cfg

@@ -0,0 +1,69 @@
+# Sonic
+# Fast, lightweight and schema-less search backend
+# Configuration file
+# Example: https://github.com/valeriansaliou/sonic/blob/master/config.cfg
+
+
+[server]
+
+log_level = "debug"
+
+
+[channel]
+
+inet = "[::1]:1491"
+tcp_timeout = 300
+
+auth_password = "${env.SONIC_SECRET}"
+
+[channel.search]
+
+query_limit_default = 10
+query_limit_maximum = 100
+query_alternates_try = 4
+
+suggest_limit_default = 5
+suggest_limit_maximum = 20
+
+list_limit_default = 100
+list_limit_maximum = 500
+
+
+[store]
+
+[store.kv]
+
+path = "./data/store/kv/"
+
+retain_word_objects = 1000
+
+[store.kv.pool]
+
+inactive_after = 1800
+
+[store.kv.database]
+
+flush_after = 900
+
+compress = true
+parallelism = 2
+max_files = 100
+max_compactions = 1
+max_flushes = 1
+write_buffer = 16384
+write_ahead_log = true
+
+[store.fst]
+
+path = "./data/store/fst/"
+
+[store.fst.pool]
+
+inactive_after = 300
+
+[store.fst.graph]
+
+consolidate_after = 180
+
+max_size = 2048
+max_words = 250000

servers/kid/docker-compose.d/docker-compose.firefish.yml

@@ -0,0 +1,53 @@
+version: '2.2'
+services:
+  firefish:
+    build: custom/firefish
+    restart: unless-stopped
+    depends_on:
+      - firefish-db
+      - firefish-redis
+      - firefish-sonic
+    env_file:
+      - /etc/ixvd/secrets/env/firefish.env
+    environment:
+      NODE_ENV: production
+      VIRTUAL_HOST: fedi.ixvd.net
+      LETSENCRYPT_HOST: fedi.ixvd.net
+      VIRTUAL_PORT: 3000
+      SONIC_HOST: firefish-sonic
+      SONIC_SECRET: "firefish"
+    volumes:
+      - /srv/firefish/data:/firefish/files
+    networks:
+      - proxy
+      - internal
+      - calcnet
+
+  firefish-redis:
+    image: docker.io/redis:7.0-alpine
+    restart: unless-stopped
+    volumes:
+      - /srv/firefish/other/redis:/data
+    networks:
+      - calcnet
+
+  firefish-sonic:
+    build: custom/sonic
+    environment:
+      SONIC_SECRET: "firefish"
+    volumes:
+      - /srv/firefish/other/sonic:/var/lib/sonic/store/
+    networks:
+      - calcnet
+
+  firefish-db:
+    image: docker.io/postgres:12.2-alpine
+    restart: unless-stopped
+    env_file:
+      - /etc/ixvd/secrets/env/firefish.env
+    volumes:
+      - /srv/firefish/other/db:/var/lib/postgresql/data
+    networks:
+      - calcnet
+networks:
+  calcnet: null

servers/kid/docker-compose.d/docker-compose.matrix.yml

@@ -0,0 +1,36 @@
+version: '2.2'
+services: 
+  conduit:
+    image: matrixconduit/matrix-conduit:latest
+    restart: unless-stopped
+    volumes:
+      - db:/var/lib/matrix-conduit/
+    environment:
+      CONDUIT_SERVER_NAME: ixvd.net
+      CONDUIT_DATABASE_PATH: /var/lib/matrix-conduit/
+      CONDUIT_DATABASE_BACKEND: rocksdb
+      CONDUIT_PORT: 6167
+      CONDUIT_MAX_REQUEST_SIZE: '20000000'
+      CONDUIT_ALLOW_REGISTRATION: 'false'
+      CONDUIT_ENABLE_LIGHTNING_BOLT: 'false'
+      CONDUIT_ALLOW_FEDERATION: 'true'
+      CONDUIT_TRUSTED_SERVERS: '["matrix.org"]'
+      CONDUIT_ADDRESS: 0.0.0.0
+      CONDUIT_CONFIG: ''
+    networks:
+      - proxy
+      - internal
+  cinny:
+    build: custom/cinny
+    restart: unless-stopped
+    networks:
+      - proxy
+      - internal
+      
+volumes:
+  db:
+    driver: local
+    driver_opts:
+      type: none
+      device: /srv/conduit/other/db
+      o: bind

servers/kid/docker-compose.d/docker-compose.search.yml

@@ -0,0 +1,11 @@
+version: '2.2'
+services:
+  whoogle:
+    image: benbusby/whoogle-search
+    restart: always
+    environment:
+      WHOOGLE_CONFIG_THEME: "dark"
+      WHOOGLE_CONFIG_NEAR: "New York"
+    networks:
+      - proxy
+      - internal

servers/kid/docker-compose.d/docker-compose.yml

@@ -0,0 +1,26 @@
+# I'm very comfortable in this version, therefore it's the standard
+version: '2.2'
+
+services:
+  # default nginx setup
+  nginx:
+    build: custom/nginx
+    environment:
+      CERTBOT_EMAIL: "webmaster@ixvd.net"
+      # CERTBOT_DOMAINS: "kid.ixvd.net,fedi.ixvd.net,matrix.ixvd.net,search.ixvd.net"
+    volumes:
+      - /srv/certbot/data:/etc/letsencrypt
+      - /srv/certbot/other/www:/var/www/certbot
+    ports:
+      - 80:80
+      - 443:443
+    networks:
+      - proxy
+
+# Here the default networks are defined
+networks:
+  proxy:
+    external: true
+  internal:
+    external: true
+  

servers/kid/setup.d/10-conduit.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+mkdir -p /srv/conduit/other/db