initial commit

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.9-slim
+
+ARG UNAME=ansible
+# should match host
+ARG UID=1000
+ARG GID=1000
+
+RUN groupadd -g $GID -o $UNAME && \
+    useradd -m -u $UID -g $GID -s /bin/sh $UNAME && \
+    apt-get update && \
+    apt-get install -y sshpass && \
+    pip3 install --no-cache-dir ansible==2.10 && \
+    rm -rf /var/lib/apt/lists/*
+
+USER $UNAME
+
+COPY . /etc/ansible
+
+ENTRYPOINT []

ansible.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec docker-compose run ansible $@

docker-compose.yml

@@ -0,0 +1,10 @@
+version: '3'
+services:
+  ansible:
+    build: .
+    command: ansible-playbook --ask-vault-pass /etc/ansible/playbook.yml
+    stdin_open: yes
+    tty: yes
+    volumes:
+      - .:/etc/ansible
+      - ~/.ssh:/home/ansible/.ssh:ro

group_vars/all.yml

@@ -0,0 +1,13 @@
+users:
+  marchambault:
+    # generated with `openssl passwd -salt <salt> -1 <plaintext>`
+    default_password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          35393364373830636539356334653263306262613038643231313762626537383637616534643237
+          3433636332383765653665643235633464343433636435300a623365666266366665363966303033
+          34656561653763636438303166363631636138616563636331323866643166323232353834376165
+          3939303664343638300a326131333663333365373036666634363235636663333465343337313638
+          39356365633332643132336438373131313239653231383735656436636332303034303466626232
+          3461626364346238666434303839373839633661616166613364
+    authorized_keys:
+      - https://github.com/badjware.keys

group_vars/k3s.yml

@@ -0,0 +1,4 @@
+k3s:
+  version: v1.21.0+k3s1
+  # disable local-path-provisioner, we use longhorn instead
+  extra_server_args: --no-deploy local-storage

group_vars/lb.yml

@@ -0,0 +1,29 @@
+haproxy:
+  routing:
+    https:
+      - src:
+          - cloud.badjware.dev
+          - code.badjware.dev
+          - drone.badjware.dev
+        dst:
+          - 192.168.20.21
+          - 192.168.20.22
+          - 192.168.20.23
+    tcp: [] # TODO, for ssh-forward
+
+letsencrypt:
+  domains:
+    - badjware.dev
+    - '*.badjware.dev'
+  email: marchambault@badjware.dev
+  digitalocean:
+    token: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          35643864626166636564363831336663363335356530316464353864643030316662633230343763
+          3439343831386632366137376137383936396164646237640a633132356332636134653832666636
+          63386235636632613666393036643737633635613139326362353166653264633536633037306632
+          3461313436326139330a366265343131366436653635623138373736353262653633666337623935
+          31653964336664313261373031613566636337643934316430306638626631633434366164306639
+          30616238613334633933343339393938326561633036633062323463636161336665373732626330
+          37386264353239353435643266333033353931336637343038353765396134333763386637653638
+          35343739666634323562

hosts

@@ -0,0 +1,63 @@
+# When adding a new host:
+# 1. create new user `useradd -m -G sudo -s /bin/bash ansible`
+# 2. configure user password `passwd ansible` (set password to badjnet/ssh/ansible)
+
+# These will throw some warnings that can be safely be ignored
+.user_config: &user_config
+  ansible_user: ansible
+  ansible_password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          34376132666239383830316437356430306535396466396537323833633137376239386464343363
+          6234303430623964353762383935323335383737666533390a643033363235383138393932393833
+          34633732646430383131643662626635373661373261323365366531316439653963353739383664
+          6139363534616231380a373931333530373339653132626238333566663362343663623532393330
+          35616230643533363032623066376536366236353335373130643262613561396131
+  ansible_become: 'yes'
+  ansible_become_password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          34376132666239383830316437356430306535396466396537323833633137376239386464343363
+          6234303430623964353762383935323335383737666533390a643033363235383138393932393833
+          34633732646430383131643662626635373661373261323365366531316439653963353739383664
+          6139363534616231380a373931333530373339653132626238333566663362343663623532393330
+          35616230643533363032623066376536366236353335373130643262613561396131
+
+all:
+  hosts:
+    pfsense:
+      ansible_host: 192.168.30.1
+  children:
+    proxmox:
+      hosts:
+        router-0:
+          ansible_host: 192.168.10.10
+      vars:
+        ansible_user: root
+    lb:
+      hosts:
+        lb:
+          ansible_host: 192.168.20.10
+    k3s:
+      children:
+        k3s-server:
+          hosts:
+            k3s-s0:
+              ansible_host: 192.168.20.20
+        k3s-agent:
+          hosts:
+            k3s-w0:
+              ansible_host: 192.168.20.21
+            k3s-w1:
+              ansible_host: 192.168.20.22
+            k3s-w2:
+              ansible_host: 192.168.20.23
+    ubuntu:
+      hosts:
+        ubiquiti:
+          ansible_host: 192.168.10.2
+        pihole:
+          ansible_host: 192.168.20.2
+      children:
+        lb:
+        k3s:
+      vars:
+        <<: *user_config

playbook.yml

@@ -0,0 +1,18 @@
+- hosts: ubuntu
+  roles:
+    - common
+
+- hosts: lb
+  roles:
+    - haproxy
+
+- hosts: proxmox
+  roles:
+    - proxmox
+
+- hosts: k3s-server
+  roles:
+    - k3s-server
+- hosts: k3s-agent
+  roles:
+    - k3s-agent

roles/common/defaults/main.yml

@@ -0,0 +1 @@
+users: {}

roles/common/tasks/main.yml

@@ -0,0 +1,45 @@
+
+- name: Perform system update
+  apt:
+    update_cache: yes
+    upgrade: 'yes'
+
+- name: Install basic packages
+  apt:
+    name:
+      - curl
+      - vim
+      - tmux
+
+- name: Set default editor to vim
+  lineinfile:
+    path: /etc/environment
+    line: EDITOR=vim
+
+- name: Disable root login
+  user:
+    name: root
+    password: '*'
+
+- name: Ensure sudoers is correctly configured
+  lineinfile:
+    path: /etc/sudoers
+    line: '%sudo ALL=(ALL:ALL) ALL'
+    regexp: '^%sudo\s+'
+    validate: '/usr/sbin/visudo -cf %s'
+
+- name: Configure users
+  include_tasks: user.yml
+  with_dict: '{{ users }}'
+
+- name: Configure hostname
+  hostname:
+    name: '{{ inventory_hostname }}'
+
+- name: Configure hostname in /etc/hosts
+  blockinfile:
+    dest: /etc/hosts
+    marker: '# {mark} ANSIBLE MANAGED BLOCK HOSTNAME'
+    block: |
+      127.0.0.1 {{ inventory_hostname }}
+      ::1       {{ inventory_hostname }}

roles/common/tasks/user.yml

@@ -0,0 +1,17 @@
+- name: Create user {{ item.key }}
+  user:
+    name: '{{ item.key }}'
+    password: '{{ item.value.default_password }}'
+    update_password: on_create
+    shell: /bin/bash
+    groups: sudo
+
+- name: '{{ item.key }}: Configure ssh public keys'
+  authorized_key:
+    user: '{{ item.key }}'
+    key: '{{ key }}'
+  loop: '{{ item.value.authorized_keys }}'
+  loop_control:
+    loop_var: key
+  when:
+    item.value.authorized_keys is defined

roles/haproxy/defaults/main.yml

@@ -0,0 +1,4 @@
+haproxy:
+  routing:
+    https: []
+    tcp: []

roles/haproxy/handlers/main.yml

@@ -0,0 +1,8 @@
+- name: Run certbot
+  command: sh /opt/certbot/certbot-run.sh
+
+- name: Restart haproxy
+  service:
+    name: haproxy
+    enabled: true
+    state: restarted

roles/haproxy/tasks/main.yml

@@ -0,0 +1,43 @@
+- name: Install haproxy and certbot
+  apt:
+    pkg:
+      - haproxy
+      - certbot
+      - python3-certbot-dns-digitalocean
+    update_cache: yes
+    state: present
+
+- name: Install haproxy.cfg
+  vars:
+    https_routing: '{{ haproxy.routing.https }}'
+    tcp_routing: '{{ haproxy.routing.tcp }}'
+  template:
+    src: haproxy.cfg
+    dest: /etc/haproxy/haproxy.cfg
+  notify: Restart haproxy
+
+- name: Create certbot configuration directory
+  file:
+    path: /opt/certbot
+    state: directory
+
+- name: Install certbot-creds.ini
+  template:
+    src: certbot-creds.ini
+    dest: /opt/certbot/certbot-creds.ini
+    mode: '0600'
+  notify: Run certbot
+
+- name: Install certbot-run.sh
+  template:
+    src: certbot-run.sh
+    dest: /opt/certbot/certbot-run.sh
+    mode: '0700'
+  notify: Run certbot
+
+- name: Install certbot-deploy.sh
+  template:
+    src: certbot-deploy.sh
+    dest: /opt/certbot/certbot-deploy.sh
+    mode: '0700'
+  notify: Run certbot

roles/haproxy/templates/certbot-creds.ini

@@ -0,0 +1 @@
+dns_digitalocean_token = {{ letsencrypt.digitalocean.token }}

roles/haproxy/templates/certbot-deploy.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+cat /etc/letsencrypt/live/{{ letsencrypt.domains[0] }}/fullchain.pem /etc/letsencrypt/live/{{ letsencrypt.domains[0] }}/privkey.pem >/etc/letsencrypt/live/{{ letsencrypt.domains[0] }}/{{ letsencrypt.domains[0] }}.pem

roles/haproxy/templates/certbot-run.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+certbot certonly \
+    --non-interactive \
+    --agree-tos \
+    --email {{ letsencrypt.email }} \
+    --deploy-hook '/opt/certbot/certbot-deploy.sh' \
+    --dns-digitalocean \
+    --dns-digitalocean-credentials /opt/certbot/certbot-creds.ini \
+    --dns-digitalocean-propagation-seconds 30 \
+    {% for domain in letsencrypt.domains %}-d '{{ domain }}' {% endfor %} \
+    $@

roles/haproxy/templates/haproxy.cfg

@@ -0,0 +1,80 @@
+global
+    daemon
+    maxconn 1024
+    log 127.0.0.1 local0
+    stats timeout 30s
+
+defaults
+    timeout connect 5s
+    timeout client 30s
+    timeout server 30s
+    log global
+
+    option forwardfor
+    option http-server-close
+
+    default-server init-addr last,none resolvers dns
+
+resolvers dns
+    parse-resolv-conf
+
+## FRONTENDS ##
+
+# haproxy stuff
+frontend http_management
+    bind *:8080
+    mode http
+
+    # redirects /status to haproxy monitor
+    monitor-uri /status
+
+    # redirects /stats to stats backend
+    acl prefixed-with-stats path_beg -i /stats
+    use_backend haproxy_stats if prefixed-with-stats
+
+    # redirects /metrics to metrics backend
+    acl prefixed-with-metrics path_beg -i /metrics
+    use_backend haproxy_metrics if prefixed-with-metrics
+
+frontend http_in
+    bind *:80
+    mode http
+
+    http-request redirect scheme https code 302
+
+frontend https_in
+    # backend is assumed to be http, perform ssl termination here
+    bind *:443 ssl crt /etc/letsencrypt/live/{{ letsencrypt.domains[0] }}/{{ letsencrypt.domains[0] }}.pem
+    # mode tcp
+
+    # request is ssl
+    # tcp-request inspect-delay 5s
+    # tcp-request content accept if { req.ssl_hello_type 1  }
+
+{% for route in https_routing %}
+    # use_backend https_{{ route.src[0]|replace('.','_') }} if { req.ssl_sni -i {% for src in route.src %}{{ src }} {% endfor %}}
+    use_backend https_{{ route.src[0]|replace('.','_') }} if { hdr_end(host) -i {% for src in route.src %}{{ src }} {% endfor %}}
+{% endfor %}
+
+## BACKENDS ##
+
+backend haproxy_stats
+    mode http
+    stats uri /stats
+    stats enable
+    stats refresh 10s
+    stats auth admin:admin
+
+backend haproxy_metrics
+    mode http
+    http-request use-service prometheus-exporter
+
+{% for route in https_routing %}
+backend https_{{ route.src[0]|replace('.','_') }}
+    # mode tcp
+    balance roundrobin
+{% for dst in route.dst %}
+    # server {{ dst }} {{ dst }}{% if ':' not in dst %}:443{% endif %} check
+    server {{ dst }} {{ dst }}{% if ':' not in dst %}:80{% endif %} check
+{% endfor %}
+{% endfor %}

roles/k3s-agent/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - role: k3s

roles/k3s-agent/tasks/main.yml

@@ -0,0 +1,14 @@
+- name: Copy k3s service file
+  register: k3s_service
+  template:
+    src: k3s.service
+    dest: /etc/systemd/system/k3s.service
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Enable k3s service
+  systemd:
+    name: k3s
+    daemon_reload: yes
+    enabled: yes

roles/k3s-agent/templates/k3s.service

@@ -0,0 +1,23 @@
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+After=network-online.target
+
+[Service]
+Type=notify
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s agent --server https://{{ hostvars[groups['k3s-server'][0]]['ansible_facts']['default_ipv4']['address'] }}:6443 --token {{ hostvars[groups['k3s-server'][0]]['token'] }} {{ k3s.extra_agent_args | default("") }}
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

roles/k3s-server/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - role: k3s

roles/k3s-server/tasks/main.yml

@@ -0,0 +1,69 @@
+- name: Copy k3s service file
+  register: k3s_service
+  template:
+    src: k3s.service
+    dest: /etc/systemd/system/k3s.service
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Enable k3s service
+  systemd:
+    name: k3s
+    daemon_reload: yes
+    enabled: yes
+
+- name: Wait for server node-token to be available
+  wait_for:
+    path: /var/lib/rancher/k3s/server/node-token
+
+- name: Read node-token from server
+  slurp:
+    src: /var/lib/rancher/k3s/server/node-token
+  register: node_token
+
+- name: Store server node-token as fact
+  set_fact:
+    token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}"
+
+- name: Create ~/.kube directory
+  file:
+    path: ~/.kube
+    state: directory
+    mode: "u=rwx,g=rx,o="
+
+- name: Copy config file to user home directory
+  copy:
+    src: /etc/rancher/k3s/k3s.yaml
+    dest: ~/.kube/config
+    remote_src: yes
+    mode: "u=rw,g=,o="
+
+- name: Set kubeconfig server ip
+  command: >-
+    k3s kubectl config set-cluster default
+      --server=https://{{ ansible_default_ipv4.address }}:6443
+      --kubeconfig ~/.kube/config
+  changed_when: no # TODO
+
+- name: Create kubectl symlink
+  file:
+    src: /usr/local/bin/k3s
+    dest: /usr/local/bin/kubectl
+    state: link
+
+- name: Create crictl symlink
+  file:
+    src: /usr/local/bin/k3s
+    dest: /usr/local/bin/crictl
+    state: link
+
+# - name: Create longhorn install directory
+#   file:
+#     path: /var/lib/rancher/k3s/server/manifests/longhorn
+#     state: directory
+
+# - name: Install longhorn
+#   get_url:
+#     url: https://raw.githubusercontent.com/longhorn/longhorn/v1.2.0/deploy/longhorn.yaml
+#     dest: /var/lib/rancher/k3s/server/manifests/longhorn/longhorn.yaml

roles/k3s-server/templates/k3s.service

@@ -0,0 +1,24 @@
+[Unit]
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+After=network-online.target
+
+[Service]
+Type=notify
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s server {{ k3s.extra_server_args | default("") }}
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

roles/k3s/files/cmdline.txt

@@ -0,0 +1 @@
+net.ifnames=0 dwc_otg.lpm_enable=0 console=serial0,115200 console=tty1 root=LABEL=writable rootfstype=ext4 elevator=deadline rootwait fixrtc cgroup_memory=1 cgroup_enable=memory

roles/k3s/handlers/main.yml

@@ -0,0 +1,2 @@
+- name: reboot
+  reboot:

roles/k3s/tasks/main.yml

@@ -0,0 +1,14 @@
+- name: Download k3s binary
+  get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ k3s.version }}/k3s-arm64
+    checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s.version }}/sha256sum-arm64.txt
+    dest: /usr/local/bin/k3s
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Enable cgroups
+  copy:
+    src: cmdline.txt
+    dest: /boot/firmware/cmdline.txt
+  notify: reboot

roles/proxmox/files/pve-no-subscription.list

@@ -0,0 +1,2 @@
+deb http://download.proxmox.com/debian/pve buster pve-no-subscription
+

roles/proxmox/handlers/main.yml

@@ -0,0 +1,2 @@
+- name: update grub
+  command: update-grub

roles/proxmox/tasks/main.yml

@@ -0,0 +1,34 @@
+- name: Enable IOMMU
+  lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_CMDLINE_LINUX_DEFAULT='
+    line: 'GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on"'
+  notify: update grub
+
+- name: Copy pve-no-subscription.list
+  copy:
+    src: pve-no-subscription.list
+    dest: /etc/apt/sources.list.d/pve-no-subscription.list
+
+- name: Stat pve-enterprise.list
+  stat:
+    path: /etc/apt/sources.list.d/pve-enterprise.list
+  register: pve_enterprise_stat
+
+- name: Backup pve-enterprise.list
+  copy:
+    remote_src: True
+    src: /etc/apt/sources.list.d/pve-enterprise.list
+    dest: /etc/apt/sources.list.d/pve-enterprise.list.bak
+  when: pve_enterprise_stat.stat.exists
+
+- name: Disable pve-enterprise.list
+  file:
+    path: /etc/apt/sources.list.d/pve-enterprise.list
+    state: absent
+
+- name: Perform dist-upgrade
+  apt:
+    update_cache: yes
+    upgrade: dist
+