pkg/adalovelace/adalovelace.go

@@ -74,7 +74,7 @@ func (p Provisioner) CheckRequirements() error {
 			Command:    "terraform",
 			Args:       []string{"version"},
 			Regex:      `Terraform (v.*)`,
-			Constraint: "1.1.x",
+			Constraint: "1.x.x",
 		},
 		{
 			Command:    "ansible-playbook",

pkg/adalovelace/templates/ansible/install.yaml

 - name: bootstrap and setup k8s
   hosts: localhost
   tasks:
+    - import_tasks: "tasks/firewall-worker.yaml"
     - import_tasks: "tasks/addons.yaml"

pkg/adalovelace/templates/ansible/playbook.yaml

@@ -10,6 +10,10 @@
       service:
         name: sshd
         state: restarted
+    - name: restart nftables
+      service:
+        name: nftables
+        state: restarted
   tasks:
     - import_tasks: "tasks/wait_nodes.yaml"
     - import_tasks: "tasks/upgrade.yaml"

pkg/adalovelace/templates/ansible/tasks/firewall-controller.yaml

+- name: copy nftables conf
+  ansible.builtin.template:
+    src: nftables.conf.j2
+    dest: /etc/nftables.conf
+  notify: restart nftables
+
+- name: enable nftables
+  service:
+    name: nftables
+    state: started
+    enabled: yes

pkg/adalovelace/templates/ansible/tasks/firewall-worker.yaml

+- name: create host endpoints
+  kubernetes.core.k8s:
+    kubeconfig: "../k0s/kubeconfig.yaml"
+    state: present
+    definition:
+      apiVersion: crd.projectcalico.org/v1
+      kind: HostEndpoint
+      metadata:
+        labels:
+          kubernetes.io/hostname: "{{ cluster.name }}-{{ item.key }}"
+        name: "{{ cluster.name }}-{{ item.key }}"
+      spec:
+        node: "{{ cluster.name }}-{{ item.key }}"
+        interfaceName: "*"
+        expectedIPs: ["{{ item.value.publicIp }}","{{ item.value.privateIp }}"]
+  loop: "{{ cluster.nodes | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  when: item.value.role == "worker"
+
+- name: apply global net policy
+  kubernetes.core.k8s:
+    kubeconfig: "../k0s/kubeconfig.yaml"
+    state: present
+    definition:
+      apiVersion: crd.projectcalico.org/v1
+      kind: GlobalNetworkPolicy
+      metadata:
+        name: default.host-firewall
+      spec:
+        egress:
+        - action: Allow
+          destination: {}
+          source: {}
+        ingress:
+        - action: Allow
+          destination:
+            nets:
+            - 127.0.0.0/8
+          source: {}
+        - action: Allow
+          destination: {}
+          protocol: ICMP
+          source: {}
+        - action: Allow
+          destination:
+            ports:
+            - 22
+          protocol: TCP
+          source: {}
+        - action: Allow
+          destination:
+            ports:
+            - 10250
+          protocol: TCP
+          source:
+            nets:
+            - "{{ cluster.network.cidr }}"
+        - action: Log
+          destination: {}
+          source: {}
+        selector: has(kubernetes.io/hostname)
+        types:
+        - Ingress
+        - Egress

pkg/adalovelace/templates/ansible/tasks/firewall.yaml

-- name: install ufw
+- name: remove ufw
   ansible.builtin.apt:
     name: ufw
-    state: present
+    state: absent
+
+- name: remove iptables file
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/iptables/rules.v4
+    - /etc/iptables/rules.v6
 
 - name: disable SSH password authentication
   lineinfile:
@@ -11,38 +19,6 @@
     state: present
   notify:
     - restart sshd
-  
-- name: allow ssh
-  community.general.ufw:
-    rule: limit
-    port: ssh
-    proto: tcp
-    comment: ssh
-
-- name: allow internal cloud traffic
-  community.general.ufw:
-    rule: allow
-    from_ip: "{{cluster.network.cloudCidr}}"
-    comment: internal cloud
-
-- name: allow internal phys traffic
-  community.general.ufw:
-    rule: allow
-    from_ip: "{{cluster.network.physCidr}}"
-    comment: internal phys
-
-- name: allow extras
-  community.general.ufw:
-    rule: allow
-    port: "{{ item.value.port }}"
-    proto: "{{ item.value.proto }}"
-    comment: "{{ item.key }}"
-  when: cluster.firewall is defined and cluster.firewall.allow is defined
-  loop: "{{ cluster.firewall.allow | dict2items }}"
-  loop_control:
-    label: "{{ item.key }}"
 
-- name: deny all traffic
-  community.general.ufw:
-    state: enabled
-    policy: deny
+- import_tasks: "tasks/firewall-controller.yaml"
+  when: '"controller" in group_names'

pkg/adalovelace/templates/ansible/tasks/nftables.conf.j2

+#!/usr/sbin/nft -f
+
+flush ruleset
+
+table inet filter {
+	chain input {
+		type filter hook input priority 0;
+        # allow connection from loopback
+        iifname lo accept;
+        # established/related connections
+        ct state established, related accept;
+        # drop invalid connections
+        ct state invalid drop;
+        # allow ssh connection
+        tcp dport 22 accept;
+        # allow k0s-api connection from load balancer
+        ip saddr {{ cluster.api.privateIp }} tcp dport 9443 counter accept;
+        # allow konnectivity connection from load balancer
+        ip saddr {{ cluster.api.privateIp }} tcp dport 8132 counter accept;
+        # allow kube-apiserver connection from load balancer
+        ip saddr {{ cluster.api.privateIp }} tcp dport 6443 counter accept;
+        {% for node in cluster.nodes | dict2items | map(attribute='value') | selectattr('role', 'equalto', 'controller') | rejectattr('publicIp', 'equalto', inventory_hostname) %}
+        # allow etcd connection from peers
+        ip saddr {{ node.privateIp }} tcp dport 2380 counter accept;
+        {% endfor %}
+        log flags all;
+        policy drop;
+	}
+	chain forward {
+		type filter hook forward priority 0;
+	}
+	chain output {
+		type filter hook output priority 0;
+	}
+}

pkg/adalovelace/templates/k0s/k0sctl.yaml

@@ -12,7 +12,7 @@ spec:
     privateAddress: {{ .PrivateIP }}
 {{ end }}
   k0s:
-    version: v1.21.6+k0s.0
+    version: v1.22.11+k0s.0
     config:
       apiVersion: k0s.k0sproject.io/v1beta1
       kind: Cluster