From a49ac0c51af2f57745db983776594805a88db5a3 Mon Sep 17 00:00:00 2001 From: Thies Mueller Date: Sat, 6 Jun 2026 00:03:38 +0200 Subject: [PATCH] commit --- inventories/tmss/hosts | 49 ++++++++++++++++++ jobs/files/Password: | 1 + jobs/files/node_exporter.crt | 25 ++++++++++ jobs/files/node_exporter.csr | 18 +++++++ jobs/files/node_exporter.key | 28 +++++++++++ jobs/files/node_exporter.yml | 5 ++ jobs/node_exporter.yml | 66 +++++++++++++++++++++++++ jobs/templates/cert-checker.yml | 5 ++ jobs/templates/node_exporter.service.j2 | 10 ++++ jobs/templates/prometheus.yml.j2 | 56 +++++++++++++++++++++ jobs/templates/rules.yml | 14 ++++++ jobs/update_prom.yml | 34 +++++++++++++ 12 files changed, 311 insertions(+) create mode 100644 inventories/tmss/hosts create mode 100644 jobs/files/Password: create mode 100644 jobs/files/node_exporter.crt create mode 100644 jobs/files/node_exporter.csr create mode 100644 jobs/files/node_exporter.key create mode 100644 jobs/files/node_exporter.yml create mode 100644 jobs/node_exporter.yml create mode 100644 jobs/templates/cert-checker.yml create mode 100644 jobs/templates/node_exporter.service.j2 create mode 100644 jobs/templates/prometheus.yml.j2 create mode 100644 jobs/templates/rules.yml create mode 100644 jobs/update_prom.yml diff --git a/inventories/tmss/hosts b/inventories/tmss/hosts new file mode 100644 index 0000000..2c690c3 --- /dev/null +++ b/inventories/tmss/hosts @@ -0,0 +1,49 @@ +[all] +monitoring +pve +docker +locations + +[locations:children] +hetzner +myloc +ethservices + +[hetzner:children] +fsn +nbg + +[fsn] +monitoring ansible_host=monitoring.tservic.es +support ansible_host=support.tservic.es +cloud ansible_host=cloud.tservic.es +helpdesk ansible_host=helpdesk.tservic.es +virt01 ansible_host=virt01.tservic.es +docker5 ansible_host=docker5.tservic.es +161hpretix ansible_host=tickets.161host.shop +unifi ansible_host=unifi.tservic.es +analytics ansible_host=analytics.tservic.es +remote ansible_host=remote.tservic.es + +[nbg] +ns1 ansible_host=ns1.xtm.es + +[myloc] +#virt02 ansible_host=virt02.tservic.es +ns0 ansible_host=ns0.xtm.es + +[ethservices] +ns2 ansible_host=ns2.xtm.es +service ansible_host=service.tservic.es +vpn ansible_host=vpn.tservic.es +dns-mgmt ansible_host=dns-mgmt.xtm.es + + +[monitoring] +monitoring + +[pve] +virt01 + +[docker] +docker5 diff --git a/jobs/files/Password: b/jobs/files/Password: new file mode 100644 index 0000000..82ce6e2 --- /dev/null +++ b/jobs/files/Password: @@ -0,0 +1 @@ + Password: \ No newline at end of file diff --git a/jobs/files/node_exporter.crt b/jobs/files/node_exporter.crt new file mode 100644 index 0000000..47d0cec --- /dev/null +++ b/jobs/files/node_exporter.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEHTCCAwWgAwIBAgIUdyYdCnJ9SmN/x/IjPKDSooUyRB0wDQYJKoZIhvcNAQEL +BQAwgbYxCzAJBgNVBAYTAkRFMRAwDgYDVQQIDAdIYW1idXJnMRAwDgYDVQQHDAdI +YW1idXJnMSgwJgYDVQQKDB9UaGllcyBNdWVsbGVyIFNlcnZpY2UgU29sdXRpb25z +MRMwEQYDVQQLDApNb25pdG9yaW5nMR4wHAYDVQQDDBVtb25pdG9yaW5nLnRzZXJ2 +aWMuZXMxJDAiBgkqhkiG9w0BCQEWFW1vbml0b3JpbmdAdHNlcnZpYy5lczAeFw0y +NjA0MjIxNDU5MTVaFw0zNjA0MTkxNDU5MTVaMIG2MQswCQYDVQQGEwJERTEQMA4G +A1UECAwHSGFtYnVyZzEQMA4GA1UEBwwHSGFtYnVyZzEoMCYGA1UECgwfVGhpZXMg +TXVlbGxlciBTZXJ2aWNlIFNvbHV0aW9uczETMBEGA1UECwwKTW9uaXRvcmluZzEe +MBwGA1UEAwwVbW9uaXRvcmluZy50c2VydmljLmVzMSQwIgYJKoZIhvcNAQkBFhVt +b25pdG9yaW5nQHRzZXJ2aWMuZXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCZAojwjIr274EYTNM7UWu6H9/04a9+y9HkOAq6twcsc248D5thEQCPSyOA +T5eq5wMjdH4B7LKkMlL/Rp4HuWWtA+EopDGZcsCGsphoszor+VyiVDc4VCT+IhUY +aSQHRC0szIgcj46wHE/hJYQlwYfRqkQWHWvzDO1r2TGrKfzwx9TBCj63Y9jj28dN +j/W1xEbwq0oW7Oh81i/ryil7Ie0DXoJYMoSwnwI1oBTQB1GIQDp4kceK0oCBOR9E +24zPGiVZNtm4iasaJaFnWQzN8RfNE2Woh5ywYT1ejYQ9fzMexTs9U4lmdl9oBlYr +bvd12pyVteycMYrLRA4ocxCEWrfFAgMBAAGjITAfMB0GA1UdDgQWBBRvOU0eImx0 +j+HJAmbpaLCUeVlGbzANBgkqhkiG9w0BAQsFAAOCAQEAhfO45AqPoVmLiKKs8Klb +zSEOazYzFm8JycZ3VdjOSIbWNnjoG51ThD8IFRc0RN3HswWmY30oC3FT6bRoo7N/ +s91CqaKNRAhDkJ9pzhT5Fy6LBXyqsESB+GLvQNWcPK75Tt0I2gNfSnnBUtIvJDpV +g6HColCnqgHeGPlhJYHUCRgXEXUkmy96qVb4Jx6HCIFL6p92sFPwsWhUf/SYiXht +AnnKZad56vwz6aSgrEeE8W/Lb2rXQGlo2+cUPUl+55WLJaBVH23PGpYjxtcN+jsl +kCBr6KbelwJdCq+5urtdLDN2mKy8hAHNAgZwkWe+ygIoTS5ChHhKhtlvy+VEVdhL +Ng== +-----END CERTIFICATE----- diff --git a/jobs/files/node_exporter.csr b/jobs/files/node_exporter.csr new file mode 100644 index 0000000..98a4193 --- /dev/null +++ b/jobs/files/node_exporter.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC/DCCAeQCAQAwgbYxCzAJBgNVBAYTAkRFMRAwDgYDVQQIDAdIYW1idXJnMRAw +DgYDVQQHDAdIYW1idXJnMSgwJgYDVQQKDB9UaGllcyBNdWVsbGVyIFNlcnZpY2Ug +U29sdXRpb25zMRMwEQYDVQQLDApNb25pdG9yaW5nMR4wHAYDVQQDDBVtb25pdG9y +aW5nLnRzZXJ2aWMuZXMxJDAiBgkqhkiG9w0BCQEWFW1vbml0b3JpbmdAdHNlcnZp +Yy5lczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJkCiPCMivbvgRhM +0ztRa7of3/Thr37L0eQ4Crq3ByxzbjwPm2ERAI9LI4BPl6rnAyN0fgHssqQyUv9G +nge5Za0D4SikMZlywIaymGizOiv5XKJUNzhUJP4iFRhpJAdELSzMiByPjrAcT+El +hCXBh9GqRBYda/MM7WvZMasp/PDH1MEKPrdj2OPbx02P9bXERvCrShbs6HzWL+vK +KXsh7QNeglgyhLCfAjWgFNAHUYhAOniRx4rSgIE5H0TbjM8aJVk22biJqxoloWdZ +DM3xF80TZaiHnLBhPV6NhD1/Mx7FOz1TiWZ2X2gGVitu93XanJW17JwxistEDihz +EIRat8UCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4IBAQBrGnq928qEJ8rfTN75WfY6 +FIF9UaJQhBShd8M7HbPbJvUf6tINi8MgCDwMhaMNKp2X3uCnzBlZkJwVtw6idlsP +1LWH1fio11nqqW+aHZBFm0SVrLE/578twvqwyMCjybEQWEqK0HQpITkB2GFTzUMW +d/JZspsNi9sMSZz0JhaD0w98DhU33PIwyuGSRYFxKnVei0n+2DV5rYhWoKXoFE1w +ZnT11Sxcjh2Jc5buqlHvP4vSDUnQvBasif7HtSxarrqQhe+IRs5o2qAFTvouLNSm +0dgnHpRVuN9p5hG+1r9M2QEK64ydfOuDUZ6ajNThwEwADgPRTTdujHUJL3lgwIUk +-----END CERTIFICATE REQUEST----- diff --git a/jobs/files/node_exporter.key b/jobs/files/node_exporter.key new file mode 100644 index 0000000..ba8385d --- /dev/null +++ b/jobs/files/node_exporter.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCZAojwjIr274EY +TNM7UWu6H9/04a9+y9HkOAq6twcsc248D5thEQCPSyOAT5eq5wMjdH4B7LKkMlL/ +Rp4HuWWtA+EopDGZcsCGsphoszor+VyiVDc4VCT+IhUYaSQHRC0szIgcj46wHE/h +JYQlwYfRqkQWHWvzDO1r2TGrKfzwx9TBCj63Y9jj28dNj/W1xEbwq0oW7Oh81i/r +yil7Ie0DXoJYMoSwnwI1oBTQB1GIQDp4kceK0oCBOR9E24zPGiVZNtm4iasaJaFn +WQzN8RfNE2Woh5ywYT1ejYQ9fzMexTs9U4lmdl9oBlYrbvd12pyVteycMYrLRA4o +cxCEWrfFAgMBAAECggEAB0t4JOPGckxr3rE29uKEXMzNPBsQZ16nRJapaEp2lJdy +P3nNAh0x5iMCb8kbpYG7BBM3OLdEsxBVJaeDvEGKB+wh7XKajMLMS/+yxXcaBGOw +l2qlcVGNq/ILCG1m0tPcxYCA0r/DYUEQS98ihDxMUeIoudcwQNeycAxcUlk/nmkN +oX5kzzpmT4TSaVrY0cvFQfGVv8MbcqHbefolw8IepyTd0fLkSBxjVImxILecle5F +iTGuwyAZmddmwPnoIYm2a+3HK4vNDxDlT6xxvcxciXNBdwweJ1DxKDDnhWyW6XO8 +8tfnhfsvSPze/ynTQOLxTxSIBmWwi4XbhhY/wtaIlwKBgQDTZD84Th8EJEAtny58 +/wpuyyk2ymkRXERSm1aOr6fjMp50UoDH+eEXa+OtXNh6s6pR1R7EG64vN/q7SnNo +DYDC8UTFIe6oQbILxbheXYUm1MpzB6pbuj80QsnowfZwRbpB85LjQ+5zlBHB+jHP +aipBj26LW1thUAyZ4tfyBRKCbwKBgQC5TGeIjU1jz3N1c8cDajFr2rdQ7NQzE9hb +6htiRMnEO4xmHY94YaPy/MdJojvh+Ed15NreAaZJLfW1RA0wZTEBQ0OjAKEbRcEF +duDshkoTULUwN51J20rCLzfaYVj3cgpoi+zx2bdhLtjtfyGCxOxv/15af+hQoL/9 +7JAxih0zCwKBgFz2iArl6CP1ITgu+c3Ehz0EIULULvKQhgdiAuXK/+UdfKuhILtQ ++5lGPCEgZJ9JZcyJEfeiSK7BSoWVDz2iOo8AdYLQEllyUq90q0jsjcLu73040MV4 +GZWNDahLrGnhcDDSwjUpybKQWmLY+ZDCmoSsZCvq6DvabUA7j0YCzwoHAoGASCBr +dIW8CAvqH1/PqY6eaiMalA97kd45q512x0uLVizvu5fqDqvDC4RFdm4F7TGdb1CD +uULfHUEckVBJ6fqZlqo+G6bVDy6ZsazSZH7pOpeBz6D2QpdvhCKpFQIrhTYIwyUk +OAfND7ESk2+W9bX9aqL2cTe0kB3iXc7FWeMu9n0CgYAz5xVC/9YzA1K89MqKle5d +G5Iuv09TEyroDmm/P/iQueaZi3QYjQR4mFEVZK+NqztxXKyZK2LJdtsTpayDCAM4 +SSuQWmRpSGjzanVlHGYcJh/UvOGf5L0xyAFIjKcpX6nnsG8B12HGXS43BjLLhNEu +0sOIcQghdMhetQYPtD7Ujw== +-----END PRIVATE KEY----- diff --git a/jobs/files/node_exporter.yml b/jobs/files/node_exporter.yml new file mode 100644 index 0000000..2d435cc --- /dev/null +++ b/jobs/files/node_exporter.yml @@ -0,0 +1,5 @@ +tls_server_config: + cert: /etc/node_exporter/cert.crt + key: /etc/node_exporter/cert.key +basic_auth_users: + monitoring: $2y$12$OOlihTG383u0nqDk4dcmUuO.KAsusfDsd04TW1/b1TUuqbWlViBSO \ No newline at end of file diff --git a/jobs/node_exporter.yml b/jobs/node_exporter.yml new file mode 100644 index 0000000..1b4330e --- /dev/null +++ b/jobs/node_exporter.yml @@ -0,0 +1,66 @@ +- hosts: all:!windows:!monitoring + + + vars: + node_exporter_version: 1.11.1 + interface_var_name: "ansible_wt0" + interface_var_ipv4: "{{ vars[interface_var_name].ipv4.address }}" + tasks: + - name: download node exporter + get_url: + url: https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz + dest: /tmp + - name: unarchive node exporter + unarchive: + remote_src: yes + src: /tmp/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz + dest: /tmp + - name: move node exporter to /usr/local/bin + copy: + src: /tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter + dest: /usr/local/bin/node_exporter + remote_src: yes + owner: root + group: root + mode: 0755 + - name: install unit file to systemd + template: + src: templates/node_exporter.service.j2 + dest: /etc/systemd/system/node_exporter.service + owner: root + group: root + mode: 0600 + - name: create config directory + file: + path: /etc/node_exporter + state: directory + owner: root + group: root + mode: 0700 + - name: copy certificates + copy: + src: files/node_exporter.crt + dest: /etc/node_exporter/cert.crt + owner: root + group: root + mode: 0600 + - name: copy certificate key + copy: + src: files/node_exporter.key + dest: /etc/node_exporter/cert.key + owner: root + group: root + mode: 0600 + - name: copy node exporter config + template: + src: files/node_exporter.yml + dest: /etc/node_exporter/config.yml + owner: root + group: root + mode: 0600 + - name: configure systemd to use service + systemd: + daemon_reload: yes + enabled: yes + state: started + name: node_exporter.service diff --git a/jobs/templates/cert-checker.yml b/jobs/templates/cert-checker.yml new file mode 100644 index 0000000..992e722 --- /dev/null +++ b/jobs/templates/cert-checker.yml @@ -0,0 +1,5 @@ +loglevel: debug +port: 8080 # Optional +intervalminutes: 2 +certificates: + - dns: auth.tservic.es \ No newline at end of file diff --git a/jobs/templates/node_exporter.service.j2 b/jobs/templates/node_exporter.service.j2 new file mode 100644 index 0000000..648118f --- /dev/null +++ b/jobs/templates/node_exporter.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Node Exporter +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/node_exporter --web.config.file="/etc/node_exporter/config.yml" --web.listen-address=":9100" + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/jobs/templates/prometheus.yml.j2 b/jobs/templates/prometheus.yml.j2 new file mode 100644 index 0000000..bce240d --- /dev/null +++ b/jobs/templates/prometheus.yml.j2 @@ -0,0 +1,56 @@ +--- +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: cert-checker + scrape_interval: 5s + static_configs: + - targets: + - cert-checker:8080 + + - job_name: "nodeexporter" + static_configs: + - targets: ["node-exporter:9100"] + + - job_name: "cadvisor" + static_configs: + - targets: ["cadvisor:8080"] +{% for host in groups['docker']%} +{% if inventory_hostname != host %} + - targets: ["{{ host }}{{hostsuffix}}:9999"] +{% endif %} +{% endfor %} + + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + - job_name: "node_exporter" + static_configs: +{% for host in groups['locations'] %} +{% if inventory_hostname != host and host not in groups['windows'] %} + - targets: ["{{ host }}{{hostsuffix}}:9100"] +{% endif %} +{% endfor %} + + + - job_name: "docker" + static_configs: +{% for host in groups['docker'] %} +{% if inventory_hostname != host %} + - targets: ["{{ host }}{{hostsuffix}}:9323"] +{% endif %} +{% endfor %} + + +rule_files: + - rules.yml + +alerting: + alertmanagers: + - static_configs: + - targets: + # Alertmanager's default port is 9093 + - localhost:9093 \ No newline at end of file diff --git a/jobs/templates/rules.yml b/jobs/templates/rules.yml new file mode 100644 index 0000000..e047d84 --- /dev/null +++ b/jobs/templates/rules.yml @@ -0,0 +1,14 @@ +groups: +- name: AllInstances + rules: + - alert: InstanceDown + # Condition for alerting + expr: up == 0 + for: 1m + # Annotation - additional informational labels to store more information + annotations: + title: 'Instance {{ $labels.instance }} down' + description: '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minute.' + # Labels - additional labels to be attached to the alert + labels: + severity: 'critical' \ No newline at end of file diff --git a/jobs/update_prom.yml b/jobs/update_prom.yml new file mode 100644 index 0000000..db9bd98 --- /dev/null +++ b/jobs/update_prom.yml @@ -0,0 +1,34 @@ +- hosts: monitoring + + tasks: + - name: deploy prometheus config + template: + src: templates/prometheus.yml.j2 + dest: /opt/containers/prometheus-grafana/prometheus/prometheus.yml + owner: root + group: root + mode: 0644 + + - name: deploy prometheus rules + copy: + src: templates/rules.yml + dest: /opt/containers/prometheus-grafana/prometheus/rules.yml + owner: root + group: root + mode: 0644 + + - name: deploy cert-checker config + copy: + src: templates/cert-checker.yaml + dest: /opt/containers/prometheus-grafana/cert-checker/config.yaml + owner: root + group: root + mode: 0644 + + - name: reload prometheus container + shell: cd /opt/containers/prometheus-grafana && docker kill prometheus && docker compose up -d + ignore_errors: true + +# - name: reload prometheus container (legacy docker) +# shell: cd /opt/containers/prometheus-grafana && docker-compose down && docker-compose up -d +# ignore_errors: true