# vim: expandtab shiftwidth=2 softtabstop=2
#
# CSLab Blackbox exporter configuration
#
# Some of these come from the standard Blackbox configuration.
#
# Before attempting to reload this, check it with:
#	/opt/prometheus/sbin/blackbox_exporter --config.file blackbox.yml --config.check
# Then:
#	systemctl reload cslab-blackbox
#
# To check a module against a target, blackbox is on port 9115:
#
#	curl 'http://localhost:9115/probe?module=WHAT&target=HOST&debug=true'

# A note on timeouts: in general, <cks> has come to think that
# Blackbox probes should leave all handling of timeouts up to
# Prometheus itself. There are times where this may not apply,
# but not many. Accordingly this configuration doesn't specify
# any.

modules:
  http_2xx:
    prober: http
    http:
      preferred_ip_protocol: "ip4"
  http_post_2xx:
    prober: http
    http:
      method: POST
  tcp_connect:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
  pop3s_banner:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
      # Note that this must use '...' instead of "...", so we can use \
      # to quote regexp special characters like '+'.
      - expect: '^\+OK '
      - send: "QUIT"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
      - expect: "^SSH-2.0-"
      # CSLab: We send a response partly so that logs on the target will
      # implicitly say whether we saw the greeting banner or not. If the
      # target logged 'Did not receive identification string' for our
      # connection, we didn't see the banner in time to send our response.
      # If we did send a response, the target will log
      # "Connection closed by <...> [preauth]". Sadly there is nothing we
      # can send that will cause nothing to be logged.
      - send: "SSH-2.0-Prometheus-Blackbox\r"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp
    icmp:
      preferred_ip_protocol: "ip4"
  # CSLab additions:
  http_redirect:
    prober: http
    http:
      preferred_ip_protocol: "ip4"
      follow_redirects: false
      valid_status_codes: [301, 302, 303, 307, 308]

  # This accepts known 2xx series responses and HTTP redirects, which
  # means it handles servers that may or may not redirect. It deliberately
  # doesn't handle HTTP 401 must authenticate replies.
  http_2xx_redirect:
    prober: http
    http:
      preferred_ip_protocol: "ip4"
      follow_redirects: false
      valid_status_codes: [200, 203, 206, 301, 302, 303, 307, 308]

  tls_connect:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      tls: true
  # Fully neurotically correct (E)SMTP servers insist that you end your
  # lines with \r\n, not just \n. Guess what sort of SMTP servers Microsoft
  # has for Office365.
  smtp_starttls:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - expect: "^220 "
        - send: "EHLO aeschylos.cs.toronto.edu\r"
        - expect: "^250-STARTTLS"
        - expect: "^250 "
        - send: "STARTTLS\r"
        - expect: "^220"
        - starttls: true
        - send: "QUIT\r"
  imaps_banner:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      tls: true
      query_response:
        - expect: '^\* OK \[CAPABILITY IMAP4rev1 '

  imap_banner_starttls:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - expect: '^\* OK \[.* STARTTLS '
        - send: "a STARTTLS"
        - expect: "^a OK "
        - starttls: true
        - send: 'b CAPABILITY'
        - expect: '^\* CAPABILITY '
        - expect: '^b OK '

  # This module is used to probe the SMTP port on machines that (currently)
  # deny us access. As a result it accepts both 220 and 451 status results.
  smtp_banner:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - expect: "^(220|451) "
        # We cannot be neat and use a 'send: "QUIT"' here, because the
        # machines that 451 us immediately drop the connection. Oh well.

  # This specifically probes the address of www.cs.toronto.edu.
  # www.cs is a CNAME, but making an A record lookup will always
  # return both the CNAME to colony and the A record for colony
  # from properly operating DNS servers, both authoritative and
  # non-authoritative, because when a server runs into a CNAME
  # during a query for another record type it restarts the query
  # with the new name. If we CNAME'd www.cs to something outside
  # the zone, authoritative DNS servers might not have the outside
  # name, but since colony.cs is in-zone an authoritative server
  # should always know it.
  dns_cslab:
    prober: dns
    # This is the exception that proves the rule for timeouts.
    # DNS probes are done along side other probes that need much longer
    # timeouts, but it is not useful waiting 40 or 50 seconds for
    # a failure here. In practice 30s is probably higher than default
    # DNS timeouts, which on Linux default to five 5s tries (25s total).
    timeout: 30s
    dns:
      query_name: "www.cs.toronto.edu"
      query_type: "A"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        # We have to look for any old TTL here, because we may be
        # querying caching/resolving DNS servers and so their TTL
        # will be counting down. Also, we don't want to hard-code
        # the TTL we currently use.
        #
        # We have to put both the match for the www.cs CNAME and the
        # match for colony.cs A into one regular expression because
        # of how Blackbox handles matches. Each separate RR is matched
        # against the regexp(s) and the probe fails if any fail to match.
        fail_if_not_matches_regexp:
          - "(www.cs.toronto.edu.\t[0-9]*\tIN\tCNAME\tcolony.cs.toronto.edu.|colony.cs.toronto.edu.\t[0-9]*\tIN\tA\t128.100.3.30)"
          # This is the old match from when we were using colony.cs
          # as the query name.
          #- "colony.cs.toronto.edu.\t[0-9]*\tIN\tA\t128.100.3.30"

  # Query that fs2 resolves to the correct IP. This is used as an internal
  # 'extremely must always work' DNS query. For historical reasons fs2 is
  # not a DNS alias like the other fileserver names, but a separate IP
  # that is more or less fixed for all time.
  dns_cslab_fs2:
    prober: dns
    timeout: 30s
    dns:
      query_name: "fs2.cs.toronto.edu"
      query_type: "A"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
          - "fs2.cs.toronto.edu.\t[0-9]*\tIN\tA\t128.100.3.132"

  # This specifically probes the SOA of cs.toronto.edu. It is present
  # in part because Blackbox version 0.13.0 and later exports the zone
  # SOA as a metric (probe_dns_serial), so if we do SOA probes we can
  # check the visible SOA across all of the servers.
  dns_cslab_soa:
    prober: dns
    timeout: 30s
    dns:
      query_name: "cs.toronto.edu"
      query_type: "SOA"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        # Require that we get back a SOA record for us, because well yes.
        # However we don't require it to have any particular content
        # because that might make life more difficult for us if we change
        # things or mutate our internal view or something.
        fail_if_not_matches_regexp:
          - "cs.toronto.edu.\t[0-9]*\tIN\tSOA\t.*"
  # SOA of cs.utoronto.ca because ITS broke that separately from
  # cs.toronto.edu.
  dns_cslab_utoronto_soa:
    prober: dns
    timeout: 30s
    dns:
      query_name: "cs.utoronto.ca"
      query_type: "SOA"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
          - "cs.utoronto.ca.\t[0-9]*\tIN\tSOA\t.*"

  # As with dns_cslab_soa, check the SOA but this time of 'sandbox.'.
  # We can only do this with internal DNS. Note that individual sandbox
  # sub-areas don't have a SOA, because they're not NS'd separately; they
  # are all rolled into the same sandbox. zone.
  dns_sandbox_soa:
    prober: dns
    timeout: 30s
    dns:
      query_name: "sandbox"
      query_type: "SOA"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
          - "sandbox.\t[0-9]*\tIN\tSOA\t.*"

  # This probes the SOA of 3.100.128.in-addr.arpa, our net-3 PTR zone.
  dns_cslab3_ptr_soa:
    prober: dns
    timeout: 30s
    dns:
      query_name: "3.100.128.in-addr.arpa"
      query_type: "SOA"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
          - "3.100.128.in-addr.arpa.\t[0-9]*\tIN\tSOA\t.*"

  # This probes the MX for cs.toronto.edu, which must be alkyone at
  # MX 0.
  dns_cslab_mx:
    prober: dns
    timeout: 30s
    dns:
      query_name: "cs.toronto.edu"
      query_type: "MX"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        # We have to look for any old TTL here, because we may be
        # querying caching/resolving DNS servers and so their TTL
        # will be counting down. Also, we don't want to hard-code
        # the TTL we currently use.
        fail_if_not_matches_regexp:
          - "cs.toronto.edu.\t[0-9]*\tIN\tMX\t0 alkyone.cs.toronto.edu."

  # Check that some things we care about resolve to IPs.
  dns_utoronto_sharepoint:
    prober: dns
    timeout: 30s
    dns:
      query_name: "utoronto.sharepoint.com"
      query_type: "A"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_none_matches_regexp:
          - ".*\t[0-9]*\tIN\tA\t.*"
  dns_dcsuoft_slack:
    prober: dns
    timeout: 30s
    dns:
      query_name: "dcsuoft.enterprise.slack.com"
      query_type: "A"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_none_matches_regexp:
          - ".*\t[0-9]*\tIN\tA\t.*"

  # Check that www.google.com resolves to an A record. We don't check for
  # what A record, because Google changes that around.
  dns_google:
    prober: dns
    timeout: 30s
    dns:
      query_name: "www.google.com"
      query_type: "A"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
          - "www.google.com.\t[0-9]*\tIN\tA\t.*"

  # Check that gmail.com. resolves to one or more MX records. Failure
  # to resolve gmail.com on our own DNS servers is basically a red alert
  # that mail is about to back up.
  dns_gmail_mx:
    prober: dns
    timeout: 30s
    dns:
      query_name: "gmail.com"
      query_type: "MX"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
          - "gmail.com.\t[0-9]*\tIN\tMX\t.*"

  # Check that amazon.ca resolves to an A record, as an additional check
  # on DNS sanity just in case resolving Google explodes for some reason
  # without general DNS problems.
  # This must be amazon.ca, not www.amazon.ca, because we can't do CNAMEs
  # very readily.
  dns_amazonca:
    prober: dns
    timeout: 30s
    dns:
      query_name: "amazon.ca"
      query_type: "A"
      preferred_ip_protocol: "ip4"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
          - "amazon.ca.\t[0-9]*\tIN\tA\t.*"

  ftp_anon:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - expect: "220"
        - send: "USER anonymous"
        - expect: "331"
        - send: "PASS anonymous"
        - expect: "230 Login successful"
        - send: "QUIT"
  # Our LDAP servers have self-signed certificates, so we must use
  # a special module that doesn't try to verify their certificates.
  # This may change someday.
  tls_insecure:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      tls: true
      tls_config:
        insecure_skip_verify: true

  # CSLab www.cs suexec CGI check
  # This is used to check https://www.cs.toronto.edu/~ops/myid.cgi
  # to make sure that it returns the magic string that demonstrates that
  # suexec for CGIs is working, because it has the right login name.
  # Sadly we have to hardcode a Blackbox module that is useful for exactly
  # one URL, because we can't parameterize it otherwise.
  suexec_uid_check:
    prober: http
    http:
      preferred_ip_protocol: "ip4"
      fail_if_body_not_matches_regexp:
        - "My Unix login is: !ops!"

  # CSLab Linux fileserver NFS mount authentication validation. We
  # should always get either '200 <IP> APPROVED' or '200 <IP>
  # CURRENTLY AUTHORIZED' if things are working right.
  linux_nfs_auth:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - expect: '^200 .* (APPROVED|CURRENTLY AUTHORIZED).*'

  # This module is exactly tcp_connect, but it is given a separate name
  # to make Prometheus alert rules easier to write because it is used for
  # services that should *not* respond.
  tcp_connect_fail:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"

  # This check verifies that we can connect to something, STARTTLS,
  # re-EHLO, and submit a MAIL FROM of cslab-email-check@cs.toronto.edu
  # It is used for additional external checks over and above our normal
  # basic 'can it do STARTTLS' checks.
  # We must use \r's to satisfy picky mail servers, like Office365.
  smtp_csfrom:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - expect: "^220 "
        - send: "EHLO aeschylos.cs.toronto.edu\r"
        - expect: "^250-STARTTLS"
        - expect: "^250 "
        - send: "STARTTLS\r"
        - expect: "^220"
        - starttls: true
        - send: "EHLO aeschylos.cs.toronto.edu\r"
        - expect: "^250 "
        - send: "MAIL FROM:<cslab-email-check@cs.toronto.edu>\r"
        - expect: "^250 "
        - send: "QUIT"

  # Because sometimes people break their TLS support without necessarily
  # breaking email at all, this is the same as smtp_csfrom but without
  # requiring TLS.
  smtp_csfrom_notls:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - expect: "^220 "
        - send: "EHLO aeschylos.cs.toronto.edu\r"
        - expect: "^250 "
        - send: "MAIL FROM:<cslab-email-check@cs.toronto.edu>\r"
        - expect: "^250 "
        - send: "QUIT\r"

  # This accepts a HTTP 401 basic authentication reply, as well as
  # HTTP 200 and HTTP 206.
  # We use this module for URLs (or entire hosts) which currently require
  # basic authentication but which might someday change to not have it on
  # exactly that URL.
  # If you want to check for a 401 being returned, check both that this
  # probe worked and that probe_http_status_code was 401.
  http_also_basic_auth:
    prober: http
    http:
      preferred_ip_protocol: "ip4"
      valid_status_codes: [200, 206, 401]

  # This checks clamd, the ClamAV daemon. The daemon uses plain TCP
  # connections passing ASCII for commands, and it has a 'PING' command
  # that makes it respond with 'PONG'.
  # rspamd.cs normally runs clamd on port 15000, but that's specified
  # by/in Prometheus.
  clamd_pingpong:
    prober: tcp
    tcp:
      preferred_ip_protocol: "ip4"
      query_response:
        - send: "PING\n"
        - expect: "PONG"

  # This checks the rspamd daemon. The daemon uses HTTP and supports
  # a /ping URL that responds with a text/plain 'pong' response.
  # rspamd.cs normally runs rspamd on port 11333.
  rspamd_pingpong:
    prober: http
    http:
      preferred_ip_protocol: "ip4"
      valid_status_codes: [200]
      fail_if_body_not_matches_regexp:
        - "^pong"
