Add Turnstile and ALTCHA captcha backends alongside reCAPTCHA

.github/dependabot.yml

@@ -14,6 +14,10 @@ updates:
       # until a Sphinx release supports it (sphinx-doc/sphinx#14455).
       - dependency-name: "docutils"
         versions: [">=0.23"]
+      # The altcha library and the browser widget share a challenge protocol
+      # that changes with the major version, so upgrade them together by hand.
+      - dependency-name: "altcha"
+        update-types: ["version-update:semver-major"]
   - package-ecosystem: github-actions
     directory: /
     schedule:

.github/workflows/ci.yml

@@ -12,11 +12,13 @@ concurrency:
 jobs:
   build:
     runs-on: ubuntu-latest
-    # The test suite mocks all external calls (reCAPTCHA, RT), so it needs the
-    # config keys to exist but not real secrets. Dummy values keep CI runnable
-    # on fork pull requests, which cannot access repository secrets.
+    # The test suite mocks all external calls (captcha verifiers, RT), so it
+    # needs the config keys to exist but not real secrets. Dummy values keep CI
+    # runnable on fork pull requests, which cannot access repository secrets.
     env:
       TOKEN: ci-token
+      TURNSTILE_SECRET: ci-turnstile-secret
+      ALTCHA_HMAC_KEY: ci-altcha-key
       RECAPTCHA_SECRET: ci-recaptcha-secret
       RT_TOKEN: ci-rt-token
     strategy:

.github/workflows/container.yml

@@ -45,15 +45,19 @@ jobs:
       - name: Smoke test the image
         env:
           TOKEN: ci-token
-          RECAPTCHA_SECRET: ci-recaptcha-secret
+          TURNSTILE_SECRET: ci-turnstile-secret
+          ALTCHA_HMAC_KEY: ci-altcha-key
           RT_TOKEN: ci-rt-token
         run: |
           docker run -d --name formsender -p 5000:5000 \
-            -e TOKEN -e RECAPTCHA_SECRET -e RT_TOKEN \
+            -e TOKEN -e TURNSTILE_SECRET -e ALTCHA_HMAC_KEY -e RT_TOKEN \
             osuosl/formsender:test
           for i in $(seq 1 10); do
             if curl -fsS http://localhost:5000/server-status; then
               echo " - server-status OK"
+              # The ALTCHA challenge endpoint exercises the captcha module
+              curl -fsS http://localhost:5000/altcha | grep -q '"parameters"'
+              echo " - altcha challenge OK"
               break
             fi
             if [ "$i" -eq 10 ]; then

Makefile

@@ -26,4 +26,5 @@ coverage:
 
 flake:
 	      flake8 request_handler.py
+	      flake8 captcha.py
 	      flake8 tests.py

README.md

@@ -10,8 +10,9 @@ REST2 API.
 
 Features:
 
-* Honeypot, shared-token, rate-limit, duplicate-detection, and reCAPTCHA checks
-  to filter out spam and abuse.
+* Honeypot, shared-token, rate-limit, duplicate-detection, and captcha checks
+  to filter out spam and abuse. Captcha backends: Cloudflare Turnstile,
+  self-hosted ALTCHA (proof of work, no third party), or Google reCAPTCHA.
 * File uploads are attached to the ticket.
 * Form fields can be mapped to RT custom fields.
 * A single image can serve multiple RT instances (one container per instance)
@@ -24,7 +25,8 @@ The file `conf.py.dist` reads its settings from environment variables. Copy it
 to `conf.py` (`cp conf.py.dist conf.py`) and supply the environment variables
 described in the [usage documentation]
 (http://formsender.readthedocs.org/en/latest/usage.html). At minimum you must
-set `TOKEN`, `RECAPTCHA_SECRET`, and `RT_TOKEN`.
+set `TOKEN`, `RT_TOKEN`, and the secret for at least one captcha provider
+(`TURNSTILE_SECRET`, `ALTCHA_HMAC_KEY`, or `RECAPTCHA_SECRET`).
 
 Deploy
 ------
@@ -36,7 +38,7 @@ Registry at `ghcr.io/osuosl/formsender`. The image runs the app under Gunicorn
 ```
 docker run -p 5000:5000 \
   -e TOKEN=... \
-  -e RECAPTCHA_SECRET=... \
+  -e TURNSTILE_SECRET=... \
   -e RT_TOKEN=... \
   -e RT_URL=https://support.example.org/REST/2.0/ \
   ghcr.io/osuosl/formsender:master

captcha.py

@@ -0,0 +1,223 @@
+"""
+Captcha backends: Cloudflare Turnstile, self-hosted ALTCHA, Google reCAPTCHA.
+See docs/source/usage.rst and docs/source/form_setup.rst.
+"""
+
+import collections
+import logging
+import time
+
+import altcha
+import requests
+
+import conf
+
+logger = logging.getLogger('formsender')
+
+TURNSTILE_VERIFY_URL = \
+    'https://challenges.cloudflare.com/turnstile/v0/siteverify'
+RECAPTCHA_VERIFY_URL = 'https://www.google.com/recaptcha/api/siteverify'
+# Seconds to wait for a hosted verifier before giving up
+VERIFY_TIMEOUT = 10
+# Key derivation functions that work with a plain iteration count and no
+# extra package. Anything else the library silently downgrades to plain SHA.
+ALTCHA_ALGORITHMS = ('PBKDF2/SHA-256', 'PBKDF2/SHA-384', 'PBKDF2/SHA-512',
+                     'SHA-256', 'SHA-384', 'SHA-512')
+
+Provider = collections.namedtuple('Provider', 'name field secret verify')
+
+# Reused so repeated verifications skip the TLS handshake
+session = requests.Session()
+
+
+def _setting(name, default=None):
+    """Read an optional conf.py setting, treating None as unset"""
+    value = getattr(conf, name, None)
+    return default if value is None else value
+
+
+def _siteverify(url, secret, token, remote_ip):
+    """POST a token to a siteverify endpoint; None if the call failed"""
+    data = {'secret': secret, 'response': token}
+    if remote_ip:
+        data['remoteip'] = remote_ip
+    try:
+        response = session.post(url, data=data, timeout=VERIFY_TIMEOUT)
+        response.raise_for_status()
+        result = response.json()
+    except (requests.RequestException, ValueError) as error:
+        logger.error('formsender: captcha verification request to %s failed: '
+                     '%s', url, error)
+        return None
+    if not isinstance(result, dict):
+        logger.error('formsender: captcha verifier %s returned %s, not an '
+                     'object', url, type(result).__name__)
+        return None
+    return result
+
+
+def allowed_hostnames():
+    """Parsed CAPTCHA_ALLOWED_HOSTNAMES, or None when the check is off"""
+    raw = _setting('CAPTCHA_ALLOWED_HOSTNAMES')
+    if raw is None:
+        return None
+    hosts = {host.strip().lower() for host in raw.split(',') if host.strip()}
+    if not hosts:
+        # Failing open here would silently disable the check after a typo
+        raise RuntimeError('CAPTCHA_ALLOWED_HOSTNAMES is set but names no '
+                           'hostnames; unset it to disable the check')
+    return hosts
+
+
+def _hostname_allowed(result):
+    """Refuse a token a hosted provider says was solved on another site"""
+    allowed = allowed_hostnames()
+    if allowed is None:
+        return True
+    hostname = (result.get('hostname') or '').lower()
+    if hostname in allowed:
+        return True
+    logger.warning('formsender: captcha solved on %r, which is not in '
+                   'CAPTCHA_ALLOWED_HOSTNAMES', hostname)
+    return False
+
+
+def _hosted_response_is_good(provider, result):
+    """Shared success handling for the two hosted verifiers"""
+    if not result or not result.get('success'):
+        logger.warning('formsender: %s rejected the response: %s', provider,
+                       (result or {}).get('error-codes')
+                       or 'no response from verifier')
+        return False
+    # reCAPTCHA fails open once its quota is spent: success is true, an error
+    # code is attached and the score is a fixed 0.9. Refuse to trust that.
+    codes = result.get('error-codes')
+    if codes:
+        logger.error('formsender: %s reported %s alongside success, so the '
+                     'verdict is not trustworthy', provider, codes)
+        return False
+    return _hostname_allowed(result)
+
+
+def verify_turnstile(token, remote_ip, controller):
+    """Verify a Cloudflare Turnstile response token"""
+    result = _siteverify(TURNSTILE_VERIFY_URL, conf.TURNSTILE_SECRET, token,
+                         remote_ip)
+    return _hosted_response_is_good('turnstile', result)
+
+
+def verify_recaptcha(token, remote_ip, controller):
+    """Verify a reCAPTCHA token, applying the score that v3 keys return"""
+    result = _siteverify(RECAPTCHA_VERIFY_URL, conf.RECAPTCHA_SECRET, token,
+                         remote_ip)
+    if not _hosted_response_is_good('recaptcha', result):
+        return False
+    score = result.get('score')
+    if score is None:
+        return True
+    try:
+        if float(score) < float(_setting('RECAPTCHA_MIN_SCORE', 0.5)):
+            logger.warning('formsender: recaptcha score %s is below '
+                           'RECAPTCHA_MIN_SCORE', score)
+            return False
+    except (TypeError, ValueError) as error:
+        logger.error('formsender: recaptcha score %r could not be compared '
+                     'with RECAPTCHA_MIN_SCORE: %s', score, error)
+        return False
+    return True
+
+
+def verify_altcha(payload, remote_ip, controller):
+    """Verify an ALTCHA proof of work, then refuse a reused challenge"""
+    try:
+        parsed = altcha.Payload.from_base64(payload)
+        result = altcha.verify_solution(parsed, conf.ALTCHA_HMAC_KEY)
+    except Exception as error:
+        # Every field here is attacker supplied, so a decode or type error is
+        # a rejected submission rather than a server error.
+        logger.warning('formsender: altcha payload was not usable: %s: %s',
+                       type(error).__name__, error)
+        return False
+    if not result.verified:
+        logger.warning('formsender: altcha rejected the solution: expired=%s '
+                       'invalid_signature=%s invalid_solution=%s error=%s',
+                       result.expired, result.invalid_signature,
+                       result.invalid_solution, result.error)
+        return False
+    params = parsed.challenge.parameters
+    # /altcha always sets an expiry, so a challenge without one is not ours
+    if not params.expires_at:
+        logger.warning('formsender: altcha challenge has no expiry')
+        return False
+    if controller.is_replayed_challenge(params.nonce, params.expires_at):
+        logger.warning('formsender: altcha challenge %s was already used',
+                       params.nonce)
+        return False
+    return True
+
+
+def altcha_algorithm():
+    """The configured ALTCHA key derivation function, validated"""
+    algorithm = _setting('ALTCHA_ALGORITHM', 'PBKDF2/SHA-256')
+    if algorithm not in ALTCHA_ALGORITHMS:
+        raise RuntimeError('ALTCHA_ALGORITHM %r is not supported; use one of '
+                           '%s' % (algorithm, ', '.join(ALTCHA_ALGORITHMS)))
+    return algorithm
+
+
+def create_altcha_challenge():
+    """Issue a signed, expiring challenge for the ALTCHA widget"""
+    expires_at = int(time.time()) + int(_setting('ALTCHA_EXPIRES', 600))
+    challenge = altcha.create_challenge(altcha_algorithm(),
+                                        int(_setting('ALTCHA_COST', 5000)),
+                                        expires_at=expires_at,
+                                        hmac_secret=conf.ALTCHA_HMAC_KEY)
+    return challenge.to_dict()
+
+
+# Checked in the order a submission is inspected
+PROVIDERS = (
+    Provider('turnstile', 'cf-turnstile-response', 'TURNSTILE_SECRET',
+             verify_turnstile),
+    Provider('altcha', 'altcha', 'ALTCHA_HMAC_KEY', verify_altcha),
+    Provider('recaptcha', 'g-recaptcha-response', 'RECAPTCHA_SECRET',
+             verify_recaptcha),
+)
+
+# Form fields carrying a captcha response; never part of the ticket body
+FIELDS = tuple(provider.field for provider in PROVIDERS)
+
+
+def configured_providers():
+    """Names of the providers whose secret is set"""
+    return [provider.name for provider in PROVIDERS
+            if _setting(provider.secret)]
+
+
+def check_configuration():
+    """Validate the captcha settings at startup so typos fail fast"""
+    providers = configured_providers()
+    if not providers:
+        raise RuntimeError('No captcha provider is configured; set at least '
+                           'one of TURNSTILE_SECRET, ALTCHA_HMAC_KEY or '
+                           'RECAPTCHA_SECRET')
+    allowed_hostnames()
+    if 'altcha' in providers:
+        altcha_algorithm()
+    return providers
+
+
+def is_valid_captcha(request, controller):
+    """Verify the submission's captcha; returns (ok, provider name or None)"""
+    for provider in PROVIDERS:
+        token = request.form.get(provider.field)
+        if not token:
+            continue
+        if not _setting(provider.secret):
+            logger.warning('formsender: form posted %s but %s is not '
+                           'configured', provider.field, provider.secret)
+            return False, provider.name
+        valid = provider.verify(token, request.remote_addr, controller)
+        return bool(valid), provider.name
+    logger.warning('formsender: submission carried no captcha response')
+    return False, None

conf.py.dist

@@ -9,7 +9,21 @@ DUPLICATE_CHECK_TIME = 3600  # seconds -- 60 seconds * 60 minutes
 MAX_CONTENT_LENGTH = 10 * 1024 * 1024  # max upload size in bytes (10 MiB)
 HOST = "0.0.0.0"
 PORT = 5000
-RECAPTCHA_SECRET = os.environ['RECAPTCHA_SECRET']
+# Captcha providers; at least one secret is required. A submission may use any
+# provider configured here, so only set the ones your forms actually use.
+TURNSTILE_SECRET = os.environ.get('TURNSTILE_SECRET')  # Cloudflare Turnstile
+ALTCHA_HMAC_KEY = os.environ.get('ALTCHA_HMAC_KEY')  # ALTCHA (self-hosted)
+RECAPTCHA_SECRET = os.environ.get('RECAPTCHA_SECRET')  # Google reCAPTCHA
+# X-Forwarded-For hops to trust. 0 unless Formsender runs behind a proxy;
+# without it the captcha verifier is told the proxy's IP, not the sender's.
+TRUSTED_PROXY_COUNT = int(os.environ.get('TRUSTED_PROXY_COUNT') or 0)
+# Comma-separated hostnames a Turnstile/reCAPTCHA token must have been solved
+# on, e.g. "osuosl.org,www.osuosl.org". Unset disables the check.
+CAPTCHA_ALLOWED_HOSTNAMES = os.environ.get('CAPTCHA_ALLOWED_HOSTNAMES')
+RECAPTCHA_MIN_SCORE = 0.5  # only applies to reCAPTCHA v3 keys (0.0 - 1.0)
+ALTCHA_ALGORITHM = 'PBKDF2/SHA-256'  # proof-of-work key derivation function
+ALTCHA_COST = 5000  # KDF iterations per attempt; raise to make bots work harder
+ALTCHA_EXPIRES = 600  # seconds a challenge stays redeemable
 URL = os.environ.get('RT_URL', "https://support.osuosl.org/REST/2.0/")
 RT_TOKEN = os.environ['RT_TOKEN']
 SENTRY_URI = os.environ.get('SENTRY_URI')

docs/source/docker.rst

@@ -20,16 +20,24 @@ the latest image:
    $ docker pull ghcr.io/osuosl/formsender:master
    $ docker run -p 5000:5000 \
        -e TOKEN=s0m3T0k3n \
-       -e RECAPTCHA_SECRET=your-recaptcha-secret \
+       -e TURNSTILE_SECRET=your-turnstile-secret \
        -e RT_TOKEN=your-rt-token \
        -e RT_URL=https://support.example.org/REST/2.0/ \
        ghcr.io/osuosl/formsender:master
 
-``TOKEN``, ``RECAPTCHA_SECRET``, and ``RT_TOKEN`` are required. ``RT_URL`` is
+``TOKEN``, ``RT_TOKEN``, and at least one captcha secret (``TURNSTILE_SECRET``,
+``ALTCHA_HMAC_KEY`` or ``RECAPTCHA_SECRET``) are required. ``RT_URL`` is
 optional and defaults to ``https://support.osuosl.org/REST/2.0/``; set it to
-point a container at a different RT instance. ``SENTRY_URI`` is also optional.
+point a container at a different RT instance. ``SENTRY_URI``,
+``CAPTCHA_ALLOWED_HOSTNAMES`` and ``TRUSTED_PROXY_COUNT`` are also optional.
 See the :ref:`usage` documentation for the full list of settings.
 
+Two settings matter when more than one container serves the same site: set
+``TRUSTED_PROXY_COUNT=1`` behind a reverse proxy so the captcha provider sees
+the sender's IP address, and give every container the same
+``ALTCHA_HMAC_KEY``, because a container cannot verify a challenge another one
+signed.
+
 
 Build the Container
 -------------------
@@ -50,7 +58,7 @@ Run the image you just built the same way as the published one:
 
    $ docker run -p 5000:5000 \
        -e TOKEN=s0m3T0k3n \
-       -e RECAPTCHA_SECRET=your-recaptcha-secret \
+       -e TURNSTILE_SECRET=your-turnstile-secret \
        -e RT_TOKEN=your-rt-token \
        formsender
 

docs/source/errorcodes.rst

@@ -23,9 +23,12 @@ Error Number   Error Message               Cause
 3              Improper Form Submission    Honeypot was not empty, token was invalid, or fields_to_join referenced a missing field
 4              Too Many Requests           Number of submissions violated CEILING variable from conf.py
 5              Duplicate Request           This request is a duplicate of an earlier request
-6              Invalid Recaptcha           The reCAPTCHA response failed verification
+6              Invalid Captcha             The captcha response was missing or failed verification
 ============   ========================    =============================================================
 
+The captcha is checked before the duplicate check, so a submission rejected
+with error 6 can be corrected and sent again without tripping error 5.
+
 Two further error conditions are not returned as redirect error codes: a request
 body larger than ``MAX_CONTENT_LENGTH`` is rejected with an HTTP ``413`` error,
 and a malformed or empty POST renders a local error page with an HTTP ``400``
@@ -86,4 +89,4 @@ logged at ``DEBUG``:
   WARNING formsender: received Invalid Email: <submission-email> from <submission-email>
   WARNING formsender: received Invalid Name:  from <submission-email>
   WARNING formsender: received Improper Form Submission: <submission-name> from <submission-email>
-  WARNING formsender: received Invalid Recaptcha: <submission-name> from <submission-email>
+  WARNING formsender: received Invalid Captcha: <submission-name> from <submission-email>