ci: auto-sort and validate systems.csv on PRs

.github/workflows/systems_csv_pr_checks.yml

@@ -0,0 +1,76 @@
+name: systems.csv PR checks
+
+# Runs whenever a pull request adds or changes systems.csv.
+#   1. sort              - re-sort the file and push the result as a separate commit
+#                          (same-repo PR branches only; forks are skipped, see below).
+#   2. validate-structure- fail on malformed rows / missing required values.
+#   3. check-new-urls    - fail if any newly added URL does not return HTTP 200-299.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - systems.csv
+
+# Cancel superseded runs when new commits are pushed to the same PR.
+concurrency:
+  group: systems-csv-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  sort:
+    name: Sort systems.csv
+    runs-on: ubuntu-latest
+    # The default GITHUB_TOKEN cannot push to a forked repository, so auto-sorting
+    # is limited to branches in this repo. Fork PRs are left untouched here and get
+    # sorted when master is next updated.
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Sort systems.csv
+        run: bash scripts/sort-systems-csv.sh systems.csv
+
+      - name: Commit sorted file if it changed
+        run: |
+          if git diff --quiet -- systems.csv; then
+            echo "systems.csv already sorted; nothing to commit."
+            exit 0
+          fi
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add systems.csv
+          git commit -m "chore: sort systems.csv"
+          git push origin "HEAD:${{ github.head_ref }}"
+
+  validate-structure:
+    name: Validate structure
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Validate systems.csv structure
+        run: node scripts/validate-systems-csv.js systems.csv
+
+  check-new-urls:
+    name: Check new URLs return 2xx
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Extract base systems.csv
+        run: |
+          git show "${{ github.event.pull_request.base.sha }}:systems.csv" > /tmp/systems.base.csv 2>/dev/null \
+            || : > /tmp/systems.base.csv
+      - name: Check newly added URLs
+        run: node scripts/check-new-urls.js /tmp/systems.base.csv systems.csv

README.md

@@ -138,7 +138,15 @@ If you would like to add a system, please fork this repository and submit a Pull
 - Create a new branch, and
 - Propose your changes by opening a [new pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests)
 
-Please keep this list alphabetized by country and system name. Alternatively, fill out [this contribution form](https://share.mobilitydata.org/gbfs-feed-contribution-form) for a Github-less contribution. 
+This list is kept sorted alphabetically, first by **Country Code** and then by **Name** (the two leftmost columns). You don't need to insert your new row in exactly the right place: when you open a pull request that changes `systems.csv`, an automated GitHub Action re-sorts the file and pushes the sorted result back to your branch as a separate `chore: sort systems.csv` commit. (This runs for pull requests opened from a branch in this repository. If you are contributing from a fork, the automatic commit is skipped — you can optionally sort the file yourself, see below.)
+
+Technically, the sort keeps the header row in place and orders the remaining rows with GNU `sort` using a comma field separator and two sort keys — column 1 (Country Code), then column 2 (Name) — under the `en_US.UTF-8` locale. It is equivalent to:
+
+```bash
+(head -n 1; LC_ALL=en_US.UTF-8 sort --field-separator=',' --key=1,1 --key=2,2) < systems.csv
+```
+
+To reproduce the exact ordering locally, run `./scripts/sort-systems-csv.sh` (on macOS, install GNU coreutils with `brew install coreutils` and run `GNUSORT=gsort ./scripts/sort-systems-csv.sh`). Alternatively, fill out [this contribution form](https://share.mobilitydata.org/gbfs-feed-contribution-form) for a Github-less contribution.
 * [systems.csv](systems.csv)
 
  Field Name | REQUIRED | Definition 

scripts/check-new-urls.js

@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+'use strict';
+//
+// Check that URLs *newly added* in this PR return an HTTP status in the 200-299 range.
+//
+// "Newly added" = a URL value present in the head systems.csv that was not present
+// anywhere in the base systems.csv. This deliberately ignores pre-existing URLs so a
+// PR is never failed for a dead link it did not introduce.
+//
+// Both URL columns are checked: "URL" (operator homepage) and "Auto-Discovery URL"
+// (the GBFS feed). Redirects are followed; the FINAL status must be 2xx.
+//
+// Usage: node scripts/check-new-urls.js <baseCsv> <headCsv>
+//   baseCsv : systems.csv from the PR base (may be empty/absent if the file is new)
+//   headCsv : systems.csv from the PR head (default: systems.csv)
+//
+const fs = require('fs');
+const http = require('http');
+const https = require('https');
+const { URL } = require('url');
+
+const baseFile = process.argv[2] || '';
+const headFile = process.argv[3] || 'systems.csv';
+
+const URL_COLUMNS = ['URL', 'Auto-Discovery URL'];
+const TIMEOUT_MS = 20000;
+const MAX_REDIRECTS = 5;
+const CONCURRENCY = 6;
+const USER_AGENT =
+  'Mozilla/5.0 (compatible; gbfs.org-ci/1.0; +https://github.com/MobilityData/gbfs)';
+
+// --- CSV parsing (same shape as validate-systems-csv.js) -------------------
+function parseCSV(text) {
+  const records = [];
+  let field = '';
+  let fields = [];
+  let inQuotes = false;
+  let i = 0;
+  let hasContent = false;
+
+  const pushField = () => { fields.push(field); field = ''; };
+  const pushRecord = () => { records.push(fields); fields = []; hasContent = false; };
+
+  while (i < text.length) {
+    const c = text[i];
+    if (inQuotes) {
+      if (c === '"') {
+        if (text[i + 1] === '"') { field += '"'; i += 2; continue; }
+        inQuotes = false; i++; continue;
+      }
+      field += c; i++; continue;
+    }
+    if (c === '"') { inQuotes = true; hasContent = true; i++; continue; }
+    if (c === ',') { pushField(); i++; continue; }
+    if (c === '\r') { i++; continue; }
+    if (c === '\n') { pushField(); pushRecord(); i++; continue; }
+    field += c; hasContent = true; i++;
+  }
+  if (hasContent || field.length > 0 || fields.length > 0) { pushField(); pushRecord(); }
+  return records;
+}
+
+// Return the set of URL values (from URL_COLUMNS) found in a CSV string.
+function urlsFrom(text) {
+  const urls = new Set();
+  if (!text || !text.trim()) return urls;
+  const records = parseCSV(text);
+  if (records.length === 0) return urls;
+  const header = records[0];
+  const idx = URL_COLUMNS.map((name) => header.indexOf(name)).filter((i) => i >= 0);
+  for (let r = 1; r < records.length; r++) {
+    const row = records[r];
+    for (const c of idx) {
+      const v = (row[c] || '').trim();
+      if (v) urls.add(v);
+    }
+  }
+  return urls;
+}
+
+// --- HTTP check ------------------------------------------------------------
+// Resolve one URL, following redirects, resolving to { url, status, ok, error }.
+function checkOnce(rawUrl, redirectsLeft) {
+  return new Promise((resolve) => {
+    let target;
+    try {
+      target = new URL(rawUrl);
+    } catch (e) {
+      resolve({ url: rawUrl, status: null, ok: false, error: `invalid URL: ${e.message}` });
+      return;
+    }
+    if (target.protocol !== 'http:' && target.protocol !== 'https:') {
+      resolve({ url: rawUrl, status: null, ok: false, error: `unsupported protocol: ${target.protocol}` });
+      return;
+    }
+
+    const lib = target.protocol === 'https:' ? https : http;
+    const req = lib.request(
+      target,
+      {
+        method: 'GET',
+        headers: { 'User-Agent': USER_AGENT, Accept: '*/*' },
+        timeout: TIMEOUT_MS,
+      },
+      (res) => {
+        const status = res.statusCode;
+        // Follow redirects to a final status.
+        if (status >= 300 && status < 400 && res.headers.location && redirectsLeft > 0) {
+          res.resume(); // discard body
+          let next;
+          try {
+            next = new URL(res.headers.location, target).toString();
+          } catch (e) {
+            resolve({ url: rawUrl, status, ok: false, error: `bad redirect target: ${res.headers.location}` });
+            return;
+          }
+          resolve(checkOnce(next, redirectsLeft - 1));
+          return;
+        }
+        res.resume(); // drain so the socket can be freed
+        resolve({ url: rawUrl, status, ok: status >= 200 && status <= 299, error: null });
+      }
+    );
+
+    req.on('timeout', () => { req.destroy(new Error(`timeout after ${TIMEOUT_MS}ms`)); });
+    req.on('error', (e) => {
+      resolve({ url: rawUrl, status: null, ok: false, error: e.message });
+    });
+    req.end();
+  });
+}
+
+// Check a URL with one retry on transient failure (network error / timeout).
+async function checkUrl(rawUrl) {
+  let result = await checkOnce(rawUrl, MAX_REDIRECTS);
+  if (!result.ok && result.status === null) {
+    result = await checkOnce(rawUrl, MAX_REDIRECTS);
+  }
+  return result;
+}
+
+async function runPool(items, worker, size) {
+  const results = new Array(items.length);
+  let next = 0;
+  async function drain() {
+    while (next < items.length) {
+      const cur = next++;
+      results[cur] = await worker(items[cur]);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(size, items.length) }, drain));
+  return results;
+}
+
+async function main() {
+  const headText = fs.readFileSync(headFile, 'utf8');
+  let baseText = '';
+  if (baseFile && fs.existsSync(baseFile)) {
+    baseText = fs.readFileSync(baseFile, 'utf8');
+  }
+
+  const baseUrls = urlsFrom(baseText);
+  const headUrls = urlsFrom(headText);
+  const newUrls = [...headUrls].filter((u) => !baseUrls.has(u)).sort();
+
+  if (newUrls.length === 0) {
+    console.log('No newly added URLs to check.');
+    process.exit(0);
+  }
+
+  console.log(`Checking ${newUrls.length} newly added URL(s) for HTTP 200-299...\n`);
+  const results = await runPool(newUrls, checkUrl, CONCURRENCY);
+
+  const failed = [];
+  for (const r of results) {
+    if (r.ok) {
+      console.log(`  ok   ${r.status}  ${r.url}`);
+    } else {
+      failed.push(r);
+      const detail = r.status !== null ? `HTTP ${r.status}` : r.error;
+      console.log(`  FAIL ${detail}  ${r.url}`);
+    }
+  }
+
+  if (failed.length > 0) {
+    console.log('');
+    for (const r of failed) {
+      const detail = r.status !== null ? `returned HTTP ${r.status}` : `error: ${r.error}`;
+      console.log(`::error::New URL ${detail}: ${r.url}`);
+    }
+    console.log(`\n${failed.length} of ${newUrls.length} new URL(s) did not return 200-299.`);
+    process.exit(1);
+  }
+
+  console.log(`\nAll ${newUrls.length} new URL(s) returned 200-299.`);
+  process.exit(0);
+}
+
+main().catch((e) => {
+  console.error(`::error::check-new-urls.js failed: ${e.stack || e.message}`);
+  process.exit(2);
+});

scripts/sort-systems-csv.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# Sort systems.csv in place: keep the header row, then sort the remaining rows
+# by column 1 (Country Code) and column 2 (Name), comma-separated, en_US.UTF-8.
+#
+# Mirrors the canonical one-liner:
+#   (head -n 1; LC_ALL=en_US.UTF-8 sort --field-separator=',' --key=1,1 --key=2,2) < systems.csv
+#
+# On Linux / GitHub Actions runners, `sort` is GNU sort, so no override is needed.
+# On macOS, install GNU coreutils (`brew install coreutils`) and run with:
+#   GNUSORT=gsort ./scripts/sort-systems-csv.sh
+#
+# Usage:
+#   ./scripts/sort-systems-csv.sh [path-to-csv]   # sort in place (default: systems.csv)
+#   ./scripts/sort-systems-csv.sh --check [path]  # exit 1 if the file is not already sorted
+#
+set -euo pipefail
+
+CHECK=0
+if [[ "${1:-}" == "--check" ]]; then
+  CHECK=1
+  shift
+fi
+
+CSV="${1:-systems.csv}"
+
+# GNU sort binary. Override with GNUSORT=gsort on macOS.
+SORT_BIN="${GNUSORT:-sort}"
+
+if [[ ! -f "$CSV" ]]; then
+  echo "error: file not found: $CSV" >&2
+  exit 2
+fi
+
+sorted="$(mktemp)"
+trap 'rm -f "$sorted"' EXIT
+
+{
+  head -n 1 "$CSV"
+  tail -n +2 "$CSV" | LC_ALL=en_US.UTF-8 "$SORT_BIN" --field-separator=',' --key=1,1 --key=2,2
+} > "$sorted"
+
+if [[ "$CHECK" -eq 1 ]]; then
+  if cmp -s "$CSV" "$sorted"; then
+    echo "$CSV is already sorted."
+    exit 0
+  fi
+  echo "error: $CSV is not sorted. Run ./scripts/sort-systems-csv.sh to fix it." >&2
+  diff -u "$CSV" "$sorted" || true
+  exit 1
+fi
+
+if cmp -s "$CSV" "$sorted"; then
+  echo "$CSV is already sorted; no changes."
+else
+  cat "$sorted" > "$CSV"
+  echo "Sorted $CSV."
+fi