crawl.yml

name: crawl

on:
  workflow_dispatch:
    inputs:
      dry_run:
        description: "Run without committing"
        type: boolean
        default: false
      limit:
        description: "Limit results per kind (0 = no limit)"
        type: string
        default: "0"
      kind:
        description: "Kind to crawl (empty = all)"
        type: choice
        options: ["", "whoami", "whois", "whatis"]
        default: ""

  # Uncomment to enable daily scheduled crawl at 03:00 UTC
  # schedule:
  #   - cron: "0 3 * * *"

permissions:
  contents: write

jobs:
  crawl:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: npm

      - name: Install
        run: npm ci

      - name: Build
        run: npm run build

      - name: Run crawler
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ARGS=""
          if [ "${{ inputs.dry_run }}" = "true" ]; then
            ARGS="$ARGS --dry-run"
          fi
          if [ "${{ inputs.limit }}" != "0" ] && [ -n "${{ inputs.limit }}" ]; then
            ARGS="$ARGS --limit ${{ inputs.limit }}"
          fi
          if [ -n "${{ inputs.kind }}" ]; then
            ARGS="$ARGS --kind ${{ inputs.kind }}"
          fi
          node dist/crawl.js $ARGS

      - name: Commit changes
        if: ${{ inputs.dry_run != 'true' }}
        run: |
          git config user.name "declared-md-bot"
          git config user.email "bot@declared-md.org"
          git add data/ invalid/
          if git diff --cached --quiet; then
            echo "No changes to commit"
            exit 0
          fi
          git commit -m "chore: crawl run $(date -u +%Y-%m-%dT%H:%M:%SZ)"
          git push