extra_tests.yml

# Syntax reference:
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions

name: Extra Tests
permissions: read-all

defaults:
  run:
    shell: bash

on:
  pull_request:
    types: [opened, synchronize]

env:
  CARGO_TERM_COLOR: always
  CC: 'clang'
  # When changing, search-and-replace the old value with the new value below,
  # particularly in `image` fields.
  #
  # It'd be nice to instead use `${{ env.CONTAINER }}`, but unfortunately it
  # doesn't work. The docs are a bit ambiguous - they show `env` as available
  # from `container`, but doesn't explicitly say anything about
  # `container.image`. We've verified experimentally that it *isn't* available
  # from there.
  # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability
  CONTAINER: 'ubuntu:22.04'
  BUILDTYPE: 'release'
  DEBIAN_FRONTEND: noninteractive

jobs:
  build_shadow:
    env:
      # used by cargo
      RUSTC_WRAPPER: sccache
      SCCACHE_CACHE_SIZE: 2G
      SCCACHE_DIR: /home/runner/.cache/sccache
      # sccache doesn't work on incrementally compiled crates, and incremental
      # compilation doesn't help us here anyway.
      CARGO_INCREMENTAL: false
      ## Increment to force eviction of previous caches, e.g. if the sccache configuration
      ## or usage has changed in a way to make the previous cache not useful.
      ## Keep in sync with same variable in `run_tests.yml`.
      SCCACHE_CACHE_VERSION: 1
    # runs in a container; just use the latest base image.
    runs-on: ubuntu-latest
    container:
      # Should match env.CONTAINER.
      image: ubuntu:22.04
    steps:
      - run: apt-get update
      - name: Checkout shadow
        uses: actions/checkout@v5
        with:
          path: shadow
          # Run on PR head instead of merge result. Running on the merge
          # result can give confusing results, and we require PR to be up to
          # date with target branch before merging, anyway.
          # See https://github.com/shadow/shadow/issues/2166
          ref: ${{ github.event.pull_request.head.sha }}
          persist-credentials: false
      - name: Set Rust toolchain
        run: ln -s shadow/ci/rust-toolchain-stable.toml rust-toolchain.toml
      - name: Install dependencies
        run: |
          cd shadow
          . ci/container_scripts/install_deps.sh
          . ci/container_scripts/install_extra_deps.sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Get month
        id: get-month
        run: |
          echo "month=$(/bin/date -u "+%Y%m")" >> $GITHUB_OUTPUT
      - name: Restore cargo registry cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: cargo-registry-${{ steps.get-month.outputs.month }}-${{ hashFiles('shadow/src/Cargo.lock') }}
          restore-keys: |
            cargo-registry-${{ steps.get-month.outputs.month }}
      - name: Get rust version
        id: get-rustv
        run: |
          echo rustv=\"$(rustc --version)\" >> $GITHUB_OUTPUT
      - name: Restore sccache cache
        uses: actions/cache@v4
        continue-on-error: false
        with:
          path: ${{ env.SCCACHE_DIR }}
          key: sccache-${{ steps.get-month.outputs.month }}-${{ steps.get-rustv.outputs.rustv }}-${{ env.CONTAINER }}-${{ env.CC }}-${{ env.BUILDTYPE }}-${{ env.SCCACHE_CACHE_VERSION}}-${{ hashFiles('shadow/src/Cargo.lock') }}
          restore-keys: |
               sccache-${{ steps.get-month.outputs.month }}-${{ steps.get-rustv.outputs.rustv }}-${{ env.CONTAINER }}-${{ env.CC }}-${{ env.BUILDTYPE }}-${{ env.SCCACHE_CACHE_VERSION}}-
      - name: Install sccache
        env:
          LINK: https://github.com/mozilla/sccache/releases/download
          SCCACHE_VERSION: v0.3.1
        run: |
          apt-get install -y curl
          SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
          mkdir -p $HOME/.local/bin
          curl -v -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
          mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
          chmod +x $HOME/.local/bin/sccache
          echo "$HOME/.local/bin" >> $GITHUB_PATH
      - name: Start sccache server
        run: sccache --start-server
      - name: Build shadow
        run: |
          cd shadow
          . ci/container_scripts/build_and_install.sh
      - name: Print sccache stats
        run: sccache --show-stats
      - name: Stop sccache server
        run: sccache --stop-server || true
      # We need to wrap in a tarball to preserve permissions.
      # We're grabbing the source directory and a lot of the build
      # directory here, since there are scripts and generated
      # outputs we reference in other jobs.
      - name: Archive shadow
        run: |
          TARFILE=shadow.tar
          tar -cf $TARFILE --files-from /dev/null
          find shadow \
            -name target -prune \
            -or -name '*.o' -prune \
            -or -print | xargs tar --no-recursion --append -f $TARFILE
          # Above command skips the whole target directories, but we
          # need the shim library, which is in one.
          # Add it explicitly:
          tar --no-recursion --append -f $TARFILE \
            shadow/build/src/shim/target/*/libshadow_shim.so
          gzip $TARFILE
      - uses: actions/upload-artifact@v4
        with:
          name: build-shadow
          path: shadow.tar.gz
          # This is meant primarily to be used by the other jobs in this
          # workflow. We don't typically need it for debugging.
          retention-days: 1

  build_tgen:
    # runs in a container; just use the latest base image.
    runs-on: ubuntu-latest
    container:
      # Should match env.CONTAINER.
      image: 'ubuntu:22.04'
    steps:
      - name: Restore cache
        id: restore-tgen-build-cache
        uses: actions/cache@v4
        with:
          path: opt/tgen
          # Sync with env.CONTAINER, and with checkout ref below.
          key: tgen-build-key-${{ env.CONTAINER }}-816d68cd3d0ff7d0ec71e8bbbae24ecd6a636117
      - name: Checkout tgen
        if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
        uses: actions/checkout@v5
        with:
          path: tgen
          repository: shadow/tgen
          # When updating, change cache key in "Restore cache" step above
          ref: 816d68cd3d0ff7d0ec71e8bbbae24ecd6a636117
          persist-credentials: false
      - name: Install dependencies
        if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
        run: |
          apt-get update
          apt-get install -y \
            automake \
            cmake \
            $CC \
            libevent-dev \
            libssl-dev \
            zlib1g-dev \
            libglib2.0-dev \
            libigraph-dev
      - name: Build tgen
        if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
        run: |
          cd tgen
          mkdir build && cd build
          cmake .. -DCMAKE_INSTALL_PREFIX="$GITHUB_WORKSPACE/opt/tgen"
          make -j$(nproc)
      - name: Install tgen
        if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
        run: |
          cd tgen/build
          make install
      # We need to wrap in a tarball to preserve permissions.
      - name: Archive tgen
        run: tar -czf tgen.tar.gz opt/tgen
      - uses: actions/upload-artifact@v4
        with:
          name: build-tgen
          path: tgen.tar.gz
          # This is meant primarily to be used by the other jobs in this
          # workflow. We don't typically need it for debugging.
          retention-days: 1

  tor:
    needs:
      - build_shadow
      - build_tgen

    # runs in a container; just use the latest base image.
    runs-on: ubuntu-latest

    container:
      # Should match env.CONTAINER.
      image: 'ubuntu:22.04'
      # the default shm-size for ubuntu:18.04, but with the size increased from
      # 65536k. github's default docker seccomp policy seems to disallow
      # process_vm_readv and process_vm_writev; disable it altogether. See
      # https://docs.docker.com/engine/security/seccomp/
      options: '--shm-size=1024g --security-opt seccomp=unconfined'

    strategy:
      matrix:
        # when changing the tor versions, make sure to also update the
        # required CI tests in the GitHub repository settings
        tor: ['tor-0.4.8.18']

    steps:
      - run: apt-get update

      - name: Download shadow
        uses: actions/download-artifact@v5
        with:
          name: build-shadow

      - name: Extract shadow
        run: tar xzf shadow.tar.gz

      - name: Install shadow runtime dependencies
        run: apt-get install -y libglib2.0-0

      - name: Download tgen
        uses: actions/download-artifact@v5
        with:
          name: build-tgen
          path: .

      - name: Extract tgen
        run: tar xzf tgen.tar.gz

      - name: Put tgen in path
        run: echo "$GITHUB_WORKSPACE/opt/tgen/bin" >> $GITHUB_PATH

      - name: Install tgen runtime dependencies
        run: |
          apt-get install -y \
            libglib2.0-dev \
            libigraph-dev

      - name: Restore tor build cache
        id: restore-tor-build-cache
        uses: actions/cache@v4
        with:
          path: opt/tor
          # sync with env.CONTAINER
          key: tor-build-keyv3-${{ env.CONTAINER }}-${{ matrix.tor }}

      - name: Install tor build deps
        if: steps.restore-tor-build-cache.outputs.cache-hit != 'true'
        run: |
          apt-get install -y \
            autoconf \
            automake \
            $CC \
            libevent-dev \
            libssl-dev \
            make \
            zlib1g-dev \
            git

      - name: Checkout tor
        if: steps.restore-tor-build-cache.outputs.cache-hit != 'true'
        run: |
          git clone https://gitlab.torproject.org/tpo/core/tor.git
          cd tor
          git checkout ${{ matrix.tor }}

      - name: Configure, build, and install tor
        if: steps.restore-tor-build-cache.outputs.cache-hit != 'true'
        run: |
          cd tor
          ./autogen.sh
          ./configure --disable-asciidoc --disable-unittests --prefix="$GITHUB_WORKSPACE/opt/tor"
          make -j$(nproc)
          make install

      - name: Install tor runtime deps
        run: |
          apt-get install -y \
            libevent-dev \
            libssl-dev \
            zlib1g

      - name: Install obfs4proxy
        run: apt-get install -y obfs4proxy

      - name: Put tor in path
        run: echo "$GITHUB_WORKSPACE/opt/tor/bin" >> $GITHUB_PATH

      - name: Install setup script dependencies
        run: apt-get install -y cmake

      - name: Test
        run: |
          cd shadow
          ./setup test -- --build-config extra --label-regex tor
      - name: Last 200 log lines
        if: failure()
        run: |
          tail -n 200 shadow/build/Testing/Temporary/LastTest.log

      - name: Compress logs
        if: failure()
        run: |
          shopt -s globstar
          tar -cJf shadow/build/Testing/Temporary{.tar.xz,/}
          for f in shadow/build/src/test/tor/**/*.data; do tar -cJf "$f.tar.xz" "$f/"; done

      - name: Upload shadow data directory
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.tor }}-shadow-data-dir
          path: shadow/build/src/test/tor/**/*.data.tar.xz

      - name: Upload shadow log file
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.tor }}-shadow-log-file
          path: shadow/build/Testing/Temporary.tar.xz

  tgen:
    needs:
      - build_shadow
      - build_tgen

    # runs in a container; just use the latest base image.
    runs-on: ubuntu-latest

    container:
      # Should match env.CONTAINER.
      image: 'ubuntu:22.04'
      # the default shm-size for ubuntu:18.04, but with the size increased from
      # 65536k. github's default docker seccomp policy seems to disallow
      # process_vm_readv and process_vm_writev; disable it altogether. See
      # https://docs.docker.com/engine/security/seccomp/
      options: '--shm-size=1024g --security-opt seccomp=unconfined'

    steps:
      - run: apt-get update

      - name: Download shadow
        uses: actions/download-artifact@v5
        with:
          name: build-shadow

      - name: Extract shadow
        run: tar xzf shadow.tar.gz

      - name: Install shadow runtime dependencies
        run: apt-get install -y libglib2.0-0

      - name: Download tgen
        uses: actions/download-artifact@v5
        with:
          name: build-tgen
          path: .

      - name: Extract tgen
        run: tar xzf tgen.tar.gz

      - name: Put tgen in path
        run: echo "$GITHUB_WORKSPACE/opt/tgen/bin" >> $GITHUB_PATH

      - name: Install tgen runtime dependencies
        run: |
          apt-get install -y \
            libglib2.0-dev \
            libigraph-dev

      - name: Install setup script dependencies
        run: apt-get install -y cmake

      - name: Test
        run: |
          cd shadow
          ./setup test -- --build-config extra --label-regex tgen

      - name: Last 200 log lines
        if: failure()
        run: |
          tail -n 200 shadow/build/Testing/Temporary/LastTest.log

      - name: Compress logs
        if: failure()
        run: |
          shopt -s globstar
          tar -cJf shadow/build/Testing/Temporary{.tar.xz,/}
          for f in shadow/build/src/test/tgen/**/*.data; do tar -cJf "$f.tar.xz" "$f/"; done

      - name: Upload shadow data directory
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: tgen-shadow-data-dir
          path: shadow/build/src/test/tgen/**/*.data.tar.xz

      - name: Upload shadow log file
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: tgen-shadow-log-file
          path: shadow/build/Testing/Temporary.tar.xz

  examples:
    needs:
      - build_shadow

    # runs in a container; just use the latest base image.
    runs-on: ubuntu-latest

    container:
      # Should match env.CONTAINER.
      image: 'ubuntu:22.04'
      # the default shm-size for ubuntu:18.04, but with the size increased from
      # 65536k. github's default docker seccomp policy seems to disallow
      # process_vm_readv and process_vm_writev; disable it altogether. See
      # https://docs.docker.com/engine/security/seccomp/
      options: '--shm-size=1024g --security-opt seccomp=unconfined'

    steps:
      - run: apt-get update

      - name: Download shadow
        uses: actions/download-artifact@v5
        with:
          name: build-shadow

      - name: Extract shadow
        run: tar xzf shadow.tar.gz

      - name: Install shadow runtime dependencies
        run: apt-get install -y libglib2.0-0

      - name: Install example dependencies
        run: |
          # if we need to build C code, we want to do so with the default compiler
          unset CC
          cd shadow

          # We install some dependencies in ~/.local/bin.
          # Put it on the path to faciliate checking if a dependency is already
          # installed.
          PATH="$PATH:$HOME/.local/bin"

          # dependencies for examples
          ./examples/apps/curl/install_deps.sh
          ./examples/apps/etcd/install_deps.sh
          ./examples/apps/http-server/install_deps.sh
          ./examples/apps/iperf-2/install_deps.sh
          ./examples/apps/jetty/install_deps.sh
          ./examples/apps/nginx/install_deps.sh
          ./examples/apps/wget2/install_deps.sh
          ./examples/docs/basic-file-transfer/install_deps.sh
          ./examples/docs/traffic-generation/install_deps.sh
          ./examples/docs/tor/install_deps.sh

      - name: Install setup script dependencies
        run: apt-get install -y cmake

      - name: Test
        run: |
          cd shadow
          PATH="$PATH:$HOME/.local/bin"
          ./setup test -- --build-config extra --label-regex example

      - name: Last 200 log lines
        if: failure()
        run: |
          tail -n 200 shadow/build/Testing/Temporary/LastTest.log

      - name: Compress logs
        if: failure()
        run: |
          shopt -s globstar
          tar -cJf shadow/build/Testing/Temporary{.tar.xz,/}
          for f in shadow/build/examples/**/*.data; do tar -cJf "$f.tar.xz" "$f/"; done

      - name: Upload shadow data directory
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: examples-shadow-data-dir
          path: shadow/build/examples/**/*.data.tar.xz

      - name: Upload shadow log file
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: examples-shadow-log-file
          path: shadow/build/Testing/Temporary.tar.xz