ci.yml

name: CI/CD Pipeline

on:
  push:
    branches: [ main, develop ]
    tags:
      - 'v*'
  pull_request:
    branches: [ main, develop ]
  schedule:
    # Run weekly on Monday at 00:00 UTC
    - cron: '0 0 * * 1'

env:
  PYTHON_DEFAULT_VERSION: '3.11'

jobs:
  # Job 1: Code Quality Checks
  code-quality:
    name: Code Quality
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
        cache: 'pip'
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install black flake8 isort mypy pylint bandit safety
    
    - name: Check code formatting with Black
      run: |
        echo "::group::Black Format Check"
        black --check snapflow tests
        echo "::endgroup::"
    
    - name: Check import sorting with isort
      run: |
        echo "::group::isort Import Check"
        isort --check-only --profile black snapflow tests
        echo "::endgroup::"
    
    - name: Lint with flake8
      run: |
        echo "::group::Flake8 Linting"
        flake8 snapflow tests --max-line-length=100 --extend-ignore=E203,W503 --statistics
        echo "::endgroup::"
    
    - name: Type check with mypy
      run: |
        echo "::group::MyPy Type Checking"
        mypy snapflow --ignore-missing-imports --no-strict-optional
        echo "::endgroup::"
      continue-on-error: true
    
    - name: Lint with pylint
      run: |
        echo "::group::Pylint Analysis"
        pylint snapflow --max-line-length=100 --disable=C0111,C0103,R0903 || true
        echo "::endgroup::"
      continue-on-error: true
    
    - name: Security check with Bandit
      run: |
        echo "::group::Bandit Security Scan"
        bandit -r snapflow -f json -o bandit-report.json || true
        bandit -r snapflow
        echo "::endgroup::"
      continue-on-error: true
    
    - name: Check dependencies for vulnerabilities
      run: |
        echo "::group::Safety Dependency Check"
        safety check --json || true
        echo "::endgroup::"
      continue-on-error: true

  # Job 2: Unit Tests
  test:
    name: Test on ${{ matrix.os }} - Python ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    needs: code-quality
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        exclude:
          # Exclude some combinations to reduce CI time
          - os: macos-latest
            python-version: '3.8'
          - os: macos-latest
            python-version: '3.9'
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    
    - name: Display Python and pip info
      run: |
        python --version
        python -m pip --version
        python -m pip list
      shell: bash
    
    - name: Install dependencies
      run: |
        echo "Installing pip..."
        python -m pip install --upgrade pip || exit 1
        echo "Installing requirements-dev.txt..."
        python -m pip install -r requirements-dev.txt || exit 1
        echo "Installing package in editable mode..."
        python -m pip install -e . || exit 1
        echo "Installation complete!"
      shell: bash
    
    - name: Install database drivers (Unix)
      if: runner.os != 'Windows'
      run: |
        echo "Installing database drivers for Unix..."
        python -m pip install psycopg2-binary>=2.9.0 PyMySQL>=1.0.0
      shell: bash
    
    - name: Install database drivers (Windows)
      if: runner.os == 'Windows'
      run: |
        echo "Installing database drivers for Windows..."
        python -m pip install PyMySQL>=1.0.0
        echo "Skipping psycopg2-binary on Windows (not needed for tests)"
      shell: bash
    
    - name: Verify pytest installation
      run: |
        echo "Verifying pytest installation..."
        python -m pip show pytest
        python -c "import pytest; print(f'pytest version: {pytest.__version__}')"
      shell: bash
    
    - name: Run tests with pytest
      run: |
        python -m pytest -v --cov=snapflow --cov-report=xml --cov-report=term-missing --cov-report=html
      shell: bash
    
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4
      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
      with:
        file: ./coverage.xml
        flags: unittests
        name: snapflow-${{ matrix.os }}-${{ matrix.python-version }}
        fail_ci_if_error: false
    
    - name: Upload test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: test-results-${{ matrix.os }}-${{ matrix.python-version }}
        path: |
          htmlcov/
          coverage.xml
        retention-days: 30

  # Job 3: Build Package
  build:
    name: Build Distribution
    runs-on: ubuntu-latest
    needs: test
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
        cache: 'pip'
    
    - name: Install build dependencies
      run: |
        python -m pip install --upgrade pip
        pip install build twine wheel
    
    - name: Build package
      run: |
        python -m build
    
    - name: Check package with twine
      run: |
        twine check dist/*
    
    - name: Upload distribution artifacts
      uses: actions/upload-artifact@v4
      with:
        name: python-packages
        path: dist/
        retention-days: 30

  # Job 4: Documentation Check
  docs:
    name: Documentation
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
    
    - name: Check README renders properly
      run: |
        pip install 'readme-renderer[md]'
        python -m readme_renderer README.md > /dev/null
      continue-on-error: true
    
    - name: Check links in documentation
      uses: gaurav-nelson/github-action-markdown-link-check@v1
      with:
        use-quiet-mode: 'yes'
        config-file: '.github/workflows/markdown-link-check-config.json'
      continue-on-error: true

  # Job 5: Security Audit
  security:
    name: Security Audit
    runs-on: ubuntu-latest
    needs: code-quality
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Run Trivy vulnerability scanner
      uses: aquasecurity/trivy-action@master
      with:
        scan-type: 'fs'
        scan-ref: '.'
        format: 'sarif'
        output: 'trivy-results.sarif'
      continue-on-error: true
    
    - name: Upload Trivy results to GitHub Security
      uses: github/codeql-action/upload-sarif@v3
      if: always()
      with:
        sarif_file: 'trivy-results.sarif'
      continue-on-error: true

  # Job 6: Release (only on tags)
  release:
    name: Release to PyPI
    runs-on: ubuntu-latest
    needs: [test, build]
    if: startsWith(github.ref, 'refs/tags/v')
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
    
    - name: Download distribution artifacts
      uses: actions/download-artifact@v4
      with:
        name: python-packages
        path: dist/
    
    - name: Publish to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets.PYPI_API_TOKEN }}
        skip-existing: true
      continue-on-error: true
    
    - name: Create GitHub Release
      uses: softprops/action-gh-release@v1
      with:
        files: dist/*
        generate_release_notes: true
        draft: false
        prerelease: false
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  # Job 7: Performance Benchmarking
  benchmark:
    name: Performance Benchmark
    runs-on: ubuntu-latest
    needs: test
    if: github.event_name == 'pull_request'
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e .
        pip install pytest-benchmark
    
    - name: Run benchmark tests
      run: |
        echo "Benchmark tests would run here"
        echo "python -m pytest tests/benchmark/ --benchmark-only"
      continue-on-error: true

  # Job 8: Notification
  notify:
    name: Notify Build Status
    runs-on: ubuntu-latest
    needs: [code-quality, test, build, docs, security]
    if: always()
    
    steps:
    - name: Check build status
      run: |
        echo "Build completed!"
        echo "Code Quality: ${{ needs.code-quality.result }}"
        echo "Tests: ${{ needs.test.result }}"
        echo "Build: ${{ needs.build.result }}"
        echo "Docs: ${{ needs.docs.result }}"
        echo "Security: ${{ needs.security.result }}"