daily-benchmark.yml

name: Daily Memory Tracker Benchmark

on:
  schedule:
    # Run daily at 23:00 UTC (EOD) to pick up all commits from the day
    - cron: '0 23 * * *'
  workflow_dispatch:
    inputs:
      target_date:
        description: 'Date to get commits from (YYYY-MM-DD, defaults to today)'
        required: false
        type: string
      environment_id:
        description: 'Environment ID'
        required: false
        default: 'gh_actions'
      server_url:
        description: 'Memory tracker server URL'
        required: false
        default: 'https://memory.python.org'
      cpython_repo:
        description: 'CPython repository URL'
        required: false
        default: 'https://github.com/python/cpython.git'
      llvm:
        description: 'LLVM version to use for JIT builds'
        required: false
        default: '19'

jobs:
  benchmark-builds:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        build_config:
          - binary_id: 'default'
            configure_flags: '-C'
            description: 'Default build'
            install_deps: 'standard'
          - binary_id: 'debug'
            configure_flags: '--with-pydebug'
            description: 'Debug build'
            install_deps: 'standard'
          - binary_id: 'jit'
            configure_flags: '--enable-experimental-jit'
            description: 'JIT build'
            install_deps: 'jit'
          - binary_id: 'lto-pgo'
            configure_flags: '--with-lto --enable-optimizations'
            description: 'LTO-PGO build'
            install_deps: 'standard'
          - binary_id: 'nogil'
            configure_flags: '--disable-gil'
            description: 'Free-threaded build'
            install_deps: 'standard'
      fail-fast: false
    
    steps:
    - name: Checkout memory tracker
      uses: actions/checkout@v4
      
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'
        
    - name: Clone CPython repository and get commits
      run: |
        git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython
        cd cpython
        git fetch --all
        
        # Determine target date
        if [ -n "${{ github.event.inputs.target_date }}" ]; then
          TARGET_DATE="${{ github.event.inputs.target_date }}"
        else
          TARGET_DATE=$(date -u +%Y-%m-%d)
        fi
        
        echo "Getting commits from date: $TARGET_DATE"
        
        # Get the last commit from the target date
        LAST_COMMIT=$(git log --since="$TARGET_DATE 00:00:00 UTC" --until="$TARGET_DATE 23:59:59 UTC" --pretty=format:"%H" -n 1)
        
        if [ -z "$LAST_COMMIT" ]; then
          echo "No commits found for date $TARGET_DATE"
          exit 1
        else
          echo "Found last commit for date $TARGET_DATE: $LAST_COMMIT"
          echo "COMMIT_RANGE=$LAST_COMMIT" >> $GITHUB_ENV
          echo "COMMIT_COUNT=1" >> $GITHUB_ENV
          
          echo "Commit to benchmark: $LAST_COMMIT"
        fi
        
    - name: Print environment variables
      run: |
        echo "=== Environment Variables ==="
        echo "COMMIT_RANGE: $COMMIT_RANGE"
        echo "COMMIT_COUNT: $COMMIT_COUNT"
        echo "Binary ID: ${{ matrix.build_config.binary_id }}"
        echo "Description: ${{ matrix.build_config.description }}"
        echo "Configure flags: ${{ matrix.build_config.configure_flags }}"
        echo "Install deps: ${{ matrix.build_config.install_deps }}"
        echo "=========================="
        
    - name: Install memory tracker worker
      run: |
        cd worker
        pip install -e .
        
    - name: Install build dependencies
      run: |
        # Install CPython dependencies using their script
        cd cpython
        sudo .github/workflows/posix-deps-apt.sh

        # Install JIT dependencies if needed
        if [ "${{ matrix.build_config.install_deps }}" = "jit" ]; then
          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ github.event.inputs.llvm || '19' }}
        fi
        
        # Install Memray dependencies  
        sudo apt-get install -y \
          python3-dev \
          libdebuginfod-dev \
          libunwind-dev \
          liblz4-dev
          
    - name: Run memory benchmark for commit range - ${{ matrix.build_config.description }}
      env:
        MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }}
      run: |
        if [ "${{ matrix.build_config.install_deps }}" = "jit" ]; then
          export PATH="$(llvm-config-${{ github.event.inputs.llvm || '19' }} --bindir):$PATH"
          export LLVM_VERSION="${{ github.event.inputs.llvm || '19' }}"
          echo "LLVM Path: $(llvm-config-${{ github.event.inputs.llvm || '19' }} --bindir)"
          echo "Clang version: $(clang-${{ github.event.inputs.llvm || '19' }} --version || echo 'clang-${{ github.event.inputs.llvm || '19' }} not found')"
        fi
        
        # Build command for commit range
        memory-tracker benchmark "$COMMIT_RANGE" \
          --repo-path ./cpython \
          --binary-id "${{ matrix.build_config.binary_id }}" \
          --environment-id "${{ github.event.inputs.environment_id || 'gh_actions' }}" \
          --api-base "${{ github.event.inputs.server_url || 'https://memory.python.org' }}" \
          --output-dir ./benchmark_results \
          --configure-flags="${{ matrix.build_config.configure_flags }}" \
          --force \
          -vv
          
    - name: Upload benchmark results (if failed)
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-logs-${{ matrix.build_config.binary_id }}
        path: |
          *.log
          ./benchmark_results/
        retention-days: 7
        
    - name: Upload benchmark results (on success)
      if: success()
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-results-${{ matrix.build_config.binary_id }}
        path: ./benchmark_results/
        retention-days: 30

  summary:
    needs: benchmark-builds
    if: always()
    runs-on: ubuntu-latest
    
    steps:
    - name: Print summary
      run: |
        echo "Daily benchmark run completed"
        echo "Benchmark jobs completed with status: ${{ needs.benchmark-builds.result }}"
        echo "Binary types benchmarked: default, debug, jit, lto-pgo, nogil"