Add JIT to daily runs and reschedule the upload

Author: pablogsal

.claude/settings.local.json

@@ -2,22 +2,18 @@ name: Daily Memory Tracker Benchmark
 
 on:
   schedule:
-    # Run daily at 02:00 UTC
-    - cron: '0 2 * * *'
+    # Run daily at 23:00 UTC (EOD) to pick up all commits from the day
+    - cron: '0 23 * * *'
   workflow_dispatch:
     inputs:
       target_date:
         description: 'Date to get commits from (YYYY-MM-DD, defaults to today)'
         required: false
         type: string
-      binary_id:
-        description: 'Binary ID to use for benchmarking'
-        required: false
-        default: 'default'
       environment_id:
         description: 'Environment ID'
         required: false
-        default: 'gcc-11'
+        default: 'gh_actions'
       server_url:
         description: 'Memory tracker server URL'
         required: false
@@ -26,6 +22,10 @@ on:
         description: 'CPython repository URL'
         required: false
         default: 'https://github.com/python/cpython.git'
+      llvm:
+        description: 'LLVM version to use for JIT builds'
+        required: false
+        default: '18'
 
 jobs:
   get-daily-commits:
@@ -75,15 +75,30 @@ jobs:
           echo "$COMMITS"
         fi
 
-  benchmark-commits:
+  benchmark-builds:
     needs: get-daily-commits
     if: needs.get-daily-commits.outputs.commit-count > 0
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        commit: ${{ fromJson(needs.get-daily-commits.outputs.commits) }}
+        build_config:
+          - binary_id: 'default'
+            configure_flags: '-C'
+            description: 'Default build'
+            install_deps: 'standard'
+          - binary_id: 'debug'
+            configure_flags: '--with-pydebug'
+            description: 'Debug build'
+            install_deps: 'standard'
+          - binary_id: 'jit'
+            configure_flags: '--enable-experimental-jit'
+            description: 'JIT build'
+            install_deps: 'jit'
+          - binary_id: 'nogil'
+            configure_flags: '--disable-gil'
+            description: 'Free-threaded build'
+            install_deps: 'standard'
       fail-fast: false
-      max-parallel: 3
 
     steps:
     - name: Checkout memory tracker
@@ -110,6 +125,11 @@ jobs:
         # Install CPython dependencies using their script
         cd cpython
         sudo .github/workflows/posix-deps-apt.sh
+
+        # Install JIT dependencies if needed
+        if [ "${{ matrix.build_config.install_deps }}" = "jit" ]; then
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ github.event.inputs.llvm || '18' }}
+        fi
         
         # Install Memray dependencies  
         sudo apt-get install -y \
@@ -118,31 +138,39 @@ jobs:
           libunwind-dev \
           liblz4-dev
           
-    - name: Run memory benchmark for commit
+    - name: Run memory benchmark for commit range - ${{ matrix.build_config.description }}
       env:
         MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }}
       run: |
-        COMMIT="${{ matrix.commit }}"
+        if [ "${{ matrix.build_config.install_deps }}" = "jit" ]; then
+          export PATH="$(llvm-config-${{ github.event.inputs.llvm || '18' }} --bindir):$PATH"
+        fi
+        
+        # Convert JSON array to A..B format directly from GitHub output
+        FIRST_COMMIT=$(echo '${{ needs.get-daily-commits.outputs.commits }}' | jq -r '.[0]')
+        LAST_COMMIT=$(echo '${{ needs.get-daily-commits.outputs.commits }}' | jq -r '.[-1]')
+        COMMIT_RANGE="${FIRST_COMMIT}..${LAST_COMMIT}"
         
-        # Build command for single commit
-        CMD="memory-tracker benchmark '$COMMIT'"
+        # Build command for commit range
+        CMD="memory-tracker benchmark '$COMMIT_RANGE'"
         CMD="$CMD --repo-path ./cpython"
-        CMD="$CMD --binary-id '${{ github.event.inputs.binary_id || 'default' }}'"
-        CMD="$CMD --environment-id '${{ github.event.inputs.environment_id || 'gcc-11' }}'"
+        CMD="$CMD --binary-id '${{ matrix.build_config.binary_id }}'"
+        CMD="$CMD --environment-id '${{ github.event.inputs.environment_id || 'gh_actions' }}'"
         CMD="$CMD --api-base '${{ github.event.inputs.server_url || 'https://memory.python.org' }}'"
         CMD="$CMD --output-dir ./benchmark_results"
+        CMD="$CMD --configure-flags='${{ matrix.build_config.configure_flags }}'"
         CMD="$CMD --force"
         CMD="$CMD -vv"
         
-        echo "Running benchmark for commit: $COMMIT"
+        echo "Running ${{ matrix.build_config.description }} benchmark for commit range with ${{ needs.get-daily-commits.outputs.commit-count }} commits"
         echo "Command: $CMD"
         eval $CMD
           
     - name: Upload benchmark results (if failed)
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: benchmark-logs-${{ matrix.commit }}
+        name: benchmark-logs-${{ matrix.build_config.binary_id }}
         path: |
           *.log
           ./benchmark_results/
@@ -152,12 +180,12 @@ jobs:
       if: success()
       uses: actions/upload-artifact@v4
       with:
-        name: benchmark-results-${{ matrix.commit }}
+        name: benchmark-results-${{ matrix.build_config.binary_id }}
         path: ./benchmark_results/
         retention-days: 30
 
   summary:
-    needs: [get-daily-commits, benchmark-commits]
+    needs: [get-daily-commits, benchmark-builds]
     if: always()
     runs-on: ubuntu-latest
 
@@ -170,5 +198,6 @@ jobs:
         if [ "${{ needs.get-daily-commits.outputs.commit-count }}" = "0" ]; then
           echo "No commits found for the target date"
         else
-          echo "Benchmark jobs completed with status: ${{ needs.benchmark-commits.result }}"
+          echo "Benchmark jobs completed with status: ${{ needs.benchmark-builds.result }}"
+          echo "Binary types benchmarked: default, debug, jit, nogil"
         fi