From 3818d87d4d032df784eb94af742fcfd5a2962bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Sat, 28 Dec 2024 08:45:42 +0100 Subject: [PATCH] chore(ci): run benchmarks for each storage in separate workers --- .github/scripts/changed-modules.sh | 64 ++++++++++++++++++++++++++++++ .github/workflows/benchmark.yml | 58 ++++++++++++++++++++++----- 2 files changed, 111 insertions(+), 11 deletions(-) create mode 100755 .github/scripts/changed-modules.sh diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh new file mode 100755 index 00000000..0e9d165d --- /dev/null +++ b/.github/scripts/changed-modules.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# How to test this script, run it with the required environment variables: +# 1. A file from the .github is modified: +# ALL_CHANGED_FILES=".github/a.txt .github/b/c/d/a.yaml" ./.github/scripts/changed-modules.sh +# The output should be: all modules. +# +# 2. A file from a module in the modules dir is modified: +# ALL_CHANGED_FILES="arangodb/go.mod" ./.github/scripts/changed-modules.sh +# The output should be: just the arangodb module. +# +# 3. A file from two modules in the modules dir are modified: +# ALL_CHANGED_FILES="arangodb/go.mod redis/go.mod" ./.github/scripts/changed-modules.sh +# The output should be: the arangodb and redis modules. + +# ROOT_DIR is the root directory of the repository. +readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) + +# modules is an array that will store the paths of all the modules in the repository. +modules=() + +# Find all go.mod files in the repository, building a list of all the available modules. +for modFile in $(find "${ROOT_DIR}" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do + modules+=("\"$(basename "$(dirname "${modFile}")")\"") +done + +# sort modules array +IFS=$'\n' modules=($(sort <<<"${modules[*]}")) +unset IFS + +# merge all modules and examples into a single array +allModules=("${modules[@]}") + +# sort allModules array +IFS=$'\n' allModules=($(sort <<<"${allModules[*]}")) +unset IFS + +# Get the list of modified files, retrieved from the environment variable ALL_CHANGED_FILES. +# On CI, this value will come from a Github Action retrieving the list of modified files from the pull request. +readonly modified_files=${ALL_CHANGED_FILES[@]} + +# Initialize variables +modified_modules=() + +# Check the modified files and determine which modules to build, following these rules: +# - if the modified files only contain files in one of the modules, include that module in the list +# - if the modified files contain any other file, include all modules in the list +for file in $modified_files; do + if [[ $file == .github/* ]]; then + modified_modules=${allModules[@]} + break + fi + + module_name=$(echo $file | cut -d'/' -f1) + if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then + modified_modules+=("\"$module_name\"") + fi +done + +# print all modules with this format: +# each module will be enclosed in double quotes +# each module will be separated by a comma +# the entire list will be enclosed in square brackets +echo "["$(IFS=,; echo "${modified_modules[*]}" | sed 's/ /,/g')"]" \ No newline at end of file diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7da4fbe6..a0ec225f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,7 +17,41 @@ permissions: name: Benchmark jobs: + detect-modules: + runs-on: ubuntu-latest + outputs: + modules: ${{ steps.set-modified-modules.outputs.modules }} + modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }} + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - id: changed-files + name: Get changed files + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4 + + - id: set-modified-modules + name: Set all modified modules + env: + ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" + run: echo "modules=$(./.github/scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + + - id: set-modified-modules-count + name: Set all modified modules count + run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT + + - name: Print out the modules to be used + run: | + echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build" + echo "${{ steps.set-modified-modules.outputs.modules }}" + Compare: + needs: + - detect-modules + strategy: + matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + fail-fast: true runs-on: ubuntu-latest services: arangodb: @@ -94,21 +128,25 @@ jobs: node-version: '18' - name: Install Azurite + if: ${{ matrix.module == 'azureblob' }} run: | docker run -d -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 --blobPort 10000 - name: Install Cloudflare Worker + if: ${{ matrix.module == 'cloudflarekv' }} run : | .github/scripts/initialize-wrangler.sh cd cloudflarekv && npx wrangler dev & npx wait-on tcp:8787 - name: Install Coherence + if: ${{ matrix.module == 'coherence' }} run: | docker run -d -p 1408:1408 -p 30000:30000 ghcr.io/oracle/coherence-ce:22.06.5 sleep 30 - name: Install etcd + if: ${{ matrix.module == 'etcd' }} run: | docker run -d --name Etcd-server \ --publish 2379:2379 \ @@ -118,38 +156,36 @@ jobs: bitnami/etcd:latest - name: Install ScyllaDb + if: ${{ matrix.module == 'scylladb' }} run: | docker run --name scylladb -p 9042:9042 -p 19042:19042 -p 9160:9160 -p 7000:7000 -p 7001:7001 -p 7199:7199 -p 9180:9180 -d scylladb/scylla:latest --broadcast-address 127.0.0.1 --listen-address 0.0.0.0 --broadcast-rpc-address 127.0.0.1 sleep 15 # Wait for ScyllaDb to initialize - name: Setup Redis + if: ${{ matrix.module == 'redis' }} uses: shogo82148/actions-setup-redis@v1 with: redis-version: '7.x' auto-start: 'false' - name: Run Redis + if: ${{ matrix.module == 'redis' }} run: | redis-server --port 6379 & - name: Run NATS + if: ${{ matrix.module == 'nats' }} run: | ./.github/scripts/gen-test-certs.sh docker run -d --name nats-jetstream -p 4443:4443 -v ./nats/testdata:/testdata -v ./tls:/tls nats:latest --jetstream -c /testdata/nats-tls.conf sleep 2 - name: Run Benchmarks + working-directory: ${{ matrix.module }} run: | set -o pipefail - for d in */ ; do - [[ $d == "tls/" ]] && continue - [[ $d == "node_modules/" ]] && continue - - cd "$d" - echo "Bench dir: $d" - go test ./... -benchmem -run=^$ -bench . | tee -a ../output.txt - cd .. - done + echo "Bench dir: ${{ matrix.module }}" + go test ./... -benchmem -run=^$ -bench . | tee -a ../${{ matrix.module }}-output.txt shell: bash env: MSSQL_DATABASE: master @@ -169,13 +205,13 @@ jobs: uses: actions/cache@v4 with: path: ./cache - key: ${{ runner.os }}-benchmark + key: ${{ runner.os }}-benchmark-${{ matrix.module }} - name: Save Benchmark Results uses: benchmark-action/github-action-benchmark@v1.20.4 with: tool: "go" - output-file-path: output.txt + output-file-path: ${{ matrix.module }}-output.txt github-token: ${{ secrets.BENCHMARK_TOKEN }} benchmark-data-dir-path: "benchmarks" alert-threshold: "300%"