diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml deleted file mode 100644 index 99d7309fd0c..00000000000 --- a/.github/workflows/pr-ci.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: PR-CI - -on: - pull_request: - types: [opened, reopened, synchronize] - -jobs: - dist-tar: - name: Build distribution tar - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "8" - cache: "maven" - - name: Build distribution tar - run: | - mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v4 - name: Upload distribution tar - with: - name: rocketmq - path: distribution/target/rocketmq*/rocketmq* - - name: Save PR number - run: | - mkdir -p ./pr - echo ${{ github.event.number }} > ./pr/NR - - uses: actions/upload-artifact@v4 - with: - name: pr - path: pr/ diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml deleted file mode 100644 index 0bc74a65ad5..00000000000 --- a/.github/workflows/pr-e2e-test.yml +++ /dev/null @@ -1,263 +0,0 @@ -name: E2E test for pull request - -# read-write repo token -# access to secrets -on: - workflow_run: - workflows: ["PR-CI"] - types: - - completed - -env: - DOCKER_REPO: apache/rocketmq-ci - -jobs: - docker: - if: > - github.repository == 'apache/rocketmq' && - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - base-image: ["ubuntu"] - java-version: ["8"] - steps: - - name: 'Download artifact' - uses: actions/github-script@v6 - with: - script: | - let artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - let matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "rocketmq" - })[0]; - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifactRmq.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/rocketmq.zip', Buffer.from(download.data)); - - run: | - unzip rocketmq.zip - mkdir rocketmq - cp -r rocketmq-* rocketmq/ - ls - - uses: actions/checkout@v3 - with: - repository: apache/rocketmq-docker.git - ref: master - path: rocketmq-docker - - name: docker-login - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and save docker images - id: build-images - run: | - cd rocketmq-docker/image-build-ci - version=${{ github.event.pull_request.number || github.ref_name }}-$(uuidgen) - mkdir versionlist - touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" - sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v4 - name: Upload distribution tar - with: - name: versionlist - path: rocketmq-docker/image-build-ci/versionlist/* - - list-version: - if: > - github.repository == 'apache/rocketmq' && - always() - name: List version - needs: [docker] - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - version-json: ${{ steps.show_versions.outputs.version-json }} - steps: - - uses: actions/download-artifact@v4 - name: Download versionlist - with: - name: versionlist - path: versionlist - - name: Show versions - id: show_versions - run: | - a=(`ls versionlist`) - printf '%s\n' "${a[@]}" | jq -R . | jq -s . - echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT - - deploy: - if: ${{ success() }} - name: Deploy RocketMQ - needs: [list-version,docker] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: Deploy rocketmq - with: - action: "deploy" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git" - chart-branch: "master" - chart-path: "./rocketmq-k8s-helm" - job-id: ${{ strategy.job-index }} - helm-values: | - nameserver: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - broker: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - proxy: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - test-e2e-grpc-java: - if: ${{ success() }} - name: Test E2E grpc java - needs: [list-version, deploy] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: e2e test - with: - action: "test" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e" - test-code-branch: "master" - test-code-path: java/e2e - test-cmd: "mvn -B test" - job-id: 0 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: '**/test_report/TEST-*.xml' - annotate_only: true - include_passed: true - detailed_summary: true - - uses: actions/upload-artifact@v4 - if: always() - name: Upload test log - with: - name: test-e2e-grpc-java-log.txt - path: testlog.txt - - test-e2e-golang: - if: ${{ success() }} - name: Test E2E golang - needs: [list-version, deploy] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: e2e test - with: - action: "test" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e" - test-code-branch: "master" - test-code-path: golang - test-cmd: | - cd ../common && mvn -Prelease -DskipTests clean package -U - cd ../rocketmq-admintools && source bin/env.sh - LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1') - wget "https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz" && \ - rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz - cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m -v - job-id: 0 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: '**/test_report/TEST-*.xml' - annotate_only: true - include_passed: true - detailed_summary: true - - uses: actions/upload-artifact@v4 - if: always() - name: Upload test log - with: - name: test-e2e-golang-log.txt - path: testlog.txt - - test-e2e-remoting-java: - if: ${{ success() }} - name: Test E2E remoting java - needs: [ list-version, deploy ] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: e2e test - with: - action: "test" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e" - test-code-branch: "master" - test-code-path: java/e2e-v4 - test-cmd: "mvn -B test" - job-id: 0 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: '**/test_report/TEST-*.xml' - annotate_only: true - include_passed: true - detailed_summary: true - - uses: actions/upload-artifact@v4 - if: always() - name: Upload test log - with: - name: test-e2e-remoting-java-log.txt - path: testlog.txt - - clean: - if: always() - name: Clean - needs: [list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: clean - with: - action: "clean" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - job-id: ${{ strategy.job-index }} \ No newline at end of file diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml deleted file mode 100644 index 6de8595724f..00000000000 --- a/.github/workflows/push-ci.yml +++ /dev/null @@ -1,348 +0,0 @@ -name: PUSH-CI - -on: - push: - branches: [master, develop] - #schedule: - # - cron: "0 18 * * *" # TimeZone: UTC 0 - -concurrency: - group: rocketmq-${{ github.ref }} - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - DOCKER_REPO: apache/rocketmq-ci - -jobs: - dist-tar: - if: github.repository == 'apache/rocketmq' - name: Build dist tar - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "8" - cache: "maven" - - name: Build distribution tar - run: | - mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v4 - name: Upload distribution tar - with: - name: rocketmq - path: distribution/target/rocketmq*/rocketmq* - - docker: - if: ${{ success() }} - name: Docker images - needs: [dist-tar] - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - base-image: ["ubuntu"] - java-version: ["8"] - steps: - - uses: actions/checkout@v3 - with: - repository: apache/rocketmq-docker.git - ref: master - path: rocketmq-docker - - uses: actions/download-artifact@v4 - name: Download distribution tar - with: - name: rocketmq - path: rocketmq - - name: docker-login - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and save docker images - id: build-images - run: | - cd rocketmq-docker/image-build-ci - version=${{ github.event.pull_request.number || github.ref_name }}-$(uuidgen) - mkdir versionlist - touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" - sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v4 - name: Upload distribution tar - with: - name: versionlist - path: rocketmq-docker/image-build-ci/versionlist/* - - list-version: - if: > - github.repository == 'apache/rocketmq' && - always() - name: List version - needs: [docker] - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - version-json: ${{ steps.show_versions.outputs.version-json }} - steps: - - uses: actions/download-artifact@v4 - name: Download versionlist - with: - name: versionlist - path: versionlist - - name: Show versions - id: show_versions - run: | - a=(`ls versionlist`) - printf '%s\n' "${a[@]}" | jq -R . | jq -s . - echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT - - deploy-e2e: - if: ${{ success() }} - name: Deploy RocketMQ For E2E - needs: [list-version,docker] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: Deploy rocketmq - with: - action: "deploy" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git" - chart-branch: "master" - chart-path: "./rocketmq-k8s-helm" - job-id: ${{ strategy.job-index }} - helm-values: | - nameserver: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - broker: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - proxy: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - - deploy-benchmark: - if: ${{ success() }} - name: Deploy RocketMQ For Benchmarking - needs: [list-version,docker] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: Deploy rocketmq - with: - action: "deploy" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git" - chart-branch: "master" - chart-path: "./rocketmq-k8s-helm" - job-id: "001-${{ strategy.job-index }}" - helm-values: | - nameserver: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - broker: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - proxy: - image: - repository: ${{env.DOCKER_REPO}} - tag: ${{ matrix.version }} - - test-e2e-grpc-java: - if: ${{ success() }} - name: Test E2E grpc java - needs: [list-version, deploy-e2e] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: e2e test - with: - action: "test" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e" - test-code-branch: "master" - test-code-path: java/e2e - test-cmd: "mvn -B test" - job-id: 0 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: '**/test_report/TEST-*.xml' - annotate_only: true - include_passed: true - detailed_summary: true - - uses: actions/upload-artifact@v4 - if: always() - name: Upload test log - with: - name: test-e2e-grpc-java-log.txt - path: testlog.txt - - test-e2e-golang: - if: ${{ success() }} - name: Test E2E golang - needs: [list-version, deploy-e2e] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: e2e test - with: - action: "test" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e" - test-code-branch: "master" - test-code-path: golang - test-cmd: | - cd ../common && mvn -Prelease -DskipTests clean package -U - cd ../rocketmq-admintools && source bin/env.sh - LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1') - wget "https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz" && \ - rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz - cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m -v - job-id: 0 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: '**/test_report/TEST-*.xml' - annotate_only: true - include_passed: true - detailed_summary: true - - uses: actions/upload-artifact@v4 - if: always() - name: Upload test log - with: - name: test-e2e-golang-log.txt - path: testlog.txt - - test-e2e-remoting-java: - if: ${{ success() }} - name: Test E2E remoting java - needs: [ list-version, deploy-e2e ] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: e2e test - with: - action: "test" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e" - test-code-branch: "master" - test-code-path: java/e2e-v4 - test-cmd: "mvn -B test" - job-id: 0 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: '**/test_report/TEST-*.xml' - annotate_only: true - include_passed: true - detailed_summary: true - - uses: actions/upload-artifact@v4 - if: always() - name: Upload test log - with: - name: test-e2e-remoting-java-log.txt - path: testlog.txt - - benchmark-test: - if: ${{ success() }} - runs-on: ubuntu-latest - name: Performance benchmark test - needs: [ list-version, deploy-benchmark ] - timeout-minutes: 60 - steps: - - uses: apache/rocketmq-test-tool/benchmark-runner@ce372e5f3906ca1891e4918b05be14608eae608e - name: Performance benchmark - with: - action: "performance-benchmark" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - job-id: "001-${{ strategy.job-index }}" - # The time to run the test, 15 minutes - test-time: "900" - # Some thresholds set in advance - min-send-tps-threshold: "12000" - max-rt-ms-threshold: "500" - avg-rt-ms-threshold: "10" - max-2c-rt-ms-threshold: "150" - avg-2c-rt-ms-threshold: "10" - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: benchmark-report - path: benchmark/ - - clean-e2e: - if: always() - name: Clean E2E - needs: [ list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java ] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: clean - with: - action: "clean" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - job-id: ${{ strategy.job-index }} - - clean-benchmark: - if: always() - name: Clean Benchmarking - needs: [ list-version, benchmark-test ] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - steps: - - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 - name: clean - with: - action: "clean" - ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - test-version: "${{ matrix.version }}" - job-id: "001-${{ strategy.job-index }}" \ No newline at end of file