From 29af91d770f0414f0f63578d1878d3558fa409d8 Mon Sep 17 00:00:00 2001 From: Amr Date: Sat, 15 Aug 2026 19:30:39 -0400 Subject: [PATCH] Run integration tests nightly rather than on every pull request The integration tests reach CATH, ECOD, RCSB, EBI and others. That is what makes them worth having: they are how we learn that an upstream service changed a URL, a format or a redirect. It is also what makes them a poor gate on pull requests, because any of those services having a bad day blocks every contributor, and a real regression then cannot be distinguished from the surrounding noise. That is not hypothetical. Every PR Build since 2025-12-19 has failed, on all five matrix jobs, whatever the pull request contained - a one-line dependency bump fails exactly as a feature branch does. Two integration tests account for it: CathDomainTest, because download.cathdb.info began redirecting http to https, and EcodInstallationTest.testVersion, which downloads a 657 MB file from a slow server once per matrix job. Master Build already excludes the module with -pl '!biojava-integrationtest'. This applies the same exclusion to pull requests and adds a scheduled workflow that runs the full suite nightly, so the coverage is kept but is no longer in anybody's way. An upstream break still gets caught, within a day, by a run whose failure means what it says. The nightly job uses a single JDK, since the pull request build already covers 11, 17 and 21 and the point here is the network paths. It carries a 90 minute timeout, because these downloads are large and the six hour default is not a useful ceiling, and it keeps surefire reports for 30 days on failure, since GitHub expires run logs after 90 and those reports are what identify which service misbehaved. --- .github/workflows/nightly.yml | 54 ++++++++++++++++++++++++++++++ .github/workflows/pull_request.yml | 20 ++++++++--- 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..07a707b7ef --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,54 @@ +name: Nightly Integration Tests + +# The integration tests reach out to CATH, ECOD, RCSB, EBI, UniProt and others. +# That makes them valuable - they are how we find out that an upstream service +# has changed a URL, a format or a redirect - but it also makes them unsuitable +# as a gate on pull requests, because an outage anywhere blocks every +# contributor. Running them on a schedule keeps the coverage while decoupling it +# from people's ability to merge. +on: + schedule: + # 03:17 UTC daily. Off the hour deliberately: scheduled jobs that ask for + # exactly midnight queue behind everybody else's. + - cron: '17 3 * * *' + # Also runnable by hand, e.g. to confirm an upstream service is back. + workflow_dispatch: + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + integrationtest: + runs-on: ubuntu-latest + # These tests download large files from servers we do not control; the + # default 6 hour limit is far more than they need and far more than we want + # to spend if one of them hangs. + timeout-minutes: 90 + strategy: + matrix: + # One JDK only. The point of this run is to exercise the network paths, + # not the language level, which the pull request build already covers + # across 11, 17 and 21. + java: [21] + fail-fast: false + name: Integration tests, JDK ${{ matrix.java }} + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: 'oracle' + java-version: ${{ matrix.java }} + - name: Build and run integration tests + run: mvn verify --no-transfer-progress + - name: Upload surefire reports + # Kept on failure so an upstream break can be diagnosed after the fact: + # GitHub expires run logs after 90 days, and these reports carry the + # stack traces that say which service misbehaved. + if: failure() + uses: actions/upload-artifact@v4 + with: + name: surefire-reports + path: '**/target/surefire-reports/**' + retention-days: 30 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index a0d31ee08a..340418cf68 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -30,8 +30,14 @@ jobs: with: distribution: 'oracle' java-version: ${{ matrix.java }} - - name: Build, test and integration test - run: mvn verify --no-transfer-progress + - name: Build and test (no integration tests) + # Integration tests are excluded here and run nightly instead, see + # nightly.yml. They depend on CATH, ECOD, RCSB, EBI and others being up + # and responsive, so running them on every pull request means a third + # party having a bad day blocks contributors, and a real regression + # cannot be told apart from the resulting noise. Master Build already + # excludes them for the same reason. + run: mvn verify -pl '!biojava-integrationtest' --no-transfer-progress # Note that 11 is not available in openjdk. So we need to do it with the Zulu distribution (see https://github.com/actions/setup-java) # When we drop 11, it will be safe to drop the copy-pasted workflow excerpt below @@ -54,5 +60,11 @@ jobs: with: distribution: 'zulu' java-version: ${{ matrix.java }} - - name: Build, test and integration test - run: mvn verify --no-transfer-progress + - name: Build and test (no integration tests) + # Integration tests are excluded here and run nightly instead, see + # nightly.yml. They depend on CATH, ECOD, RCSB, EBI and others being up + # and responsive, so running them on every pull request means a third + # party having a bad day blocks contributors, and a real regression + # cannot be told apart from the resulting noise. Master Build already + # excludes them for the same reason. + run: mvn verify -pl '!biojava-integrationtest' --no-transfer-progress