From 0458c050ce6b2ef94be0f9871a0043b3ec9c9d44 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 19 Sep 2021 13:41:39 +0200 Subject: [PATCH 1/5] Adding trivy integration test Testing against a recent version of the juiceshop from dockerhub Signed-off-by: Sebastian --- tests/integration/scanner/trivy.test.js | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/integration/scanner/trivy.test.js diff --git a/tests/integration/scanner/trivy.test.js b/tests/integration/scanner/trivy.test.js new file mode 100644 index 0000000000..59ee5b6907 --- /dev/null +++ b/tests/integration/scanner/trivy.test.js @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2021 iteratec GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +const { scan } = require("../helpers"); + +jest.retryTimes(3); + +test( + "trivy scans vulnerable juiceshop demo target", + async () => { + const { categories, severities, count } = await scan( + "trivy-juice-shop", + "trivy", + ["bkimminich/juice-shop:v12.9.0"], + 90 + ); + + expect(count).toBe(32); + expect(categories).toMatchInlineSnapshot(` +Object { + "NPM Package Vulnerability": 32, +} +`); + expect(severities).toMatchInlineSnapshot(` +Object { + "high": 22, + "low": 1, + "medium": 9, +} +`); + }, + 3 * 60 * 1000 +); + +test( + "Invalid argument should be marked as errored", + async () => { + await expect( + scan( + "trivy-invalidArg", + "trivy", + ["--invalidArg", "not/a-valid-image:v0.0.0"], + 90 + ) + ).rejects.toThrow("HTTP request failed"); + }, + 3 * 60 * 1000 +); From e153c96a590f60ac40196dae9a61e85c43bfdf64 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 19 Sep 2021 13:42:01 +0200 Subject: [PATCH 2/5] Adding trivy integration tests to ci Signed-off-by: Sebastian --- .github/workflows/ci.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7e0aa9b760..2dd354a5c0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -976,6 +976,19 @@ jobs: cd tests/integration/ npx jest --ci --color scanner/sslyze.test.js + # ---- Trivy Integration Tests ---- + + - name: "trivy Integration Tests" + run: | + kubectl -n integration-tests delete scans --all + helm -n integration-tests install trivy ./scanners/trivy/ \ + --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ + --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-typo3scan" \ + --set="parser.env[0].name=CRASH_ON_FAILED_VALIDATION" \ + --set-string="parser.env[0].value=true" + cd tests/integration/ + npx jest --ci --color scanner/trivy.test.js + # ---- Typo3scan Integration Tests ---- - name: "typo3scan Integration Tests" From 5f83677cecca2854d5c1cbc36e3d21ab68c23820 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 19 Sep 2021 14:08:20 +0200 Subject: [PATCH 3/5] Fixing wrong parser name in ci Signed-off-by: Sebastian --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2dd354a5c0..70c494dc5b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -983,7 +983,7 @@ jobs: kubectl -n integration-tests delete scans --all helm -n integration-tests install trivy ./scanners/trivy/ \ --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-typo3scan" \ + --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-trivy" \ --set="parser.env[0].name=CRASH_ON_FAILED_VALIDATION" \ --set-string="parser.env[0].value=true" cd tests/integration/ From 0c88c0fdb2d12b7a01af7d3baf6afde521fe472b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 21 Sep 2021 09:49:16 +0200 Subject: [PATCH 4/5] Downgrading scanned juiceshop image to v10.2.0 This should ensure a more stable test result due to the older version Signed-off-by: Sebastian --- tests/integration/scanner/trivy.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/scanner/trivy.test.js b/tests/integration/scanner/trivy.test.js index 59ee5b6907..d5d15abb1f 100644 --- a/tests/integration/scanner/trivy.test.js +++ b/tests/integration/scanner/trivy.test.js @@ -12,7 +12,7 @@ test( const { categories, severities, count } = await scan( "trivy-juice-shop", "trivy", - ["bkimminich/juice-shop:v12.9.0"], + ["bkimminich/juice-shop:v10.2.0"], 90 ); From dbe44d7f274f852dc3230214fe593d370c131fd1 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 21 Sep 2021 09:50:06 +0200 Subject: [PATCH 5/5] Refactored test results to use GreaterThanOrEqual This will not let the tests failed when new vulnerabilities are found Signed-off-by: Sebastian --- tests/integration/scanner/trivy.test.js | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/integration/scanner/trivy.test.js b/tests/integration/scanner/trivy.test.js index d5d15abb1f..722a879c6a 100644 --- a/tests/integration/scanner/trivy.test.js +++ b/tests/integration/scanner/trivy.test.js @@ -16,19 +16,12 @@ test( 90 ); - expect(count).toBe(32); - expect(categories).toMatchInlineSnapshot(` -Object { - "NPM Package Vulnerability": 32, -} -`); - expect(severities).toMatchInlineSnapshot(` -Object { - "high": 22, - "low": 1, - "medium": 9, -} -`); + expect(count).toBeGreaterThanOrEqual(134); + expect(categories["Image Vulnerability"]).toBeGreaterThanOrEqual(26); + expect(categories["NPM Package Vulnerability"]).toBeGreaterThanOrEqual(108); + expect(severities["high"]).toBeGreaterThanOrEqual(82); + expect(severities["medium"]).toBeGreaterThanOrEqual(47); + expect(severities["low"]).toBeGreaterThanOrEqual(5); }, 3 * 60 * 1000 );