From 084d8975ded012ce82a6fc576816f8fda53840bf Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 26 Nov 2025 10:20:41 +0100 Subject: [PATCH 1/9] Rewrite git-repo-scanner in go Signed-off-by: Patrick --- scanners/git-repo-scanner/.helm-docs.gotmpl | 6 +- scanners/git-repo-scanner/README.md | 5 +- .../github-secureCodeBox-scan/findings.yaml | 558 ++++++++------ .../github-secureCodeBox-scan/scan.yaml | 2 +- .../examples/gitlab-group-scan/scan.yaml | 2 +- .../git-repo-scanner.test.js | 5 +- .../git-scanner-test-findings.json | 688 ++++++++++++------ .../git-repo-scanner/parser/parser.test.js | 580 ++++++++++----- .../git-repo-scanner/scanner/.dockerignore | 4 +- scanners/git-repo-scanner/scanner/Dockerfile | 33 +- .../scanner/git_repo_scanner/__main__.py | 188 ----- .../git_repo_scanner/abstract_scanner.py | 61 -- .../git_repo_scanner/github_scanner.py | 190 ----- .../git_repo_scanner/gitlab_scanner.py | 154 ---- scanners/git-repo-scanner/scanner/go.mod | 21 + scanners/git-repo-scanner/scanner/go.sum | 38 + .../git-repo-scanner/scanner/go.sum.license | 3 + .../scanner/internal/config/config.go | 212 ++++++ .../scanner/internal/config/config_test.go | 232 ++++++ .../scanner/internal/duration/parser.go | 57 ++ .../scanner/internal/duration/parser_test.go | 147 ++++ .../git_repo_scanner/git_repo_scanner.go | 73 ++ .../git_repo_scanner/github_repo_scanner.go | 299 ++++++++ .../git_repo_scanner/gitlab_repo_scanner.go | 317 ++++++++ .../scanner/internal/output/writer.go | 59 ++ scanners/git-repo-scanner/scanner/main.go | 102 +++ .../git-repo-scanner/scanner/requirements.txt | 8 - .../scanner/tests/__init__.py | 0 .../scanner/tests/git_repo_scanner_test.py | 420 ----------- .../templates/git-repo-scanner-scan-type.yaml | 6 +- .../tests/__snapshot__/scanner_test.yaml.snap | 3 - 31 files changed, 2812 insertions(+), 1661 deletions(-) delete mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py delete mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py delete mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py delete mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py create mode 100644 scanners/git-repo-scanner/scanner/go.mod create mode 100644 scanners/git-repo-scanner/scanner/go.sum create mode 100644 scanners/git-repo-scanner/scanner/go.sum.license create mode 100644 scanners/git-repo-scanner/scanner/internal/config/config.go create mode 100644 scanners/git-repo-scanner/scanner/internal/config/config_test.go create mode 100644 scanners/git-repo-scanner/scanner/internal/duration/parser.go create mode 100644 scanners/git-repo-scanner/scanner/internal/duration/parser_test.go create mode 100644 scanners/git-repo-scanner/scanner/internal/git_repo_scanner/git_repo_scanner.go create mode 100644 scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go create mode 100644 scanners/git-repo-scanner/scanner/internal/git_repo_scanner/gitlab_repo_scanner.go create mode 100644 scanners/git-repo-scanner/scanner/internal/output/writer.go create mode 100644 scanners/git-repo-scanner/scanner/main.go delete mode 100644 scanners/git-repo-scanner/scanner/requirements.txt delete mode 100644 scanners/git-repo-scanner/scanner/tests/__init__.py delete mode 100644 scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py diff --git a/scanners/git-repo-scanner/.helm-docs.gotmpl b/scanners/git-repo-scanner/.helm-docs.gotmpl index 430ba846e5..285100b5bd 100644 --- a/scanners/git-repo-scanner/.helm-docs.gotmpl +++ b/scanners/git-repo-scanner/.helm-docs.gotmpl @@ -25,7 +25,7 @@ usecase: "Discover Git repositories" {{- define "extra.chartAboutSection" -}} ## What is Git-Repo-Scanner? -Git-Repo-Scanner is a small Python script which discovers repositories on GitHub or GitLab. The main purpose of this scanner +Git-Repo-Scanner is a small Go project which discovers repositories on GitHub or GitLab. The main purpose of this scanner is to provide a cascading input for the [gitleaks](https://www.securecodebox.io/docs/scanners/gitleaks) and [semgrep](https://www.securecodebox.io/docs/scanners/semgrep) scanners. {{- end }} @@ -36,9 +36,9 @@ The scanner options can be divided into two groups for Gitlab and GitHub. You ca repository type with the option: ```bash ---git-type github +--git-type GitHub or ---git-type Gitlab +--git-type GitLab ``` #### GitHub diff --git a/scanners/git-repo-scanner/README.md b/scanners/git-repo-scanner/README.md index 1415a9501a..6a1c22b55b 100644 --- a/scanners/git-repo-scanner/README.md +++ b/scanners/git-repo-scanner/README.md @@ -52,9 +52,9 @@ The scanner options can be divided into two groups for Gitlab and GitHub. You ca repository type with the option: ```bash ---git-type github +--git-type GitHub or ---git-type Gitlab +--git-type GitLab ``` #### GitHub @@ -146,4 +146,3 @@ Code of secureCodeBox is licensed under the [Apache License 2.0][scb-license]. [scb-mastodon]: https://infosec.exchange/@secureCodeBox [scb-slack]: https://owasp.org/slack/invite [scb-license]: https://github.com/secureCodeBox/secureCodeBox/blob/master/LICENSE - diff --git a/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/findings.yaml b/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/findings.yaml index ab51e98e35..485e03b5ec 100644 --- a/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/findings.yaml +++ b/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/findings.yaml @@ -11,11 +11,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "144957631", - "web_url": "https://github.com/secureCodeBox/ansible-role-securecodebox-openshift", + "archived": true, + "created_at": "2018-08-16T08:11:15Z", "full_name": "secureCodeBox/ansible-role-securecodebox-openshift", - "owner_type": "Organization", + "id": "144957631", + "last_activity_at": "2023-01-28T10:22:09Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "openshift", @@ -24,11 +27,8 @@ "security-tools", "security", ], - "owner_name": "secureCodeBox", - "created_at": "2018-08-16T08:11:15Z", - "last_activity_at": "2021-02-26T14:43:24Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/ansible-role-securecodebox-openshift", }, }, { @@ -39,11 +39,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "142870794", - "web_url": "https://github.com/secureCodeBox/integration-pipeline-jenkins-examples", + "archived": true, + "created_at": "2018-07-30T12:13:41Z", "full_name": "secureCodeBox/integration-pipeline-jenkins-examples", - "owner_type": "Organization", + "id": "142870794", + "last_activity_at": "2023-01-28T10:22:08Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "security", @@ -53,11 +56,8 @@ "jenkinsfile", "demo", ], - "owner_name": "secureCodeBox", - "created_at": "2018-07-30T12:13:41Z", - "last_activity_at": "2021-02-26T14:42:45Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/integration-pipeline-jenkins-examples", }, }, { @@ -68,17 +68,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "214418800", - "web_url": "https://github.com/secureCodeBox/swagger-petstore-openshift", + "archived": false, + "created_at": "2019-10-11T11:28:15Z", "full_name": "secureCodeBox/swagger-petstore-openshift", - "owner_type": "Organization", + "id": "214418800", + "last_activity_at": "2019-10-11T11:37:41Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-10-11T11:28:15Z", - "last_activity_at": "2019-10-11T11:37:41Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/swagger-petstore-openshift", }, }, { @@ -89,17 +89,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "180568880", - "web_url": "https://github.com/secureCodeBox/ruby-scanner-scaffolding", + "archived": true, + "created_at": "2019-04-10T11:39:04Z", "full_name": "secureCodeBox/ruby-scanner-scaffolding", - "owner_type": "Organization", + "id": "180568880", + "last_activity_at": "2023-01-28T10:22:10Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-04-10T11:39:04Z", - "last_activity_at": "2021-02-26T14:42:14Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/ruby-scanner-scaffolding", }, }, { @@ -110,17 +110,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "141462466", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-amass", + "archived": true, + "created_at": "2018-07-18T16:38:18Z", "full_name": "secureCodeBox/scanner-infrastructure-amass", - "owner_type": "Organization", + "id": "141462466", + "last_activity_at": "2023-06-22T01:51:32Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2018-07-18T16:38:18Z", - "last_activity_at": "2021-02-26T14:41:40Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-amass", }, }, { @@ -131,17 +131,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "251007807", - "web_url": "https://github.com/secureCodeBox/zap-extensions", + "archived": false, + "created_at": "2020-03-29T10:40:12Z", "full_name": "secureCodeBox/zap-extensions", - "owner_type": "Organization", + "id": "251007807", + "last_activity_at": "2020-03-29T10:40:13Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2020-03-29T10:40:12Z", - "last_activity_at": "2020-03-29T10:40:13Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/zap-extensions", }, }, { @@ -152,17 +152,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "171298120", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ssh", + "archived": true, + "created_at": "2019-02-18T14:23:57Z", "full_name": "secureCodeBox/scanner-infrastructure-ssh", - "owner_type": "Organization", + "id": "171298120", + "last_activity_at": "2023-01-28T10:22:09Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-02-18T14:23:57Z", - "last_activity_at": "2021-02-26T14:40:57Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ssh", }, }, { @@ -173,11 +173,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "128396681", - "web_url": "https://github.com/secureCodeBox/scanner-webserver-nikto", + "archived": true, + "created_at": "2018-04-06T13:13:14Z", "full_name": "secureCodeBox/scanner-webserver-nikto", - "owner_type": "Organization", + "id": "128396681", + "last_activity_at": "2024-08-10T17:59:05Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "nikto", @@ -187,11 +190,8 @@ "security-tools", "microservice", ], - "owner_name": "secureCodeBox", - "created_at": "2018-04-06T13:13:14Z", - "last_activity_at": "2021-02-26T14:40:31Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-webserver-nikto", }, }, { @@ -202,11 +202,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "134673181", - "web_url": "https://github.com/secureCodeBox/scanner-webapplication-arachni", + "archived": true, + "created_at": "2018-05-24T06:47:00Z", "full_name": "secureCodeBox/scanner-webapplication-arachni", - "owner_type": "Organization", + "id": "134673181", + "last_activity_at": "2023-03-29T14:00:28Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "arachni", @@ -216,11 +219,8 @@ "security-tools", "microservice", ], - "owner_name": "secureCodeBox", - "created_at": "2018-05-24T06:47:00Z", - "last_activity_at": "2021-02-26T14:40:03Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-webapplication-arachni", }, }, { @@ -231,17 +231,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "180543766", - "web_url": "https://github.com/secureCodeBox/scanner-cms-wpscan", + "archived": true, + "created_at": "2019-04-10T09:03:38Z", "full_name": "secureCodeBox/scanner-cms-wpscan", - "owner_type": "Organization", + "id": "180543766", + "last_activity_at": "2023-04-25T07:15:25Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-04-10T09:03:38Z", - "last_activity_at": "2021-02-26T14:39:25Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-cms-wpscan", }, }, { @@ -252,11 +252,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "124402117", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-nmap", + "archived": true, + "created_at": "2018-03-08T14:20:36Z", "full_name": "secureCodeBox/scanner-infrastructure-nmap", - "owner_type": "Organization", + "id": "124402117", + "last_activity_at": "2025-04-08T19:31:01Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "nmap", @@ -266,11 +269,8 @@ "security-tools", "microservice", ], - "owner_name": "secureCodeBox", - "created_at": "2018-03-08T14:20:36Z", - "last_activity_at": "2021-06-11T21:49:14Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-nmap", }, }, { @@ -281,11 +281,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "133507929", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-sslyze", + "archived": true, + "created_at": "2018-05-15T11:43:11Z", "full_name": "secureCodeBox/scanner-infrastructure-sslyze", - "owner_type": "Organization", + "id": "133507929", + "last_activity_at": "2023-01-28T10:22:08Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "sslyze", @@ -295,11 +298,8 @@ "security-tools", "microservice", ], - "owner_name": "secureCodeBox", - "created_at": "2018-05-15T11:43:11Z", - "last_activity_at": "2021-02-26T14:38:12Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-sslyze", }, }, { @@ -310,17 +310,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "223956455", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ncrack", + "archived": true, + "created_at": "2019-11-25T13:34:16Z", "full_name": "secureCodeBox/scanner-infrastructure-ncrack", - "owner_type": "Organization", + "id": "223956455", + "last_activity_at": "2023-01-28T10:22:10Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-11-25T13:34:16Z", - "last_activity_at": "2021-02-26T14:37:34Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ncrack", }, }, { @@ -331,17 +331,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "277835641", - "web_url": "https://github.com/secureCodeBox/zaproxy", + "archived": false, + "created_at": "2020-07-07T14:14:16Z", "full_name": "secureCodeBox/zaproxy", - "owner_type": "Organization", + "id": "277835641", + "last_activity_at": "2024-01-30T22:45:22Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2020-07-07T14:14:16Z", - "last_activity_at": "2020-07-07T14:14:18Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/zaproxy", }, }, { @@ -352,11 +352,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "249731346", - "web_url": "https://github.com/secureCodeBox/secureCodeBox-v2", + "archived": true, + "created_at": "2020-03-24T14:33:08Z", "full_name": "secureCodeBox/secureCodeBox-v2", - "owner_type": "Organization", + "id": "249731346", + "last_activity_at": "2024-01-30T22:40:47Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "securecodebox", @@ -367,11 +370,8 @@ "scanning", "hacktoberfest", ], - "owner_name": "secureCodeBox", - "created_at": "2020-03-24T14:33:08Z", - "last_activity_at": "2020-11-05T15:40:55Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/secureCodeBox-v2", }, }, { @@ -382,38 +382,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "203588805", - "web_url": "https://github.com/secureCodeBox/securecodebox.github.io", - "full_name": "secureCodeBox/securecodebox.github.io", - "owner_type": "Organization", + "archived": true, + "created_at": "2018-03-20T15:48:39Z", + "full_name": "secureCodeBox/nodejs-scanner-scaffolding", + "id": "126042943", + "last_activity_at": "2023-01-28T10:22:08Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-08-21T13:21:09Z", - "last_activity_at": "2020-10-16T11:40:25Z", - "visibility": "public", - "archived": false, - }, - }, - { - "name": "GitHub Repo", - "description": "A GitHub repository", - "category": "Git Repository", - "osi_layer": "APPLICATION", - "severity": "INFORMATIONAL", - "attributes": - { - "id": "126042943", - "web_url": "https://github.com/secureCodeBox/nodejs-scanner-scaffolding", - "full_name": "secureCodeBox/nodejs-scanner-scaffolding", "owner_type": "Organization", - "owner_id": "34573705", "topics": [], - "owner_name": "secureCodeBox", - "created_at": "2018-03-20T15:48:39Z", - "last_activity_at": "2021-02-26T14:36:53Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/nodejs-scanner-scaffolding", }, }, { @@ -424,11 +403,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "128920739", - "web_url": "https://github.com/secureCodeBox/scanner-webapplication-zap", + "archived": true, + "created_at": "2018-04-10T11:17:29Z", "full_name": "secureCodeBox/scanner-webapplication-zap", - "owner_type": "Organization", + "id": "128920739", + "last_activity_at": "2024-10-03T05:13:23Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "zap", @@ -438,11 +420,8 @@ "security-tools", "microservice", ], - "owner_name": "secureCodeBox", - "created_at": "2018-04-10T11:17:29Z", - "last_activity_at": "2021-02-26T14:36:02Z", "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/scanner-webapplication-zap", }, }, { @@ -453,17 +432,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "123422137", - "web_url": "https://github.com/secureCodeBox/engine", + "archived": true, + "created_at": "2018-03-01T10:50:05Z", "full_name": "secureCodeBox/engine", - "owner_type": "Organization", + "id": "123422137", + "last_activity_at": "2023-01-28T10:22:08Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2018-03-01T10:50:05Z", - "last_activity_at": "2021-02-26T14:35:25Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": true, + "web_url": "https://github.com/secureCodeBox/engine", }, }, { @@ -474,17 +453,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "327336031", - "web_url": "https://github.com/secureCodeBox/gitleaks", - "full_name": "secureCodeBox/gitleaks", - "owner_type": "Organization", + "archived": false, + "created_at": "2020-10-12T09:58:26Z", + "full_name": "secureCodeBox/kubeaudit", + "id": "303349727", + "last_activity_at": "2024-01-30T22:38:13Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2021-01-06T14:27:46Z", - "last_activity_at": "2021-03-06T20:23:36Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/kubeaudit", }, }, { @@ -495,17 +474,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "357207085", - "web_url": "https://github.com/secureCodeBox/django-DefectDojo", + "archived": false, + "created_at": "2021-04-12T13:36:31Z", "full_name": "secureCodeBox/django-DefectDojo", - "owner_type": "Organization", + "id": "357207085", + "last_activity_at": "2024-01-30T22:35:01Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2021-04-12T13:36:31Z", - "last_activity_at": "2021-12-14T14:46:54Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/django-DefectDojo", }, }, { @@ -516,17 +495,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "204701677", - "web_url": "https://github.com/secureCodeBox/ssh_scan", + "archived": false, + "created_at": "2019-08-27T12:46:48Z", "full_name": "secureCodeBox/ssh_scan", - "owner_type": "Organization", + "id": "204701677", + "last_activity_at": "2022-01-25T02:32:59Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-08-27T12:46:48Z", - "last_activity_at": "2021-06-22T12:11:47Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/ssh_scan", }, }, { @@ -537,17 +516,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "222679857", - "web_url": "https://github.com/secureCodeBox/nikto", + "archived": false, + "created_at": "2019-11-19T11:25:21Z", "full_name": "secureCodeBox/nikto", - "owner_type": "Organization", + "id": "222679857", + "last_activity_at": "2021-08-25T14:24:37Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2019-11-19T11:25:21Z", - "last_activity_at": "2021-08-25T14:24:37Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/nikto", }, }, { @@ -558,17 +537,17 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "409468006", - "web_url": "https://github.com/secureCodeBox/sslyze", + "archived": false, + "created_at": "2021-09-23T06:03:50Z", "full_name": "secureCodeBox/sslyze", - "owner_type": "Organization", + "id": "409468006", + "last_activity_at": "2021-09-23T06:03:51Z", "owner_id": "34573705", - "topics": [], "owner_name": "secureCodeBox", - "created_at": "2021-09-23T06:03:50Z", - "last_activity_at": "2021-09-23T06:03:51Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/sslyze", }, }, { @@ -579,25 +558,60 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "327269915", - "web_url": "https://github.com/secureCodeBox/defectdojo-client-java", - "full_name": "secureCodeBox/defectdojo-client-java", + "archived": false, + "created_at": "2019-08-21T13:21:09Z", + "full_name": "secureCodeBox/securecodebox.github.io", + "id": "203588805", + "last_activity_at": "2022-04-19T13:33:36Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/securecodebox.github.io", + }, + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": + { + "archived": true, + "created_at": "2020-09-02T13:39:10Z", + "full_name": "secureCodeBox/documentation", + "id": "292293538", + "last_activity_at": "2024-01-20T09:04:19Z", "owner_id": "34573705", - "topics": - [ - "defectdojo", - "owasp", - "client-library", - "java", - "gradle", - "hacktoberfest", - ], "owner_name": "secureCodeBox", - "created_at": "2021-01-06T09:59:17Z", - "last_activity_at": "2021-10-20T08:45:43Z", + "owner_type": "Organization", + "topics": + ["securecodebox", "docusaurus", "documentation", "hacktoberfest"], "visibility": "public", + "web_url": "https://github.com/secureCodeBox/documentation", + }, + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": + { "archived": false, + "created_at": "2023-03-21T19:51:07Z", + "full_name": "secureCodeBox/www-community", + "id": "617150115", + "last_activity_at": "2024-03-14T15:30:35Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/www-community", }, }, { @@ -608,17 +622,80 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "288212154", - "web_url": "https://github.com/secureCodeBox/telemetry", - "full_name": "secureCodeBox/telemetry", + "archived": false, + "created_at": "2023-10-13T14:15:48Z", + "full_name": "secureCodeBox/landscape", + "id": "704559693", + "last_activity_at": "2024-07-11T19:08:33Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/landscape", + }, + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": + { + "archived": false, + "created_at": "2024-08-09T14:48:36Z", + "full_name": "secureCodeBox/DevSecOps-MaturityModel", + "id": "840369455", + "last_activity_at": "2024-08-09T14:48:36Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/DevSecOps-MaturityModel", + }, + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": + { + "archived": false, + "created_at": "2021-01-06T14:27:46Z", + "full_name": "secureCodeBox/gitleaks", + "id": "327336031", + "last_activity_at": "2024-01-30T22:39:59Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-08-17T15:09:19Z", - "last_activity_at": "2021-12-06T14:24:34Z", + "owner_type": "Organization", + "topics": [], "visibility": "public", + "web_url": "https://github.com/secureCodeBox/gitleaks", + }, + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": + { "archived": false, + "created_at": "2025-04-18T19:36:30Z", + "full_name": "secureCodeBox/scb-cascades-demo", + "id": "968815564", + "last_activity_at": "2025-04-24T11:47:52Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scb-cascades-demo", }, }, { @@ -629,18 +706,67 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "292293538", - "web_url": "https://github.com/secureCodeBox/documentation", - "full_name": "secureCodeBox/documentation", + "archived": false, + "created_at": "2024-12-29T17:07:02Z", + "full_name": "secureCodeBox/scan-throttler", + "id": "909750535", + "last_activity_at": "2025-08-19T14:02:18Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scan-throttler", + }, + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": + { + "archived": false, + "created_at": "2021-01-06T09:59:17Z", + "full_name": "secureCodeBox/defectdojo-client-java", + "id": "327269915", + "last_activity_at": "2025-10-20T08:29:33Z", "owner_id": "34573705", - "topics": - ["securecodebox", "docusaurus", "documentation", "hacktoberfest"], "owner_name": "secureCodeBox", - "created_at": "2020-09-02T13:39:10Z", - "last_activity_at": "2021-12-15T13:55:43Z", + "owner_type": "Organization", + "topics": + [ + "defectdojo", + "owasp", + "client-library", + "java", + "gradle", + "hacktoberfest", + ], "visibility": "public", + "web_url": "https://github.com/secureCodeBox/defectdojo-client-java", + }, + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": + { "archived": false, + "created_at": "2020-08-17T15:09:19Z", + "full_name": "secureCodeBox/telemetry", + "id": "288212154", + "last_activity_at": "2025-11-20T07:43:05Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/telemetry", }, }, { @@ -651,11 +777,14 @@ "severity": "INFORMATIONAL", "attributes": { - "id": "80711933", - "web_url": "https://github.com/secureCodeBox/secureCodeBox", + "archived": false, + "created_at": "2017-02-02T09:48:05Z", "full_name": "secureCodeBox/secureCodeBox", - "owner_type": "Organization", + "id": "80711933", + "last_activity_at": "2025-11-24T10:04:46Z", "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", "topics": [ "security", @@ -667,14 +796,11 @@ "kubernetes", "kubernetes-operator", "owasp", - "owasp-zap", "hacktoberfest", + "zaproxy", ], - "owner_name": "secureCodeBox", - "created_at": "2017-02-02T09:48:05Z", - "last_activity_at": "2021-12-21T09:48:07Z", "visibility": "public", - "archived": false, + "web_url": "https://github.com/secureCodeBox/secureCodeBox", }, }, ] diff --git a/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml b/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml index b747b5a4e9..eabc173087 100644 --- a/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml +++ b/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml @@ -10,7 +10,7 @@ spec: scanType: "git-repo-scanner" parameters: - "--git-type" - - "github" + - "GitHub" - "--organization" - "secureCodeBox" cascades: diff --git a/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml b/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml index a390b3d1d2..5e510c5ec6 100644 --- a/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml +++ b/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml @@ -10,7 +10,7 @@ spec: scanType: "git-repo-scanner" parameters: - "--git-type" - - "gitlab" + - "GitLab" - "--url" - "https://gitlab.your-company.com" - "--access-token" diff --git a/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js index 7815274085..be5c7f164d 100644 --- a/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js +++ b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js @@ -8,12 +8,11 @@ test.skip( "git-repo-scanner should find at least 1 repository in the GitHub secureCodeBox organisation", async () => { // This integration tests runs about 30min because of the GitHub Public API call rate limit. - // If you want to speed up you need to add an valid access token like: ['--git-type', 'github', '--organization', 'secureCodeBox', '--access-token', '23476VALID2345TOKEN'], + // If you want to speed up you need to add an valid access token like: ['--git-type', 'GitHub', '--organization', 'secureCodeBox', '--access-token', '23476VALID2345TOKEN'], const { count } = await scan( "git-repo-scanner-dummy-scan", "git-repo-scanner", - ["--git-type", "github", "--organization", "secureCodeBox"], - 90, + ["--git-type", "GitHub", "--organization", "secureCodeBox"], ); // There must be >= 28 Repositories found in the GitHub secureCodeBox organisation. expect(count).toBeGreaterThanOrEqual(28); diff --git a/scanners/git-repo-scanner/parser/__testFiles__/git-scanner-test-findings.json b/scanners/git-repo-scanner/parser/__testFiles__/git-scanner-test-findings.json index 435b7428f7..6ca11b3fa5 100644 --- a/scanners/git-repo-scanner/parser/__testFiles__/git-scanner-test-findings.json +++ b/scanners/git-repo-scanner/parser/__testFiles__/git-scanner-test-findings.json @@ -6,15 +6,23 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 80711933, - "web_url": "https://github.com/secureCodeBox/secureCodeBox", - "full_name": "secureCodeBox/secureCodeBox", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-08-16T08:11:15Z", + "full_name": "secureCodeBox/ansible-role-securecodebox-openshift", + "id": "144957631", + "last_activity_at": "2023-01-28T10:22:09Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2017-02-02T09:48:05Z", - "last_activity_at": "2020-10-23T08:59:27Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "openshift", + "ansible-role", + "ansible", + "security-tools", + "security" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/ansible-role-securecodebox-openshift" } }, { @@ -24,15 +32,24 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 123422137, - "web_url": "https://github.com/secureCodeBox/engine", - "full_name": "secureCodeBox/engine", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-07-30T12:13:41Z", + "full_name": "secureCodeBox/integration-pipeline-jenkins-examples", + "id": "142870794", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-03-01T10:50:05Z", - "last_activity_at": "2020-10-07T08:07:32Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "security", + "security-automation", + "security-testing", + "jenkins-pipeline", + "jenkinsfile", + "demo" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/integration-pipeline-jenkins-examples" } }, { @@ -42,15 +59,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 124402117, - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-nmap", - "full_name": "secureCodeBox/scanner-infrastructure-nmap", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2019-10-11T11:28:15Z", + "full_name": "secureCodeBox/swagger-petstore-openshift", + "id": "214418800", + "last_activity_at": "2019-10-11T11:37:41Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-03-08T14:20:36Z", - "last_activity_at": "2020-09-14T15:40:40Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/swagger-petstore-openshift" } }, { @@ -60,15 +79,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 126042943, - "web_url": "https://github.com/secureCodeBox/nodejs-scanner-scaffolding", - "full_name": "secureCodeBox/nodejs-scanner-scaffolding", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2019-04-10T11:39:04Z", + "full_name": "secureCodeBox/ruby-scanner-scaffolding", + "id": "180568880", + "last_activity_at": "2023-01-28T10:22:10Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-03-20T15:48:39Z", - "last_activity_at": "2020-07-16T10:37:40Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/ruby-scanner-scaffolding" } }, { @@ -78,15 +99,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 128396681, - "web_url": "https://github.com/secureCodeBox/scanner-webserver-nikto", - "full_name": "secureCodeBox/scanner-webserver-nikto", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-07-18T16:38:18Z", + "full_name": "secureCodeBox/scanner-infrastructure-amass", + "id": "141462466", + "last_activity_at": "2023-06-22T01:51:32Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-04-06T13:13:14Z", - "last_activity_at": "2020-06-25T10:11:41Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-amass" } }, { @@ -96,15 +119,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 128920739, - "web_url": "https://github.com/secureCodeBox/scanner-webapplication-zap", - "full_name": "secureCodeBox/scanner-webapplication-zap", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2020-03-29T10:40:12Z", + "full_name": "secureCodeBox/zap-extensions", + "id": "251007807", + "last_activity_at": "2020-03-29T10:40:13Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-04-10T11:17:29Z", - "last_activity_at": "2020-10-07T14:05:09Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/zap-extensions" } }, { @@ -114,15 +139,44 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 133507929, - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-sslyze", - "full_name": "secureCodeBox/scanner-infrastructure-sslyze", + "archived": true, + "created_at": "2019-02-18T14:23:57Z", + "full_name": "secureCodeBox/scanner-infrastructure-ssh", + "id": "171298120", + "last_activity_at": "2023-01-28T10:22:09Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", - "owner_id": 34573705, + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ssh" + } + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": { + "archived": true, + "created_at": "2018-04-06T13:13:14Z", + "full_name": "secureCodeBox/scanner-webserver-nikto", + "id": "128396681", + "last_activity_at": "2024-08-10T17:59:05Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-05-15T11:43:11Z", - "last_activity_at": "2020-07-16T10:52:54Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "nikto", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-webserver-nikto" } }, { @@ -132,15 +186,24 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 134673181, - "web_url": "https://github.com/secureCodeBox/scanner-webapplication-arachni", + "archived": true, + "created_at": "2018-05-24T06:47:00Z", "full_name": "secureCodeBox/scanner-webapplication-arachni", - "owner_type": "Organization", - "owner_id": 34573705, + "id": "134673181", + "last_activity_at": "2023-03-29T14:00:28Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-05-24T06:47:00Z", - "last_activity_at": "2020-10-10T10:29:42Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "arachni", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-webapplication-arachni" } }, { @@ -150,15 +213,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 141462466, - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-amass", - "full_name": "secureCodeBox/scanner-infrastructure-amass", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2019-04-10T09:03:38Z", + "full_name": "secureCodeBox/scanner-cms-wpscan", + "id": "180543766", + "last_activity_at": "2023-04-25T07:15:25Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-07-18T16:38:18Z", - "last_activity_at": "2020-03-17T18:59:35Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-cms-wpscan" } }, { @@ -168,15 +233,24 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 142870794, - "web_url": "https://github.com/secureCodeBox/integration-pipeline-jenkins-examples", - "full_name": "secureCodeBox/integration-pipeline-jenkins-examples", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-03-08T14:20:36Z", + "full_name": "secureCodeBox/scanner-infrastructure-nmap", + "id": "124402117", + "last_activity_at": "2025-04-08T19:31:01Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-07-30T12:13:41Z", - "last_activity_at": "2020-09-27T18:59:24Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "nmap", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-nmap" } }, { @@ -186,15 +260,24 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 144957631, - "web_url": "https://github.com/secureCodeBox/ansible-role-securecodebox-openshift", - "full_name": "secureCodeBox/ansible-role-securecodebox-openshift", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-05-15T11:43:11Z", + "full_name": "secureCodeBox/scanner-infrastructure-sslyze", + "id": "133507929", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-08-16T08:11:15Z", - "last_activity_at": "2019-04-17T13:36:12Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "sslyze", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-sslyze" } }, { @@ -204,15 +287,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 161506648, - "web_url": "https://github.com/secureCodeBox/django-DefectDojo", - "full_name": "secureCodeBox/django-DefectDojo", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2019-11-25T13:34:16Z", + "full_name": "secureCodeBox/scanner-infrastructure-ncrack", + "id": "223956455", + "last_activity_at": "2023-01-28T10:22:10Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2018-12-12T15:21:02Z", - "last_activity_at": "2019-01-09T08:41:31Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ncrack" } }, { @@ -222,15 +307,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 171298120, - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ssh", - "full_name": "secureCodeBox/scanner-infrastructure-ssh", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2020-07-07T14:14:16Z", + "full_name": "secureCodeBox/zaproxy", + "id": "277835641", + "last_activity_at": "2024-01-30T22:45:22Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-02-18T14:23:57Z", - "last_activity_at": "2020-06-25T10:11:16Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/zaproxy" } }, { @@ -240,15 +327,25 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 180543766, - "web_url": "https://github.com/secureCodeBox/scanner-cms-wpscan", - "full_name": "secureCodeBox/scanner-cms-wpscan", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2020-03-24T14:33:08Z", + "full_name": "secureCodeBox/secureCodeBox-v2", + "id": "249731346", + "last_activity_at": "2024-01-30T22:40:47Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-04-10T09:03:38Z", - "last_activity_at": "2020-06-25T10:12:29Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "securecodebox", + "security-tools", + "penetration-testers", + "devsecops", + "kubernetes-operator", + "scanning", + "hacktoberfest" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/secureCodeBox-v2" } }, { @@ -258,15 +355,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 180568880, - "web_url": "https://github.com/secureCodeBox/ruby-scanner-scaffolding", - "full_name": "secureCodeBox/ruby-scanner-scaffolding", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-03-20T15:48:39Z", + "full_name": "secureCodeBox/nodejs-scanner-scaffolding", + "id": "126042943", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-04-10T11:39:04Z", - "last_activity_at": "2020-03-11T14:20:03Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/nodejs-scanner-scaffolding" } }, { @@ -276,15 +375,24 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 203588805, - "web_url": "https://github.com/secureCodeBox/securecodebox.github.io", - "full_name": "secureCodeBox/securecodebox.github.io", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-04-10T11:17:29Z", + "full_name": "secureCodeBox/scanner-webapplication-zap", + "id": "128920739", + "last_activity_at": "2024-10-03T05:13:23Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-08-21T13:21:09Z", - "last_activity_at": "2020-10-16T11:40:25Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "zap", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-webapplication-zap" } }, { @@ -294,15 +402,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 204489733, - "web_url": "https://github.com/secureCodeBox/gatsby-gh-pages-action", - "full_name": "secureCodeBox/gatsby-gh-pages-action", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-03-01T10:50:05Z", + "full_name": "secureCodeBox/engine", + "id": "123422137", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-08-26T14:11:02Z", - "last_activity_at": "2019-08-26T14:11:05Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/engine" } }, { @@ -312,15 +422,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 204701677, - "web_url": "https://github.com/secureCodeBox/ssh_scan", - "full_name": "secureCodeBox/ssh_scan", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2020-10-12T09:58:26Z", + "full_name": "secureCodeBox/kubeaudit", + "id": "303349727", + "last_activity_at": "2024-01-30T22:38:13Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-08-27T12:46:48Z", - "last_activity_at": "2019-08-27T12:53:11Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/kubeaudit" } }, { @@ -330,15 +442,37 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 214418800, - "web_url": "https://github.com/secureCodeBox/swagger-petstore-openshift", - "full_name": "secureCodeBox/swagger-petstore-openshift", + "archived": false, + "created_at": "2021-04-12T13:36:31Z", + "full_name": "secureCodeBox/django-DefectDojo", + "id": "357207085", + "last_activity_at": "2024-01-30T22:35:01Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", - "owner_id": 34573705, + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/django-DefectDojo" + } + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": { + "archived": false, + "created_at": "2019-08-27T12:46:48Z", + "full_name": "secureCodeBox/ssh_scan", + "id": "204701677", + "last_activity_at": "2022-01-25T02:32:59Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-10-11T11:28:15Z", - "last_activity_at": "2019-10-11T11:37:41Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/ssh_scan" } }, { @@ -348,15 +482,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 222679857, - "web_url": "https://github.com/secureCodeBox/nikto", + "archived": false, + "created_at": "2019-11-19T11:25:21Z", "full_name": "secureCodeBox/nikto", - "owner_type": "Organization", - "owner_id": 34573705, + "id": "222679857", + "last_activity_at": "2021-08-25T14:24:37Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-11-19T11:25:21Z", - "last_activity_at": "2020-03-21T12:43:04Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/nikto" } }, { @@ -366,15 +502,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 223956455, - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ncrack", - "full_name": "secureCodeBox/scanner-infrastructure-ncrack", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2021-09-23T06:03:50Z", + "full_name": "secureCodeBox/sslyze", + "id": "409468006", + "last_activity_at": "2021-09-23T06:03:51Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2019-11-25T13:34:16Z", - "last_activity_at": "2020-07-19T11:16:33Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/sslyze" } }, { @@ -384,15 +522,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 249731346, - "web_url": "https://github.com/secureCodeBox/secureCodeBox-v2", - "full_name": "secureCodeBox/secureCodeBox-v2", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2019-08-21T13:21:09Z", + "full_name": "secureCodeBox/securecodebox.github.io", + "id": "203588805", + "last_activity_at": "2022-04-19T13:33:36Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-03-24T14:33:08Z", - "last_activity_at": "2020-10-22T08:39:01Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/securecodebox.github.io" } }, { @@ -402,15 +542,22 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 251007807, - "web_url": "https://github.com/secureCodeBox/zap-extensions", - "full_name": "secureCodeBox/zap-extensions", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": true, + "created_at": "2020-09-02T13:39:10Z", + "full_name": "secureCodeBox/documentation", + "id": "292293538", + "last_activity_at": "2024-01-20T09:04:19Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-03-29T10:40:12Z", - "last_activity_at": "2020-03-29T10:40:13Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [ + "securecodebox", + "docusaurus", + "documentation", + "hacktoberfest" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/documentation" } }, { @@ -420,15 +567,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 277835641, - "web_url": "https://github.com/secureCodeBox/zaproxy", - "full_name": "secureCodeBox/zaproxy", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2023-03-21T19:51:07Z", + "full_name": "secureCodeBox/www-community", + "id": "617150115", + "last_activity_at": "2024-03-14T15:30:35Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-07-07T14:14:16Z", - "last_activity_at": "2020-07-07T14:14:18Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/www-community" } }, { @@ -438,15 +587,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 285890805, - "web_url": "https://github.com/secureCodeBox/static-export", - "full_name": "secureCodeBox/static-export", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2023-10-13T14:15:48Z", + "full_name": "secureCodeBox/landscape", + "id": "704559693", + "last_activity_at": "2024-07-11T19:08:33Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-08-07T17:58:52Z", - "last_activity_at": "2020-08-12T12:53:05Z", - "visibility": "private" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/landscape" } }, { @@ -456,15 +607,17 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 288212154, - "web_url": "https://github.com/secureCodeBox/telemetry", - "full_name": "secureCodeBox/telemetry", - "owner_type": "Organization", - "owner_id": 34573705, + "archived": false, + "created_at": "2024-08-09T14:48:36Z", + "full_name": "secureCodeBox/DevSecOps-MaturityModel", + "id": "840369455", + "last_activity_at": "2024-08-09T14:48:36Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-08-17T15:09:19Z", - "last_activity_at": "2020-09-01T10:08:23Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/DevSecOps-MaturityModel" } }, { @@ -474,15 +627,37 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 292293538, - "web_url": "https://github.com/secureCodeBox/documentation", - "full_name": "secureCodeBox/documentation", + "archived": false, + "created_at": "2021-01-06T14:27:46Z", + "full_name": "secureCodeBox/gitleaks", + "id": "327336031", + "last_activity_at": "2024-01-30T22:39:59Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", - "owner_id": 34573705, + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/gitleaks" + } + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": { + "archived": false, + "created_at": "2025-04-18T19:36:30Z", + "full_name": "secureCodeBox/scb-cascades-demo", + "id": "968815564", + "last_activity_at": "2025-04-24T11:47:52Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-09-02T13:39:10Z", - "last_activity_at": "2020-10-21T14:28:35Z", - "visibility": "public" + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scb-cascades-demo" } }, { @@ -492,15 +667,44 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 292573194, - "web_url": "https://github.com/secureCodeBox/ui", - "full_name": "secureCodeBox/ui", + "archived": false, + "created_at": "2024-12-29T17:07:02Z", + "full_name": "secureCodeBox/scan-throttler", + "id": "909750535", + "last_activity_at": "2025-08-19T14:02:18Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", - "owner_id": 34573705, + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scan-throttler" + } + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": { + "archived": false, + "created_at": "2021-01-06T09:59:17Z", + "full_name": "secureCodeBox/defectdojo-client-java", + "id": "327269915", + "last_activity_at": "2025-10-20T08:29:33Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-09-03T13:08:22Z", - "last_activity_at": "2020-10-07T14:38:02Z", - "visibility": "private" + "owner_type": "Organization", + "topics": [ + "defectdojo", + "owasp", + "client-library", + "java", + "gradle", + "hacktoberfest" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/defectdojo-client-java" } }, { @@ -510,15 +714,49 @@ "osi_layer": "APPLICATION", "severity": "INFORMATIONAL", "attributes": { - "id": 299249650, - "web_url": "https://github.com/secureCodeBox/internal", - "full_name": "secureCodeBox/internal", + "archived": false, + "created_at": "2020-08-17T15:09:19Z", + "full_name": "secureCodeBox/telemetry", + "id": "288212154", + "last_activity_at": "2025-11-20T07:43:05Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", "owner_type": "Organization", - "owner_id": 34573705, + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/telemetry" + } + }, + { + "name": "GitHub Repo", + "description": "A GitHub repository", + "category": "Git Repository", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + "attributes": { + "archived": false, + "created_at": "2017-02-02T09:48:05Z", + "full_name": "secureCodeBox/secureCodeBox", + "id": "80711933", + "last_activity_at": "2025-11-24T10:04:46Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", - "created_at": "2020-09-28T08:58:53Z", - "last_activity_at": "2020-10-21T15:11:56Z", - "visibility": "private" + "owner_type": "Organization", + "topics": [ + "security", + "security-automation", + "security-tools", + "security-testing", + "securecodebox", + "devsecops", + "kubernetes", + "kubernetes-operator", + "owasp", + "hacktoberfest", + "zaproxy" + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/secureCodeBox" } } ] diff --git a/scanners/git-repo-scanner/parser/parser.test.js b/scanners/git-repo-scanner/parser/parser.test.js index 108df857f6..5b54ada567 100644 --- a/scanners/git-repo-scanner/parser/parser.test.js +++ b/scanners/git-repo-scanner/parser/parser.test.js @@ -32,15 +32,23 @@ test("should properly parse git-scanner json file", async () => { [ { "attributes": { - "created_at": "2017-02-02T09:48:05Z", - "full_name": "secureCodeBox/secureCodeBox", - "id": 80711933, - "last_activity_at": "2020-10-23T08:59:27Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-08-16T08:11:15Z", + "full_name": "secureCodeBox/ansible-role-securecodebox-openshift", + "id": "144957631", + "last_activity_at": "2023-01-28T10:22:09Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "openshift", + "ansible-role", + "ansible", + "security-tools", + "security", + ], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/secureCodeBox", + "web_url": "https://github.com/secureCodeBox/ansible-role-securecodebox-openshift", }, "category": "Git Repository", "description": "A GitHub repository", @@ -50,15 +58,24 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-03-01T10:50:05Z", - "full_name": "secureCodeBox/engine", - "id": 123422137, - "last_activity_at": "2020-10-07T08:07:32Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-07-30T12:13:41Z", + "full_name": "secureCodeBox/integration-pipeline-jenkins-examples", + "id": "142870794", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "security", + "security-automation", + "security-testing", + "jenkins-pipeline", + "jenkinsfile", + "demo", + ], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/engine", + "web_url": "https://github.com/secureCodeBox/integration-pipeline-jenkins-examples", }, "category": "Git Repository", "description": "A GitHub repository", @@ -68,15 +85,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-03-08T14:20:36Z", - "full_name": "secureCodeBox/scanner-infrastructure-nmap", - "id": 124402117, - "last_activity_at": "2020-09-14T15:40:40Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2019-10-11T11:28:15Z", + "full_name": "secureCodeBox/swagger-petstore-openshift", + "id": "214418800", + "last_activity_at": "2019-10-11T11:37:41Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-nmap", + "web_url": "https://github.com/secureCodeBox/swagger-petstore-openshift", }, "category": "Git Repository", "description": "A GitHub repository", @@ -86,15 +105,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-03-20T15:48:39Z", - "full_name": "secureCodeBox/nodejs-scanner-scaffolding", - "id": 126042943, - "last_activity_at": "2020-07-16T10:37:40Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2019-04-10T11:39:04Z", + "full_name": "secureCodeBox/ruby-scanner-scaffolding", + "id": "180568880", + "last_activity_at": "2023-01-28T10:22:10Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/nodejs-scanner-scaffolding", + "web_url": "https://github.com/secureCodeBox/ruby-scanner-scaffolding", }, "category": "Git Repository", "description": "A GitHub repository", @@ -104,15 +125,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-04-06T13:13:14Z", - "full_name": "secureCodeBox/scanner-webserver-nikto", - "id": 128396681, - "last_activity_at": "2020-06-25T10:11:41Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-07-18T16:38:18Z", + "full_name": "secureCodeBox/scanner-infrastructure-amass", + "id": "141462466", + "last_activity_at": "2023-06-22T01:51:32Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-webserver-nikto", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-amass", }, "category": "Git Repository", "description": "A GitHub repository", @@ -122,15 +145,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-04-10T11:17:29Z", - "full_name": "secureCodeBox/scanner-webapplication-zap", - "id": 128920739, - "last_activity_at": "2020-10-07T14:05:09Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2020-03-29T10:40:12Z", + "full_name": "secureCodeBox/zap-extensions", + "id": "251007807", + "last_activity_at": "2020-03-29T10:40:13Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-webapplication-zap", + "web_url": "https://github.com/secureCodeBox/zap-extensions", }, "category": "Git Repository", "description": "A GitHub repository", @@ -140,15 +165,44 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-05-15T11:43:11Z", - "full_name": "secureCodeBox/scanner-infrastructure-sslyze", - "id": 133507929, - "last_activity_at": "2020-07-16T10:52:54Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2019-02-18T14:23:57Z", + "full_name": "secureCodeBox/scanner-infrastructure-ssh", + "id": "171298120", + "last_activity_at": "2023-01-28T10:22:09Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-sslyze", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ssh", + }, + "category": "Git Repository", + "description": "A GitHub repository", + "name": "GitHub Repo", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + }, + { + "attributes": { + "archived": true, + "created_at": "2018-04-06T13:13:14Z", + "full_name": "secureCodeBox/scanner-webserver-nikto", + "id": "128396681", + "last_activity_at": "2024-08-10T17:59:05Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [ + "nikto", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice", + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scanner-webserver-nikto", }, "category": "Git Repository", "description": "A GitHub repository", @@ -158,13 +212,22 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { + "archived": true, "created_at": "2018-05-24T06:47:00Z", "full_name": "secureCodeBox/scanner-webapplication-arachni", - "id": 134673181, - "last_activity_at": "2020-10-10T10:29:42Z", - "owner_id": 34573705, + "id": "134673181", + "last_activity_at": "2023-03-29T14:00:28Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "arachni", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice", + ], "visibility": "public", "web_url": "https://github.com/secureCodeBox/scanner-webapplication-arachni", }, @@ -176,15 +239,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-07-18T16:38:18Z", - "full_name": "secureCodeBox/scanner-infrastructure-amass", - "id": 141462466, - "last_activity_at": "2020-03-17T18:59:35Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2019-04-10T09:03:38Z", + "full_name": "secureCodeBox/scanner-cms-wpscan", + "id": "180543766", + "last_activity_at": "2023-04-25T07:15:25Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-amass", + "web_url": "https://github.com/secureCodeBox/scanner-cms-wpscan", }, "category": "Git Repository", "description": "A GitHub repository", @@ -194,15 +259,24 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-07-30T12:13:41Z", - "full_name": "secureCodeBox/integration-pipeline-jenkins-examples", - "id": 142870794, - "last_activity_at": "2020-09-27T18:59:24Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-03-08T14:20:36Z", + "full_name": "secureCodeBox/scanner-infrastructure-nmap", + "id": "124402117", + "last_activity_at": "2025-04-08T19:31:01Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "nmap", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice", + ], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/integration-pipeline-jenkins-examples", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-nmap", }, "category": "Git Repository", "description": "A GitHub repository", @@ -212,15 +286,24 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-08-16T08:11:15Z", - "full_name": "secureCodeBox/ansible-role-securecodebox-openshift", - "id": 144957631, - "last_activity_at": "2019-04-17T13:36:12Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-05-15T11:43:11Z", + "full_name": "secureCodeBox/scanner-infrastructure-sslyze", + "id": "133507929", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "sslyze", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice", + ], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/ansible-role-securecodebox-openshift", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-sslyze", }, "category": "Git Repository", "description": "A GitHub repository", @@ -230,15 +313,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2018-12-12T15:21:02Z", - "full_name": "secureCodeBox/django-DefectDojo", - "id": 161506648, - "last_activity_at": "2019-01-09T08:41:31Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2019-11-25T13:34:16Z", + "full_name": "secureCodeBox/scanner-infrastructure-ncrack", + "id": "223956455", + "last_activity_at": "2023-01-28T10:22:10Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/django-DefectDojo", + "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ncrack", }, "category": "Git Repository", "description": "A GitHub repository", @@ -248,15 +333,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-02-18T14:23:57Z", - "full_name": "secureCodeBox/scanner-infrastructure-ssh", - "id": 171298120, - "last_activity_at": "2020-06-25T10:11:16Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2020-07-07T14:14:16Z", + "full_name": "secureCodeBox/zaproxy", + "id": "277835641", + "last_activity_at": "2024-01-30T22:45:22Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ssh", + "web_url": "https://github.com/secureCodeBox/zaproxy", }, "category": "Git Repository", "description": "A GitHub repository", @@ -266,15 +353,25 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-04-10T09:03:38Z", - "full_name": "secureCodeBox/scanner-cms-wpscan", - "id": 180543766, - "last_activity_at": "2020-06-25T10:12:29Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2020-03-24T14:33:08Z", + "full_name": "secureCodeBox/secureCodeBox-v2", + "id": "249731346", + "last_activity_at": "2024-01-30T22:40:47Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "securecodebox", + "security-tools", + "penetration-testers", + "devsecops", + "kubernetes-operator", + "scanning", + "hacktoberfest", + ], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-cms-wpscan", + "web_url": "https://github.com/secureCodeBox/secureCodeBox-v2", }, "category": "Git Repository", "description": "A GitHub repository", @@ -284,15 +381,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-04-10T11:39:04Z", - "full_name": "secureCodeBox/ruby-scanner-scaffolding", - "id": 180568880, - "last_activity_at": "2020-03-11T14:20:03Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-03-20T15:48:39Z", + "full_name": "secureCodeBox/nodejs-scanner-scaffolding", + "id": "126042943", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/ruby-scanner-scaffolding", + "web_url": "https://github.com/secureCodeBox/nodejs-scanner-scaffolding", }, "category": "Git Repository", "description": "A GitHub repository", @@ -302,15 +401,24 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-08-21T13:21:09Z", - "full_name": "secureCodeBox/securecodebox.github.io", - "id": 203588805, - "last_activity_at": "2020-10-16T11:40:25Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-04-10T11:17:29Z", + "full_name": "secureCodeBox/scanner-webapplication-zap", + "id": "128920739", + "last_activity_at": "2024-10-03T05:13:23Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "zap", + "security", + "security-scanner", + "security-automation", + "security-tools", + "microservice", + ], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/securecodebox.github.io", + "web_url": "https://github.com/secureCodeBox/scanner-webapplication-zap", }, "category": "Git Repository", "description": "A GitHub repository", @@ -320,15 +428,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-08-26T14:11:02Z", - "full_name": "secureCodeBox/gatsby-gh-pages-action", - "id": 204489733, - "last_activity_at": "2019-08-26T14:11:05Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2018-03-01T10:50:05Z", + "full_name": "secureCodeBox/engine", + "id": "123422137", + "last_activity_at": "2023-01-28T10:22:08Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/gatsby-gh-pages-action", + "web_url": "https://github.com/secureCodeBox/engine", }, "category": "Git Repository", "description": "A GitHub repository", @@ -338,15 +448,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-08-27T12:46:48Z", - "full_name": "secureCodeBox/ssh_scan", - "id": 204701677, - "last_activity_at": "2019-08-27T12:53:11Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2020-10-12T09:58:26Z", + "full_name": "secureCodeBox/kubeaudit", + "id": "303349727", + "last_activity_at": "2024-01-30T22:38:13Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/ssh_scan", + "web_url": "https://github.com/secureCodeBox/kubeaudit", }, "category": "Git Repository", "description": "A GitHub repository", @@ -356,15 +468,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-10-11T11:28:15Z", - "full_name": "secureCodeBox/swagger-petstore-openshift", - "id": 214418800, - "last_activity_at": "2019-10-11T11:37:41Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2021-04-12T13:36:31Z", + "full_name": "secureCodeBox/django-DefectDojo", + "id": "357207085", + "last_activity_at": "2024-01-30T22:35:01Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/swagger-petstore-openshift", + "web_url": "https://github.com/secureCodeBox/django-DefectDojo", }, "category": "Git Repository", "description": "A GitHub repository", @@ -374,13 +488,35 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { + "archived": false, + "created_at": "2019-08-27T12:46:48Z", + "full_name": "secureCodeBox/ssh_scan", + "id": "204701677", + "last_activity_at": "2022-01-25T02:32:59Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/ssh_scan", + }, + "category": "Git Repository", + "description": "A GitHub repository", + "name": "GitHub Repo", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + }, + { + "attributes": { + "archived": false, "created_at": "2019-11-19T11:25:21Z", "full_name": "secureCodeBox/nikto", - "id": 222679857, - "last_activity_at": "2020-03-21T12:43:04Z", - "owner_id": 34573705, + "id": "222679857", + "last_activity_at": "2021-08-25T14:24:37Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", "web_url": "https://github.com/secureCodeBox/nikto", }, @@ -392,15 +528,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2019-11-25T13:34:16Z", - "full_name": "secureCodeBox/scanner-infrastructure-ncrack", - "id": 223956455, - "last_activity_at": "2020-07-19T11:16:33Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2021-09-23T06:03:50Z", + "full_name": "secureCodeBox/sslyze", + "id": "409468006", + "last_activity_at": "2021-09-23T06:03:51Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/scanner-infrastructure-ncrack", + "web_url": "https://github.com/secureCodeBox/sslyze", }, "category": "Git Repository", "description": "A GitHub repository", @@ -410,15 +548,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-03-24T14:33:08Z", - "full_name": "secureCodeBox/secureCodeBox-v2", - "id": 249731346, - "last_activity_at": "2020-10-22T08:39:01Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2019-08-21T13:21:09Z", + "full_name": "secureCodeBox/securecodebox.github.io", + "id": "203588805", + "last_activity_at": "2022-04-19T13:33:36Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/secureCodeBox-v2", + "web_url": "https://github.com/secureCodeBox/securecodebox.github.io", }, "category": "Git Repository", "description": "A GitHub repository", @@ -428,15 +568,22 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-03-29T10:40:12Z", - "full_name": "secureCodeBox/zap-extensions", - "id": 251007807, - "last_activity_at": "2020-03-29T10:40:13Z", - "owner_id": 34573705, + "archived": true, + "created_at": "2020-09-02T13:39:10Z", + "full_name": "secureCodeBox/documentation", + "id": "292293538", + "last_activity_at": "2024-01-20T09:04:19Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [ + "securecodebox", + "docusaurus", + "documentation", + "hacktoberfest", + ], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/zap-extensions", + "web_url": "https://github.com/secureCodeBox/documentation", }, "category": "Git Repository", "description": "A GitHub repository", @@ -446,15 +593,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-07-07T14:14:16Z", - "full_name": "secureCodeBox/zaproxy", - "id": 277835641, - "last_activity_at": "2020-07-07T14:14:18Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2023-03-21T19:51:07Z", + "full_name": "secureCodeBox/www-community", + "id": "617150115", + "last_activity_at": "2024-03-14T15:30:35Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/zaproxy", + "web_url": "https://github.com/secureCodeBox/www-community", }, "category": "Git Repository", "description": "A GitHub repository", @@ -464,15 +613,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-08-07T17:58:52Z", - "full_name": "secureCodeBox/static-export", - "id": 285890805, - "last_activity_at": "2020-08-12T12:53:05Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2023-10-13T14:15:48Z", + "full_name": "secureCodeBox/landscape", + "id": "704559693", + "last_activity_at": "2024-07-11T19:08:33Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", - "visibility": "private", - "web_url": "https://github.com/secureCodeBox/static-export", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/landscape", }, "category": "Git Repository", "description": "A GitHub repository", @@ -482,15 +633,17 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-08-17T15:09:19Z", - "full_name": "secureCodeBox/telemetry", - "id": 288212154, - "last_activity_at": "2020-09-01T10:08:23Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2024-08-09T14:48:36Z", + "full_name": "secureCodeBox/DevSecOps-MaturityModel", + "id": "840369455", + "last_activity_at": "2024-08-09T14:48:36Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/telemetry", + "web_url": "https://github.com/secureCodeBox/DevSecOps-MaturityModel", }, "category": "Git Repository", "description": "A GitHub repository", @@ -500,15 +653,57 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-09-02T13:39:10Z", - "full_name": "secureCodeBox/documentation", - "id": 292293538, - "last_activity_at": "2020-10-21T14:28:35Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2021-01-06T14:27:46Z", + "full_name": "secureCodeBox/gitleaks", + "id": "327336031", + "last_activity_at": "2024-01-30T22:39:59Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", + "topics": [], "visibility": "public", - "web_url": "https://github.com/secureCodeBox/documentation", + "web_url": "https://github.com/secureCodeBox/gitleaks", + }, + "category": "Git Repository", + "description": "A GitHub repository", + "name": "GitHub Repo", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + }, + { + "attributes": { + "archived": false, + "created_at": "2025-04-18T19:36:30Z", + "full_name": "secureCodeBox/scb-cascades-demo", + "id": "968815564", + "last_activity_at": "2025-04-24T11:47:52Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scb-cascades-demo", + }, + "category": "Git Repository", + "description": "A GitHub repository", + "name": "GitHub Repo", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + }, + { + "attributes": { + "archived": false, + "created_at": "2024-12-29T17:07:02Z", + "full_name": "secureCodeBox/scan-throttler", + "id": "909750535", + "last_activity_at": "2025-08-19T14:02:18Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/scan-throttler", }, "category": "Git Repository", "description": "A GitHub repository", @@ -518,15 +713,24 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-09-03T13:08:22Z", - "full_name": "secureCodeBox/ui", - "id": 292573194, - "last_activity_at": "2020-10-07T14:38:02Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2021-01-06T09:59:17Z", + "full_name": "secureCodeBox/defectdojo-client-java", + "id": "327269915", + "last_activity_at": "2025-10-20T08:29:33Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", - "visibility": "private", - "web_url": "https://github.com/secureCodeBox/ui", + "topics": [ + "defectdojo", + "owasp", + "client-library", + "java", + "gradle", + "hacktoberfest", + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/defectdojo-client-java", }, "category": "Git Repository", "description": "A GitHub repository", @@ -536,15 +740,49 @@ test("should properly parse git-scanner json file", async () => { }, { "attributes": { - "created_at": "2020-09-28T08:58:53Z", - "full_name": "secureCodeBox/internal", - "id": 299249650, - "last_activity_at": "2020-10-21T15:11:56Z", - "owner_id": 34573705, + "archived": false, + "created_at": "2020-08-17T15:09:19Z", + "full_name": "secureCodeBox/telemetry", + "id": "288212154", + "last_activity_at": "2025-11-20T07:43:05Z", + "owner_id": "34573705", "owner_name": "secureCodeBox", "owner_type": "Organization", - "visibility": "private", - "web_url": "https://github.com/secureCodeBox/internal", + "topics": [], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/telemetry", + }, + "category": "Git Repository", + "description": "A GitHub repository", + "name": "GitHub Repo", + "osi_layer": "APPLICATION", + "severity": "INFORMATIONAL", + }, + { + "attributes": { + "archived": false, + "created_at": "2017-02-02T09:48:05Z", + "full_name": "secureCodeBox/secureCodeBox", + "id": "80711933", + "last_activity_at": "2025-11-24T10:04:46Z", + "owner_id": "34573705", + "owner_name": "secureCodeBox", + "owner_type": "Organization", + "topics": [ + "security", + "security-automation", + "security-tools", + "security-testing", + "securecodebox", + "devsecops", + "kubernetes", + "kubernetes-operator", + "owasp", + "hacktoberfest", + "zaproxy", + ], + "visibility": "public", + "web_url": "https://github.com/secureCodeBox/secureCodeBox", }, "category": "Git Repository", "description": "A GitHub repository", diff --git a/scanners/git-repo-scanner/scanner/.dockerignore b/scanners/git-repo-scanner/scanner/.dockerignore index d4ea27fec2..ddb1ff28e6 100644 --- a/scanners/git-repo-scanner/scanner/.dockerignore +++ b/scanners/git-repo-scanner/scanner/.dockerignore @@ -2,6 +2,4 @@ # # SPDX-License-Identifier: Apache-2.0 -**/.pytest_cache -**/__pycache__ -/tests +/test diff --git a/scanners/git-repo-scanner/scanner/Dockerfile b/scanners/git-repo-scanner/scanner/Dockerfile index d03a8a91aa..989b145ed4 100644 --- a/scanners/git-repo-scanner/scanner/Dockerfile +++ b/scanners/git-repo-scanner/scanner/Dockerfile @@ -2,10 +2,29 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM docker.io/python:3-alpine -COPY . /scripts/ -RUN pip install -r /scripts/requirements.txt -RUN adduser -S -H -u 1001 gitreposcanner -USER 1001 -WORKDIR /scripts -ENTRYPOINT ["python", "-m", "git_repo_scanner"] +# Build the pull-secret-extractor binary +FROM --platform=$BUILDPLATFORM golang:1.25.4 AS builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY internal/ internal/ + +# Build +ARG TARGETOS TARGETARCH +RUN GOOS="$TARGETOS" GOARCH="$TARGETARCH" CGO_ENABLED=0 go build -a -o git-repo-scanner main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/git-repo-scanner . + +ENTRYPOINT ["/git-repo-scanner"] diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py deleted file mode 100644 index 093c0acf97..0000000000 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py +++ /dev/null @@ -1,188 +0,0 @@ -# SPDX-FileCopyrightText: the secureCodeBox authors -# -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import json -import logging -import sys -from datetime import datetime, timedelta, timezone -from pathlib import Path - -import github -import gitlab -import pytz - -# https://pypi.org/project/pytimeparse/ -from pytimeparse.timeparse import timeparse - -from git_repo_scanner.abstract_scanner import AbstractScanner -from git_repo_scanner.github_scanner import GitHubScanner -from git_repo_scanner.gitlab_scanner import GitLabScanner - -log_format = "%(asctime)s - %(levelname)-7s - %(name)s - %(message)s" -logging.basicConfig(level=logging.INFO, format=log_format) -logger = logging.getLogger("git_repo_scanner") - -now_utc = datetime.now(timezone.utc) - - -def main(): - args = get_parser_args() - - if not args.git_type: - logger.info("Argument error: No git type specified") - sys.exit(1) - - findings = process(args) - - logger.info("Write findings to file...") - write_findings_to_file(args, findings) - logger.info("Finished!") - - -def process(args): - scanner: AbstractScanner - - if args.git_type == "gitlab": - scanner = GitLabScanner( - url=args.url, - access_token=args.access_token, - group=args.group, - ignored_groups=args.ignore_groups, - ignore_repos=args.ignore_repos, - obey_rate_limit=args.obey_rate_limit, - annotate_latest_commit_id=args.annotate_latest_commit_id, - ) - elif args.git_type == "github": - scanner = GitHubScanner( - url=args.url, - access_token=args.access_token, - organization=args.organization, - ignore_repos=args.ignore_repos, - obey_rate_limit=args.obey_rate_limit, - annotate_latest_commit_id=args.annotate_latest_commit_id, - ) - else: - logger.info("Argument error: Unknown git type") - sys.exit(1) - - try: - return scanner.process( - args.activity_since_duration, args.activity_until_duration - ) - except argparse.ArgumentError as e: - logger.error(f"Argument error: {e}") - sys.exit(1) - except gitlab.exceptions.GitlabAuthenticationError: - logger.info("No permission. Check your access token.") - sys.exit(1) - except github.GithubException as e: - logger.error(f'Github API Exception: {e.status} -> {e.data["message"]}') - sys.exit(2) - except gitlab.GitlabError as e: - logger.error(f"Gitlab API Exception: {e}") - sys.exit(2) - except Exception as e: - logger.error(f"Unexpected error: {e}") - sys.exit(3) - - -def write_findings_to_file(args, findings): - Path(args.file_output).mkdir(parents=True, exist_ok=True) - with open(f"{args.file_output}/git-repo-scanner-findings.json", "w") as out: - json.dump(findings, out) - - -def parse_duration_as_datetime(val: str): - try: - parsed = timeparse(val) - if parsed is None: - raise argparse.ArgumentTypeError(f"Not a valid duration: {val}.") - delta = timedelta(seconds=parsed) - return now_utc - delta - except Exception: - raise argparse.ArgumentTypeError(f"Not a valid duration: {val}.") - - -def get_parser_args(args=None): - parser = argparse.ArgumentParser( - prog="git_repo_scanner", - description="Scan public or private git repositories of organizations or groups", - ) - parser.add_argument( - "--git-type", - help="Repository type can be github or GitLab", - choices=["github", "gitlab"], - required=True, - ) - parser.add_argument( - "--file-output", help="The path of the output file", required=True - ), - parser.add_argument( - "--url", help="The GitLab url or a GitHub enterprise api url.", required=False - ) - parser.add_argument( - "--access-token", help="An access token for authentication", required=False - ) - parser.add_argument( - "--organization", - help="The name of the GitHub organization to scan", - required=False, - ) - parser.add_argument( - "--group", help="The id of the GitLab group to scan", type=int, required=False - ) - parser.add_argument( - "--ignore-repos", - help="A list of repo ids to ignore", - action="extend", - nargs="+", - type=int, - default=[], - required=False, - ) - parser.add_argument( - "--ignore-groups", - help="A list of GitLab group ids to ignore", - action="extend", - nargs="+", - type=int, - default=[], - required=False, - ) - parser.add_argument( - "--obey-rate-limit", - help="True to obey the rate limit of the GitLab or GitHub server (default), otherwise False", - type=bool, - default=True, - required=False, - ) - parser.add_argument( - "--annotate-latest-commit-id", - help="Annotate the results with the latest commit hash of the main branch of the repository. " - "Will result in up to two extra API hits per repository", - type=bool, - default=False, - required=False, - ) - parser.add_argument( - "--activity-since-duration", - help="Return git repo findings with repo activity (e.g. commits) more recent than a specific " - "date expressed by a duration (now - duration)", - type=parse_duration_as_datetime, - required=False, - ) - parser.add_argument( - "--activity-until-duration", - help="Return git repo findings with repo activity (e.g. commits) older than a specific date " - "expressed by a duration (now - duration)", - type=parse_duration_as_datetime, - required=False, - ) - - return parser.parse_args(args) - - -if __name__ == "__main__": - main() diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py deleted file mode 100644 index 175b247379..0000000000 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-FileCopyrightText: the secureCodeBox authors -# -# SPDX-License-Identifier: Apache-2.0 - -import abc -from datetime import datetime -from typing import Dict, List, Optional - -FINDING = Dict[str, any] - - -class AbstractScanner(abc.ABC): - @property - @abc.abstractmethod - def git_type(self) -> str: - raise NotImplementedError() - - @abc.abstractmethod - def process( - self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None - ) -> List[FINDING]: - raise NotImplementedError() - - def _create_finding( - self, - repo_id: str, - web_url: str, - full_name: str, - owner_type: str, - owner_id: str, - owner_name: str, - created_at: str, - last_activity_at: str, - visibility: str, - archived: bool, - topics: list, - last_commit_id: str = None, - ) -> FINDING: - finding = { - "name": f"{self.git_type} Repo", - "description": f"A {self.git_type} repository", - "category": "Git Repository", - "osi_layer": "APPLICATION", - "severity": "INFORMATIONAL", - "attributes": { - "id": repo_id, - "web_url": web_url, - "full_name": full_name, - "owner_type": owner_type, - "owner_id": owner_id, - "topics": topics, - "owner_name": owner_name, - "created_at": created_at, - "last_activity_at": last_activity_at, - "visibility": visibility, - "archived": archived, - }, - } - if last_commit_id is not None: - finding["attributes"]["last_commit_id"] = last_commit_id - return finding diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py deleted file mode 100644 index 2a02ca43f5..0000000000 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py +++ /dev/null @@ -1,190 +0,0 @@ -# SPDX-FileCopyrightText: the secureCodeBox authors -# -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import logging -import time -from calendar import timegm -from datetime import datetime, timezone -from typing import Optional, List - -import github -from github.Organization import Organization -from github.PaginatedList import PaginatedList -from github.Repository import Repository - -from git_repo_scanner.abstract_scanner import AbstractScanner, FINDING - - -class GitHubScanner(AbstractScanner): - LOGGER = logging.getLogger("git_repo_scanner") - - def __init__( - self, - url: Optional[str], - access_token: Optional[str], - organization: str, - ignore_repos: List[int], - obey_rate_limit: bool = True, - annotate_latest_commit_id: bool = False, - ) -> None: - super().__init__() - if not organization: - raise argparse.ArgumentError( - None, "Organization required for GitHub connection." - ) - if url and not access_token: - raise argparse.ArgumentError( - None, "Access token required for GitHub connection." - ) - - self._url = url - self._access_token = access_token - self._organization = organization - self._ignore_repos = ignore_repos - self._obey_rate_limit = obey_rate_limit - self._annotate_latest_commit_id = annotate_latest_commit_id - self._gh: Optional[github.Github] = None - - @property - def git_type(self) -> str: - return "GitHub" - - def process( - self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None - ) -> List[FINDING]: - self._setup() - return self._process_repos(start_time, end_time) - - def _process_repos( - self, start_time: Optional[datetime], end_time: Optional[datetime] - ): - findings = [] - org: Organization = self._gh.get_organization(self._organization) - - repos: PaginatedList[Repository] = org.get_repos( - type="all", sort="pushed", direction="asc" - ) - - if start_time: - repos = org.get_repos(type="all", sort="pushed", direction="desc") - - for i in range(repos.totalCount): - self._process_repos_page(findings, repos.get_page(i), start_time, end_time) - return findings - - def _process_repos_page( - self, - findings: List[FINDING], - repos: List[Repository], - start_time: Optional[datetime] = None, - end_time: Optional[datetime] = None, - ): - repo: Repository - for repo in repos: - if repo.id not in self._ignore_repos: - self.LOGGER.info( - f"{len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}" - ) - - if (start_time or end_time) and not self._check_repo_is_in_time_frame( - repo.pushed_at, start_time, end_time - ): - break - - findings.append(self._create_finding_from_repo(repo)) - self._respect_github_ratelimit() - - def _check_repo_is_in_time_frame( - self, - pushed_at: datetime, - start_time: Optional[datetime] = None, - end_time: Optional[datetime] = None, - ): - # Explicitly set timezone of pushed_at, as it is not set by the library (but is in UTC) - pushed_at = pushed_at.replace(tzinfo=timezone.utc) - if start_time: - if pushed_at > start_time: - return True - else: - self.LOGGER.info( - f"Reached activity limit! Ignoring all repos with activity since `{start_time}`." - ) - return False - elif end_time: - if pushed_at < end_time: - return True - else: - self.LOGGER.info( - f"Reached activity limit! Ignoring all repos with activity until `{end_time}`." - ) - return False - - def _respect_github_ratelimit(self): - if self._obey_rate_limit: - api_limit = self._gh.get_rate_limit().core - reset_timestamp = timegm(api_limit.reset.timetuple()) - # add 5 seconds to be sure the rate limit has been reset - seconds_until_reset = reset_timestamp - timegm(time.gmtime()) + 5 - sleep_time = seconds_until_reset / api_limit.remaining - - self.LOGGER.info( - "Checking Rate-Limit (" - + str(self._obey_rate_limit) - + ") [remainingApiCalls: " - + str(api_limit.remaining) - + ", seconds_until_reset: " - + str(seconds_until_reset) - + ", sleepTime: " - + str(sleep_time) - + "]" - ) - time.sleep(sleep_time) - - def _setup(self): - if self._url: - self._setup_with_url() - else: - self._setup_without_url() - - def _setup_without_url(self): - if self._access_token: - self._gh = github.Github(self._access_token) - else: - self._gh = github.Github() - - def _setup_with_url(self): - if self._access_token: - self._gh = github.Github( - base_url=self._url, login_or_token=self._access_token - ) - else: - raise argparse.ArgumentError( - None, "Access token required for github enterprise authentication." - ) - - def _create_finding_from_repo(self, repo: Repository) -> FINDING: - latest_commit: str = None - if self._annotate_latest_commit_id: - try: - latest_commit = repo.get_commits()[0].sha - except Exception: - self.LOGGER.warn( - "Could not identify the latest commit ID - repository without commits?" - ) - latest_commit = "" - return super()._create_finding( - str(repo.id), - repo.html_url, - repo.full_name, - repo.owner.type, - str(repo.owner.id), - repo.owner.name, - repo.created_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - repo.updated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - "private" if repo.private else "public", - repo.archived, - repo.get_topics(), - latest_commit, - ) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py deleted file mode 100644 index 70140bca41..0000000000 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: the secureCodeBox authors -# -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import logging -from datetime import datetime -from typing import List, Optional - -import gitlab -from gitlab.v4.objects import Project, ProjectManager - -from git_repo_scanner.abstract_scanner import AbstractScanner, FINDING - -logger = logging.getLogger("git_repo_scanner") - - -class GitLabScanner(AbstractScanner): - LOGGER = logging.getLogger("git_repo_scanner") - - def __init__( - self, - url: str, - access_token: str, - group: Optional[int], - ignored_groups: List[int], - ignore_repos: List[int], - obey_rate_limit: bool = True, - annotate_latest_commit_id: bool = False, - ) -> None: - super().__init__() - if not url: - raise argparse.ArgumentError(None, "URL required for GitLab connection.") - if not access_token: - raise argparse.ArgumentError( - None, "Access token required for GitLab authentication." - ) - - self._url = url - self._access_token = access_token - self._group = group - self._ignored_groups = ignored_groups - self._ignore_repos = ignore_repos - self._obey_rate_limit = obey_rate_limit - self._annotate_latest_commit_id = annotate_latest_commit_id - self._gl: Optional[gitlab.Gitlab] = None - - @property - def git_type(self) -> str: - return "GitLab" - - def process( - self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None - ) -> List[FINDING]: - self._authenticate() - - projects: List[Project] = self._get_projects(start_time, end_time) - return self._process_projects(projects) - - def _group_project_to_project(self, group_project): - # The GitLab API library gives us a GroupProject object, which has limited functionality. - # This function turns the GroupProject into a "real" project, which allows us to get the - # list of commits and include the SHA1 of the latest commit in the output later - return self._gl.projects.get(group_project.id, lazy=True) - - def _get_projects( - self, start_time: Optional[datetime], end_time: Optional[datetime] - ): - logger.info( - f"Get GitLab repositories with last activity between {start_time} and {end_time}." - ) - - project_manager: ProjectManager = self._gl.projects - options = dict( - all=True, - order_by="last_activity_at", - sort="desc", - obey_rate_limit=self._obey_rate_limit, - max_retries=12, - ) - if start_time is not None: - options["last_activity_after"] = start_time - if end_time is not None: - options["last_activity_before"] = end_time - - if self._group: - options["include_subgroups"] = True - project_manager = self._gl.groups.get(self._group).projects - - return project_manager.list(**options) - - def _process_projects(self, projects: List[Project]) -> List[FINDING]: - project_count = len(projects) - return [ - self._create_finding_from_project(project, i, project_count) - for i, project in enumerate(projects) - if self._is_not_ignored(project) - ] - - def _authenticate(self): - logger.info("Start GitLab authentication") - try: - self._gl = gitlab.Gitlab(self._url, private_token=self._access_token) - self._gl.auth() - except gitlab.exceptions.GitlabAuthenticationError: - self._gl = gitlab.Gitlab(self._url, oauth_token=self._access_token) - self._gl.auth() - - logger.info("GitLab authentication succeeded") - - def _is_not_ignored(self, project: Project) -> bool: - id_project = project.id - kind = project.namespace["kind"] - id_namespace = project.namespace["id"] - if id_project in self._ignore_repos: - return False - if kind == "group" and id_namespace in self._ignored_groups: - return False - return True - - def _create_finding_from_project( - self, project: Project, index: int, total: int - ) -> FINDING: - logger.info( - f"({index + 1}/{total}) Add finding for repo {project.name} with last activity at " - f"{datetime.fromisoformat(project.last_activity_at)}" - ) - - # Retrieve the latest commit ID - latest_commit_id: str = None - if self._annotate_latest_commit_id: - try: - latest_commit_id = ( - self._group_project_to_project(project).commits.list()[0].id - ) - except Exception as e: - logger.warn( - "Could not identify the latest commit ID - repository without commits?" - ) - latest_commit_id = "" - return super()._create_finding( - project.id, - project.web_url, - project.path_with_namespace, - project.namespace["kind"], - project.namespace["id"], - project.namespace["name"], - project.created_at, - project.last_activity_at, - project.visibility, - project.archived, - project.topics, - latest_commit_id, - ) diff --git a/scanners/git-repo-scanner/scanner/go.mod b/scanners/git-repo-scanner/scanner/go.mod new file mode 100644 index 0000000000..89efde301c --- /dev/null +++ b/scanners/git-repo-scanner/scanner/go.mod @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +module github.com/secureCodeBox/scanners/git-repo-scanner/scanner + +go 1.25.4 + +require ( + github.com/google/go-github/v79 v79.0.0 + gitlab.com/gitlab-org/api/client-go v0.160.1 + golang.org/x/oauth2 v0.30.0 +) + +require ( + github.com/google/go-querystring v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/time v0.12.0 // indirect +) diff --git a/scanners/git-repo-scanner/scanner/go.sum b/scanners/git-repo-scanner/scanner/go.sum new file mode 100644 index 0000000000..eb2dcdd019 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/go.sum @@ -0,0 +1,38 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v79 v79.0.0 h1:MdodQojuFPBhmtwHiBcIGLw/e/wei2PvFX9ndxK0X4Y= +github.com/google/go-github/v79 v79.0.0/go.mod h1:OAFbNhq7fQwohojb06iIIQAB9CBGYLq999myfUFnrS4= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gitlab.com/gitlab-org/api/client-go v0.160.1 h1:7kEgo1yQ3ZMRps/2JbXzqbRb4Rs8n2ECkAv+6MadJw8= +gitlab.com/gitlab-org/api/client-go v0.160.1/go.mod h1:YqKcnxyV9OPAL5U99mpwBVEgBPz1PK/3qwqq/3h6bao= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scanners/git-repo-scanner/scanner/go.sum.license b/scanners/git-repo-scanner/scanner/go.sum.license new file mode 100644 index 0000000000..c95bc37185 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/go.sum.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: the secureCodeBox authors + +SPDX-License-Identifier: Apache-2.0 diff --git a/scanners/git-repo-scanner/scanner/internal/config/config.go b/scanners/git-repo-scanner/scanner/internal/config/config.go new file mode 100644 index 0000000000..4e4ef5a192 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/config/config.go @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "flag" + "fmt" + "log" + "strconv" + "strings" + "time" + + "github.com/secureCodeBox/scanners/git-repo-scanner/scanner/internal/duration" +) + +type Config struct { + GitType string + FileOutput string + URL string + AccessToken string + Organization string + Group *int + IgnoreRepos []int64 + IgnoreGroups []int + ObeyRateLimit bool + AnnotateLatestCommitID bool + ActivitySinceDuration *time.Duration + ActivityUntilDuration *time.Duration +} + +func ParseFlags(logger *log.Logger) (*Config, error) { + config := &Config{} + + // Define flags + flag.StringVar(&config.GitType, "git-type", "", "Repository type can be GitHub or GitLab") + flag.StringVar(&config.FileOutput, "file-output", "", "The path of the output file") + flag.StringVar(&config.URL, "url", "", "The GitLab url or a GitHub enterprise api url") + flag.StringVar(&config.AccessToken, "access-token", "", "An access token for authentication") + flag.StringVar(&config.Organization, "organization", "", "The name of the GitHub organization to scan") + + var groupStr string + flag.StringVar(&groupStr, "group", "", "The id of the GitLab group to scan") + + var ignoreReposStr string + flag.StringVar(&ignoreReposStr, "ignore-repos", "", "Comma-separated list of repo ids to ignore") + + var ignoreGroupsStr string + flag.StringVar(&ignoreGroupsStr, "ignore-groups", "", "Comma-separated list of GitLab group ids to ignore") + + flag.BoolVar(&config.ObeyRateLimit, "obey-rate-limit", true, + "True to obey the rate limit of the GitLab or GitHub server (default), otherwise false") + flag.BoolVar(&config.AnnotateLatestCommitID, "annotate-latest-commit-id", false, + "Annotate the results with the latest commit hash of the main branch of the repository") + + var activitySinceStr string + var activityUntilStr string + flag.StringVar(&activitySinceStr, "activity-since-duration", "", + "Return git repo findings with repo activity more recent than a specific duration (e.g., '7d', '2w', '1h')") + flag.StringVar(&activityUntilStr, "activity-until-duration", "", + "Return git repo findings with repo activity older than a specific duration (e.g., '7d', '2w', '1h')") + + flag.Parse() + + if err := config.validate(); err != nil { + flag.Usage() + return nil, err + } + + if err := config.parseOptionalFields(groupStr, ignoreReposStr, ignoreGroupsStr, activitySinceStr, activityUntilStr); err != nil { + return nil, err + } + + return config, nil +} + +func (c *Config) validate() error { + if c.GitType == "" { + return fmt.Errorf("--git-type is required") + } + + if c.GitType != "GitHub" && c.GitType != "GitLab" { + return fmt.Errorf("invalid git-type: %s. Must be 'GitHub' or 'GitLab'", c.GitType) + } + + if c.FileOutput == "" { + return fmt.Errorf("--file-output is required") + } + + // Validate GitLab specific requirements + if c.GitType == "GitLab" && c.URL == "" { + return fmt.Errorf("--url is required for GitLab") + } + + // Validate GitHub specific requirements + if c.GitType == "GitHub" && c.Organization == "" { + return fmt.Errorf("--organization is required for GitHub") + } + + return nil +} + +func (c *Config) parseOptionalFields(groupStr, ignoreReposStr, ignoreGroupsStr, activitySinceStr, activityUntilStr string) error { + if groupStr != "" { + group, err := strconv.Atoi(groupStr) + if err != nil { + return fmt.Errorf("invalid group id: %s", groupStr) + } + c.Group = &group + } + + if ignoreReposStr != "" { + repos, err := parseIntListAsInt64(ignoreReposStr) + if err != nil { + return fmt.Errorf("invalid repo ids in ignore-repos: %w", err) + } + c.IgnoreRepos = repos + } + + if ignoreGroupsStr != "" { + groups, err := parseIntListAsInt(ignoreGroupsStr) + if err != nil { + return fmt.Errorf("invalid group ids in ignore-groups: %w", err) + } + c.IgnoreGroups = groups + } + + if activitySinceStr != "" { + d, err := duration.Parse(activitySinceStr) + if err != nil { + return fmt.Errorf("invalid activity-since-duration: %w", err) + } + c.ActivitySinceDuration = &d + } + + if activityUntilStr != "" { + d, err := duration.Parse(activityUntilStr) + if err != nil { + return fmt.Errorf("invalid activity-until-duration: %w", err) + } + c.ActivityUntilDuration = &d + } + + return nil +} + +func parseIntListAsInt64(s string) ([]int64, error) { + var result []int64 + parts := strings.SplitSeq(s, ",") + + for part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + id, err := strconv.ParseInt(part, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid id: %s", part) + } + result = append(result, id) + } + + return result, nil +} + +func parseIntListAsInt(s string) ([]int, error) { + var result []int + parts := strings.SplitSeq(s, ",") + + for part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + id, err := strconv.Atoi(part) + if err != nil { + return nil, fmt.Errorf("invalid id: %s", part) + } + result = append(result, id) + } + + return result, nil +} + +// GetTimeFrame returns the start and end times based on duration configuration +func (c *Config) GetTimeFrame() (*time.Time, *time.Time, error) { + if c.ActivitySinceDuration == nil && c.ActivityUntilDuration == nil { + return nil, nil, nil + } + + now := time.Now().UTC() + var startTime, endTime *time.Time + + if c.ActivitySinceDuration != nil { + t := now.Add(-*c.ActivitySinceDuration) + startTime = &t + } + + if c.ActivityUntilDuration != nil { + t := now.Add(-*c.ActivityUntilDuration) + endTime = &t + } + + if startTime != nil && endTime != nil && startTime.After(*endTime) { + return nil, nil, fmt.Errorf("activity-since-duration must be greater than activity-until-duration") + } + + return startTime, endTime, nil +} diff --git a/scanners/git-repo-scanner/scanner/internal/config/config_test.go b/scanners/git-repo-scanner/scanner/internal/config/config_test.go new file mode 100644 index 0000000000..1776c6ea34 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/config/config_test.go @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "strings" + "testing" + "time" +) + +func TestConfig_Validate(t *testing.T) { + tests := []struct { + name string + config Config + wantErr bool + errMsg string + }{ + { + name: "missing git-type", + config: Config{FileOutput: "output.json"}, + wantErr: true, + errMsg: "--git-type is required", + }, + { + name: "invalid git-type", + config: Config{GitType: "Bitbucket", FileOutput: "output.json"}, + wantErr: true, + errMsg: "invalid git-type", + }, + { + name: "GitLab missing URL", + config: Config{GitType: "GitLab", FileOutput: "output.json"}, + wantErr: true, + errMsg: "--url is required for GitLab", + }, + { + name: "GitHub missing organization", + config: Config{GitType: "GitHub", FileOutput: "output.json"}, + wantErr: true, + errMsg: "--organization is required for GitHub", + }, + { + name: "valid GitHub config", + config: Config{ + GitType: "GitHub", + FileOutput: "output.json", + Organization: "test-org", + }, + wantErr: false, + }, + { + name: "valid GitLab config", + config: Config{ + GitType: "GitLab", + FileOutput: "output.json", + URL: "https://gitlab.com", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.validate() + if (err != nil) != tt.wantErr { + t.Errorf("validate() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { + t.Errorf("error message = %v, want to contain %v", err.Error(), tt.errMsg) + } + }) + } +} + +func TestConfig_ParseOptionalFields(t *testing.T) { + tests := []struct { + name string + groupStr string + ignoreReposStr string + ignoreGroupsStr string + activitySinceStr string + activityUntilStr string + wantErr bool + validateResult func(t *testing.T, c *Config) + }{ + { + name: "valid group ID", + groupStr: "123", + wantErr: false, + validateResult: func(t *testing.T, c *Config) { + if c.Group == nil || *c.Group != 123 { + t.Errorf("expected group to be 123, got %v", c.Group) + } + }, + }, + { + name: "invalid group ID", + groupStr: "abc", + wantErr: true, + }, + { + name: "valid ignore repos list", + ignoreReposStr: "1,2,3,456", + wantErr: false, + validateResult: func(t *testing.T, c *Config) { + expected := []int64{1, 2, 3, 456} + if len(c.IgnoreRepos) != len(expected) { + t.Errorf("expected %d repos, got %d", len(expected), len(c.IgnoreRepos)) + } + for i, v := range expected { + if c.IgnoreRepos[i] != v { + t.Errorf("repo[%d]: expected %d, got %d", i, v, c.IgnoreRepos[i]) + } + } + }, + }, + { + name: "ignore repos with spaces", + ignoreReposStr: " 1 , 2 , 3 ", + wantErr: false, + validateResult: func(t *testing.T, c *Config) { + if len(c.IgnoreRepos) != 3 { + t.Errorf("expected 3 repos, got %d", len(c.IgnoreRepos)) + } + }, + }, + { + name: "invalid ignore repos", + ignoreReposStr: "1,abc,3", + wantErr: true, + }, + { + name: "valid activity durations", + activitySinceStr: "7d", + activityUntilStr: "1d", + wantErr: false, + validateResult: func(t *testing.T, c *Config) { + if c.ActivitySinceDuration == nil { + t.Error("ActivitySinceDuration should not be nil") + } + if c.ActivityUntilDuration == nil { + t.Error("ActivityUntilDuration should not be nil") + } + }, + }, + { + name: "invalid activity-since duration", + activitySinceStr: "invalid", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Config{} + err := c.parseOptionalFields(tt.groupStr, tt.ignoreReposStr, tt.ignoreGroupsStr, + tt.activitySinceStr, tt.activityUntilStr) + + if (err != nil) != tt.wantErr { + t.Errorf("parseOptionalFields() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr && tt.validateResult != nil { + tt.validateResult(t, c) + } + }) + } +} + +func TestConfig_GetTimeFrame(t *testing.T) { + sevenDays := 7 * 24 * time.Hour + oneDay := 24 * time.Hour + + tests := []struct { + name string + activitySinceDuration *time.Duration + activityUntilDuration *time.Duration + wantErr bool + errMsg string + }{ + { + name: "no durations returns nil", + wantErr: false, + }, + { + name: "valid range: 7 days since, 1 day until", + activitySinceDuration: &sevenDays, + activityUntilDuration: &oneDay, + wantErr: false, + }, + { + name: "invalid range: since < until", + activitySinceDuration: &oneDay, + activityUntilDuration: &sevenDays, + wantErr: true, + errMsg: "activity-since-duration must be greater than activity-until-duration", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Config{ + ActivitySinceDuration: tt.activitySinceDuration, + ActivityUntilDuration: tt.activityUntilDuration, + } + + start, end, err := c.GetTimeFrame() + + if (err != nil) != tt.wantErr { + t.Errorf("GetTimeFrame() error = %v, wantErr %v", err, tt.wantErr) + } + + if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { + t.Errorf("error message = %v, want to contain %v", err.Error(), tt.errMsg) + } + + // Basic validation of the logic + if err == nil { + if tt.activitySinceDuration == nil && tt.activityUntilDuration == nil { + if start != nil || end != nil { + t.Error("expected both start and end to be nil when no durations set") + } + } + if start != nil && end != nil && start.After(*end) { + t.Error("start time should be before end time") + } + } + }) + } +} diff --git a/scanners/git-repo-scanner/scanner/internal/duration/parser.go b/scanners/git-repo-scanner/scanner/internal/duration/parser.go new file mode 100644 index 0000000000..bdce224709 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/duration/parser.go @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package duration + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +var durationMultipliers = map[string]time.Duration{ + "s": time.Second, + "m": time.Minute, + "h": time.Hour, + "d": 24 * time.Hour, + "w": 7 * 24 * time.Hour, + "mo": 30 * 24 * time.Hour, + "y": 365 * 24 * time.Hour, +} + +// parseDuration parses duration strings like "7d", "2w", "1h30m" +func Parse(s string) (time.Duration, error) { + // First try standard Go duration parsing + if d, err := time.ParseDuration(s); err == nil { + return d, nil + } + + // Handle common suffixes + s = strings.ToLower(strings.TrimSpace(s)) + + multipliers := map[string]time.Duration{ + "s": time.Second, + "m": time.Minute, + "h": time.Hour, + "d": 24 * time.Hour, + "w": 7 * 24 * time.Hour, + "mo": 30 * 24 * time.Hour, + "y": 365 * 24 * time.Hour, + } + + // Try to parse with custom suffixes + for suffix, multiplier := range multipliers { + if strings.HasSuffix(s, suffix) { + numStr := strings.TrimSuffix(s, suffix) + num, err := strconv.ParseFloat(numStr, 64) + if err != nil { + continue + } + return time.Duration(float64(multiplier) * num), nil + } + } + + return 0, fmt.Errorf("unable to parse duration: %s", s) +} diff --git a/scanners/git-repo-scanner/scanner/internal/duration/parser_test.go b/scanners/git-repo-scanner/scanner/internal/duration/parser_test.go new file mode 100644 index 0000000000..23d7e4b826 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/duration/parser_test.go @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package duration + +import ( + "testing" + "time" +) + +func TestParse(t *testing.T) { + tests := []struct { + name string + input string + expected time.Duration + wantErr bool + }{ + { + name: "standard_seconds", + input: "30s", + expected: 30 * time.Second, + wantErr: false, + }, + { + name: "standard_minutes", + input: "15m", + expected: 15 * time.Minute, + wantErr: false, + }, + { + name: "standard_hours", + input: "2h", + expected: 2 * time.Hour, + wantErr: false, + }, + { + name: "standard_combined", + input: "1h30m", + expected: 90 * time.Minute, + wantErr: false, + }, + { + name: "custom_days", + input: "7d", + expected: 7 * 24 * time.Hour, + wantErr: false, + }, + { + name: "custom_weeks", + input: "2w", + expected: 14 * 24 * time.Hour, + wantErr: false, + }, + { + name: "custom_months", + input: "1mo", + expected: 30 * 24 * time.Hour, + wantErr: false, + }, + { + name: "custom_years", + input: "1y", + expected: 365 * 24 * time.Hour, + wantErr: false, + }, + { + name: "decimal_days", + input: "1.5d", + expected: 36 * time.Hour, + wantErr: false, + }, + { + name: "decimal_weeks", + input: "0.5w", + expected: 84 * time.Hour, + wantErr: false, + }, + { + name: "uppercase_days", + input: "7D", + expected: 7 * 24 * time.Hour, + wantErr: false, + }, + { + name: "uppercase_months", + input: "2MO", + expected: 60 * 24 * time.Hour, + wantErr: false, + }, + { + name: "invalid_format", + input: "invalid", + expected: 0, + wantErr: true, + }, + { + name: "empty_string", + input: "", + expected: 0, + wantErr: true, + }, + { + name: "number_only", + input: "42", + expected: 0, + wantErr: true, + }, + { + name: "invalid_suffix", + input: "7x", + expected: 0, + wantErr: true, + }, + { + name: "non_numeric_prefix", + input: "abcd", + expected: 0, + wantErr: true, + }, + { + name: "mixed_invalid", + input: "1d2w", // Not supported format + expected: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Parse(tt.input) + + if tt.wantErr { + if err == nil { + t.Errorf("Parse(%q) expected error, got nil", tt.input) + } + } else { + if err != nil { + t.Errorf("Parse(%q) unexpected error: %v", tt.input, err) + } + if got != tt.expected { + t.Errorf("Parse(%q) = %v, want %v", tt.input, got, tt.expected) + } + } + }) + } +} diff --git a/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/git_repo_scanner.go b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/git_repo_scanner.go new file mode 100644 index 0000000000..d33bdbcc52 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/git_repo_scanner.go @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package gitreposcanner + +import ( + "time" +) + +type Finding struct { + Name string `json:"name"` + Description string `json:"description"` + Category string `json:"category"` + OSILayer string `json:"osi_layer"` + Severity string `json:"severity"` + Attributes map[string]any `json:"attributes"` +} + +type GitType string + +// GitRepoScanner defines the interface that all scanners must implement +type GitRepoScanner interface { + GitType() GitType + Process(startTime, endTime *time.Time) ([]Finding, error) +} + +// BaseScanner provides common functionality for scanner implementations +type BaseScanner struct{} + +func (b *BaseScanner) CreateFinding( + gitType GitType, + repoID string, + webURL string, + fullName string, + ownerType string, + ownerID string, + ownerName string, + createdAt string, + lastActivityAt string, + visibility string, + archived bool, + topics []string, + lastCommitID *string, +) Finding { + finding := Finding{ + Name: string(gitType) + " Repo", + Description: "A " + string(gitType) + " repository", + Category: "Git Repository", + OSILayer: "APPLICATION", + Severity: "INFORMATIONAL", + Attributes: map[string]any{ + "id": repoID, + "web_url": webURL, + "full_name": fullName, + "owner_type": ownerType, + "owner_id": ownerID, + "topics": topics, + "owner_name": ownerName, + "created_at": createdAt, + "last_activity_at": lastActivityAt, + "visibility": visibility, + "archived": archived, + }, + } + + if lastCommitID != nil { + attributes := finding.Attributes + attributes["last_commit_id"] = *lastCommitID + } + + return finding +} diff --git a/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go new file mode 100644 index 0000000000..bd8c33e2a3 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go @@ -0,0 +1,299 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package gitreposcanner + +import ( + "context" + "fmt" + "log" + "net/http" + "time" + + "github.com/google/go-github/v79/github" + "golang.org/x/oauth2" +) + +const GitHub GitType = "GitHub" + +// GitHubRepoScanner implements the GitRepoScanner interface for GitHub repositories +type GitHubRepoScanner struct { + BaseScanner + url string + accessToken string + organization string + ignoreRepos map[int64]bool + obeyRateLimit bool + annotateLatestCommitID bool + client *github.Client + logger *log.Logger + ctx context.Context +} + +func NewGitHubScanner( + url string, + accessToken string, + organization string, + ignoreRepos []int64, + obeyRateLimit bool, + annotateLatestCommitID bool, + logger *log.Logger, +) (*GitHubRepoScanner, error) { + if organization == "" { + return nil, fmt.Errorf("organization required for GitHub connection") + } + if url != "" && accessToken == "" { + return nil, fmt.Errorf("access token required for GitHub connection") + } + + ignoreMap := make(map[int64]bool) + for _, id := range ignoreRepos { + ignoreMap[id] = true + } + + if logger == nil { + logger = log.New(log.Writer(), "git_repo_scanner: ", log.LstdFlags) + } + + return &GitHubRepoScanner{ + url: url, + accessToken: accessToken, + organization: organization, + ignoreRepos: ignoreMap, + obeyRateLimit: obeyRateLimit, + annotateLatestCommitID: annotateLatestCommitID, + logger: logger, + ctx: context.Background(), + }, nil +} + +func (g *GitHubRepoScanner) GitType() GitType { + return GitHub +} + +func (g *GitHubRepoScanner) Process(startTime, endTime *time.Time) ([]Finding, error) { + if err := g.setup(); err != nil { + return nil, fmt.Errorf("failed to setup GitHub client: %w", err) + } + return g.processRepos(startTime, endTime) +} + +func (g *GitHubRepoScanner) setup() error { + if g.url != "" { + return g.setupWithURL() + } + return g.setupWithoutURL() +} + +func (g *GitHubRepoScanner) setupWithoutURL() error { + if g.accessToken != "" { + tc := g.createTokenClient() + g.client = github.NewClient(tc) + } else { + g.client = github.NewClient(nil) + } + return nil +} + +func (g *GitHubRepoScanner) setupWithURL() error { + if g.accessToken == "" { + return fmt.Errorf("access token required for github enterprise authentication") + } + + tc := g.createTokenClient() + + var err error + g.client, err = github.NewClient(tc).WithEnterpriseURLs(g.url, g.url) + return err +} + +// The go-github library does not directly handle authentication. +// Instead, when creating a new client, pass an http.Client that can handle authentication for you. +// https://pkg.go.dev/github.com/google/go-github/github#hdr-Authentication +func (g *GitHubRepoScanner) createTokenClient() *http.Client { + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: g.accessToken}) + return oauth2.NewClient(g.ctx, ts) +} + +func (g *GitHubRepoScanner) processRepos(startTime, endTime *time.Time) ([]Finding, error) { + var findings []Finding + + org, _, err := g.client.Organizations.Get(g.ctx, g.organization) + if err != nil { + return nil, fmt.Errorf("failed to get organization: %w", err) + } + + opts := &github.RepositoryListByOrgOptions{ + Type: "all", + Sort: "pushed", + Direction: "asc", + ListOptions: github.ListOptions{ + PerPage: 100, + }, + } + + // If start time is specified, reverse the sort order + if startTime != nil { + opts.Direction = "desc" + } + + // Paginate through all repositories + for { + repos, resp, err := g.client.Repositories.ListByOrg(g.ctx, org.GetLogin(), opts) + if err != nil { + return nil, fmt.Errorf("failed to list repositories: %w", err) + } + + findingsForRepo, shouldContinue, err := g.processReposPage(repos, startTime, endTime) + if err != nil { + return nil, err + } + findings = append(findings, findingsForRepo...) + + if !shouldContinue || resp.NextPage == 0 { + break + } + + opts.Page = resp.NextPage + } + + return findings, nil +} + +func (g *GitHubRepoScanner) processReposPage( + repos []*github.Repository, + startTime, endTime *time.Time, +) ([]Finding, bool, error) { + var findings []Finding + + for _, repo := range repos { + if g.ignoreRepos[repo.GetID()] { + continue + } + + if (startTime != nil || endTime != nil) && !g.checkRepoIsInTimeFrame(repo.GetPushedAt().Time, startTime, endTime) { + return findings, false, nil // Stop processing further pages + } + + finding, err := g.createFindingFromRepo(repo) + if err != nil { + g.logger.Printf("Warning: failed to create finding for repo %s: %v", repo.GetName(), err) + continue + } + + findings = append(findings, finding) + + // Respect rate limit after processing each repo + if err := g.respectGitHubRateLimit(); err != nil { + return findings, false, err + } + } + + return findings, true, nil // Continue to next page +} + +func (g *GitHubRepoScanner) checkRepoIsInTimeFrame( + pushedAt time.Time, + startTime, endTime *time.Time, +) bool { + // GitHub API returns timestamps in UTC + pushedAt = pushedAt.UTC() + + if startTime != nil && pushedAt.Before(*startTime) { + g.logger.Printf("Reached activity limit! Ignoring all repos with activity before `%s`.", + startTime.Format(time.RFC3339)) + return false + } + + if endTime != nil && pushedAt.After(*endTime) { + g.logger.Printf("Reached activity limit! Ignoring all repos with activity after `%s`.", + endTime.Format(time.RFC3339)) + return false + } + + return true +} + +func (g *GitHubRepoScanner) respectGitHubRateLimit() error { + if !g.obeyRateLimit { + return nil + } + + rate, _, err := g.client.RateLimit.Get(g.ctx) + if err != nil { + return fmt.Errorf("failed to get rate limit: %w", err) + } + + core := rate.GetCore() + remaining := core.Remaining + reset := core.Reset.Time + + secondsUntilReset := time.Until(reset).Seconds() + 5 // add 5 seconds buffer + + if remaining > 0 { + sleepTime := secondsUntilReset / float64(remaining) + + if sleepTime > 0 { + time.Sleep(time.Duration(sleepTime * float64(time.Second))) + } + } else { + // No remaining API calls, wait until reset + g.logger.Printf("Rate limit exhausted. Waiting until %s", reset.Format(time.RFC3339)) + time.Sleep(time.Until(reset) + 5*time.Second) + } + + return nil +} + +func (g *GitHubRepoScanner) createFindingFromRepo(repo *github.Repository) (Finding, error) { + var latestCommitID *string + + if g.annotateLatestCommitID { + commits, _, err := g.client.Repositories.ListCommits(g.ctx, + repo.GetOwner().GetLogin(), + repo.GetName(), + &github.CommitsListOptions{ + ListOptions: github.ListOptions{PerPage: 1}, + }) + + if err != nil { + g.logger.Printf("Warning: Could not identify the latest commit ID - repository without commits?") + empty := "" + latestCommitID = &empty + } else if len(commits) > 0 { + sha := commits[0].GetSHA() + latestCommitID = &sha + } + } + + topics, _, err := g.client.Repositories.ListAllTopics(g.ctx, + repo.GetOwner().GetLogin(), + repo.GetName()) + + var topicList []string + if err == nil && topics != nil { + topicList = topics + } + + visibility := "public" + if repo.GetPrivate() { + visibility = "private" + } + + return g.CreateFinding( + g.GitType(), + fmt.Sprintf("%d", repo.GetID()), + repo.GetHTMLURL(), + repo.GetFullName(), + repo.GetOwner().GetType(), + fmt.Sprintf("%d", repo.GetOwner().GetID()), + repo.GetOwner().GetLogin(), + repo.GetCreatedAt().Format("2006-01-02T15:04:05Z"), + repo.GetUpdatedAt().Format("2006-01-02T15:04:05Z"), + visibility, + repo.GetArchived(), + topicList, + latestCommitID, + ), nil +} diff --git a/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/gitlab_repo_scanner.go b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/gitlab_repo_scanner.go new file mode 100644 index 0000000000..21d106e2fb --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/gitlab_repo_scanner.go @@ -0,0 +1,317 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package gitreposcanner + +import ( + "fmt" + "log" + "time" + + gitlab "gitlab.com/gitlab-org/api/client-go" + "golang.org/x/oauth2" +) + +const GitLab GitType = "GitLab" + +// GitLabScanner implements the GitRepoScanner interface for GitLab repositories +type GitLabRepoScanner struct { + BaseScanner + url string + accessToken string + group *int + ignoredGroups map[int]bool + ignoreRepos map[int]bool + obeyRateLimit bool + annotateLatestCommitID bool + client *gitlab.Client + logger *log.Logger +} + +func NewGitLabScanner( + url string, + accessToken string, + group *int, + ignoredGroups []int, + ignoreRepos []int, + obeyRateLimit bool, + annotateLatestCommitID bool, + logger *log.Logger, +) (*GitLabRepoScanner, error) { + if url == "" { + return nil, fmt.Errorf("URL required for GitLab connection") + } + if accessToken == "" { + return nil, fmt.Errorf("access token required for GitLab authentication") + } + + ignoredGroupsMap := make(map[int]bool) + for _, id := range ignoredGroups { + ignoredGroupsMap[id] = true + } + + ignoreReposMap := make(map[int]bool) + for _, id := range ignoreRepos { + ignoreReposMap[id] = true + } + + if logger == nil { + logger = log.New(log.Writer(), "git_repo_scanner: ", log.LstdFlags) + } + + return &GitLabRepoScanner{ + url: url, + accessToken: accessToken, + group: group, + ignoredGroups: ignoredGroupsMap, + ignoreRepos: ignoreReposMap, + obeyRateLimit: obeyRateLimit, + annotateLatestCommitID: annotateLatestCommitID, + logger: logger, + }, nil +} + +func (g *GitLabRepoScanner) GitType() GitType { + return GitLab +} + +func (g *GitLabRepoScanner) Process(startTime, endTime *time.Time) ([]Finding, error) { + if err := g.authenticate(); err != nil { + return nil, fmt.Errorf("failed to authenticate: %w", err) + } + + projects, err := g.getProjects(startTime, endTime) + if err != nil { + return nil, fmt.Errorf("failed to get projects: %w", err) + } + + return g.processProjects(projects) +} + +func (g *GitLabRepoScanner) authenticate() error { + g.logger.Println("Start GitLab authentication") + + var err error + // Try private token authentication first + g.client, err = gitlab.NewClient(g.accessToken, gitlab.WithBaseURL(g.url)) + if err != nil { + return fmt.Errorf("failed to create GitLab client: %w", err) + } + + // Test authentication by getting current user + _, _, err = g.client.Users.CurrentUser() + if err != nil { + // Try OAuth token if private token fails + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: g.accessToken}) + g.client, err = gitlab.NewAuthSourceClient(gitlab.OAuthTokenSource{TokenSource: ts}, gitlab.WithBaseURL(g.url)) + if err != nil { + return fmt.Errorf("failed to create GitLab OAuth client: %w", err) + } + + // Test OAuth authentication + _, _, err = g.client.Users.CurrentUser() + if err != nil { + return fmt.Errorf("GitLab authentication failed: %w", err) + } + } + + g.logger.Println("GitLab authentication succeeded") + return nil +} + +func (g *GitLabRepoScanner) getProjects(startTime, endTime *time.Time) ([]*gitlab.Project, error) { + g.logger.Printf("Get GitLab repositories with last activity between %v and %v", startTime, endTime) + + var allProjects []*gitlab.Project + + listOptions := &gitlab.ListProjectsOptions{ + OrderBy: gitlab.Ptr("last_activity_at"), + Sort: gitlab.Ptr("desc"), + ListOptions: gitlab.ListOptions{ + PerPage: 100, + }, + } + + if startTime != nil { + listOptions.LastActivityAfter = startTime + } + if endTime != nil { + listOptions.LastActivityBefore = endTime + } + + if g.group != nil { + groupOptions := &gitlab.ListGroupProjectsOptions{ + OrderBy: listOptions.OrderBy, + Sort: listOptions.Sort, + IncludeSubGroups: gitlab.Ptr(true), + ListOptions: listOptions.ListOptions, + } + + // Paginate through all group projects + for { + projects, resp, err := g.client.Groups.ListGroupProjects(*g.group, groupOptions) + if err != nil { + return nil, fmt.Errorf("failed to list group projects: %w", err) + } + + allProjects = append(allProjects, projects...) + + if resp.NextPage == 0 { + break + } + groupOptions.Page = resp.NextPage + + if g.obeyRateLimit { + g.respectRateLimit(resp) + } + } + } else { + // List all projects accessible to the user + for { + projects, resp, err := g.client.Projects.ListProjects(listOptions) + if err != nil { + return nil, fmt.Errorf("failed to list projects: %w", err) + } + + allProjects = append(allProjects, projects...) + + if resp.NextPage == 0 { + break + } + listOptions.Page = resp.NextPage + + if g.obeyRateLimit { + g.respectRateLimit(resp) + } + } + } + + return allProjects, nil +} + +func (g *GitLabRepoScanner) respectRateLimit(resp *gitlab.Response) { + if !g.obeyRateLimit || resp == nil { + return + } + + // GitLab provides rate limit info in headers + remaining := resp.Header.Get("RateLimit-Remaining") + reset := resp.Header.Get("RateLimit-Reset") + + if remaining != "" && reset != "" { + g.logger.Printf("Rate limit - Remaining: %s, Reset: %s", remaining, reset) + + var remainingInt int + fmt.Sscanf(remaining, "%d", &remainingInt) + if remainingInt < 10 { + time.Sleep(time.Second) + } + } +} + +func (g *GitLabRepoScanner) processProjects(projects []*gitlab.Project) ([]Finding, error) { + projectCount := len(projects) + findings := make([]Finding, 0, projectCount) + + for i, project := range projects { + if !g.isNotIgnored(project) { + continue + } + + finding, err := g.createFindingFromProject(project, i, projectCount) + if err != nil { + g.logger.Printf("Warning: failed to create finding for project %s: %v", + project.Name, err) + continue + } + + findings = append(findings, finding) + } + + return findings, nil +} + +func (g *GitLabRepoScanner) isNotIgnored(project *gitlab.Project) bool { + if g.ignoreRepos[project.ID] { + return false + } + + if project.Namespace != nil && project.Namespace.Kind == "group" { + if g.ignoredGroups[project.Namespace.ID] { + return false + } + } + + return true +} + +func (g *GitLabRepoScanner) createFindingFromProject( + project *gitlab.Project, + index int, + total int, +) (Finding, error) { + g.logger.Printf("(%d/%d) Add finding for repo %s with last activity at %s", + index+1, total, project.Name, project.LastActivityAt.String()) + + var latestCommitID *string + + if g.annotateLatestCommitID { + // Get the latest commit + commits, _, err := g.client.Commits.ListCommits(project.ID, &gitlab.ListCommitsOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 1, + Page: 1, + }, + }) + + if err != nil || len(commits) == 0 { + g.logger.Printf("Warning: Could not identify the latest commit ID - repository without commits?") + empty := "" + latestCommitID = &empty + } else { + latestCommitID = &commits[0].ID + } + } + + // Determine owner info from namespace + ownerType := "" + ownerID := "" + ownerName := "" + if project.Namespace != nil { + ownerType = project.Namespace.Kind + ownerID = fmt.Sprintf("%d", project.Namespace.ID) + ownerName = project.Namespace.Name + } + + createdAt := "" + if project.CreatedAt != nil { + createdAt = project.CreatedAt.Format("2006-01-02T15:04:05Z") + } + + lastActivityAt := "" + if project.LastActivityAt != nil { + lastActivityAt = project.LastActivityAt.Format("2006-01-02T15:04:05Z") + } + + topics := []string{} + if project.Topics != nil { + topics = project.Topics + } + + return g.CreateFinding( + g.GitType(), + fmt.Sprintf("%d", project.ID), + project.WebURL, + project.PathWithNamespace, + ownerType, + ownerID, + ownerName, + createdAt, + lastActivityAt, + string(project.Visibility), + project.Archived, + topics, + latestCommitID, + ), nil +} diff --git a/scanners/git-repo-scanner/scanner/internal/output/writer.go b/scanners/git-repo-scanner/scanner/internal/output/writer.go new file mode 100644 index 0000000000..9d7989ee47 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/internal/output/writer.go @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package output + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +const defaultOutputFileName = "git-repo-scanner-findings.json" + +func WriteFindings(fileOutput string, findings any) error { + outputPath, err := resolveOutputPath(fileOutput) + if err != nil { + return err + } + + var data []byte + data, err = json.MarshalIndent(findings, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal findings: %w", err) + } + + if err := os.WriteFile(outputPath, data, 0644); err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + return nil +} + +func resolveOutputPath(fileOutput string) (string, error) { + // Create directory if it doesn't exist + dir := filepath.Dir(fileOutput) + if dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + } + + // Check if fileOutput is a directory + fileInfo, err := os.Stat(fileOutput) + if err == nil && fileInfo.IsDir() { + return filepath.Join(fileOutput, defaultOutputFileName), nil + } + + // If no extension, treat as directory + if filepath.Ext(fileOutput) == "" { + if err := os.MkdirAll(fileOutput, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + return filepath.Join(fileOutput, defaultOutputFileName), nil + } + + return fileOutput, nil +} diff --git a/scanners/git-repo-scanner/scanner/main.go b/scanners/git-repo-scanner/scanner/main.go new file mode 100644 index 0000000000..61ed640530 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/main.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: the secureCodeBox authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "log" + "os" + "time" + + "github.com/secureCodeBox/scanners/git-repo-scanner/scanner/internal/config" + gitreposcanner "github.com/secureCodeBox/scanners/git-repo-scanner/scanner/internal/git_repo_scanner" + "github.com/secureCodeBox/scanners/git-repo-scanner/scanner/internal/output" +) + +var ( + logger = log.New(os.Stdout, "git-repo-scanner - ", log.LstdFlags) +) + +func main() { + config, err := config.ParseFlags(logger) + if err != nil { + logger.Fatalf("Error parsing flags: %v", err) + } + + findings, err := process(config) + if err != nil { + logger.Fatalf("Error processing: %v", err) + } + + logger.Println("Write findings to file...") + if err := output.WriteFindings(config.FileOutput, findings); err != nil { + logger.Fatalf("Failed to write findings: %v", err) + } + logger.Println("Finished!") +} + +func process(config *config.Config) ([]gitreposcanner.Finding, error) { + var scanner gitreposcanner.GitRepoScanner + var err error + + var startTime, endTime *time.Time + now := time.Now().UTC() + + if config.ActivitySinceDuration != nil { + t := now.Add(-*config.ActivitySinceDuration) + startTime = &t + } + + if config.ActivityUntilDuration != nil { + t := now.Add(-*config.ActivityUntilDuration) + endTime = &t + } + + if startTime != nil && endTime != nil && startTime.After(*endTime) { + return nil, fmt.Errorf("activity-since-duration must be greater than activity-until-duration") + } + + switch config.GitType { + case "GitLab": + // Convert int64 slice to int slice for GitLab + ignoreRepos := make([]int, len(config.IgnoreRepos)) + for i, id := range config.IgnoreRepos { + ignoreRepos[i] = int(id) + } + + scanner, err = gitreposcanner.NewGitLabScanner( + config.URL, + config.AccessToken, + config.Group, + config.IgnoreGroups, + ignoreRepos, + config.ObeyRateLimit, + config.AnnotateLatestCommitID, + logger, + ) + if err != nil { + return nil, fmt.Errorf("failed to create GitLab scanner: %w", err) + } + + case "GitHub": + scanner, err = gitreposcanner.NewGitHubScanner( + config.URL, + config.AccessToken, + config.Organization, + config.IgnoreRepos, + config.ObeyRateLimit, + config.AnnotateLatestCommitID, + logger, + ) + if err != nil { + return nil, fmt.Errorf("failed to create GitHub scanner: %w", err) + } + + default: + return nil, fmt.Errorf("unknown git type: %s", config.GitType) + } + + return scanner.Process(startTime, endTime) +} diff --git a/scanners/git-repo-scanner/scanner/requirements.txt b/scanners/git-repo-scanner/scanner/requirements.txt deleted file mode 100644 index 92af8e9fbc..0000000000 --- a/scanners/git-repo-scanner/scanner/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: the secureCodeBox authors -# -# SPDX-License-Identifier: Apache-2.0 - -PyGithub == 2.8.1 -python-gitlab == 7.0.0 -pytimeparse == 1.1.8 -pytz == 2025.2 diff --git a/scanners/git-repo-scanner/scanner/tests/__init__.py b/scanners/git-repo-scanner/scanner/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py deleted file mode 100644 index 3ac4331e92..0000000000 --- a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py +++ /dev/null @@ -1,420 +0,0 @@ -# SPDX-FileCopyrightText: the secureCodeBox authors -# -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import datetime -import unittest -from datetime import timezone -from unittest.mock import MagicMock, Mock -from unittest.mock import patch - -import gitlab -from gitlab.v4.objects import Project, ProjectManager - -from git_repo_scanner.__main__ import get_parser_args -from git_repo_scanner.github_scanner import GitHubScanner -from git_repo_scanner.gitlab_scanner import GitLabScanner - - -class GitRepoScannerTests(unittest.TestCase): - @property - def wrong_output_msg(self) -> str: - return "Test finding output" - - def prepare_gitlab_commitlist_mock(self, mock_gptp, mock_commitmanager): - mock_gptp.side_effect = self._mock_group_project_to_project - mock_commitmanager.return_value = [Mock(id="deadbeef")] - - def _mock_group_project_to_project(self, project): - return project - - @patch("gitlab.v4.objects.ProjectCommitManager.list") - @patch("git_repo_scanner.gitlab_scanner.GitLabScanner._group_project_to_project") - def test_process_gitlab_projects_with_no_ignore_list( - self, mock_gptp, mock_commitmanager - ): - # given - scanner = GitLabScanner( - "url", "token", None, [], [], annotate_latest_commit_id=True - ) - projects = assemble_projects() - self.prepare_gitlab_commitlist_mock(mock_gptp, mock_commitmanager) - # when - findings = scanner._process_projects(projects) - # then - self.assertEqual(3, len(findings), msg="There should be exactly 3 findings") - self.assertEqual(findings[0]["name"], "GitLab Repo", msg=self.wrong_output_msg) - self.assertEqual( - findings[0]["attributes"]["web_url"], "url1", msg=self.wrong_output_msg - ) - self.assertEqual( - findings[1]["attributes"]["web_url"], "url2", msg=self.wrong_output_msg - ) - self.assertEqual( - findings[2]["attributes"]["web_url"], "url3", msg=self.wrong_output_msg - ) - self.assertEqual(findings[0]["attributes"]["last_commit_id"], "deadbeef") - self.assertEqual(findings[1]["attributes"]["archived"], False) - self.assertEqual(findings[2]["attributes"]["archived"], True) - self.assertEqual(findings[0]["attributes"]["topics"], []) - self.assertEqual(findings[2]["attributes"]["topics"], ["outdated"]) - mock_gptp.assert_called() - mock_commitmanager.assert_called() - - @patch("gitlab.v4.objects.ProjectCommitManager.list") - @patch("git_repo_scanner.gitlab_scanner.GitLabScanner._group_project_to_project") - def test_process_gitlab_projects_without_annotating_commit_id( - self, mock_gptp, mock_commitmanager - ): - # given - scanner = GitLabScanner( - "url", "token", None, [], [], annotate_latest_commit_id=False - ) - projects = assemble_projects() - self.prepare_gitlab_commitlist_mock(mock_gptp, mock_commitmanager) - # when - findings = scanner._process_projects(projects) - # then - self.assertEqual(3, len(findings), msg="There should be exactly 3 findings") - self.assertEqual(findings[0]["name"], "GitLab Repo", msg=self.wrong_output_msg) - self.assertEqual( - findings[0]["attributes"]["web_url"], "url1", msg=self.wrong_output_msg - ) - self.assertEqual( - findings[1]["attributes"]["web_url"], "url2", msg=self.wrong_output_msg - ) - self.assertEqual( - findings[2]["attributes"]["web_url"], "url3", msg=self.wrong_output_msg - ) - self.assertFalse("last_commit_id" in findings[0]["attributes"]) - mock_gptp.assert_not_called() - mock_commitmanager.assert_not_called() - - @patch("gitlab.v4.objects.ProjectCommitManager.list") - @patch("git_repo_scanner.gitlab_scanner.GitLabScanner._group_project_to_project") - def test_process_gitlab_projects_with_ignore_group( - self, mock_gptp, mock_commitmanager - ): - # given - scanner = GitLabScanner( - "url", "token", None, [33], [], annotate_latest_commit_id=True - ) - projects = assemble_projects() - self.prepare_gitlab_commitlist_mock(mock_gptp, mock_commitmanager) - # when - findings = scanner._process_projects(projects) - # then - self.assertEqual(2, len(findings), msg="There should be exactly 2 findings") - self.assertEqual( - findings[0]["attributes"]["web_url"], "url1", msg=self.wrong_output_msg - ) - self.assertEqual( - findings[1]["attributes"]["web_url"], "url2", msg=self.wrong_output_msg - ) - self.assertEqual(findings[0]["attributes"]["last_commit_id"], "deadbeef") - mock_gptp.assert_called() - mock_commitmanager.assert_called() - - @patch("gitlab.v4.objects.ProjectCommitManager.list") - @patch("git_repo_scanner.gitlab_scanner.GitLabScanner._group_project_to_project") - def test_process_gitlab_projects_with_ignore_project( - self, mock_gptp, mock_commitmanager - ): - # given - scanner = GitLabScanner( - "url", "token", None, [], [1], annotate_latest_commit_id=True - ) - projects = assemble_projects() - self.prepare_gitlab_commitlist_mock(mock_gptp, mock_commitmanager) - # when - findings = scanner._process_projects(projects) - # then - self.assertEqual(2, len(findings), msg="There should be exactly 2 findings") - self.assertEqual( - findings[0]["attributes"]["web_url"], "url2", msg=self.wrong_output_msg - ) - self.assertEqual( - findings[1]["attributes"]["web_url"], "url3", msg=self.wrong_output_msg - ) - self.assertEqual(findings[0]["attributes"]["last_commit_id"], "deadbeef") - mock_gptp.assert_called() - mock_commitmanager.assert_called() - - @patch("github.Github") - @patch("github.Organization") - @patch("github.PaginatedList") - def test_process_github_repos_with_no_ignore_list( - self, github_mock, org_mock, pag_mock - ): - # given - scanner = GitHubScanner( - "url", "token", "org", [], False, annotate_latest_commit_id=True - ) - repos = assemble_repos() - create_mocks(github_mock, org_mock, pag_mock, repos) - scanner._gh = github_mock - # when - findings = scanner._process_repos(None, None) - # then - org_mock.get_repos.assert_called_with( - type="all", sort="pushed", direction="asc" - ) - self.assertEqual(6, len(findings), msg="There should be exactly 6 findings") - for finding in findings: - self.assertEqual(finding["name"], "GitHub Repo", msg=self.wrong_output_msg) - self.assertEqual(finding["attributes"]["last_commit_id"], "deadbeef") - - @patch("github.Github") - @patch("github.Organization") - @patch("github.PaginatedList") - def test_process_github_repos_without_annotating_commit_ids( - self, github_mock, org_mock, pag_mock - ): - # given - scanner = GitHubScanner( - "url", "token", "org", [], False, annotate_latest_commit_id=False - ) - repos = assemble_repos() - create_mocks(github_mock, org_mock, pag_mock, repos) - scanner._gh = github_mock - # when - findings = scanner._process_repos(None, None) - # then - org_mock.get_repos.assert_called_with( - type="all", sort="pushed", direction="asc" - ) - self.assertEqual(6, len(findings), msg="There should be exactly 6 findings") - self.assertFalse(findings[0]["attributes"]["archived"]) - self.assertFalse(findings[1]["attributes"]["archived"]) - self.assertTrue(findings[2]["attributes"]["archived"]) - self.assertFalse(findings[3]["attributes"]["archived"]) - self.assertFalse(findings[4]["attributes"]["archived"]) - self.assertTrue(findings[5]["attributes"]["archived"]) - self.assertEqual(findings[0]["attributes"]["topics"], []) - self.assertEqual(findings[2]["attributes"]["topics"], ["outdated"]) - for finding in findings: - self.assertEqual(finding["name"], "GitHub Repo", msg=self.wrong_output_msg) - self.assertFalse("last_commit_id" in finding["attributes"]) - - @patch("github.Github") - @patch("github.Organization") - @patch("github.PaginatedList") - def test_process_github_repos_with_ignore_repos( - self, github_mock, org_mock, pag_mock - ): - # given - scanner = GitHubScanner( - "url", "token", "org", [1], False, annotate_latest_commit_id=True - ) - repos = assemble_repos() - create_mocks(github_mock, org_mock, pag_mock, repos) - scanner._gh = github_mock - # when - findings = scanner._process_repos(None, None) - # then - github_mock.get_organization.assert_called_with("org") - self.assertEqual(4, len(findings), msg="There should be exactly 4 findings") - self.assertEqual(findings[0]["attributes"]["last_commit_id"], "deadbeef") - - def test_setup_github_with_url_and_no_token_should_exit(self): - # when - with self.assertRaises(argparse.ArgumentError) as cm: - GitHubScanner("url", None, "org", []) - # then - self.assertEqual( - cm.exception.args[1], - "Access token required for GitHub connection.", - msg="Process should exit", - ) - - -def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, org=None): - args = [ - "--git-type", - "someType", - "--file-output", - "out", - "--obey-rate-limit", - False, - "--ignore-repos", - str(ignore_projects), - "--ignore-groups", - str(ignore_groups), - ] - if url: - args.append("--url") - args.append(url) - if access_token: - args.append("--access-token") - args.append(access_token) - if org: - args.append("--organization") - args.append(org) - - return get_parser_args(args) - - -def create_mocks(github_mock, org_mock, pag_mock, repos): - pag_mock.totalCount = 2 - pag_mock.get_page = MagicMock(return_value=repos) - org_mock.get_repos = MagicMock(return_value=pag_mock) - github_mock.get_organization = MagicMock(return_value=org_mock) - - -def assemble_projects(): - created = datetime.datetime(2020, 10, 10, tzinfo=timezone.utc).isoformat() - updated = datetime.datetime(2020, 11, 10, tzinfo=timezone.utc).isoformat() - project1 = assemble_project( - p_id=1, - name="name1", - url="url1", - path="path1", - date_created=created, - date_updated=updated, - visibility="private", - o_id=11, - o_kind="group", - o_name="name11", - ) - project2 = assemble_project( - p_id=2, - name="name2", - url="url2", - path="path2", - date_created=created, - date_updated=updated, - visibility="private", - o_id=22, - o_kind="user", - o_name="name22", - ) - project3 = assemble_project( - p_id=3, - name="name3", - url="url3", - path="path3", - date_created=created, - date_updated=updated, - visibility="private", - o_id=33, - o_kind="group", - o_name="name33", - archived=True, - topics=["outdated"], - ) - return [project1, project2, project3] - - -def assemble_project( - p_id, - name, - url, - path, - date_created, - date_updated, - visibility, - o_id, - o_kind, - o_name, - archived=False, - topics=[], -): - project = Project(ProjectManager(gitlab), {}) - project.id = p_id - project.name = name - project.web_url = url - project.path_with_namespace = path - project.created_at = date_created - project.last_activity_at = date_updated - project.visibility = visibility - project.namespace = {"kind": o_kind, "id": o_id, "name": o_name} - project.archived = archived - project.topics = topics - return project - - -def assemble_repos(): - date = datetime.datetime(2020, 5, 17, tzinfo=timezone.utc) - project1 = assemble_repository( - p_id=1, - name="name1", - url="url1", - path="path1", - date_created=date, - date_updated=date, - date_pushed=date, - visibility=True, - o_id=11, - o_kind="organization", - o_name="name11", - ) - project2 = assemble_repository( - p_id=2, - name="name2", - url="url2", - path="path2", - date_created=date, - date_updated=date, - date_pushed=date, - visibility=False, - o_id=22, - o_kind="organization", - o_name="name22", - ) - project3 = assemble_repository( - p_id=3, - name="name3", - url="url3", - path="path3", - date_created=date, - date_updated=date, - date_pushed=date, - visibility=False, - o_id=33, - o_kind="organization", - o_name="name33", - archived=True, - topics=["outdated"], - ) - return [project1, project2, project3] - - -def assemble_repository( - p_id, - name, - url, - path, - date_created: datetime, - date_updated: datetime, - date_pushed: datetime, - visibility: bool, - o_id, - o_kind, - o_name, - archived=False, - topics=[], -): - - repo = Mock() - owner = Mock() - owner.type = o_kind - owner.id = o_id - owner.name = o_name - repo.id = p_id - repo.name = name - repo.html_url = url - repo.full_name = path - repo.created_at = date_created - repo.pushed_at = date_pushed - repo.updated_at = date_updated - repo.private = visibility - repo.owner = owner - repo.get_commits = lambda: [Mock(sha="deadbeef")] - repo.get_topics = lambda: topics - repo.archived = archived - return repo - - -if __name__ == "__main__": - unittest.main() diff --git a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml index 62f9d2ff3f..f056faf2b1 100644 --- a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml +++ b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml @@ -25,7 +25,7 @@ spec: restartPolicy: OnFailure affinity: {{- toYaml .Values.scanner.affinity | nindent 12 }} - tolerations: + tolerations: {{- toYaml .Values.scanner.tolerations | nindent 12 }} {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -38,9 +38,7 @@ spec: image: "{{ .Values.scanner.image.repository }}:{{ .Values.scanner.image.tag | default .Chart.Version }}" imagePullPolicy: {{ .Values.scanner.image.pullPolicy }} command: - - "python" - - "-m" - - "git_repo_scanner" + - "/git-repo-scanner" - "--file-output" - "/home/securecodebox" resources: diff --git a/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap b/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap index 79d7837be9..6f132db42a 100644 --- a/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap +++ b/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap @@ -40,9 +40,6 @@ matches the snapshot: foo: bar containers: - command: - - python - - -m - - git_repo_scanner - --file-output - /home/securecodebox env: From acf6f27efb6a241bf9a853d5297854bd3ea8bde9 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 10 Dec 2025 09:37:08 +0100 Subject: [PATCH 2/9] Normalize git-type Signed-off-by: Patrick --- .../git-repo-scanner/docs/README.ArtifactHub.md | 3 +-- .../docs/README.DockerHub-Scanner.md | 3 +-- .../examples/github-secureCodeBox-scan/scan.yaml | 2 +- .../examples/gitlab-group-scan/scan.yaml | 2 +- .../integration-tests/git-repo-scanner.test.js | 2 +- .../scanner/internal/config/config.go | 13 +++++++++++++ 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scanners/git-repo-scanner/docs/README.ArtifactHub.md b/scanners/git-repo-scanner/docs/README.ArtifactHub.md index b97cfb120a..1f621ae355 100644 --- a/scanners/git-repo-scanner/docs/README.ArtifactHub.md +++ b/scanners/git-repo-scanner/docs/README.ArtifactHub.md @@ -61,7 +61,7 @@ repository type with the option: ```bash --git-type github or ---git-type Gitlab +--git-type gitlab ``` #### GitHub @@ -168,4 +168,3 @@ Code of secureCodeBox is licensed under the [Apache License 2.0][scb-license]. [scb-mastodon]: https://infosec.exchange/@secureCodeBox [scb-slack]: https://owasp.org/slack/invite [scb-license]: https://github.com/secureCodeBox/secureCodeBox/blob/master/LICENSE - diff --git a/scanners/git-repo-scanner/docs/README.DockerHub-Scanner.md b/scanners/git-repo-scanner/docs/README.DockerHub-Scanner.md index b3d288b888..85824d81a0 100644 --- a/scanners/git-repo-scanner/docs/README.DockerHub-Scanner.md +++ b/scanners/git-repo-scanner/docs/README.DockerHub-Scanner.md @@ -64,7 +64,7 @@ repository type with the option: ```bash --git-type github or ---git-type Gitlab +--git-type gitlab ``` #### GitHub @@ -124,4 +124,3 @@ As for any pre-built image usage, it is the image user's responsibility to ensur [scb-mastodon]: https://infosec.exchange/@secureCodeBox [scb-slack]: https://owasp.org/slack/invite [scb-license]: https://github.com/secureCodeBox/secureCodeBox/blob/master/LICENSE - diff --git a/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml b/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml index eabc173087..b747b5a4e9 100644 --- a/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml +++ b/scanners/git-repo-scanner/examples/github-secureCodeBox-scan/scan.yaml @@ -10,7 +10,7 @@ spec: scanType: "git-repo-scanner" parameters: - "--git-type" - - "GitHub" + - "github" - "--organization" - "secureCodeBox" cascades: diff --git a/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml b/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml index 5e510c5ec6..a390b3d1d2 100644 --- a/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml +++ b/scanners/git-repo-scanner/examples/gitlab-group-scan/scan.yaml @@ -10,7 +10,7 @@ spec: scanType: "git-repo-scanner" parameters: - "--git-type" - - "GitLab" + - "gitlab" - "--url" - "https://gitlab.your-company.com" - "--access-token" diff --git a/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js index be5c7f164d..1dcccb8e0f 100644 --- a/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js +++ b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js @@ -12,7 +12,7 @@ test.skip( const { count } = await scan( "git-repo-scanner-dummy-scan", "git-repo-scanner", - ["--git-type", "GitHub", "--organization", "secureCodeBox"], + ["--git-type", "github", "--organization", "secureCodeBox"], ); // There must be >= 28 Repositories found in the GitHub secureCodeBox organisation. expect(count).toBeGreaterThanOrEqual(28); diff --git a/scanners/git-repo-scanner/scanner/internal/config/config.go b/scanners/git-repo-scanner/scanner/internal/config/config.go index 4e4ef5a192..c4a103e16b 100644 --- a/scanners/git-repo-scanner/scanner/internal/config/config.go +++ b/scanners/git-repo-scanner/scanner/internal/config/config.go @@ -63,6 +63,8 @@ func ParseFlags(logger *log.Logger) (*Config, error) { flag.Parse() + config.GitType = normalizeGitType(config.GitType) + if err := config.validate(); err != nil { flag.Usage() return nil, err @@ -75,6 +77,17 @@ func ParseFlags(logger *log.Logger) (*Config, error) { return config, nil } +func normalizeGitType(gitType string) string { + switch strings.ToLower(gitType) { + case "github": + return "GitHub" + case "gitlab": + return "GitLab" + default: + return gitType + } +} + func (c *Config) validate() error { if c.GitType == "" { return fmt.Errorf("--git-type is required") From b3dd0d22f3fc855aae359d3848fa39fd0d8d24d0 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 10 Dec 2025 09:47:39 +0100 Subject: [PATCH 3/9] Implement SonarQube Feedback Signed-off-by: Patrick --- .../scanner/internal/config/config.go | 3 +-- .../scanner/internal/config/config_test.go | 20 ++++++++++--------- scanners/git-repo-scanner/scanner/main.go | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/internal/config/config.go b/scanners/git-repo-scanner/scanner/internal/config/config.go index c4a103e16b..e11c5b7767 100644 --- a/scanners/git-repo-scanner/scanner/internal/config/config.go +++ b/scanners/git-repo-scanner/scanner/internal/config/config.go @@ -7,7 +7,6 @@ package config import ( "flag" "fmt" - "log" "strconv" "strings" "time" @@ -30,7 +29,7 @@ type Config struct { ActivityUntilDuration *time.Duration } -func ParseFlags(logger *log.Logger) (*Config, error) { +func ParseFlags() (*Config, error) { config := &Config{} // Define flags diff --git a/scanners/git-repo-scanner/scanner/internal/config/config_test.go b/scanners/git-repo-scanner/scanner/internal/config/config_test.go index 1776c6ea34..d32171d4fe 100644 --- a/scanners/git-repo-scanner/scanner/internal/config/config_test.go +++ b/scanners/git-repo-scanner/scanner/internal/config/config_test.go @@ -10,7 +10,9 @@ import ( "time" ) -func TestConfig_Validate(t *testing.T) { +const fileOutput = "output.json" + +func TestValidateConfig(t *testing.T) { tests := []struct { name string config Config @@ -19,25 +21,25 @@ func TestConfig_Validate(t *testing.T) { }{ { name: "missing git-type", - config: Config{FileOutput: "output.json"}, + config: Config{FileOutput: fileOutput}, wantErr: true, errMsg: "--git-type is required", }, { name: "invalid git-type", - config: Config{GitType: "Bitbucket", FileOutput: "output.json"}, + config: Config{GitType: "Bitbucket", FileOutput: fileOutput}, wantErr: true, errMsg: "invalid git-type", }, { name: "GitLab missing URL", - config: Config{GitType: "GitLab", FileOutput: "output.json"}, + config: Config{GitType: "GitLab", FileOutput: fileOutput}, wantErr: true, errMsg: "--url is required for GitLab", }, { name: "GitHub missing organization", - config: Config{GitType: "GitHub", FileOutput: "output.json"}, + config: Config{GitType: "GitHub", FileOutput: fileOutput}, wantErr: true, errMsg: "--organization is required for GitHub", }, @@ -45,7 +47,7 @@ func TestConfig_Validate(t *testing.T) { name: "valid GitHub config", config: Config{ GitType: "GitHub", - FileOutput: "output.json", + FileOutput: fileOutput, Organization: "test-org", }, wantErr: false, @@ -54,7 +56,7 @@ func TestConfig_Validate(t *testing.T) { name: "valid GitLab config", config: Config{ GitType: "GitLab", - FileOutput: "output.json", + FileOutput: fileOutput, URL: "https://gitlab.com", }, wantErr: false, @@ -74,7 +76,7 @@ func TestConfig_Validate(t *testing.T) { } } -func TestConfig_ParseOptionalFields(t *testing.T) { +func TestParseOptionalFields(t *testing.T) { tests := []struct { name string groupStr string @@ -169,7 +171,7 @@ func TestConfig_ParseOptionalFields(t *testing.T) { } } -func TestConfig_GetTimeFrame(t *testing.T) { +func TestGetTimeFrame(t *testing.T) { sevenDays := 7 * 24 * time.Hour oneDay := 24 * time.Hour diff --git a/scanners/git-repo-scanner/scanner/main.go b/scanners/git-repo-scanner/scanner/main.go index 61ed640530..168d8cd0c1 100644 --- a/scanners/git-repo-scanner/scanner/main.go +++ b/scanners/git-repo-scanner/scanner/main.go @@ -20,7 +20,7 @@ var ( ) func main() { - config, err := config.ParseFlags(logger) + config, err := config.ParseFlags() if err != nil { logger.Fatalf("Error parsing flags: %v", err) } From 8733246e20b3f3137b153e4e26abcb9b39d04bef Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 10 Dec 2025 10:06:23 +0100 Subject: [PATCH 4/9] Fix helm unittest Signed-off-by: Patrick --- .../git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap | 1 + 1 file changed, 1 insertion(+) diff --git a/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap b/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap index 6f132db42a..82db039ed9 100644 --- a/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap +++ b/scanners/git-repo-scanner/tests/__snapshot__/scanner_test.yaml.snap @@ -40,6 +40,7 @@ matches the snapshot: foo: bar containers: - command: + - /git-repo-scanner - --file-output - /home/securecodebox env: From efb9ee0d267700c737874668eecdff959720d7d4 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 11 Dec 2025 09:35:45 +0100 Subject: [PATCH 5/9] Run Go tests in ci Signed-off-by: Patrick --- .github/workflows/ci.yaml | 6 ++++++ scanners/git-repo-scanner/Taskfile.yaml | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92e44c47e5..d4a724ba3e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -496,11 +496,17 @@ jobs: chmod +x ./kubectl/kubectl && sudo mv ./kubectl/kubectl /usr/local/bin/kubectl chmod +x ./helm/helm && sudo mv ./helm/helm /usr/local/bin/helm + - name: Go Setup + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + with: + go-version-file: "scanners/git-repo-scanner/scanner/go.mod" + - name: Verify tools run: | kind version kubectl version || true helm version + go version - name: Unit Tests working-directory: ./scanners/${{ matrix.unit }}/ diff --git a/scanners/git-repo-scanner/Taskfile.yaml b/scanners/git-repo-scanner/Taskfile.yaml index a7b2ce668b..5bcdd049a9 100644 --- a/scanners/git-repo-scanner/Taskfile.yaml +++ b/scanners/git-repo-scanner/Taskfile.yaml @@ -11,4 +11,11 @@ includes: vars: scannerName: git-repo-scanner -tasks: {} +tasks: + test:unit: + desc: Run unit tests for git-repo-scanner + deps: + - scanner:test:setup + cmds: + - bun test {{ .TASKFILE_DIR }}/parser/ + - cd {{ .TASKFILE_DIR }}/scanner && go test ./... From 2c9bbc53100183a1f298c019e4150f0ce8ad74d4 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 11 Dec 2025 09:54:00 +0100 Subject: [PATCH 6/9] Fix Taskfile Signed-off-by: Patrick --- scanners/git-repo-scanner/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/scanners/git-repo-scanner/Taskfile.yaml b/scanners/git-repo-scanner/Taskfile.yaml index 5bcdd049a9..448dbe5da2 100644 --- a/scanners/git-repo-scanner/Taskfile.yaml +++ b/scanners/git-repo-scanner/Taskfile.yaml @@ -8,6 +8,7 @@ includes: scanner: taskfile: ../Taskfile.yaml flatten: true + excludes: [test:unit] vars: scannerName: git-repo-scanner From f112b059abd4e9efd886e9ae721a6dbfa8cce433 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 11 Dec 2025 10:04:20 +0100 Subject: [PATCH 7/9] Fix Taskfile deps Signed-off-by: Patrick --- scanners/git-repo-scanner/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/git-repo-scanner/Taskfile.yaml b/scanners/git-repo-scanner/Taskfile.yaml index 448dbe5da2..aa7524c73f 100644 --- a/scanners/git-repo-scanner/Taskfile.yaml +++ b/scanners/git-repo-scanner/Taskfile.yaml @@ -16,7 +16,7 @@ tasks: test:unit: desc: Run unit tests for git-repo-scanner deps: - - scanner:test:setup + - test:setup cmds: - bun test {{ .TASKFILE_DIR }}/parser/ - cd {{ .TASKFILE_DIR }}/scanner && go test ./... From ffa283e01431135cc4343b8f52d100014c79381e Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 15 Dec 2025 10:44:07 +0100 Subject: [PATCH 8/9] Fix rate limit calculation Signed-off-by: Patrick --- .../git_repo_scanner/github_repo_scanner.go | 89 ++++++++++++++----- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go index bd8c33e2a3..39ecd97676 100644 --- a/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go +++ b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go @@ -29,6 +29,8 @@ type GitHubRepoScanner struct { client *github.Client logger *log.Logger ctx context.Context + lastRateLimitCheck time.Time + requestsSinceCheck int } func NewGitHubScanner( @@ -65,6 +67,8 @@ func NewGitHubScanner( annotateLatestCommitID: annotateLatestCommitID, logger: logger, ctx: context.Background(), + lastRateLimitCheck: time.Time{}, + requestsSinceCheck: 0, }, nil } @@ -76,7 +80,20 @@ func (g *GitHubRepoScanner) Process(startTime, endTime *time.Time) ([]Finding, e if err := g.setup(); err != nil { return nil, fmt.Errorf("failed to setup GitHub client: %w", err) } - return g.processRepos(startTime, endTime) + + findings, err := g.processRepos(startTime, endTime) + + // Log remaining API calls at the end + if g.obeyRateLimit { + rate, _, rateLimitErr := g.client.RateLimit.Get(g.ctx) + if rateLimitErr == nil { + core := rate.GetCore() + g.logger.Printf("Scan complete. Rate limit status: %d/%d remaining, resets at %s", + core.Remaining, core.Limit, core.Reset.Time.Format(time.RFC3339)) + } + } + + return findings, err } func (g *GitHubRepoScanner) setup() error { @@ -116,10 +133,15 @@ func (g *GitHubRepoScanner) createTokenClient() *http.Client { return oauth2.NewClient(g.ctx, ts) } +func (g *GitHubRepoScanner) trackAPICall() { + g.requestsSinceCheck++ +} + func (g *GitHubRepoScanner) processRepos(startTime, endTime *time.Time) ([]Finding, error) { var findings []Finding org, _, err := g.client.Organizations.Get(g.ctx, g.organization) + g.trackAPICall() if err != nil { return nil, fmt.Errorf("failed to get organization: %w", err) } @@ -141,6 +163,7 @@ func (g *GitHubRepoScanner) processRepos(startTime, endTime *time.Time) ([]Findi // Paginate through all repositories for { repos, resp, err := g.client.Repositories.ListByOrg(g.ctx, org.GetLogin(), opts) + g.trackAPICall() if err != nil { return nil, fmt.Errorf("failed to list repositories: %w", err) } @@ -176,6 +199,8 @@ func (g *GitHubRepoScanner) processReposPage( return findings, false, nil // Stop processing further pages } + g.logger.Printf("Processing repository: %s", repo.GetFullName()) + finding, err := g.createFindingFromRepo(repo) if err != nil { g.logger.Printf("Warning: failed to create finding for repo %s: %v", repo.GetName(), err) @@ -220,7 +245,17 @@ func (g *GitHubRepoScanner) respectGitHubRateLimit() error { return nil } + // Only check rate limit every 50 API calls OR every 30 seconds + if g.requestsSinceCheck < 50 && time.Since(g.lastRateLimitCheck) < 30*time.Second { + return nil + } + + // Reset counters + g.requestsSinceCheck = 0 + g.lastRateLimitCheck = time.Now() + rate, _, err := g.client.RateLimit.Get(g.ctx) + g.trackAPICall() if err != nil { return fmt.Errorf("failed to get rate limit: %w", err) } @@ -228,19 +263,39 @@ func (g *GitHubRepoScanner) respectGitHubRateLimit() error { core := rate.GetCore() remaining := core.Remaining reset := core.Reset.Time + limit := core.Limit - secondsUntilReset := time.Until(reset).Seconds() + 5 // add 5 seconds buffer - - if remaining > 0 { - sleepTime := secondsUntilReset / float64(remaining) + // Determine threshold based on whether authenticated or not + var lowThreshold int + if limit <= 60 { + lowThreshold = 10 + g.logger.Printf("Warning: Using unauthenticated GitHub API (60 requests/hour limit). Consider providing an access token for better performance.") + } else { + lowThreshold = 100 + } - if sleepTime > 0 { - time.Sleep(time.Duration(sleepTime * float64(time.Second))) + if remaining < lowThreshold { + secondsUntilReset := time.Until(reset).Seconds() + 5 + if remaining > 0 { + sleepTime := secondsUntilReset / float64(remaining) + + maxSleep := 10.0 + if sleepTime > maxSleep { + sleepTime = maxSleep + g.logger.Printf("Rate limit low (%d/%d remaining), sleeping %.1fs (capped)", + remaining, limit, sleepTime) + } else if sleepTime > 1 { + g.logger.Printf("Rate limit low (%d/%d remaining), sleeping %.1fs between requests", + remaining, limit, sleepTime) + } + + if sleepTime > 0 { + time.Sleep(time.Duration(sleepTime * float64(time.Second))) + } + } else { + g.logger.Printf("Rate limit exhausted. Waiting until %s", reset.Format(time.RFC3339)) + time.Sleep(time.Until(reset) + 5*time.Second) } - } else { - // No remaining API calls, wait until reset - g.logger.Printf("Rate limit exhausted. Waiting until %s", reset.Format(time.RFC3339)) - time.Sleep(time.Until(reset) + 5*time.Second) } return nil @@ -256,6 +311,7 @@ func (g *GitHubRepoScanner) createFindingFromRepo(repo *github.Repository) (Find &github.CommitsListOptions{ ListOptions: github.ListOptions{PerPage: 1}, }) + g.trackAPICall() if err != nil { g.logger.Printf("Warning: Could not identify the latest commit ID - repository without commits?") @@ -267,15 +323,6 @@ func (g *GitHubRepoScanner) createFindingFromRepo(repo *github.Repository) (Find } } - topics, _, err := g.client.Repositories.ListAllTopics(g.ctx, - repo.GetOwner().GetLogin(), - repo.GetName()) - - var topicList []string - if err == nil && topics != nil { - topicList = topics - } - visibility := "public" if repo.GetPrivate() { visibility = "private" @@ -293,7 +340,7 @@ func (g *GitHubRepoScanner) createFindingFromRepo(repo *github.Repository) (Find repo.GetUpdatedAt().Format("2006-01-02T15:04:05Z"), visibility, repo.GetArchived(), - topicList, + repo.Topics, latestCommitID, ), nil } From 3fdaef47bfe3e6dc62a4999bf89938593d585482 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 15 Dec 2025 11:32:45 +0100 Subject: [PATCH 9/9] Update rate limit check interval Signed-off-by: Patrick --- .../internal/git_repo_scanner/github_repo_scanner.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go index 39ecd97676..ebc39a67bd 100644 --- a/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go +++ b/scanners/git-repo-scanner/scanner/internal/git_repo_scanner/github_repo_scanner.go @@ -245,12 +245,17 @@ func (g *GitHubRepoScanner) respectGitHubRateLimit() error { return nil } - // Only check rate limit every 50 API calls OR every 30 seconds - if g.requestsSinceCheck < 50 && time.Since(g.lastRateLimitCheck) < 30*time.Second { + // Determine check interval based on authentication + // For unauthenticated (60/hour), check more frequently + checkInterval := 50 + if g.accessToken == "" { + checkInterval = 5 + } + + if g.requestsSinceCheck < checkInterval && time.Since(g.lastRateLimitCheck) < 30*time.Second { return nil } - // Reset counters g.requestsSinceCheck = 0 g.lastRateLimitCheck = time.Now()