From 025d86270a6828630e7607fb99c60c810cd17bf2 Mon Sep 17 00:00:00 2001 From: Larry Singleton <166439969+larrysingleton007@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:01:13 -0500 Subject: [PATCH 1/2] fix: Automate versioned documentation publication Fixes #6762 Signed-off-by: Larry Singleton <166439969+larrysingleton007@users.noreply.github.com> --- .github/workflows/release.yml | 28 ++ docs/project/release-process.md | 20 +- .../scripts/release/publish_versioned_docs.py | 342 ++++++++++++++++++ .../release/test_publish_versioned_docs.py | 246 +++++++++++++ 4 files changed, 632 insertions(+), 4 deletions(-) create mode 100644 infra/scripts/release/publish_versioned_docs.py create mode 100644 infra/scripts/release/test_publish_versioned_docs.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 24a76628fec..e7b87553817 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,10 @@ on: required: true default: true type: boolean + secrets: + GITBOOK_TOKEN: + description: 'Token used to publish versioned documentation' + required: true jobs: get_dry_release_versions: @@ -196,3 +200,27 @@ jobs: run: | git checkout -B stable origin/${GITHUB_REF#refs/heads/} git push origin stable --force + + publish_versioned_docs: + name: Publish versioned documentation + if: github.event.inputs.dry_run == 'false' + runs-on: ubuntu-latest + needs: [get_dry_release_versions, release] + permissions: + contents: write + env: + GITHUB_TOKEN: ${{ github.event.inputs.token }} + GITBOOK_TOKEN: ${{ secrets.GITBOOK_TOKEN }} + GITBOOK_ORG_ID: ${{ vars.GITBOOK_ORG_ID }} + GITBOOK_SITE_ID: ${{ vars.GITBOOK_SITE_ID }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Publish released documentation + run: | + python infra/scripts/release/publish_versioned_docs.py \ + --version "${{ needs.get_dry_release_versions.outputs.next_version }}" diff --git a/docs/project/release-process.md b/docs/project/release-process.md index 2cddca508cf..fd13c024474 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -110,9 +110,19 @@ We verify the building the wheels and Docker images in **your fork** of Feast, n - [Maven repo (feast-datatypes, feast-serving-client)](https://mvnrepository.com/artifact/dev.feast) ### 4. (for minor releases) Post-release steps -#### 4a: Creating a new branch -Create a new branch based on master (i.e. v0.22-branch) and push to the main Feast repo. This will be where -cherry-picks go for future patch releases and where documentation will point. +#### 4a: Versioned branch and documentation publication + +The `publish_versioned_docs` release job creates the `v.-branch` maintenance branch +at the released tag, imports that branch into a versioned GitBook space, and makes the new space +the public default. It verifies the final GitBook state before succeeding. + +The repository must define `GITBOOK_ORG_ID` and `GITBOOK_SITE_ID` variables and a scoped +`GITBOOK_TOKEN` secret with permission to create and update spaces, update the site, and publish +it. The release token must be able to create the maintenance branch. + +The job is safe to rerun. It reuses matching branches and spaces, but it will not move an existing +maintenance branch that points somewhere other than the released tag. It also leaves the previous +documentation version as the public default until the new branch has imported successfully. #### 4b: Adding a high level summary in the GitHub release notes By default, Semantic Release will pull in messages from commits (features vs fixes, etc). But this is hard to digest, @@ -120,7 +130,9 @@ so it helps to have a high level overview. See https://github.com/feast-dev/feas #### 4c: Update documentation -In the Feast Gitbook: +If the automated documentation job fails, use its error as the recovery point and complete the +remaining steps in the Feast GitBook: + 1. Create a new space within the Feast collection 2. Go to the overflow menu on the top -> Synchronize with Git 1. Specify GitHub as the provider diff --git a/infra/scripts/release/publish_versioned_docs.py b/infra/scripts/release/publish_versioned_docs.py new file mode 100644 index 00000000000..cf982909645 --- /dev/null +++ b/infra/scripts/release/publish_versioned_docs.py @@ -0,0 +1,342 @@ +"""Publish a released Feast minor version as the default GitBook documentation. + +The release workflow supplies the released semantic version and credentials through +environment variables. The operation is idempotent: matching branches and GitBook +spaces are reused, while conflicting release branches fail without being moved. +""" + +import argparse +import json +import os +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, NamedTuple + + +GITHUB_API_URL = "https://api.github.com" +GITBOOK_API_URL = "https://api.gitbook.com/v1" +GITHUB_API_VERSION = "2022-11-28" + + +class ApiError(RuntimeError): + def __init__(self, status: int, message: str): + super().__init__(message) + self.status = status + + +class VersionInfo(NamedTuple): + version: str + tag: str + branch: str + + @classmethod + def parse(cls, value: str) -> "VersionInfo": + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)", value.strip()) + if match is None: + raise ValueError(f"expected a stable semantic version, got {value!r}") + major, minor, patch = match.groups() + version = f"{major}.{minor}.{patch}" + return cls(version, f"v{version}", f"v{major}.{minor}-branch") + + +class JsonApi: + def __init__( + self, + base_url: str, + token: str, + extra_headers: dict[str, str] | None = None, + ) -> None: + self.base_url = base_url.rstrip("/") + self.token = token + self.extra_headers = extra_headers or {} + + def request( + self, + method: str, + path: str, + payload: dict[str, Any] | None = None, + expected: tuple[int, ...] = (200,), + ) -> Any: + body = json.dumps(payload).encode() if payload is not None else None + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.token}", + "User-Agent": "feast-release-docs", + **self.extra_headers, + } + if body is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{self.base_url}{path}", data=body, headers=headers, method=method + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + status = response.status + response_body = response.read() + except urllib.error.HTTPError as error: + status = error.code + response_body = error.read() + except urllib.error.URLError as error: + raise RuntimeError( + f"request to {self.base_url} failed: {error.reason}" + ) from error + + if status not in expected: + detail = response_body.decode(errors="replace").strip() + raise ApiError(status, f"{method} {path} returned HTTP {status}: {detail}") + if not response_body: + return None + return json.loads(response_body) + + +class GitHubPublisher: + def __init__(self, api: JsonApi, repository: str) -> None: + self.api = api + self.repository = repository + self.repository_path = urllib.parse.quote(repository, safe="/") + + def _commit_sha(self, tag: str) -> str: + encoded_tag = urllib.parse.quote(tag, safe="") + commit = self.api.request( + "GET", f"/repos/{self.repository_path}/commits/{encoded_tag}" + ) + return commit["sha"] + + def _branch_sha(self, branch: str) -> str | None: + encoded_branch = urllib.parse.quote(branch, safe="") + try: + result = self.api.request( + "GET", f"/repos/{self.repository_path}/branches/{encoded_branch}" + ) + except ApiError as error: + if error.status == 404: + return None + raise + return result["commit"]["sha"] + + def ensure_release_branch(self, tag: str, branch: str) -> str: + tag_sha = self._commit_sha(tag) + branch_sha = self._branch_sha(branch) + if branch_sha is None: + self.api.request( + "POST", + f"/repos/{self.repository_path}/git/refs", + {"ref": f"refs/heads/{branch}", "sha": tag_sha}, + expected=(201,), + ) + print(f"Created {branch} at {tag_sha}") + return tag_sha + if branch_sha != tag_sha: + raise RuntimeError( + f"refusing to move existing {branch}: {branch_sha} != released {tag_sha}" + ) + print(f"Verified existing {branch} at {tag_sha}") + return tag_sha + + +class GitBookPublisher: + def __init__( + self, + api: JsonApi, + organization_id: str, + site_id: str, + repository: str, + sync_timeout: int = 180, + sync_poll_interval: int = 5, + ) -> None: + self.api = api + self.organization_id = urllib.parse.quote(organization_id, safe="") + self.site_id = urllib.parse.quote(site_id, safe="") + self.repository = repository + self.sync_timeout = sync_timeout + self.sync_poll_interval = sync_poll_interval + + @property + def _site_path(self) -> str: + return f"/orgs/{self.organization_id}/sites/{self.site_id}" + + def _list_site_spaces(self, default: bool | None = None) -> list[dict[str, Any]]: + query = {"limit": "1000"} + if default is not None: + query["default"] = str(default).lower() + result = self.api.request( + "GET", f"{self._site_path}/site-spaces?{urllib.parse.urlencode(query)}" + ) + return result["items"] + + @staticmethod + def _space_title(site_space: dict[str, Any]) -> str | None: + return site_space.get("space", {}).get("title") or site_space.get("title") + + def _find_site_space(self, title: str) -> dict[str, Any] | None: + return next( + ( + site_space + for site_space in self._list_site_spaces() + if self._space_title(site_space) == title + ), + None, + ) + + def _duplicate_default_space(self, title: str) -> dict[str, Any]: + defaults = self._list_site_spaces(default=True) + if len(defaults) != 1: + raise RuntimeError( + f"expected one default GitBook site space, found {len(defaults)}" + ) + source_id = urllib.parse.quote(defaults[0]["id"], safe="") + site_space = self.api.request( + "POST", + f"{self._site_path}/site-spaces/{source_id}/duplicate", + {"draft": True}, + expected=(201,), + ) + space_id = urllib.parse.quote(site_space["space"]["id"], safe="") + self.api.request("PATCH", f"/spaces/{space_id}", {"title": title}) + site_space["space"]["title"] = title + return site_space + + def _import_branch(self, space_id: str, branch: str) -> None: + encoded_space_id = urllib.parse.quote(space_id, safe="") + self.api.request( + "POST", + f"/spaces/{encoded_space_id}/git/import", + { + "url": f"https://github.com/{self.repository}", + "ref": f"refs/heads/{branch}", + "force": True, + }, + expected=(204,), + ) + + def _wait_for_import(self, space_id: str, branch: str) -> None: + encoded_space_id = urllib.parse.quote(space_id, safe="") + expected_suffix = f"/tree/{branch}" + deadline = time.monotonic() + self.sync_timeout + while True: + info = self.api.request("GET", f"/spaces/{encoded_space_id}/git/info") + operation = info.get("operation") or {} + state = operation.get("state") + branch_matches = info.get("url", "").rstrip("/").endswith(expected_suffix) + if branch_matches and state == "success": + return + if branch_matches and state in {"failure", "timeout"}: + detail = operation.get("error") or state + raise RuntimeError(f"GitBook import for {branch} failed: {detail}") + if time.monotonic() >= deadline: + raise RuntimeError( + f"timed out waiting for GitBook to import {branch} after " + f"{self.sync_timeout}s" + ) + time.sleep(self.sync_poll_interval) + + def _wait_for_default(self, site_space_id: str, branch: str) -> None: + deadline = time.monotonic() + self.sync_timeout + while True: + defaults = self._list_site_spaces(default=True) + if ( + len(defaults) == 1 + and defaults[0]["id"] == site_space_id + and self._space_title(defaults[0]) == branch + and not defaults[0].get("draft", False) + ): + return + if time.monotonic() >= deadline: + raise RuntimeError( + f"GitBook did not publish {branch} as the default space after " + f"{self.sync_timeout}s" + ) + time.sleep(self.sync_poll_interval) + + def publish_version(self, branch: str) -> None: + site_space = self._find_site_space(branch) + if site_space is None: + site_space = self._duplicate_default_space(branch) + print(f"Created draft GitBook space for {branch}") + else: + print(f"Reusing GitBook space for {branch}") + + site_space_id = site_space["id"] + space_id = site_space["space"]["id"] + self._import_branch(space_id, branch) + self._wait_for_import(space_id, branch) + + encoded_site_space_id = urllib.parse.quote(site_space_id, safe="") + self.api.request( + "PATCH", + f"{self._site_path}/site-spaces/{encoded_site_space_id}", + {"path": branch, "draft": False}, + ) + self.api.request("PATCH", self._site_path, {"defaultSiteSpace": site_space_id}) + self.api.request("POST", f"{self._site_path}/publish") + self._wait_for_default(site_space_id, branch) + print(f"Published {branch} as the default GitBook documentation") + + +def publish_release( + version: VersionInfo, + github: GitHubPublisher, + gitbook: GitBookPublisher, +) -> None: + github.ensure_release_branch(version.tag, version.branch) + gitbook.publish_version(version.branch) + + +def _required_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise SystemExit(f"error: {name} is required") + return value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--version", required=True, help="released version, such as 0.66.0" + ) + parser.add_argument( + "--repository", + default=os.environ.get("GITHUB_REPOSITORY", "feast-dev/feast"), + help="GitHub owner/repository", + ) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--sync-timeout", type=int, default=180) + args = parser.parse_args() + + try: + version = VersionInfo.parse(args.version) + except ValueError as error: + raise SystemExit(f"error: {error}") from error + + if args.dry_run: + print( + f"Would publish {version.tag} from {version.branch} as the default " + "GitBook documentation" + ) + return + + github_api = JsonApi( + GITHUB_API_URL, + _required_env("GITHUB_TOKEN"), + { + "X-GitHub-Api-Version": GITHUB_API_VERSION, + "Accept": "application/vnd.github+json", + }, + ) + gitbook_api = JsonApi(GITBOOK_API_URL, _required_env("GITBOOK_TOKEN")) + github = GitHubPublisher(github_api, args.repository) + gitbook = GitBookPublisher( + gitbook_api, + _required_env("GITBOOK_ORG_ID"), + _required_env("GITBOOK_SITE_ID"), + args.repository, + sync_timeout=args.sync_timeout, + ) + publish_release(version, github, gitbook) + + +if __name__ == "__main__": + main() diff --git a/infra/scripts/release/test_publish_versioned_docs.py b/infra/scripts/release/test_publish_versioned_docs.py new file mode 100644 index 00000000000..536ecd7153e --- /dev/null +++ b/infra/scripts/release/test_publish_versioned_docs.py @@ -0,0 +1,246 @@ +import importlib.util +import sys +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("publish_versioned_docs.py") +SPEC = importlib.util.spec_from_file_location("publish_versioned_docs", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +publisher = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = publisher +SPEC.loader.exec_module(publisher) + + +class FakeApi: + def __init__(self, responses): + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None, expected=(200,)): + self.calls.append((method, path, payload, expected)) + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + +def site_space(identifier, title, *, default=False, draft=False): + return { + "id": identifier, + "title": title, + "default": default, + "draft": draft, + "space": {"id": f"space-{identifier}", "title": title}, + } + + +def successful_import(branch): + return { + "url": f"https://github.com/feast-dev/feast/tree/{branch}", + "operation": {"state": "success", "direction": "import"}, + } + + +class VersionInfoTests(unittest.TestCase): + def test_parses_release_version(self): + self.assertEqual( + publisher.VersionInfo.parse("v0.66.0"), + ("0.66.0", "v0.66.0", "v0.66-branch"), + ) + + def test_rejects_prerelease_version(self): + with self.assertRaisesRegex(ValueError, "stable semantic version"): + publisher.VersionInfo.parse("0.66.0-rc1") + + +class GitHubPublisherTests(unittest.TestCase): + def test_creates_missing_release_branch_at_tag(self): + api = FakeApi( + [ + {"sha": "release-sha"}, + publisher.ApiError(404, "missing"), + {"ref": "refs/heads/v0.66-branch"}, + ] + ) + github = publisher.GitHubPublisher(api, "feast-dev/feast") + + self.assertEqual( + github.ensure_release_branch("v0.66.0", "v0.66-branch"), + "release-sha", + ) + self.assertEqual( + api.calls[-1], + ( + "POST", + "/repos/feast-dev/feast/git/refs", + {"ref": "refs/heads/v0.66-branch", "sha": "release-sha"}, + (201,), + ), + ) + + def test_reuses_matching_release_branch(self): + api = FakeApi([{"sha": "release-sha"}, {"commit": {"sha": "release-sha"}}]) + github = publisher.GitHubPublisher(api, "feast-dev/feast") + + github.ensure_release_branch("v0.66.0", "v0.66-branch") + + self.assertEqual(len(api.calls), 2) + + def test_refuses_to_move_conflicting_release_branch(self): + api = FakeApi([{"sha": "release-sha"}, {"commit": {"sha": "different-sha"}}]) + github = publisher.GitHubPublisher(api, "feast-dev/feast") + + with self.assertRaisesRegex(RuntimeError, "refusing to move"): + github.ensure_release_branch("v0.66.0", "v0.66-branch") + + self.assertEqual(len(api.calls), 2) + + +class GitBookPublisherTests(unittest.TestCase): + def _publisher(self, responses): + api = FakeApi(responses) + gitbook = publisher.GitBookPublisher( + api, + "org", + "site", + "feast-dev/feast", + sync_timeout=0, + sync_poll_interval=0, + ) + return gitbook, api + + def test_creates_imports_and_publishes_new_version(self): + old_default = site_space("old", "v0.64-branch", default=True) + new_space = site_space("new", "v0.64-branch", draft=True) + new_default = site_space("new", "v0.66-branch", default=True) + gitbook, api = self._publisher( + [ + {"items": [old_default]}, + {"items": [old_default]}, + new_space, + {}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [new_default]}, + ] + ) + + gitbook.publish_version("v0.66-branch") + + self.assertIn( + ( + "POST", + "/orgs/org/sites/site/site-spaces/old/duplicate", + {"draft": True}, + (201,), + ), + api.calls, + ) + self.assertIn( + ( + "PATCH", + "/orgs/org/sites/site", + {"defaultSiteSpace": "new"}, + (200,), + ), + api.calls, + ) + + def test_reuses_existing_version_space(self): + target = site_space("new", "v0.66-branch") + new_default = site_space("new", "v0.66-branch", default=True) + gitbook, api = self._publisher( + [ + {"items": [target]}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [new_default]}, + ] + ) + + gitbook.publish_version("v0.66-branch") + + self.assertFalse(any(call[1].endswith("/duplicate") for call in api.calls)) + + def test_failed_import_does_not_change_public_default(self): + target = site_space("new", "v0.66-branch", draft=True) + gitbook, api = self._publisher( + [ + {"items": [target]}, + None, + { + "url": "https://github.com/feast-dev/feast/tree/v0.66-branch", + "operation": { + "state": "failure", + "direction": "import", + "error": "invalid content", + }, + }, + ] + ) + + with self.assertRaisesRegex(RuntimeError, "invalid content"): + gitbook.publish_version("v0.66-branch") + + self.assertFalse(any(call[0] == "PATCH" for call in api.calls)) + + def test_fails_when_default_space_readback_does_not_match(self): + target = site_space("new", "v0.66-branch") + old_default = site_space("old", "v0.64-branch", default=True) + gitbook, _api = self._publisher( + [ + {"items": [target]}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [old_default]}, + ] + ) + + with self.assertRaisesRegex(RuntimeError, "did not publish"): + gitbook.publish_version("v0.66-branch") + + def test_waits_for_default_space_readback_to_match(self): + target = site_space("new", "v0.66-branch") + old_default = site_space("old", "v0.64-branch", default=True) + new_default = site_space("new", "v0.66-branch", default=True) + api = FakeApi( + [ + {"items": [target]}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [old_default]}, + {"items": [new_default]}, + ] + ) + gitbook = publisher.GitBookPublisher( + api, + "org", + "site", + "feast-dev/feast", + sync_timeout=1, + sync_poll_interval=0, + ) + + gitbook.publish_version("v0.66-branch") + + default_reads = [ + call for call in api.calls if call[0] == "GET" and "default=true" in call[1] + ] + self.assertEqual(len(default_reads), 2) + + +if __name__ == "__main__": + unittest.main() From 686bfd3a225a3d673cadd7429ed1d54308c98297 Mon Sep 17 00:00:00 2001 From: Larry Singleton <166439969+larrysingleton007@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:45:35 -0500 Subject: [PATCH 2/2] fix: Add standalone documentation publishing Signed-off-by: Larry Singleton <166439969+larrysingleton007@users.noreply.github.com> --- .github/workflows/release.yml | 38 ++++++++++++++++++++++++++++++++- docs/project/release-process.md | 6 ++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7b87553817..f3a18478da8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,11 @@ on: required: true default: true type: boolean + publish_docs_version: + description: 'Publish docs for an existing release (for example, 0.66.0)' + required: false + default: "" + type: string workflow_call: inputs: dry_run: @@ -35,6 +40,11 @@ on: required: true default: true type: boolean + publish_docs_version: + description: 'Publish docs for an existing release (for example, 0.66.0)' + required: false + default: "" + type: string secrets: GITBOOK_TOKEN: description: 'Token used to publish versioned documentation' @@ -42,7 +52,7 @@ on: jobs: get_dry_release_versions: - if: github.repository == 'feast-dev/feast' + if: github.repository == 'feast-dev/feast' && inputs.publish_docs_version == '' runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ github.event.inputs.token }} @@ -224,3 +234,29 @@ jobs: run: | python infra/scripts/release/publish_versioned_docs.py \ --version "${{ needs.get_dry_release_versions.outputs.next_version }}" + + publish_existing_version_docs: + name: Publish documentation for an existing release + if: github.repository == 'feast-dev/feast' && inputs.publish_docs_version != '' + runs-on: ubuntu-latest + permissions: + contents: write + env: + GITHUB_TOKEN: ${{ inputs.token }} + GITBOOK_TOKEN: ${{ secrets.GITBOOK_TOKEN }} + GITBOOK_ORG_ID: ${{ vars.GITBOOK_ORG_ID }} + GITBOOK_SITE_ID: ${{ vars.GITBOOK_SITE_ID }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Publish released documentation + run: | + args=(--version "${{ inputs.publish_docs_version }}") + if [[ "${{ inputs.dry_run }}" == "true" ]]; then + args+=(--dry-run) + fi + python infra/scripts/release/publish_versioned_docs.py "${args[@]}" diff --git a/docs/project/release-process.md b/docs/project/release-process.md index fd13c024474..7a7ab4d3657 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -124,6 +124,12 @@ The job is safe to rerun. It reuses matching branches and spaces, but it will no maintenance branch that points somewhere other than the released tag. It also leaves the previous documentation version as the public default until the new branch has imported successfully. +To publish documentation independently of a release, run the `release` workflow from `master` and +set `publish_docs_version` to an existing release such as `0.66.0`. When this input is set, the +release jobs are skipped and only the versioned documentation job runs. Keep `dry_run` enabled for +the first run, then disable it to create or verify the maintenance branch and publish its GitBook +space. The standalone job does not move `stable` or create a release. + #### 4b: Adding a high level summary in the GitHub release notes By default, Semantic Release will pull in messages from commits (features vs fixes, etc). But this is hard to digest, so it helps to have a high level overview. See https://github.com/feast-dev/feast/releases for the releases.