From 246a8765689f595297865a4f9b5917000afa3931 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 20:34:09 +0800 Subject: [PATCH] chore: Use a local Docker builder image for Rust protobuf generation (#320) * feat: dockerize rust protobuf generation in make and CI Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/6438cfba-52ed-4c85-b8fe-1268cc55755a Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com> * fix: unify make all and CI generation with local builder image Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/6438cfba-52ed-4c85-b8fe-1268cc55755a Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com> * fix: only use custom Docker image for rust target, go/java/cpp use protoc-all directly Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/6e6885ee-2919-46f5-9dc5-301c3303888d Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com> * chore: fix README wording and add --user flag to rust docker run Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/3983f532-e5e5-45e4-953d-27bc551ec13d Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com> * fix: make cargo/rustup dirs world-writable so --user flag works in container Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/1bddab48-992c-481b-937f-6698dd9cc067 Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ---- Makefile | 34 +++++++++++++++------------------- README.md | 6 +++--- docker/protoc-all/Dockerfile | 19 +++++++++++++++++++ scripts/generate-rust.sh | 7 +++++++ xtask/src/main.rs | 5 ++++- 6 files changed, 48 insertions(+), 27 deletions(-) create mode 100644 docker/protoc-all/Dockerfile create mode 100755 scripts/generate-rust.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcb7fa7..97d7f80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,10 +37,6 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Run cargo check run: cargo check --workspace --all-targets - - uses: arduino/setup-protoc@v3 - with: - version: '21.12' - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Generate protobuf run: | make all diff --git a/Makefile b/Makefile index 09ff81b..b83acc1 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,25 @@ -.PHONY: all rust go go-deps java cpp check-protoc +.PHONY: all rust go go-deps java cpp build-builder-image -BUILDER_CONTAINER=namely/protoc-all:1.51_2 -PROTOC_VERSION=3.21.12 +PROTOC_CONTAINER=namely/protoc-all:1.51_2 +BUILDER_CONTAINER=greptime/protoc-all-local:latest +BUILDER_DOCKERFILE=./docker/protoc-all/Dockerfile +BUILDER_CONTEXT=./docker/protoc-all all: rust go java cpp -check-protoc: - @if ! command -v protoc >/dev/null 2>&1; then \ - echo "Error: protoc is not installed. Please install protoc $(PROTOC_VERSION)."; \ - exit 1; \ - fi - @CURRENT_PROTOC_VERSION=$$(protoc --version | awk '{print $$2}'); \ - if [ "$$CURRENT_PROTOC_VERSION" != "$(PROTOC_VERSION)" ]; then \ - echo "Error: Required protoc version is $(PROTOC_VERSION), but found $$CURRENT_PROTOC_VERSION."; \ - echo "Please install protoc $(PROTOC_VERSION) to ensure generated code is consistent."; \ - exit 1; \ - fi +build-builder-image: + docker build -f ${BUILDER_DOCKERFILE} -t ${BUILDER_CONTAINER} ${BUILDER_CONTEXT} -rust: check-protoc - cargo run --manifest-path xtask/Cargo.toml -- generate-rust +rust: build-builder-image + docker run --rm -t -w /greptime-proto \ + --user $(shell id -u):$(shell id -g) \ + --entrypoint ./scripts/generate-rust.sh \ + -v ${PWD}:/greptime-proto ${BUILDER_CONTAINER} go: go-deps docker run --rm -t -w /greptime-proto \ --entrypoint ./scripts/generate-go.sh \ - -v ${PWD}:/greptime-proto ${BUILDER_CONTAINER} + -v ${PWD}:/greptime-proto ${PROTOC_CONTAINER} go-deps: go mod download @@ -31,9 +27,9 @@ go-deps: java: docker run --rm -t -w /greptime-proto \ --entrypoint ./scripts/generate-java.sh \ - -v ${PWD}:/greptime-proto ${BUILDER_CONTAINER} + -v ${PWD}:/greptime-proto ${PROTOC_CONTAINER} cpp: docker run --rm -t -w /greptime-proto \ --entrypoint ./scripts/generate-cpp.sh \ - -v ${PWD}:/greptime-proto ${BUILDER_CONTAINER} + -v ${PWD}:/greptime-proto ${PROTOC_CONTAINER} diff --git a/README.md b/README.md index cfacbdf..c72e36e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ GreptimeDB protobuf definitions and pre-generated Rust bindings. ### Requirement - Rust consumers do not need `protoc`. -- Maintainers need [google/protobuf][protobuf] v3 to regenerate Rust bindings after `.proto` changes. +- Maintainers need Docker to regenerate checked-in bindings after `.proto` changes. ### Command @@ -29,7 +29,7 @@ GreptimeDB protobuf definitions and pre-generated Rust bindings. make java ``` - The compilation for Go and Java will use builder container `namely/protoc-all`. + Rust generation uses the local builder image built from `docker/protoc-all/Dockerfile`. Go, Java, and C++ generation uses `namely/protoc-all:1.51_2` directly. ## Usage @@ -53,7 +53,7 @@ use greptime_proto::prometheus::remote::*; When working in this repository, regenerate the checked-in Rust bindings after `.proto` changes with: ```console -cargo run --manifest-path xtask/Cargo.toml -- generate-rust +make rust ``` ### Go diff --git a/docker/protoc-all/Dockerfile b/docker/protoc-all/Dockerfile new file mode 100644 index 0000000..3f6fce5 --- /dev/null +++ b/docker/protoc-all/Dockerfile @@ -0,0 +1,19 @@ +FROM rust:1.88.0-bookworm AS rust + +FROM namely/protoc-all:1.51_2 + +COPY --from=rust /usr/local/cargo /usr/local/cargo +COPY --from=rust /usr/local/rustup /usr/local/rustup + +ENV CARGO_HOME=/usr/local/cargo +ENV RUSTUP_HOME=/usr/local/rustup +ENV PATH=/usr/local/cargo/bin:${PATH} + +RUN rm -f /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rustc --version \ + && cargo --version \ + && protoc --version \ + && chmod -R a+rwX /usr/local/cargo /usr/local/rustup diff --git a/scripts/generate-rust.sh b/scripts/generate-rust.sh new file mode 100755 index 0000000..452b951 --- /dev/null +++ b/scripts/generate-rust.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env bash + +set -e + +export PROTOC_INCLUDE=/opt/include + +cargo run --manifest-path xtask/Cargo.toml -- generate-rust diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1c446cf..44c2fea 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -102,7 +102,10 @@ fn generate_rust() -> Result<(), Box> { .iter() .map(|path| repo_root.join(path)) .collect(); - let include_dirs = [repo_root.join("proto")]; + let mut include_dirs = vec![repo_root.join("proto")]; + if let Some(protoc_include) = env::var_os("PROTOC_INCLUDE") { + include_dirs.push(PathBuf::from(protoc_include)); + } tonic_prost_build::configure() .out_dir(&generated_dir)