chore: Use a local Docker builder image for Rust protobuf generation (#320)

* feat: dockerize rust protobuf generation in make and CI

Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/6438cfba-52ed-4c85-b8fe-1268cc55755a

Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com>

* fix: unify make all and CI generation with local builder image

Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/6438cfba-52ed-4c85-b8fe-1268cc55755a

Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com>

* fix: only use custom Docker image for rust target, go/java/cpp use protoc-all directly

Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/6e6885ee-2919-46f5-9dc5-301c3303888d

Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com>

* chore: fix README wording and add --user flag to rust docker run

Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/3983f532-e5e5-45e4-953d-27bc551ec13d

Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com>

* fix: make cargo/rustup dirs world-writable so --user flag works in container

Agent-Logs-Url: https://github.com/GreptimeTeam/greptime-proto/sessions/1bddab48-992c-481b-937f-6698dd9cc067

Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: WenyXu <32535939+WenyXu@users.noreply.github.com>

.github/workflows/ci.yml

@@ -37,10 +37,6 @@ jobs:
         uses: Swatinem/rust-cache@v2
       - name: Run cargo check
         run: cargo check --workspace --all-targets
-      - uses: arduino/setup-protoc@v3
-        with:
-          version: '21.12'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Generate protobuf
         run: |
           make all

Makefile

@@ -1,29 +1,25 @@
-.PHONY: all rust go go-deps java cpp check-protoc
+.PHONY: all rust go go-deps java cpp build-builder-image
 
-BUILDER_CONTAINER=namely/protoc-all:1.51_2
+PROTOC_CONTAINER=namely/protoc-all:1.51_2
-PROTOC_VERSION=3.21.12
+BUILDER_CONTAINER=greptime/protoc-all-local:latest
+BUILDER_DOCKERFILE=./docker/protoc-all/Dockerfile
+BUILDER_CONTEXT=./docker/protoc-all
 
 all: rust go java cpp
 
-check-protoc:
+build-builder-image:
-	@if ! command -v protoc >/dev/null 2>&1; then \
+	docker build -f ${BUILDER_DOCKERFILE} -t ${BUILDER_CONTAINER} ${BUILDER_CONTEXT}
-		echo "Error: protoc is not installed. Please install protoc $(PROTOC_VERSION)."; \
-		exit 1; \
-	fi
-	@CURRENT_PROTOC_VERSION=$$(protoc --version | awk '{print $$2}'); \
-	if [ "$$CURRENT_PROTOC_VERSION" != "$(PROTOC_VERSION)" ]; then \
-		echo "Error: Required protoc version is $(PROTOC_VERSION), but found $$CURRENT_PROTOC_VERSION."; \
-		echo "Please install protoc $(PROTOC_VERSION) to ensure generated code is consistent."; \
-		exit 1; \
-	fi
 
-rust: check-protoc
+rust: build-builder-image
-	cargo run --manifest-path xtask/Cargo.toml -- generate-rust
+	docker run --rm -t -w /greptime-proto \
+		--user $(shell id -u):$(shell id -g) \
+		--entrypoint ./scripts/generate-rust.sh \
+		-v ${PWD}:/greptime-proto ${BUILDER_CONTAINER}
 
 go: go-deps
 	docker run --rm -t -w /greptime-proto \
 		--entrypoint ./scripts/generate-go.sh \
-		-v ${PWD}:/greptime-proto ${BUILDER_CONTAINER}
+		-v ${PWD}:/greptime-proto ${PROTOC_CONTAINER}
 
 go-deps:
 	go mod download
@@ -31,9 +27,9 @@ go-deps:
 java:
 	docker run --rm -t -w /greptime-proto \
 		--entrypoint ./scripts/generate-java.sh \
-		-v ${PWD}:/greptime-proto ${BUILDER_CONTAINER}
+		-v ${PWD}:/greptime-proto ${PROTOC_CONTAINER}
 
 cpp:
 	docker run --rm -t -w /greptime-proto \
 		--entrypoint ./scripts/generate-cpp.sh \
-		-v ${PWD}:/greptime-proto ${BUILDER_CONTAINER}
+		-v ${PWD}:/greptime-proto ${PROTOC_CONTAINER}

README.md

@@ -7,7 +7,7 @@ GreptimeDB protobuf definitions and pre-generated Rust bindings.
 ### Requirement
 
 - Rust consumers do not need `protoc`.
-- Maintainers need [google/protobuf][protobuf] v3 to regenerate Rust bindings after `.proto` changes.
+- Maintainers need Docker to regenerate checked-in bindings after `.proto` changes.
 
 ### Command
 
@@ -29,7 +29,7 @@ GreptimeDB protobuf definitions and pre-generated Rust bindings.
   make java
   ```
 
-  The compilation for Go and Java will use builder container `namely/protoc-all`.
+  Rust generation uses the local builder image built from `docker/protoc-all/Dockerfile`. Go, Java, and C++ generation uses `namely/protoc-all:1.51_2` directly.
 
 ## Usage
 
@@ -53,7 +53,7 @@ use greptime_proto::prometheus::remote::*;
 When working in this repository, regenerate the checked-in Rust bindings after `.proto` changes with:
 
 ```console
-cargo run --manifest-path xtask/Cargo.toml -- generate-rust
+make rust
 ```
 
 ### Go

docker/protoc-all/Dockerfile

@@ -0,0 +1,19 @@
+FROM rust:1.88.0-bookworm AS rust
+
+FROM namely/protoc-all:1.51_2
+
+COPY --from=rust /usr/local/cargo /usr/local/cargo
+COPY --from=rust /usr/local/rustup /usr/local/rustup
+
+ENV CARGO_HOME=/usr/local/cargo
+ENV RUSTUP_HOME=/usr/local/rustup
+ENV PATH=/usr/local/cargo/bin:${PATH}
+
+RUN rm -f /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends build-essential \
+    && rm -rf /var/lib/apt/lists/* \
+    && rustc --version \
+    && cargo --version \
+    && protoc --version \
+    && chmod -R a+rwX /usr/local/cargo /usr/local/rustup

scripts/generate-rust.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+
+set -e
+
+export PROTOC_INCLUDE=/opt/include
+
+cargo run --manifest-path xtask/Cargo.toml -- generate-rust

xtask/src/main.rs

@@ -102,7 +102,10 @@ fn generate_rust() -> Result<(), Box<dyn Error>> {
         .iter()
         .map(|path| repo_root.join(path))
         .collect();
-    let include_dirs = [repo_root.join("proto")];
+    let mut include_dirs = vec![repo_root.join("proto")];
+    if let Some(protoc_include) = env::var_os("PROTOC_INCLUDE") {
+        include_dirs.push(PathBuf::from(protoc_include));
+    }
 
     tonic_prost_build::configure()
         .out_dir(&generated_dir)