From 1b23524b2168073474004b4fae7318b30198bd34 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 3 Feb 2026 09:54:13 +0200 Subject: [PATCH 1/4] refactor(docker): update demo to use GHCR images - Change default KAFKA_VERSION from local to edge - Update docker-compose to use ghcr.io/aiven/inkless - Add image version options documentation - Update Ctrl+C instructions (run make destroy to clean up) - Add monitoring section to demo README - Simplify Dockerfile COPY paths --- .../docker-compose-files/inkless/Makefile | 16 ++++--- .../docker-compose-files/inkless/README.md | 44 ++++++++++++++++--- .../inkless/docker-compose.demo.yml | 2 +- .../inkless/docker-compose.yml | 2 +- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/docker/examples/docker-compose-files/inkless/Makefile b/docker/examples/docker-compose-files/inkless/Makefile index 9d0510f52c1..33f9e37e2fa 100644 --- a/docker/examples/docker-compose-files/inkless/Makefile +++ b/docker/examples/docker-compose-files/inkless/Makefile @@ -1,30 +1,34 @@ DOCKER := docker -KAFKA_VERSION ?= 4.1.0-inkless-SNAPSHOT +# Default to edge (latest development build from main). Other options: +# KAFKA_VERSION=latest (latest stable release) +# KAFKA_VERSION=4.1.0-0.33 (specific Kafka+Inkless version) +# KAFKA_VERSION=local (locally built image, requires: make docker_build from repo root) +KAFKA_VERSION ?= edge .PHONY: in-memory in-memory: - docker compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml up + KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml up $(MAKE) destroy .PHONY: s3-local s3-local: - $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.s3-local.yml up + KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.s3-local.yml up $(MAKE) destroy .PHONY: gcs-local gcs-local: - $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.gcs-local.yml up + KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.gcs-local.yml up $(MAKE) destroy .PHONY: azure-local azure-local: - $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.azure-local.yml up + KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.azure-local.yml up $(MAKE) destroy .PHONY: s3-aws s3-aws: - $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.s3-aws.yml up + KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.s3-aws.yml up $(MAKE) destroy .PHONY: destroy diff --git a/docker/examples/docker-compose-files/inkless/README.md b/docker/examples/docker-compose-files/inkless/README.md index 35fdc6107d6..2e97cbba2e1 100644 --- a/docker/examples/docker-compose-files/inkless/README.md +++ b/docker/examples/docker-compose-files/inkless/README.md @@ -2,19 +2,34 @@ Contains examples of how to run Inkless with Postgres as batch coordinator and different object storage back-ends using Docker Compose. -Running the make task for a setup will start the necessary services and run the Inkless demo, including topic creation, producer/consumer perf clients, and monitoring with Prometheus and Grafana. +Running a make task will start all services and run the Inkless demo, including topic creation, producer/consumer perf clients, and monitoring with Prometheus and Grafana. Press `Ctrl+C` to stop, then run `make destroy` to clean up all containers and networks. -## Run the demo +## Prerequisites -### in-memory +- Docker and Docker Compose v2 +- Available ports: 9092 (Kafka), 3000 (Grafana), 9001 (MinIO), 5432 (PostgreSQL) +- ~2GB disk space for images -This setup uses an in-memory object storage and Postgres as a metadata store. +## Services and credentials -``` -make in-memory +| Service | URL | Credentials | +|---------|-----|-------------| +| Grafana | http://localhost:3000 | `admin` / `admin` | +| MinIO Console | http://localhost:9001 | `minioadmin` / `minioadmin` | +| Kafka Bootstrap | localhost:9092 | - | +| PostgreSQL | localhost:5432 | `admin` / `admin` | + +## Run the demo + +By default, the demo pulls the `edge` image from GHCR. To use a different image: + +```bash +make s3-local KAFKA_VERSION=latest # Latest stable release +make s3-local KAFKA_VERSION=4.1.0-0.33 # Specific version +make s3-local KAFKA_VERSION=local # Locally built (requires: make docker_build from repo root) ``` -### S3 local +### S3 local (recommended) This setup uses MinIO as a local S3-compatible object storage and Postgres as a metadata store. @@ -63,3 +78,18 @@ Then run make s3-aws ``` +## Monitoring + +### View logs (in a separate terminal) + +```bash +docker compose logs -f broker # Kafka broker logs +docker compose logs -f producer-1 # Producer performance stats +docker compose logs -f consumer # Consumer performance stats +``` + +### Grafana dashboards + +Access Grafana at http://localhost:3000 and navigate to: +- **Kafka Inkless** - Inkless-specific metrics (batch sizes, object storage operations) +- **Kafka Clients** - Producer and consumer performance metrics diff --git a/docker/examples/docker-compose-files/inkless/docker-compose.demo.yml b/docker/examples/docker-compose-files/inkless/docker-compose.demo.yml index 1fb320a44bd..9b4c53e38c1 100644 --- a/docker/examples/docker-compose-files/inkless/docker-compose.demo.yml +++ b/docker/examples/docker-compose-files/inkless/docker-compose.demo.yml @@ -1,6 +1,6 @@ services: create_topic: - image: aivenoy/kafka:${KAFKA_VERSION} + image: ghcr.io/aiven/inkless:${KAFKA_VERSION} restart: "no" depends_on: broker: diff --git a/docker/examples/docker-compose-files/inkless/docker-compose.yml b/docker/examples/docker-compose-files/inkless/docker-compose.yml index 2a1dd0739c2..32edb613668 100644 --- a/docker/examples/docker-compose-files/inkless/docker-compose.yml +++ b/docker/examples/docker-compose-files/inkless/docker-compose.yml @@ -1,6 +1,6 @@ services: broker: &base-broker - image: aivenoy/kafka:${KAFKA_VERSION} + image: ghcr.io/aiven/inkless:${KAFKA_VERSION} restart: unless-stopped ports: - "9092:9092" From 2f4b6608a7b232532c88a6b91b64eb6e96525cd0 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 3 Feb 2026 11:29:17 +0200 Subject: [PATCH 2/4] docs(inkless): add releases documentation and update quickstart - Add Quick Start decision tree to README.md - Update QUICKSTART.md: - Add prerequisites section - Add demo backends and services table - Add binary usage guide with sample configs - Add monitoring instructions - Fix podman compose references --- .../docker-compose-files/inkless/README.md | 98 +++------------ docs/inkless/QUICKSTART.md | 116 ++++++++++++++++-- docs/inkless/README.md | 13 ++ 3 files changed, 131 insertions(+), 96 deletions(-) diff --git a/docker/examples/docker-compose-files/inkless/README.md b/docker/examples/docker-compose-files/inkless/README.md index 2e97cbba2e1..15dee68e599 100644 --- a/docker/examples/docker-compose-files/inkless/README.md +++ b/docker/examples/docker-compose-files/inkless/README.md @@ -1,95 +1,25 @@ -# Inkless demo +# Inkless Docker Compose Demo -Contains examples of how to run Inkless with Postgres as batch coordinator and different object storage back-ends using Docker Compose. +Docker Compose examples for running Inkless with different storage backends. -Running a make task will start all services and run the Inkless demo, including topic creation, producer/consumer perf clients, and monitoring with Prometheus and Grafana. Press `Ctrl+C` to stop, then run `make destroy` to clean up all containers and networks. +See the [Quickstart guide](../../../../docs/inkless/QUICKSTART.md#dockerized-demo) for complete setup instructions. -## Prerequisites +## Quick reference -- Docker and Docker Compose v2 -- Available ports: 9092 (Kafka), 3000 (Grafana), 9001 (MinIO), 5432 (PostgreSQL) -- ~2GB disk space for images +| Command | Description | +|-----------------------|---------------------------------------------------| +| `make s3-local` | MinIO S3-compatible storage (default) | +| `make gcs-local` | Fake GCS server | +| `make azure-local` | Azurite Azure-compatible storage | +| `make s3-aws` | Real AWS S3 (requires AWS credentials) | +| `make destroy` | Manual cleanup (automatic on Ctrl+C) | -## Services and credentials +## Image version -| Service | URL | Credentials | -|---------|-----|-------------| -| Grafana | http://localhost:3000 | `admin` / `admin` | -| MinIO Console | http://localhost:9001 | `minioadmin` / `minioadmin` | -| Kafka Bootstrap | localhost:9092 | - | -| PostgreSQL | localhost:5432 | `admin` / `admin` | - -## Run the demo - -By default, the demo pulls the `edge` image from GHCR. To use a different image: +By default, the demo pulls the `edge` image. Override with `KAFKA_VERSION`: ```bash make s3-local KAFKA_VERSION=latest # Latest stable release make s3-local KAFKA_VERSION=4.1.0-0.33 # Specific version -make s3-local KAFKA_VERSION=local # Locally built (requires: make docker_build from repo root) -``` - -### S3 local (recommended) - -This setup uses MinIO as a local S3-compatible object storage and Postgres as a metadata store. - -``` -make s3-local -``` - -### GCS local - -This setup uses `fake-gcs-server` as a local GCS-compatible object storage and Postgres as a metadata store. - -``` -make gcs-local +make s3-local KAFKA_VERSION=local # Locally built image ``` - -### Azure local - -This setup uses `azurite` as a local Azure-compatible object storage and Postgres as a metadata store. - -``` -make azure-local -``` - -### AWS S3 - -This example demonstrates how to use Inkless with remote data and metadata stores, -using a Remote Postgres instance for metadata and AWS S3 for data storage. - -#### Prerequisites - -Set the environment variables: - -For AWS S3: - -```properties -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_SESSION_TOKEN= -AWS_REGION= -AWS_S3_BUCKET_NAME= -``` - -Then run - -```bash -make s3-aws -``` - -## Monitoring - -### View logs (in a separate terminal) - -```bash -docker compose logs -f broker # Kafka broker logs -docker compose logs -f producer-1 # Producer performance stats -docker compose logs -f consumer # Consumer performance stats -``` - -### Grafana dashboards - -Access Grafana at http://localhost:3000 and navigate to: -- **Kafka Inkless** - Inkless-specific metrics (batch sizes, object storage operations) -- **Kafka Clients** - Producer and consumer performance metrics diff --git a/docs/inkless/QUICKSTART.md b/docs/inkless/QUICKSTART.md index 9dc4df55284..422c970b85b 100644 --- a/docs/inkless/QUICKSTART.md +++ b/docs/inkless/QUICKSTART.md @@ -1,5 +1,14 @@ # Quickstart +## Prerequisites + +Before running Inkless, ensure you have: + +- **Docker** and **Docker Compose v2** installed +- **Available ports:** 9092 (Kafka), 3000 (Grafana), 9001 (MinIO), 5432 (PostgreSQL) +- **Disk space:** ~2GB for demo images +- **For building from source:** JDK 17, Gradle (included via wrapper) + ## Dockerized demo Run: @@ -10,23 +19,97 @@ make demo make demo DOCKER=podman ``` -It will pull the Docker image, and start the local demo with two producers, one consumer, the PostgreSQL-backed control plane, and MinIO as the object storage. -Grafana with metrics will be accessible at http://localhost:3000 (login `admin`, password `admin`). -Minio will be accessible at http://localhost:9001 (login `minioadmin`, password `minioadmin`). +It will pull the Docker image and start the local demo with two producers, one consumer, the PostgreSQL-backed control plane, and MinIO as the object storage. Press `Ctrl+C` to stop; containers and networks are cleaned up automatically. + +### Demo services + +| Service | URL | Credentials | +|------------------|-----------------------|-------------------------| +| Grafana | http://localhost:3000 | `admin` / `admin` | +| MinIO Console | http://localhost:9001 | `minioadmin` / `minioadmin` | +| Kafka Bootstrap | localhost:9092 | - | +| PostgreSQL | localhost:5432 | `admin` / `admin` | -If you prefer to build the image locally, you can run: +### Demo backends + +The default backend is `s3-local` (MinIO). You can choose different storage backends: + +```shell +make demo DEMO=s3-local # MinIO S3-compatible storage (default) +make demo DEMO=gcs-local # Fake GCS server +make demo DEMO=azure-local # Azurite Azure-compatible storage +make demo DEMO=s3-aws # Real AWS S3 (requires AWS credentials) +``` + +#### AWS S3 backend + +For `s3-aws`, set the following environment variables before running: + +```shell +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +export AWS_SESSION_TOKEN= # if using temporary credentials +export AWS_REGION= +export AWS_S3_BUCKET_NAME= + +make demo DEMO=s3-aws +``` + +### Using a locally built image + +By default, the demo pulls the `edge` image from GHCR. If the image is not available or you want to test local changes: ```shell -# make clean # to rebuild the binaries make docker_build +make demo KAFKA_VERSION=local ``` -and then run the demo. +### Monitoring the demo -The logs will print the producer and consumer stats. -To observe the metrics, go to the Kafka Inkless and Kafka Clients dashboards in Grafana. +The logs will print the producer and consumer stats. To observe the metrics, access Grafana at http://localhost:3000 and navigate to: -For more details, see the [demo docs](./../../docker/examples/docker-compose-files/inkless/README.md). +- **Kafka Inkless** - Inkless-specific metrics (batch sizes, object storage operations) +- **Kafka Clients** - Producer and consumer performance metrics + +**View service logs (in a separate terminal):** +```shell +cd docker/examples/docker-compose-files/inkless +docker compose logs -f broker # Kafka broker logs (or: podman compose logs -f broker) +docker compose logs -f producer-1 # Producer performance stats +docker compose logs -f consumer # Consumer performance stats +``` + + +## Using prebuilt binaries + +If you prefer to run Inkless without Docker, download prebuilt binaries from [GitHub Releases](https://github.com/aiven/inkless/releases). See [Releases](./RELEASES.md#binary-distributions) for download instructions. + +### Configure and run + +Sample configurations are provided in `config/inkless/`: + +| Config | Control Plane | Storage Backend | +|--------|---------------|-----------------| +| `single-broker-pg-0.properties` | PostgreSQL | S3 (MinIO) | +| `single-broker-pg-0-gcs.properties` | PostgreSQL | GCS | +| `single-broker-pg-0-azure.properties` | PostgreSQL | Azure Blob | + +1. **Copy and edit a sample config** for your environment: + ```shell + cp config/inkless/single-broker-pg-0.properties config/server.properties + # Edit config/server.properties with your storage and database settings + ``` + +2. **Format storage and start broker:** + ```shell + # Format storage directory (first time only) + bin/kafka-storage.sh format -t $(bin/kafka-storage.sh random-uuid) -c config/server.properties + + # Start the broker + bin/kafka-server-start.sh config/server.properties + ``` + +See [configs.rst](./configs.rst) for the complete configuration reference and [Versioning Strategy](./VERSIONING-STRATEGY.md) for choosing the right version. ## Topic creation @@ -45,9 +128,18 @@ make build_release To run docker image: ```shell -docker run -it --entrypoint bash aivenoy/kafka:4.1.0-inkless-SNAPSHOT -4e3b0026634e:/$ cd /opt/ -java/ kafka/ +# Use locally built image (after running: make docker_build) +docker run -it --entrypoint bash ghcr.io/aiven/inkless:local + +# Or use latest edge build from GHCR +docker run -it --entrypoint bash ghcr.io/aiven/inkless:edge + +# Or use latest stable release +docker run -it --entrypoint bash ghcr.io/aiven/inkless:latest +``` + +Example session: +```shell 4e3b0026634e:/$ cd /opt/kafka/ 4e3b0026634e:/opt/kafka$ bin/kafka-topics.sh ``` diff --git a/docs/inkless/README.md b/docs/inkless/README.md index f8320b68e33..e333998359c 100644 --- a/docs/inkless/README.md +++ b/docs/inkless/README.md @@ -14,6 +14,19 @@ Each release on version branches represents stable progress of the Inkless imple > **Note:** Any S3-compatible storage backend should work in principle. MinIO has been used for integration testing, though we are evaluating alternatives given [MinIO's recent changes](https://github.com/minio/minio/commit/27742d469462e1561c776f88ca7a1f26816d69e2) to their Docker image and single-node deployment support. - PostgreSQL (for batch coordinator) or in-memory mode for testing +## Quick Start + +Choose your path based on what you want to do: + +| Goal | Command / Action | +|------------------------------|---------------------------------------------------------------| +| **Try Inkless in 5 minutes** | `make demo` ([details](QUICKSTART.md#dockerized-demo)) | +| **Use prebuilt Docker image**| `docker pull ghcr.io/aiven/inkless:latest` | +| **Use prebuilt binaries** | [Binary usage guide](QUICKSTART.md#using-prebuilt-binaries) | +| **Build from source** | `make docker_build` ([details](QUICKSTART.md#using-a-locally-built-image))| + +See [Releases](RELEASES.md) for available versions. + ## Documentation Index ### Getting Started From 9399e8dfc8335ef29c22061255e75401da091c96 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 3 Feb 2026 13:40:40 +0200 Subject: [PATCH 3/4] chore(inkless:docker): add link to repository --- docker/inkless/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/inkless/Dockerfile b/docker/inkless/Dockerfile index ff0297bb32c..41ef8bf5644 100644 --- a/docker/inkless/Dockerfile +++ b/docker/inkless/Dockerfile @@ -51,11 +51,12 @@ ARG kafka_version COPY resources/distributions/kafka_2.13-$kafka_version.tgz /kafka.tgz -LABEL org.label-schema.name="kafka" \ - org.label-schema.description="Apache Kafka" \ +LABEL org.label-schema.name="inkless" \ + org.label-schema.description="Inkless - Apache Kafka with object storage" \ org.label-schema.build-date="${build_date}" \ - org.label-schema.vcs-url="https://github.com/apache/kafka" \ - maintainer="Apache Kafka" + org.label-schema.vcs-url="https://github.com/aiven/inkless" \ + org.opencontainers.image.source="https://github.com/aiven/inkless" \ + maintainer="Aiven" RUN set -eux ; \ apk update ; \ From dbb2c04183a5eb6bc49ec74a289a9cbc752fce46 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 3 Feb 2026 14:43:15 +0200 Subject: [PATCH 4/4] fixup! docs(inkless): add releases documentation and update quickstart --- docker/examples/docker-compose-files/inkless/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/examples/docker-compose-files/inkless/README.md b/docker/examples/docker-compose-files/inkless/README.md index 15dee68e599..7b7446e13aa 100644 --- a/docker/examples/docker-compose-files/inkless/README.md +++ b/docker/examples/docker-compose-files/inkless/README.md @@ -16,10 +16,12 @@ See the [Quickstart guide](../../../../docs/inkless/QUICKSTART.md#dockerized-dem ## Image version -By default, the demo pulls the `edge` image. Override with `KAFKA_VERSION`: +By default, the demo pulls the `edge` image (latest development build). Override with `KAFKA_VERSION`: ```bash make s3-local KAFKA_VERSION=latest # Latest stable release -make s3-local KAFKA_VERSION=4.1.0-0.33 # Specific version -make s3-local KAFKA_VERSION=local # Locally built image +make s3-local KAFKA_VERSION=4.1.0-0.33 # Specific Kafka+Inkless version +make s3-local KAFKA_VERSION=local # Locally built image (requires: make docker_build) ``` + +See [Releases](../../../../docs/inkless/RELEASES.md#docker-images) for all available tags.