Question 85402 in Docker Question posted in
The official documentation can be found here.

How to have multiple cluster in kafka by using docker?

I’m trying to up multiple clusters with three nodes in kafka with using docker and I followed this approach in my docker-compose file but this is wrong and I can’t up Kafka those I wrote in my docker-compose file
please tell me the true docker-compose file or true approach for that to up multiple clusters (with three nodes) in Kafka by using docker

version: '2'
services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 22101
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS:      localhost:22000:23000;localhost:32000:33000;localhost:42000:43000
    network_mode: host
      extra_hosts:
        - "moby:127.0.0.1"

  zookeeper-2:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_CLIENT_PORT: 32181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
    network_mode: host
    extra_hosts:
      - "moby:127.0.0.1"

  zookeeper-3:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_CLIENT_PORT: 42181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
    network_mode: host
    extra_hosts:
      - "moby:127.0.0.1"

kafka-1:
  image: confluentinc/cp-zookeeper:latest
  network_mode: host
  depends_on:
    - zookeeper-1
    - zookeeper-2
    - zookeeper-3
  environment:
  KAFKA_BROKER_ID: 1
  KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
  KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19092
kafka-2:
   image: confluentinc/cp-zookeeper:latest
   network_mode: host
   depends_on:
     - zookeeper-1
     - zookeeper-2
     - zookeeper-3
   environment:
   KAFKA_BROKER_ID: 2
   KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
   KAFKA_ADVERTISED_LISTENER: PLAINTEXT://localhost:29092
 extra_hosts:
   - "moby:127.0.0.1"
kafka-3:
  image: confluentinc/cp-zookeeper:latest
  network_mode: host
  depends_on:
    - zookeeper-1
    - zookeeper-2
    - zookeeper-3
  environment:
    KAFKA_BROKER_ID: 3
    KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
    KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092
extra_hosts:
  - "moby:127.0.0.1"
Tags:

2

Answers


  1. Chosen as BEST ANSWER
    0

    I solved my problem by following this link:

    https://github.com/conduktor/kafka-stack-docker-compose

    my question was about how can I have multiple nodes in Kafka by using docker-compose and I used this Item : Single Zookeeper / Multiple Kafka , from that repository and ran this command to use it:

    docker-compose -f zk-single-Kafka-single.yml up

    and after that , you can use this command for checking which containers are up:

    sudo docker ps

    and name of outputs are:

    kafka-stack-docker-compose_kafka2_1

    kafka-stack-docker-compose_kafka3_1

    kafka-stack-docker-compose_kafka1_1

    zoo1

    with their ID and STATUS and ...


  2. 0

    Your configuration is wrong:
    Each zk node has a process (named QuorumPeerMain) which listen to clients on a default port 2181 and communicate with each other on port 2888:3888. So when try to define a zookeeper quorum define it as zk1:[port],zk2:[port],...

     zookeeper-1:
      ZOOKEEPER_CLIENT_PORT: 2181
...
...
 zookeeper-2:
      ZOOKEEPER_CLIENT_PORT: 2181
...
...
 zookeeper-3:
      ZOOKEEPER_CLIENT_PORT: 2181

    and in Kafka properties:

    KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181

    Follow this link: Guide to Setting Up Apache Kafka Using Docker

    Login or Signup to reply.
Please signup or login to give your own answer.

Back To Top
Search