demo version

master/srv/pillar/placement.sls

@@ -5,7 +5,6 @@ placement:
     - hosts:
       - h5
       - h6
-      - h7
       - h8
     - version: 3.11.1
     - first_cqlsh_host: h5
@@ -13,14 +12,12 @@ placement:
     - hosts:
       - h5
       - h6
-      - h7
       - h8
     - version: 3.4.11
     - host_zooid_mapping:
       - h5 : 1 
       - h6 : 2
-      - h7 : 3
-      - h8 : 4
+      - h8 : 3
   - kafka:
     - replication_factor : 3
     - partition_factor: 6
@@ -28,6 +25,6 @@ placement:
   - spark:
     - slaves:
       - h5
-      - h6
+      - h8
     - master: h7
     - image : nherbaut/spark-wordcount:latest

master/srv/pillar/top.sls

@@ -3,5 +3,6 @@ base:
     - urlsource
     - influxdb
     - placement
+    - mine_functions
   
  

master/srv/salt/experiment_data/init.sls

@@ -17,4 +17,7 @@ bible:
     - template: jinja
     - mode: 0755
 
-
+/root/killall.sh:
+  file.managed:
+    - source: salt://experiment_data/killall.sh
+    - mode: 0755

master/srv/salt/experiment_data/killall.sh

+salt "*" cmd.run 'docker rm -f $(docker ps -qa)' && salt "*" cmd.run 'service telegraf stop'

master/srv/salt/init-components/init.sls

@@ -43,7 +43,7 @@ docker exec kafka bash -c '/opt/kafka*/bin/kafka-topics.sh --create --zookeeper
 
 {% set spark_master_ip  = salt['mine.get'](grains.id,"datapath_ip")[grains.id][0] %}
 
-docker exec -d spark bash -c '/opt/spark/bin/spark-submit  --class org.apache.spark.examples.streaming.DirectKafkaWordCount   --master spark://{{spark_master_ip}}:7077  --deploy-mode client /opt/wordcount/spark-wordcount-1.0-jar-with-dependencies.jar  {{ cassandraspec|join(",") }} {{ topic_name }} {{ kafkaspec|join(",") }}':
+docker exec -d spark bash -c '/opt/spark/bin/spark-submit  --class org.apache.spark.examples.streaming.DirectKafkaWordCount   --master spark://{{spark_master_ip}}:7077  --conf spark.block.manager.port=40000 --conf spark.driver.port=41000 --conf spark.executor.port=42000 --conf spark.fileserver.port=43000 --conf spark.broadcast.port=44000 --conf spark.blockManager.port=45000 --conf spark.port.maxretries=4  --deploy-mode client /opt/wordcount/spark-wordcount-1.0-jar-with-dependencies.jar  {{ cassandraspec|join(",") }} {{ topic_name }} {{ kafkaspec|join(",") }}':
   cmd.run
 
 

master/srv/salt/spark/init.sls

@@ -17,6 +17,7 @@ spark_master:
     - hostname: spark
     - image: {{ salt['pillar.get']('placement:spark:image') }}
     - port_bindings:
+      - 4040:4040
       - 7077:7077
       - 6066:6066
       - 8080:8080
@@ -24,12 +25,17 @@ spark_master:
       - 41000-41004:41000-41004
       - 42000-42004:42000-42004
       - 43000-43004:43000-43004
+      - 44000-44004:44000-44004
+      - 45000-45004:45000-45004
     - hostname: {{ grains.id }}
     - environment:
       - SPARK_BLOCK_MANAGER_PORT: 40000
+      - SPARK_BROADCAST_PORT: 44000
       - SPARK_DRIVER_PORT: 41000
       - SPARK_EXECUTOR_PORT: 42000
       - SPARK_FILESERVER_PORT: 43000
+      - SPARK_BROADCAST_PORT: 44000
+      - SPART_BLOCKMANAGER_PORT: 45000
       - SPARK_PORT_MAXRETRIES: 4
       - SPARK_MASTER_HOST : {{ master_hostname }}
       - SPARK_MASTER_IP : 0.0.0.0 
@@ -37,8 +43,8 @@ spark_master:
       - SPARK_PUBLIC_DNS : {{ grains.id }}
       - SPARK_LOCAL_HOSTNAME: {{ grains.id }}
       - SPARK_IDENT_STRING : {{ grains.id }}
-#    - command: bash -c 'tail -f `SPARK_MASTER_IP=0.0.0.0 SPARK_MASTER_HOST={{ grains.id }} /opt/spark/sbin/start-master.sh --port  |sed -rn "s/.*(      \/opt.*out)/\1/p"`'
-     - command: bash -c 'sleep 5000'
+    - command: bash -c 'tail -f `SPARK_MASTER_IP=0.0.0.0 SPARK_MASTER_HOST={{ grains.id }} /opt/spark/sbin/start-master.sh  |sed -rn "s/.*(\/opt.*out)/\1/p"`'
+#    - command: bash -c 'sleep 5000'
 
 
 #since salt do not support add-host yet, work around it
@@ -69,7 +75,16 @@ spark_slave:
       - 6066:6066
       - 8081:8081
       - 2606:2606
+      - 45000:45004
     - environment:
+      - SPARK_BLOCK_MANAGER_PORT: 40000
+      - SPARK_BROADCAST_PORT: 44000
+      - SPARK_DRIVER_PORT: 41000
+      - SPARK_EXECUTOR_PORT: 42000
+      - SPARK_FILESERVER_PORT: 43000
+      - SPARK_BROADCAST_PORT: 44000
+      - SPART_BLOCKMANAGER_PORT: 45000
+      - SPARK_PORT_MAXRETRIES: 4
       - SPARK_DRIVER_HOST : {{ grains.id }}
       - SPARK_PUBLIC_DNS : {{ grains.id }}
       - SPARK_LOCAL_HOSTNAME: {{ grains.id }}

master/srv/salt/top.sls

@@ -4,7 +4,10 @@ base:
     - openssh
     - docker
     - cassandra
+    - zookeeper
+    - spark
     - nftables
     - telegraf
   "salt":
     - tickstack
+    - experiment_data

master/srv/salt/zookeeper/init.sls

-include:
-  - docker
-
 {# set zookeeper_image="nherbaut/netcont:sleep" #}
 {# set kafka_image="nherbaut/netcont:sleep" #}
 
@@ -23,13 +20,14 @@ include:
   {% endfor %}
 
 
-zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}:
+zookeeper_image_pull:{{ salt['pillar.get']("placement:zookeeper:version")}}:
   docker_image.present:
-    - require:
-      - sls: docker
+    - name: zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}
+
+zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}:
   docker_container.running:
     - name: zoo
-    - image: {{ zookeeper_image }}
+    - image: zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}
     - port_bindings:
       - {{ salt['mine.get'](grains.id,"datapath_ip")[grains.id][0] }}:2181:2181
       - {{ salt['mine.get'](grains.id,"datapath_ip")[grains.id][0] }}:2888:2888
@@ -37,12 +35,12 @@ zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}:
     - environment:
       - ZOO_MY_ID: {{ salt['pillar.get']("placement:zookeeper:host_zooid_mapping:%s"%grains.id)}} 
       - ZOO_SERVERS: {{ zoospec|join(" ")}}
+    - require:
+      - docker_image: zookeeper_image_pull:{{ salt['pillar.get']("placement:zookeeper:version")}}
 
 kafka:
   docker_image.present:
     - name: {{ kafka_image }}
-    - require:
-      - sls: docker
   docker_container.running:
     - name: kafka
     - image: {{ kafka_image }}