Commit f8a339c6 authored by Nicolas Herbaut's avatar Nicolas Herbaut
Browse files

demo version

parent fb8c4dea
......@@ -5,7 +5,6 @@ placement:
- hosts:
- h5
- h6
- h7
- h8
- version: 3.11.1
- first_cqlsh_host: h5
......@@ -13,14 +12,12 @@ placement:
- hosts:
- h5
- h6
- h7
- h8
- version: 3.4.11
- host_zooid_mapping:
- h5 : 1
- h6 : 2
- h7 : 3
- h8 : 4
- h8 : 3
- kafka:
- replication_factor : 3
- partition_factor: 6
......@@ -28,6 +25,6 @@ placement:
- spark:
- slaves:
- h5
- h6
- h8
- master: h7
- image : nherbaut/spark-wordcount:latest
......@@ -3,5 +3,6 @@ base:
- urlsource
- influxdb
- placement
- mine_functions
......@@ -17,4 +17,7 @@ bible:
- template: jinja
- mode: 0755
/root/killall.sh:
file.managed:
- source: salt://experiment_data/killall.sh
- mode: 0755
salt "*" cmd.run 'docker rm -f $(docker ps -qa)' && salt "*" cmd.run 'service telegraf stop'
......@@ -43,7 +43,7 @@ docker exec kafka bash -c '/opt/kafka*/bin/kafka-topics.sh --create --zookeeper
{% set spark_master_ip = salt['mine.get'](grains.id,"datapath_ip")[grains.id][0] %}
docker exec -d spark bash -c '/opt/spark/bin/spark-submit --class org.apache.spark.examples.streaming.DirectKafkaWordCount --master spark://{{spark_master_ip}}:7077 --deploy-mode client /opt/wordcount/spark-wordcount-1.0-jar-with-dependencies.jar {{ cassandraspec|join(",") }} {{ topic_name }} {{ kafkaspec|join(",") }}':
docker exec -d spark bash -c '/opt/spark/bin/spark-submit --class org.apache.spark.examples.streaming.DirectKafkaWordCount --master spark://{{spark_master_ip}}:7077 --conf spark.block.manager.port=40000 --conf spark.driver.port=41000 --conf spark.executor.port=42000 --conf spark.fileserver.port=43000 --conf spark.broadcast.port=44000 --conf spark.blockManager.port=45000 --conf spark.port.maxretries=4 --deploy-mode client /opt/wordcount/spark-wordcount-1.0-jar-with-dependencies.jar {{ cassandraspec|join(",") }} {{ topic_name }} {{ kafkaspec|join(",") }}':
cmd.run
......
......@@ -17,6 +17,7 @@ spark_master:
- hostname: spark
- image: {{ salt['pillar.get']('placement:spark:image') }}
- port_bindings:
- 4040:4040
- 7077:7077
- 6066:6066
- 8080:8080
......@@ -24,12 +25,17 @@ spark_master:
- 41000-41004:41000-41004
- 42000-42004:42000-42004
- 43000-43004:43000-43004
- 44000-44004:44000-44004
- 45000-45004:45000-45004
- hostname: {{ grains.id }}
- environment:
- SPARK_BLOCK_MANAGER_PORT: 40000
- SPARK_BROADCAST_PORT: 44000
- SPARK_DRIVER_PORT: 41000
- SPARK_EXECUTOR_PORT: 42000
- SPARK_FILESERVER_PORT: 43000
- SPARK_BROADCAST_PORT: 44000
- SPART_BLOCKMANAGER_PORT: 45000
- SPARK_PORT_MAXRETRIES: 4
- SPARK_MASTER_HOST : {{ master_hostname }}
- SPARK_MASTER_IP : 0.0.0.0
......@@ -37,8 +43,8 @@ spark_master:
- SPARK_PUBLIC_DNS : {{ grains.id }}
- SPARK_LOCAL_HOSTNAME: {{ grains.id }}
- SPARK_IDENT_STRING : {{ grains.id }}
# - command: bash -c 'tail -f `SPARK_MASTER_IP=0.0.0.0 SPARK_MASTER_HOST={{ grains.id }} /opt/spark/sbin/start-master.sh --port |sed -rn "s/.*( \/opt.*out)/\1/p"`'
- command: bash -c 'sleep 5000'
- command: bash -c 'tail -f `SPARK_MASTER_IP=0.0.0.0 SPARK_MASTER_HOST={{ grains.id }} /opt/spark/sbin/start-master.sh |sed -rn "s/.*(\/opt.*out)/\1/p"`'
# - command: bash -c 'sleep 5000'
#since salt do not support add-host yet, work around it
......@@ -69,7 +75,16 @@ spark_slave:
- 6066:6066
- 8081:8081
- 2606:2606
- 45000:45004
- environment:
- SPARK_BLOCK_MANAGER_PORT: 40000
- SPARK_BROADCAST_PORT: 44000
- SPARK_DRIVER_PORT: 41000
- SPARK_EXECUTOR_PORT: 42000
- SPARK_FILESERVER_PORT: 43000
- SPARK_BROADCAST_PORT: 44000
- SPART_BLOCKMANAGER_PORT: 45000
- SPARK_PORT_MAXRETRIES: 4
- SPARK_DRIVER_HOST : {{ grains.id }}
- SPARK_PUBLIC_DNS : {{ grains.id }}
- SPARK_LOCAL_HOSTNAME: {{ grains.id }}
......
......@@ -4,7 +4,10 @@ base:
- openssh
- docker
- cassandra
- zookeeper
- spark
- nftables
- telegraf
"salt":
- tickstack
- experiment_data
include:
- docker
{# set zookeeper_image="nherbaut/netcont:sleep" #}
{# set kafka_image="nherbaut/netcont:sleep" #}
......@@ -23,13 +20,14 @@ include:
{% endfor %}
zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}:
zookeeper_image_pull:{{ salt['pillar.get']("placement:zookeeper:version")}}:
docker_image.present:
- require:
- sls: docker
- name: zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}
zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}:
docker_container.running:
- name: zoo
- image: {{ zookeeper_image }}
- image: zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}
- port_bindings:
- {{ salt['mine.get'](grains.id,"datapath_ip")[grains.id][0] }}:2181:2181
- {{ salt['mine.get'](grains.id,"datapath_ip")[grains.id][0] }}:2888:2888
......@@ -37,12 +35,12 @@ zookeeper:{{ salt['pillar.get']("placement:zookeeper:version")}}:
- environment:
- ZOO_MY_ID: {{ salt['pillar.get']("placement:zookeeper:host_zooid_mapping:%s"%grains.id)}}
- ZOO_SERVERS: {{ zoospec|join(" ")}}
- require:
- docker_image: zookeeper_image_pull:{{ salt['pillar.get']("placement:zookeeper:version")}}
kafka:
docker_image.present:
- name: {{ kafka_image }}
- require:
- sls: docker
docker_container.running:
- name: kafka
- image: {{ kafka_image }}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment