Commit 7aaab621 authored by Millian Poquet's avatar Millian Poquet
Browse files

[env] recipe cleanup + add batsim_g5k

parent 7906a943
Batsim environment recipes
==========================
This is a Kameleon recipe for Batsim experiments. It provides several
recipes to build Docker images (can also be configured for other format
like VDI or QCOW2) to build, test and run Batsim.
This directory contains several recipes that uses Batsim:
- [batsim_ci](batsim_ci.yaml) contains all the dependencies to build
batsim install it and run tests. This recipe is updated each time
dependencies changes. As it is used by the Continuous integration system,
Batsim, to run tests and to build the documentation.
This recipe is updated each time dependencies changes.
As it is used by the Continuous integration system,
it is always up-to-date. It is accessible on Docker Hub at
[oarteam/batsimi_ci](https://hub.docker.com/r/oarteam/batsim_ci/).
- [batsim](batsim.yaml) contains a complete appliance with batsim installed
and tested. It is accessible on Docker Hub at
[oarteam/batsim](https://hub.docker.com/r/oarteam/batsim/).
- [batsim_g5k](batsim_g5k.yaml) builds an appliance that can be deployed on
Grid'5000 thanks to kadeploy. This recipe can be used as a base to setup
reproducible experiments with Batsim.
Build images
------------
It is recommended to run these images directly from docker, but you can build
them from scratch if necessary
If you only want to execute an existing image, executing it directly from
docker is recommended.
All the images can be built from scratch if necessary
First, you need to install Kameleon:
[http://kameleon.imag.fr/installation.html](http://kameleon.imag.fr/installation.html)
You can now run Kameleon, for example:
```bash
kameleon build batsim.yaml
kameleon build batsim_ci.yaml
```
You can find the generated images in ``build/batsim/``.
If you want to use the [batsim_g5k](batsim_g5k.yaml) image,
some configuration is needed:
- select the base Grid'5000 debian image you want to use
(images can be found in ``/grid5000/images`` on Grid'5000 nodes),
move the base image if needed then modify the ``rootfs_archive_url``
variable accordingly in [batsim_g5k.yaml](batsim_g5k.yaml)
- specify your Grid'5000 user name (``g5k_user``) and user id
(``g5k_user_uid``) in [batsim_g5k.yaml](batsim_g5k.yaml)
- once a batsim_g5K image has been generated, you may need to modify the
yaml description file so that the paths are fine on Grid'5000
Run the image with docker
-------------------------
......
......@@ -48,23 +48,19 @@ setup:
#
# Simulator and schedulers install
#
- configure_apt_testing
- configure_apt_repo
- install_common_deps:
- install_build_tools:
- exec_in: |
apt-get -y -t testing install cmake gcc clang g++
apt-get -y install cmake gcc clang g++
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-6 100
- simgrid_install:
- commit_hash: $${simgrid_commit}
- simgrid:
- batsim_dependencies
#- reprozip:
# - reprozip_version: 1.0.8
export:
- "@base"
- docker_export
......
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: <MY RECIPE DESCRIPTION>
#
#==============================================================================
---
extend: default/from_image/from_tarball.yaml
global:
# This is the backend you have imported to switch to an other backend BCKD do:
#
# kameleon template import default/from_image/from_tarball.yaml --global backend:BCKB
#
# Then, uncomment and update the following variable.
# backend: qemu
# To see the variables that you can override, use the following command:
#
# kameleon info batsim_g5k.yaml
# Be sure to have enough memory to build in parallel
qemu_memory_size: 2048
# select the grid5000 image version
kaenv_version: 2017011212
rootfs_archive_url: file:///home/carni/dl/jessie-x64-nfs-$${kaenv_version}.tgz
distrib: debian
release: 8
# fix the debian repository to allow reproducibility
apt_repository: http://snapshot.debian.org/archive/debian/20170116T092404Z
# export in tar Gzipped format for Kadeploy
appliance_formats: tar.gz
# disable zerofree (no necessary for tar.gz format)
zerofree: false
g5k_user: mpoquet
g5k_user_uid: 11941 # gid=8000(users)
# This is where your image will be stored
g5k_images_path: /home/carni/my_g5k_images
# Your image version: be sure to increment this when build a new version
g5k_image_version: 7
simgrid_remote: https://github.com/mpoquet/simgrid.git
simgrid_commit: 587483ebe7882e # Fri Jan 6 15:45:05 2017
batsim_commit: 7906a94
batsched_commit: c2bbc1f
execo_remote: https://github.com/mpoquet/execo.git
execo_commit: 406f4fa06
work_dir: /workdir
root_password: root
bootstrap:
- "@base"
setup:
- create_workdir:
- create_it:
- exec_in: mkdir /$${work_dir}
- set_permission:
- exec_in: chown $${g5k_user_uid}:8000 /$${work_dir}
- exec_in: chmod 775 /$${work_dir}
- set_root_password:
- set_root_password:
- exec_in: echo -n 'root:$${root_password}' | chpasswd
- configure_apt_repo
- fix_libcurl_error:
- update_curl:
- exec_in: apt-get upgrade -y curl libcurl3 git
- install_common_deps:
- install_build_tools:
- exec_in: |
apt-get -y install cmake gcc clang g++ git
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-6 100
- simgrid
- batsim_dependencies:
- build
- experiment_management
- batsim
- batsched
- result_analysis
export:
- "@base"
- create_kaenv:
- create_env_file:
- write_local:
- $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.yaml
- |
---
name: $${kameleon_recipe_name}
version: $${g5k_image_version}
description: My image uuid=$${kameleon_short_uuid}
author: $${g5k_user}
visibility: shared
destructive: false
os: linux
image:
file: $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.tar.gz
kind: tar
compression: gzip
postinstalls:
- archive: server:///grid5000/postinstalls/debian-x64-nfs-2.6-post.tgz
compression: gzip
script: traitement.ash /rambin
boot:
kernel: "/vmlinuz"
initrd: "/initrd.img"
filesystem: ext4
partition_type: 131
multipart: false
- copy_applicance:
- exec_local: cp -av $${appliance_filename}.tar.gz $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.tar.gz
- exec_local: |
echo -e "To import the environment to Kadeploy on the frontal execute:\nkaenv3 -a $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.yaml"
- exec_local: |
echo -e "To backup the build cache in the same environment:\n cp -av $${kameleon_cwd}/$${kameleon_recipe_name}-cache.tar.gz $${g5k_images_path}/$${kameleon_recipe_name}-cache_$${kameleon_short_uuid}.tar.gz"
......@@ -8,10 +8,10 @@
- |
deb $${apt_repository} $${release} main
deb-src $${apt_repository} $${release} main
deb $${apt_repository} $${release}-backports main
deb-src $${apt_repository} $${release}-backports main
deb http://security.debian.org/ $${release}/updates main
deb-src http://security.debian.org/ $${release}/updates main
......@@ -30,10 +30,8 @@
# Deactivate the check to make the cache system works after a while...
- exec_in: apt-get -o Acquire::Check-Valid-Until=false -y --force-yes update
- on_setup_clean:
- exec_in: apt-get -y --force-yes autoremove
- exec_in: apt-get -y --force-yes autoclean
- exec_in: apt-get -y --force-yes clean
- exec_in: apt-get -y --force-yes purge
- exec_in: rm -rf /var/lib/apt/lists/*
- exec_in: apt-get -y --force-yes autoremove || true
- exec_in: apt-get -y --force-yes autoclean || true
- exec_in: apt-get -y --force-yes clean || true
- exec_in: apt-get -y --force-yes purge || true
- exec_in: rm -rf /var/lib/apt/lists/* || true
# A C++ Batch Scheduler for Batsim
#
# You have to specify the desired "commit_hash" to be built.
# Optionally you can specify the working directory.
#
# Usage example:
# global:
# batsched_commit: 1a2e210e
# setup:
# - commit_hash: $${batsched_commit}
# # Optional
# - work_dir: /tmp
#############
# Variables #
#############
- batsched_remote: https://gitlab.inria.fr/batsim/batsched.git
- batsched_commit: c2bbc1f
- work_dir: /root
# Same dependencies as Batsim
- get_it:
- exec_in: |
mkdir -p $${work_dir}
cd $${work_dir}
git clone git@gitlab.inria.fr:batsim/batsched.git
- exec_in: |
cd $${work_dir}/batsched
git checkout $${commit_hash}
mkdir -p build
cd build
cmake -DCMAKE_BUILD_TYPE=DEBUG ..
make
##########
# Script #
##########
- build_and_install:
- exec_in: git clone $${batsched_remote} $${work_dir}/batsched
- exec_in: cd $${work_dir}/batsched && git checkout $${batsched_commit}
- exec_in: mkdir $${work_dir}/batsched/build
- exec_in: >
cd $${work_dir}/batsched/build &&
cmake -DCMAKE_BUILD_TYPE=DEBUG .. &&
make &&
make install
# Build and install Batsim
#############
# Variables #
#############
- batsim_remote: https://github.com/oar-team/batsim.git
- batsim_commit: 7906a94
- work_dir: /root
- install_prefix: /usr
##########
# Script #
##########
- clone:
- exec_in: git clone $${batsim_remote} $${work_dir}/batsim
- exec_in: |
cd $${work_dir}/batsim
git checkout $${batsim_commit}
- make_and_install:
- exec_in: |
mkdir $${work_dir}/batsim/build
cd $${work_dir}/batsim/build
cmake .. -DCMAKE_CXX_COMPILER=/usr/bin/g++-6 \
-DCMAKE_INSTALL_PREFIX=$${install_prefix} \
-DCMAKE_BUILD_TYPE=Debug \
-Denable_warnings=ON \
-Dtreat_warnings_as_errors=ON \
-Dignore_assertions=OFF
- exec_in: |
cd $${work_dir}/batsim/build
make -j $(nproc)
- exec_in: |
cd $${work_dir}/batsim/build
make install
- clone:
- exec_in: |
cd $${work_dir}
git clone https://github.com/oar-team/batsim.git
- exec_in: |
cd $${work_dir}/batsim
git checkout $${batsim_commit}
mkdir build
- make_and_install:
- exec_in: |
cd $${work_dir}/batsim/build
cmake .. -DCMAKE_CXX_COMPILER=/usr/bin/g++-6
make -j$(nproc)
make install
# Build and install most Batsim dependencies (not Simgrid)
#############
# Variables #
#############
- work_dir: /root
- rapidjson_remote: https://github.com/miloyip/rapidjson.git
- rapidjson_commit: 3d5848a
- redox_remote: https://github.com/hmartiro/redox.git
- redox_commit: 520fe0c2bfb
- execo_remote: https://github.com/mpoquet/execo.git
- execo_commit: 406f4fa06
- install_prefix: /usr
# Need testing to be available
- install dependencies:
- exec_in: |
apt-get -y -t testing install \
libboost-all-dev \
libgmp-dev \
redis-server \
libhiredis-dev \
libev-dev \
libzmq3-dev \
python3-yaml \
python3-zmq \
libssl-dev 2>&1
##########
# Script #
##########
# Install rapidjson (not available in jessie)
- exec_in: |
cd $${work_dir}
git clone https://github.com/miloyip/rapidjson.git
- exec_in: |
cd $${work_dir}/rapidjson
git checkout 3d5848a # v1.02
- exec_in: |
cd $${work_dir}/rapidjson
cp -a include/rapidjson /usr/include
- build:
# Build dependencies packaged in debian
- exec_in: |
apt-get -y install libboost-all-dev \
libgmp-dev \
libhiredis-dev \
libev-dev \
libzmq3-dev \
libssl-dev 2>&1
# Install redox
- exec_in: |
cd $${work_dir}
git clone https://github.com/hmartiro/redox.git
- exec_in: |
cd $${work_dir}/redox
git checkout 520fe0c2bfb
- exec_in: |
mkdir $${work_dir}/redox/build
cd $${work_dir}/redox/build
cmake .. -DCMAKE_INSTALL_PREFIX=../install
make -j $(nproc)
make install
cd ../install
mv -f lib64 lib
cp -r ./* /usr/
- install_test_dependencies:
- exec_in: |
apt install -y -t testing \
python3-pip valgrind python3-pandas \
r-base r-base-dev
- exec_in: pip3 install sortedcontainers redis async_timeout
- exec_in: |
cd $${work_dir}
git clone $${execo_remote} execo
- exec_in: |
cd $${work_dir}/execo
git checkout $${execo_commit}
- exec_in: |
cd $${work_dir}/execo
pip3 install -e .
# install R packages
# Install rapidjson from source
- exec_in: git clone $${rapidjson_remote} $${work_dir}/rapidjson
- exec_in: cd $${work_dir}/rapidjson && git checkout $${rapidjson_commit}
- exec_in: |
R --vanilla <<EOF
install.packages(c('ggplot2','scales','dplyr'), repos='http://cran.us.r-project.org')
q()
EOF
cp -a $${work_dir}/rapidjson/include/rapidjson $${install_prefix}/include
- install_doc_dependencies:
- exec_in: apt install -y -t testing doxygen graphviz
# Install redox from source
- exec_in: git clone $${redox_remote} $${work_dir}/redox
- exec_in: cd $${work_dir}/redox && git checkout $${redox_commit}
- exec_in: >
mkdir $${work_dir}/redox/build &&
cd $${work_dir}/redox/build &&
cmake .. -DCMAKE_INSTALL_PREFIX=$${work_dir}/redox/install
- exec_in: cd $${work_dir}/redox/build && make -j $(nproc)
- exec_in: cd $${work_dir}/redox/build && make install
- exec_in: mv $${work_dir}/redox/install/lib64 $${work_dir}/redox/install/lib
- exec_in: cp -r $${work_dir}/redox/install/* $${install_prefix}/
# Dependencies for managing experiments with tools/experiment scripts
- experiment_management:
- exec_in: apt install -y python3-pip psmisc
- exec_in: |
pip3 install sortedcontainers redis async_timeout pandas PyYAML pyzmq
- exec_in: git clone $${execo_remote} $${work_dir}/execo
- exec_in: cd $${work_dir}/execo && git checkout $${execo_commit}
- exec_in: cd $${work_dir}/execo && pip3 install -e .
# Dependencies for building Batsim documentation
- doc:
- exec_in: apt install -y doxygen graphviz
# Configure debian to use a given source repository
#############
# Variables #
#############
- apt_repository: http://snapshot.debian.org/archive/debian/20170116T092404Z
# this can be "testing" "sid" or directly the release name like "stretch"
- debian_version: sid
- enable_contrib: false
- enable_nonfree: false
# Change this to enable it only when specified (-t option of apt)
- force_default_version: true
##########
# Script #
##########
- add_debian_version:
- append_in:
- /etc/apt/sources.list
- deb $${apt_repository} $${debian_version} main
- add_contribs_and_nonfree_sources:
- exec_in: |
if [ $${enable_contrib} = true ]; then
REPO_CONTRIB="contrib"
fi
- exec_in: |
if [ $${enable_nonfree} = true ]; then
REPO_NONFREE="non-free"
fi
- exec_in: perl -pi -e "s/$${debian_version} main$/$${debian_version} main $REPO_CONTRIB $REPO_NONFREE/" /etc/apt/sources.list
- make_it_not_optional:
- exec_in: |
if [ $${force_default_version} = true ]; then
echo 'APT::Default-Release "$${debian_version}";' >> /etc/apt/apt.conf.d/00default-release
fi
- update_repositories:
# Deactivate the check to make the cache system work after a while...
- exec_in: apt-get -o Acquire::Check-Valid-Until=false -y --force-yes update
- on_setup_clean:
- exec_in: apt-get -y --force-yes autoremove || true
- exec_in: apt-get -y --force-yes autoclean || true
- exec_in: apt-get -y --force-yes clean || true
- exec_in: apt-get -y --force-yes purge || true
- exec_in: rm -rf /var/lib/apt/lists/* || true
- enable_contrib: false
- enable_nonfree: false
- add_testing:
- append_in:
- /etc/apt/sources.list
- |
deb $${apt_repository} testing main
- add_contribs_and_nonfree_sources:
- exec_in: |
if [ $${enable_contrib} = true ]; then
REPO_CONTRIB="contrib"
fi
- exec_in: |
if [ $${enable_nonfree} = true ]; then
REPO_NONFREE="non-free"
fi
- exec_in: perl -pi -e "s/testing main$/testing main $REPO_CONTRIB $REPO_NONFREE/" /etc/apt/sources.list
#- make_it_not_optional:
# - write_in:
# - /etc/apt/apt.conf.d/00default-release
# - |
# APT::Default-Release "jessie";
- update_repositories:
# Deactivate the check to make the cache system works after a while...
- exec_in: apt-get -o Acquire::Check-Valid-Until=false -y --force-yes update
- expe_batsim_home: /root
- install_dependencies:
# For python scripts
- exec_in: apt-get install -y python3-pip git
- exec_in: pip3 install sortedcontainers docopt
# For prv2tit.pl
- exec_in: apt-get install -y libswitch-perl
- import_expe_batsim:
- exec_in: |
cd $${expe_batsim_home}
git clone https://github.com/oar-team/batsim-experiments.git
cd batsim-experiments
git checkout $${batsim_experiments_commit}
# Tools to analyse simulation results. Mainly R and R packages.
#############
# Variables #
#############
- r_mirror: http://cran.us.r-project.org
##########
# Script #
##########
- install_r:
- exec_in: |
apt-get -y install r-base r-base-dev
# R packages
- install_r_packages:
- exec_in: |
R --vanilla <<EOF
install.packages(c('ggplot2','scales','dplyr'), repos='$${r_mirror}')
q()
EOF
# Build and install SimGrid, including its dependencies
#############
# Variables #
#############
- work_dir: /root
- install_prefix: /usr
- simgrid_remote: https://github.com/simgrid/simgrid.git
- simgrid_commit: e96681fb8
##########
# Script #
##########
- install_dependencies:
- exec_in: apt-get -y --force-yes install libboost-all-dev python3 2>&1
- install_simgrid:
- exec_in: git clone $${simgrid_remote} $${work_dir}/simgrid
- exec_in: cd $${work_dir}/simgrid && git checkout $${simgrid_commit}
- exec_in: >
mkdir $${work_dir}/simgrid/build &&
cd $${work_dir}/simgrid/build &&
cmake -DPYTHON_EXECUTABLE=/usr/bin/python3 \
-DCMAKE_INSTALL_PREFIX=$${install_prefix} \
-Denable_documentation=OFF \
-Denable_compile_optimizations=OFF \
-Denable_debug=ON ../
- exec_in: cd $${work_dir}/simgrid/build && make -j $(nproc)
- exec_in: cd $${work_dir}/simgrid/build && make install
# Simgrid simulator installation from source
#
# You have to specify the desired "commit_hash" to be built.
# Optionally you can specify the install prefix and the working directory.
#
# Usage example:
# global:
# simgrid_commit: 20e851bd58a0b31
# setup:
# - simgrid_install:
# - commit_hash: $${simgrid_commit}
# # Optional
# - install_prefix: /usr
# - work_dir: /tmp