[env] recipe cleanup + add batsim_g5k

environments/README.md

 Batsim environment recipes
 ==========================
 
-This is a Kameleon recipe for Batsim experiments. It provides several
-recipes to build Docker images (can also be configured for other format
-like VDI or QCOW2) to build, test and run Batsim.
+This directory contains several recipes that uses Batsim:
 
 - [batsim_ci](batsim_ci.yaml) contains all the dependencies to build
-  batsim install it and run tests. This recipe is updated each time
-  dependencies changes. As it is used by the Continuous integration system,
+  Batsim, to run tests and to build the documentation.
+  This recipe is updated each time dependencies changes.
+  As it is used by the Continuous integration system,
   it is always up-to-date. It is accessible on Docker Hub at
   [oarteam/batsimi_ci](https://hub.docker.com/r/oarteam/batsim_ci/).
 
-- [batsim](batsim.yaml) contains a complete appliance with batsim installed
-  and tested. It is accessible on Docker Hub at
-  [oarteam/batsim](https://hub.docker.com/r/oarteam/batsim/).
+- [batsim_g5k](batsim_g5k.yaml) builds an appliance that can be deployed on
+  Grid'5000 thanks to kadeploy. This recipe can be used as a base to setup
+  reproducible experiments with Batsim.
 
 Build images
 ------------
 
-It is recommended to run these images directly from docker, but you can build
-them from scratch if necessary
+If you only want to execute an existing image, executing it directly from
+docker is recommended.
+All the images can be built from scratch if necessary
 
 First, you need to install Kameleon:
 [http://kameleon.imag.fr/installation.html](http://kameleon.imag.fr/installation.html)
 
 You can now run Kameleon, for example:
 ```bash
-  kameleon build batsim.yaml
+  kameleon build batsim_ci.yaml
 ```
 You can find the generated images in ``build/batsim/``.
 
+If you want to use the [batsim_g5k](batsim_g5k.yaml) image,
+some configuration is needed:
+- select the base Grid'5000 debian image you want to use
+  (images can be found in ``/grid5000/images`` on Grid'5000 nodes),
+  move the base image if needed then modify the ``rootfs_archive_url``
+  variable accordingly in [batsim_g5k.yaml](batsim_g5k.yaml)
+- specify your Grid'5000 user name (``g5k_user``) and user id
+  (``g5k_user_uid``) in [batsim_g5k.yaml](batsim_g5k.yaml)
+- once a batsim_g5K image has been generated, you may need to modify the
+  yaml description file so that the paths are fine on Grid'5000
+
 Run the image with docker
 -------------------------
 

environments/batsim_ci.yaml

@@ -48,23 +48,19 @@ setup:
   #
   # Simulator and schedulers install
   #
-  - configure_apt_testing
+  - configure_apt_repo
 
   - install_common_deps:
 
     - install_build_tools:
       - exec_in: |
-          apt-get -y -t testing install cmake gcc clang g++
+          apt-get -y install cmake gcc clang g++
           update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-6 100
 
-  - simgrid_install:
-    - commit_hash: $${simgrid_commit}
+  - simgrid:
 
   - batsim_dependencies
 
-  #- reprozip:
-  #  - reprozip_version: 1.0.8
-
 export:
   - "@base"
   - docker_export

environments/batsim_g5k.yaml

+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: <MY RECIPE DESCRIPTION>
+#
+#==============================================================================
+---
+extend: default/from_image/from_tarball.yaml
+
+global:
+  # This is the backend you have imported to switch to an other backend BCKD do:
+  #
+  #   kameleon template import default/from_image/from_tarball.yaml --global backend:BCKB
+  #
+  # Then, uncomment and update the following variable.
+
+  # backend: qemu
+
+  # To see the variables that you can override, use the following command:
+  #
+  #   kameleon info batsim_g5k.yaml
+
+  # Be sure to have enough memory to build in parallel
+  qemu_memory_size: 2048
+
+  # select the grid5000 image version
+  kaenv_version: 2017011212
+  rootfs_archive_url: file:///home/carni/dl/jessie-x64-nfs-$${kaenv_version}.tgz
+  distrib: debian
+  release: 8
+
+  # fix the debian repository to allow reproducibility
+  apt_repository: http://snapshot.debian.org/archive/debian/20170116T092404Z
+
+  # export in tar Gzipped format for Kadeploy
+  appliance_formats: tar.gz
+  # disable zerofree (no necessary for tar.gz format)
+  zerofree: false
+
+  g5k_user: mpoquet
+  g5k_user_uid: 11941 # gid=8000(users)
+
+  # This is where your image will be stored
+  g5k_images_path: /home/carni/my_g5k_images
+
+  # Your image version: be sure to increment this when build a new version
+  g5k_image_version: 7
+
+  simgrid_remote: https://github.com/mpoquet/simgrid.git
+  simgrid_commit: 587483ebe7882e # Fri Jan 6 15:45:05 2017
+
+  batsim_commit: 7906a94
+  batsched_commit: c2bbc1f
+
+  execo_remote: https://github.com/mpoquet/execo.git
+  execo_commit: 406f4fa06
+
+  work_dir: /workdir
+  root_password: root
+
+bootstrap:
+  - "@base"
+
+setup:
+  - create_workdir:
+    - create_it:
+      - exec_in: mkdir /$${work_dir}
+    - set_permission:
+      - exec_in: chown $${g5k_user_uid}:8000 /$${work_dir}
+      - exec_in: chmod 775 /$${work_dir}
+  - set_root_password:
+    - set_root_password:
+      - exec_in: echo -n 'root:$${root_password}' | chpasswd
+
+  - configure_apt_repo
+
+  - fix_libcurl_error:
+    - update_curl:
+      - exec_in: apt-get upgrade -y curl libcurl3 git
+
+  - install_common_deps:
+    - install_build_tools:
+      - exec_in: |
+          apt-get -y install cmake gcc clang g++ git
+          update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-6 100
+
+  - simgrid
+  - batsim_dependencies:
+    - build
+    - experiment_management
+  - batsim
+  - batsched
+  - result_analysis
+
+export:
+  - "@base"
+  - create_kaenv:
+    - create_env_file:
+      - write_local:
+        - $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.yaml
+        - |
+            ---
+            name: $${kameleon_recipe_name}
+            version: $${g5k_image_version}
+            description: My image uuid=$${kameleon_short_uuid}
+            author: $${g5k_user}
+            visibility: shared
+            destructive: false
+            os: linux
+            image:
+              file: $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.tar.gz
+              kind: tar
+              compression: gzip
+            postinstalls:
+            - archive: server:///grid5000/postinstalls/debian-x64-nfs-2.6-post.tgz
+              compression: gzip
+              script: traitement.ash /rambin
+            boot:
+              kernel: "/vmlinuz"
+              initrd: "/initrd.img"
+            filesystem: ext4
+            partition_type: 131
+            multipart: false
+    - copy_applicance:
+      - exec_local: cp -av $${appliance_filename}.tar.gz $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.tar.gz
+      - exec_local: |
+          echo -e "To import the environment to Kadeploy on the frontal execute:\nkaenv3 -a $${g5k_images_path}/$${kameleon_recipe_name}_$${kameleon_short_uuid}.yaml"
+      - exec_local: |
+          echo -e "To backup the build cache in the same environment:\n cp -av $${kameleon_cwd}/$${kameleon_recipe_name}-cache.tar.gz $${g5k_images_path}/$${kameleon_recipe_name}-cache_$${kameleon_short_uuid}.tar.gz"
+

environments/default/steps/setup/debian/configure_apt.yaml

@@ -8,10 +8,10 @@
     - |
       deb $${apt_repository} $${release} main
       deb-src $${apt_repository} $${release} main
-      
+
       deb $${apt_repository} $${release}-backports main
       deb-src $${apt_repository} $${release}-backports main
-      
+
       deb http://security.debian.org/ $${release}/updates main
       deb-src http://security.debian.org/ $${release}/updates main
 
@@ -30,10 +30,8 @@
   # Deactivate the check to make the cache system works after a while...
   - exec_in: apt-get -o Acquire::Check-Valid-Until=false -y --force-yes update
   - on_setup_clean:
-    - exec_in: apt-get -y --force-yes autoremove
-    - exec_in: apt-get -y --force-yes autoclean
-    - exec_in: apt-get -y --force-yes clean
-    - exec_in: apt-get -y --force-yes purge
-    - exec_in: rm -rf /var/lib/apt/lists/*
-
-
+    - exec_in: apt-get -y --force-yes autoremove || true
+    - exec_in: apt-get -y --force-yes autoclean || true
+    - exec_in: apt-get -y --force-yes clean || true
+    - exec_in: apt-get -y --force-yes purge || true
+    - exec_in: rm -rf /var/lib/apt/lists/* || true

environments/steps/batsched.yaml

 # A C++ Batch Scheduler for Batsim
-#
-# You have to specify the desired "commit_hash" to be built.
-# Optionally you can specify the working directory.
-#
-# Usage example:
-# global:
-#   batsched_commit: 1a2e210e
-# setup:
-#   - commit_hash: $${batsched_commit}
-#   # Optional
-#   - work_dir: /tmp
 
+#############
+# Variables #
+#############
+- batsched_remote: https://gitlab.inria.fr/batsim/batsched.git
+- batsched_commit: c2bbc1f
 - work_dir: /root
 
-# Same dependencies as Batsim
-- get_it:
-  - exec_in: |
-      mkdir -p $${work_dir}
-      cd $${work_dir}
-      git clone git@gitlab.inria.fr:batsim/batsched.git
-  - exec_in: |
-      cd $${work_dir}/batsched
-      git checkout $${commit_hash}
-      mkdir -p build
-      cd build
-      cmake -DCMAKE_BUILD_TYPE=DEBUG ..
-      make
+##########
+# Script #
+##########
+- build_and_install:
+  - exec_in: git clone $${batsched_remote} $${work_dir}/batsched
+  - exec_in: cd $${work_dir}/batsched && git checkout $${batsched_commit}
+  - exec_in: mkdir $${work_dir}/batsched/build
+  - exec_in: >
+      cd $${work_dir}/batsched/build &&
+      cmake -DCMAKE_BUILD_TYPE=DEBUG .. &&
+      make &&
       make install

environments/steps/batsim.yaml

+# Build and install Batsim
+
+#############
+# Variables #
+#############
+- batsim_remote: https://github.com/oar-team/batsim.git
+- batsim_commit: 7906a94
+- work_dir: /root
+- install_prefix: /usr
+
+##########
+# Script #
+##########
+- clone:
+  - exec_in: git clone $${batsim_remote} $${work_dir}/batsim
+  - exec_in: |
+      cd $${work_dir}/batsim
+      git checkout $${batsim_commit}
+- make_and_install:
+  - exec_in: |
+      mkdir $${work_dir}/batsim/build
+      cd $${work_dir}/batsim/build
+      cmake .. -DCMAKE_CXX_COMPILER=/usr/bin/g++-6 \
+               -DCMAKE_INSTALL_PREFIX=$${install_prefix} \
+               -DCMAKE_BUILD_TYPE=Debug \
+               -Denable_warnings=ON \
+               -Dtreat_warnings_as_errors=ON \
+               -Dignore_assertions=OFF
+  - exec_in: |
+      cd $${work_dir}/batsim/build
+      make -j $(nproc)
+  - exec_in: |
+      cd $${work_dir}/batsim/build
+      make install

environments/steps/batsim_build_and_install.yaml

-- clone:
-  - exec_in: |
-      cd $${work_dir}
-      git clone https://github.com/oar-team/batsim.git
-  - exec_in: |
-      cd $${work_dir}/batsim
-      git checkout $${batsim_commit}
-      mkdir build
-- make_and_install:
-  - exec_in: |
-      cd $${work_dir}/batsim/build
-      cmake .. -DCMAKE_CXX_COMPILER=/usr/bin/g++-6
-      make -j$(nproc)
-      make install

environments/steps/batsim_dependencies.yaml

+# Build and install most Batsim dependencies (not Simgrid)
+
+#############
+# Variables #
+#############
 - work_dir: /root
+- rapidjson_remote: https://github.com/miloyip/rapidjson.git
+- rapidjson_commit: 3d5848a
+- redox_remote: https://github.com/hmartiro/redox.git
+- redox_commit: 520fe0c2bfb
+- execo_remote: https://github.com/mpoquet/execo.git
+- execo_commit: 406f4fa06
+- install_prefix: /usr
 
-# Need testing to be available
-- install dependencies:
-  - exec_in: |
-      apt-get -y -t testing install \
-      libboost-all-dev \
-      libgmp-dev \
-      redis-server \
-      libhiredis-dev \
-      libev-dev \
-      libzmq3-dev \
-      python3-yaml \
-      python3-zmq \
-      libssl-dev 2>&1
+##########
+# Script #
+##########
 
-  # Install rapidjson (not available in jessie)
-  - exec_in: |
-      cd $${work_dir}
-      git clone https://github.com/miloyip/rapidjson.git
-  - exec_in: |
-      cd $${work_dir}/rapidjson
-      git checkout 3d5848a # v1.02
-  - exec_in: |
-      cd $${work_dir}/rapidjson
-      cp -a include/rapidjson /usr/include
+- build:
+  # Build dependencies packaged in debian
+  - exec_in: |
+      apt-get -y install libboost-all-dev \
+                         libgmp-dev \
+                         libhiredis-dev \
+                         libev-dev \
+                         libzmq3-dev \
+                         libssl-dev 2>&1
 
-  # Install redox
-  - exec_in: |
-      cd $${work_dir}
-      git clone https://github.com/hmartiro/redox.git
-  - exec_in: |
-      cd $${work_dir}/redox
-      git checkout 520fe0c2bfb
-  - exec_in: |
-      mkdir $${work_dir}/redox/build
-      cd $${work_dir}/redox/build
-      cmake .. -DCMAKE_INSTALL_PREFIX=../install
-      make -j $(nproc)
-      make install
-      cd ../install
-      mv -f lib64 lib
-      cp -r ./* /usr/
-
-- install_test_dependencies:
-  - exec_in: |
-      apt install -y -t testing \
-      python3-pip valgrind python3-pandas \
-      r-base r-base-dev
-  - exec_in: pip3 install sortedcontainers redis async_timeout
-  - exec_in: |
-      cd $${work_dir}
-      git clone $${execo_remote} execo
-  - exec_in: |
-      cd $${work_dir}/execo
-      git checkout $${execo_commit}
-  - exec_in: |
-      cd $${work_dir}/execo
-      pip3 install -e .
-  # install R packages
+  # Install rapidjson from source
+  - exec_in: git clone $${rapidjson_remote} $${work_dir}/rapidjson
+  - exec_in: cd $${work_dir}/rapidjson && git checkout $${rapidjson_commit}
   - exec_in: |
-      R --vanilla <<EOF
-      install.packages(c('ggplot2','scales','dplyr'), repos='http://cran.us.r-project.org')
-      q()
-      EOF
+      cp -a $${work_dir}/rapidjson/include/rapidjson $${install_prefix}/include
 
-- install_doc_dependencies:
-  - exec_in: apt install -y -t testing doxygen graphviz
+  # Install redox from source
+  - exec_in: git clone $${redox_remote} $${work_dir}/redox
+  - exec_in: cd $${work_dir}/redox && git checkout $${redox_commit}
+  - exec_in: >
+      mkdir $${work_dir}/redox/build &&
+      cd $${work_dir}/redox/build &&
+      cmake .. -DCMAKE_INSTALL_PREFIX=$${work_dir}/redox/install
+  - exec_in: cd $${work_dir}/redox/build && make -j $(nproc)
+  - exec_in: cd $${work_dir}/redox/build && make install
+  - exec_in: mv $${work_dir}/redox/install/lib64 $${work_dir}/redox/install/lib
+  - exec_in: cp -r $${work_dir}/redox/install/* $${install_prefix}/
 
+# Dependencies for managing experiments with tools/experiment scripts
+- experiment_management:
+  - exec_in: apt install -y python3-pip psmisc
+  - exec_in: |
+      pip3 install sortedcontainers redis async_timeout pandas PyYAML pyzmq
+  - exec_in: git clone $${execo_remote} $${work_dir}/execo
+  - exec_in: cd $${work_dir}/execo && git checkout $${execo_commit}
+  - exec_in: cd $${work_dir}/execo && pip3 install -e .
 
+# Dependencies for building Batsim documentation
+- doc:
+  - exec_in: apt install -y doxygen graphviz

environments/steps/configure_apt_repo.yaml

+# Configure debian to use a given source repository
+
+#############
+# Variables #
+#############
+- apt_repository: http://snapshot.debian.org/archive/debian/20170116T092404Z
+# this can be "testing" "sid" or directly the release name like "stretch"
+- debian_version: sid
+- enable_contrib: false
+- enable_nonfree: false
+# Change this to enable it only when specified (-t option of apt)
+- force_default_version: true
+
+##########
+# Script #
+##########
+- add_debian_version:
+  - append_in:
+    - /etc/apt/sources.list
+    - deb $${apt_repository} $${debian_version} main
+
+- add_contribs_and_nonfree_sources:
+  - exec_in: |
+      if [ $${enable_contrib} = true ]; then
+        REPO_CONTRIB="contrib"
+      fi
+  - exec_in: |
+      if [ $${enable_nonfree} = true ]; then
+        REPO_NONFREE="non-free"
+      fi
+  - exec_in: perl -pi -e "s/$${debian_version} main$/$${debian_version} main $REPO_CONTRIB $REPO_NONFREE/" /etc/apt/sources.list
+
+- make_it_not_optional:
+  - exec_in: |
+      if [ $${force_default_version} = true ]; then
+      echo 'APT::Default-Release "$${debian_version}";' >> /etc/apt/apt.conf.d/00default-release
+      fi
+
+- update_repositories:
+  # Deactivate the check to make the cache system work after a while...
+  - exec_in: apt-get -o Acquire::Check-Valid-Until=false -y --force-yes update
+  - on_setup_clean:
+    - exec_in: apt-get -y --force-yes autoremove || true
+    - exec_in: apt-get -y --force-yes autoclean || true
+    - exec_in: apt-get -y --force-yes clean || true
+    - exec_in: apt-get -y --force-yes purge || true
+    - exec_in: rm -rf /var/lib/apt/lists/* || true

environments/steps/configure_apt_testing.yaml

-- enable_contrib: false
-- enable_nonfree: false
-- add_testing:
-  - append_in:
-    - /etc/apt/sources.list
-    - |
-
-      deb $${apt_repository} testing main
-
-- add_contribs_and_nonfree_sources:
-  - exec_in: |
-      if [ $${enable_contrib} = true ]; then
-        REPO_CONTRIB="contrib"
-      fi
-  - exec_in: |
-      if [ $${enable_nonfree} = true ]; then
-        REPO_NONFREE="non-free"
-      fi
-  - exec_in: perl -pi -e "s/testing main$/testing main $REPO_CONTRIB $REPO_NONFREE/" /etc/apt/sources.list
-
-#- make_it_not_optional:
-#  - write_in:
-#    - /etc/apt/apt.conf.d/00default-release
-#    - |
-#      APT::Default-Release "jessie";
-
-- update_repositories:
-  # Deactivate the check to make the cache system works after a while...
-  - exec_in: apt-get -o Acquire::Check-Valid-Until=false -y --force-yes update

environments/steps/import_batsim_experiment_tools.yaml

-- expe_batsim_home: /root
-- install_dependencies:
-  # For python scripts
-  - exec_in: apt-get install -y python3-pip git
-  - exec_in: pip3 install sortedcontainers docopt
-  # For prv2tit.pl
-  - exec_in: apt-get install -y libswitch-perl
-
-- import_expe_batsim:
-  - exec_in: |
-      cd $${expe_batsim_home}
-      git clone https://github.com/oar-team/batsim-experiments.git
-      cd batsim-experiments
-      git checkout $${batsim_experiments_commit}
-

environments/steps/result_analysis.yaml

+# Tools to analyse simulation results. Mainly R and R packages.
+
+#############
+# Variables #
+#############
+- r_mirror: http://cran.us.r-project.org
+
+##########
+# Script #
+##########
+- install_r:
+  - exec_in: |
+      apt-get -y install r-base r-base-dev
+
+# R packages
+- install_r_packages:
+  - exec_in: |
+      R --vanilla <<EOF
+      install.packages(c('ggplot2','scales','dplyr'), repos='$${r_mirror}')
+      q()
+      EOF

environments/steps/simgrid.yaml

+# Build and install SimGrid, including its dependencies
+
+#############
+# Variables #
+#############
+- work_dir: /root
+- install_prefix: /usr
+- simgrid_remote: https://github.com/simgrid/simgrid.git
+- simgrid_commit: e96681fb8
+
+##########
+# Script #
+##########
+- install_dependencies:
+  - exec_in: apt-get -y --force-yes install libboost-all-dev python3 2>&1
+
+- install_simgrid:
+  - exec_in: git clone $${simgrid_remote} $${work_dir}/simgrid
+  - exec_in: cd $${work_dir}/simgrid && git checkout $${simgrid_commit}
+  - exec_in: >
+      mkdir $${work_dir}/simgrid/build &&
+      cd $${work_dir}/simgrid/build &&
+      cmake -DPYTHON_EXECUTABLE=/usr/bin/python3 \
+            -DCMAKE_INSTALL_PREFIX=$${install_prefix} \
+            -Denable_documentation=OFF \
+            -Denable_compile_optimizations=OFF \
+            -Denable_debug=ON ../
+  - exec_in: cd $${work_dir}/simgrid/build && make -j $(nproc)
+  - exec_in: cd $${work_dir}/simgrid/build && make install

environments/steps/simgrid_install.yaml

-# Simgrid simulator installation from source
-#
-# You have to specify the desired "commit_hash" to be built.
-# Optionally you can specify the install prefix and the working directory.
-#
-# Usage example:
-# global:
-#   simgrid_commit: 20e851bd58a0b31
-# setup:
-# - simgrid_install:
-#   - commit_hash: $${simgrid_commit}
-#   # Optional
-#   - install_prefix: /usr
-#   - work_dir: /tmp