Skip to content

GitLab

Commit fa4c5db7 authored by Michael Mercier's avatar Michael Mercier
Browse files

Use docker backend: layer for batsim above batsim_ci

parent cdc87693
Hide whitespace changes
Inline Side-by-side
environments/batsim.yaml
View file @ fa4c5db7
......@@ -6,30 +6,42 @@
#
#==============================================================================
---
extend: batsim_ci.yaml
extend: default/from_image/from_docker
global:
batsim_commit: f1e835a7e1935 # Tue Dec 6 16:31:12 2016 +0100
backend: docker
from_docker_image: oarteam/batsim_ci:latest
to_docker_image: oarteam/batsim:latest
dir: /root
batsim_commit: cdc8769388b82e5 # Thu Dec 8 18:38:49 2016
bootstrap:
- "@base"
setup:
- "@base"
- build_and_install_batsim:
- do_it:
- exec_in: |
cd $$simctn_dir
cd $${dir}
git clone https://github.com/oar-team/batsim.git
- exec_in: |
cd $$simctn_dir/batsim
git checkout $$batsim_commit
cd $${dir}/batsim
git checkout $${batsim_commit}
mkdir build
- exec_in: |
cd $$simctn_dir/batsim/build
cd $${dir}/batsim/build
cmake .. -DCMAKE_CXX_COMPILER=/usr/bin/g++-6
make -j$(nproc)
make install
export:
- "@base"
- tag_and_push_image:
- do_tag:
- exec_local: |
docker tag oarteam/$${kameleon_recipe_name}:latest oarteam/$${kameleon_recipe_name}:$(date --iso-8601)
- do_push:
- exec_local: |
docker push oarteam/$${kameleon_recipe_name}:latest
docker push oarteam/$${kameleon_recipe_name}:$(date --iso-8601)
environments/batsim_ci.yaml
View file @ fa4c5db7
......@@ -21,7 +21,7 @@ global:
qemu_memory_size : 2048
appliance_formats: tar.gz
appliance_tar_compression_level: fast
setup_packages: git cmake build-essential e2fsprogs $$setup_packages
setup_packages: git cmake build-essential e2fsprogs $${setup_packages}
default_keyboard_layout: fr
default_timezone: Europe/Paris
root_password: root
......@@ -54,7 +54,7 @@ setup:
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-6 100
- simgrid_install:
- commit_hash: $$simgrid_commit
- commit_hash: $${simgrid_commit}
- batsim:
# Need testing to be available
......@@ -68,16 +68,16 @@ setup:
libev-dev \
python-yaml \
libssl-dev 2>&1
# Install rapidjson (not available in jessy)
# Install rapidjson (not available in jessie)
- exec_in: |
cd $$simctn_dir
cd $${simctn_dir}
git clone https://github.com/miloyip/rapidjson.git
- exec_in: |
cd $$simctn_dir/rapidjson
cd $${simctn_dir}/rapidjson
git checkout 3d5848a # v1.02
cp -a include/rapidjson /usr/include
- exec_in: |
cd $$simctn_dir
cd $${simctn_dir}
git clone https://github.com/hmartiro/redox.git
cd redox
git checkout 64becf28f3dd
......@@ -93,7 +93,7 @@ setup:
- exec_in: apt install -y -t testing python-pip valgrind python-pandas
- exec_in: pip2 install sortedcontainers redis
- exec_in: |
cd $$simctn_dir
cd $${simctn_dir}
git clone https://github.com/mickours/execo.git
cd execo
git checkout $${execo_commit}
......@@ -107,12 +107,5 @@ setup:
export:
- "@base"
- docker_export:
- do_export:
- exec_local: |
docker import $${appliance_filename}.tar.gz oarteam/$${kameleon_recipe_name}:latest
- exec_local: |
docker tag oarteam/$${kameleon_recipe_name}:latest oarteam/$${kameleon_recipe_name}:$(date --iso-8601)
- exec_local: |
docker push oarteam/$${kameleon_recipe_name}:latest
docker push oarteam/$${kameleon_recipe_name}:$(date --iso-8601)
- docker_export
environments/batsim_mickours.yaml
View file @ fa4c5db7
......@@ -6,9 +6,13 @@
#
#==============================================================================
extend: batsim_oarteam.yaml
extend: default/from_image/from_docker
global:
backend: docker
from_docker_image: oarteam/batsim_ci:latest
to_docker_image: oarteam/batsim:latest
simctn_dir: /root
locality_sched_commit: 12df12a
oar_commit: 2c7c4df
......@@ -19,24 +23,25 @@ bootstrap:
- "@base"
setup:
- "@base"
- apt_init:
- do_update:
- exec_in: apt update -y
#
# Batsim schedulers
#
- locality_scheduler:
- commit_hash: $$locality_sched_commit
- commit_hash: $${locality_sched_commit}
- oar_sched:
- install dependencies:
- exec_in: apt-get -y --force-yes install python3-pip 2>&1
- exec_in: apt -y --force-yes install python3-pip 2>&1
- install_oar3:
- exec_in: |
cd $$simctn_dir
cd $${simctn_dir}
git clone https://github.com/oar-team/oar3.git
- exec_in: |
cd $$simctn_dir/oar3
git checkout $$oar_commit
cd $${simctn_dir}/oar3
git checkout $${oar_commit}
pip3 install -e .
#
......@@ -49,11 +54,11 @@ setup:
- exec_in: apt-get -y --force-yes install vite 2>&1
#- get_sources:
#- - exec_in: |
#- cd $$simctn_dir
#- cd $${simctn_dir}
#- svn checkout svn://scm.gforge.inria.fr/svnroot/vite/
#- install_it:
#- - exec_in: |
#- cd $$simctn_dir/vite/trunk
#- cd $${simctn_dir}/vite/trunk
#- mkdir build
#- cd build
#- cmake ..
......@@ -70,7 +75,7 @@ setup:
# Experiment inputs and launch scripts
#
- import_batsim_experiment_tools:
- expe_batsim_home: $$simctn_dir
- expe_batsim_home: $${simctn_dir}
#
# Personal customization
......
environments/debian8_workload_generation.yaml
View file @ fa4c5db7
......@@ -12,7 +12,7 @@ extend: grid5000/kameleon-g5k/virtualbox/jessie-x64-nfs.yaml
global:
virtualbox_memory_size: 2048
g5k_home: /opt/
g5k_tgz_path: $$g5k_home/my_g5k_images/debian8_workload_generation_nfs.tgz
g5k_tgz_path: $${g5k_home}/my_g5k_images/debian8_workload_generation_nfs.tgz
bootstrap:
- "@base"
......@@ -21,9 +21,9 @@ setup:
- "@base"
- extrae
- NAS_benchmark:
- NAS_home: $$g5k_home
- NAS_home: $${g5k_home}
- import_batsim_experiment_tools:
- expe_batsim_home: $$g5k_home
- expe_batsim_home: $${g5k_home}
export:
- "@base"
environments/default/from_image/from_docker.yaml 0 → 100644
View file @ fa4c5db7
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: Simple recipe that use a docker image directly. It is made to
# conserve docker layers for the exported image.
# No setup section is provided: You have to implement yours
#
#==============================================================================
---
extend: ../steps/backend/$${backend}.yaml
# Loads some helpful aliases (this files are located in steps/aliases/ directory)
aliases: defaults.yaml
# Custom shell environement (this files are located in steps/env/ directory)
env:
- bashrc
- functions.sh
global:
ssh_config_file: $${kameleon_cwd}/ssh_config
backend: docker
# You should specify this in the global of your recipe
from_docker_image: "ubuntu:latest"
to_docker_image: "$${kameleon_recipe_name}:latest"
bootstrap:
- prepare_docker_image:
- pull_and_tag_image:
- exec_out: |
if ! docker tag $${from_docker_image} "$${docker_image}:base"; then
echo "Importing docker base image to build from"
docker pull $${from_docker_image}
fi
- start_docker
export:
- tag_docker_image:
- commit_container:
- exec_out: docker commit $(cat MAIN_CONTAINER_ID) $${to_docker_image}
environments/default/from_image/from_tarball.yaml 0 → 100644
View file @ fa4c5db7
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: Simple recipe that only import a tarball images like provided by
# here http://kameleon.imag.fr/rootfs/
# It also work for kadeploy images
#
#==============================================================================
---
extend: base.yaml
global:
rootfs_archive_url: http://kameleon.imag.fr/rootfs/$${arch}/$${distrib}$${release}.tar.xz
bootstrap:
- "@base"
export:
- "@base"
environments/default/steps/backend/docker.yaml 0 → 100644
View file @ fa4c5db7
# Docker backend recipe
---
extend: chroot.yaml
checkpoint: docker.yaml
global:
## Docker options
docker_image: $${kameleon_recipe_name}_$${kameleon_short_uuid}
docker_hostname: kameleon-$${kameleon_short_uuid}
# rootfs options
rootfs: $${kameleon_cwd}/rootfs
# Shell session from where we launch exec_out commands. There is often a
# local bash session, but it can be a remote shell on other machines or on
# any shell. (eg. bash, chroot, fakechroot, ssh, tmux, lxc...)
out_context:
cmd: bash
workdir: $${kameleon_cwd}
proxy_cache: localhost
# Shell session that allows us to connect to the building machine in order to
# configure it and setup additional programs
in_context:
cmd: LC_ALL=POSIX docker exec -i $(cat MAIN_CONTAINER_ID) /bin/bash
interactive_cmd: LC_ALL=POSIX docker exec -it $(cat MAIN_CONTAINER_ID) /bin/bash
workdir: /root/kameleon_workdir
proxy_cache: 172.17.0.1
bootstrap:
- prepare_docker
- start_docker
setup:
- "@base"
export:
- tag_docker_image:
- commit_container:
- exec_out: docker commit $(cat MAIN_CONTAINER_ID) $${docker_image}:$${kameleon_short_uuid}
- save_appliance_docker
- "@base"
environments/default/steps/bootstrap/prepare_docker.yaml 0 → 100644
View file @ fa4c5db7
- insecure_ssh_key: $${kameleon_cwd}/insecure_ssh_key
- clean_containers:
- on_checkpoint: redo
- on_export_clean:
- exec_out: echo "Stopping trailing containers"
- exec_out: touch CONTAINERS_TO_CLEAN
- exec_out: cat CONTAINERS_TO_CLEAN | xargs -I {} docker kill {}
- exec_out: echo "Removing trailing containers"
- exec_out: cat CONTAINERS_TO_CLEAN | xargs -I {} docker rm {}
- exec_out: rm -f CONTAINERS_TO_CLEAN
- download_rootfs_archive:
- download_file_local:
- $${rootfs_archive_url}
- $${rootfs_archive_download_path}
- import_to_docker:
- check_cmd_out: docker
- exec_out: echo "Importing $${docker_image} to docker..."
- exec_out: xzcat $${rootfs_archive_download_path} | docker import - $${docker_image}:base
- exec_out: rm -f $${rootfs_archive_download_path}
environments/default/steps/bootstrap/start_docker.yaml 0 → 100644
View file @ fa4c5db7
- clean_containers:
- on_checkpoint: redo
- on_export_clean:
- exec_out: echo "Stopping trailing containers"
- exec_out: touch CONTAINERS_TO_CLEAN
- exec_out: cat CONTAINERS_TO_CLEAN | xargs -I {} docker kill {}
- exec_out: echo "Removing trailing containers"
- exec_out: cat CONTAINERS_TO_CLEAN | xargs -I {} docker rm {}
- exec_out: rm -f CONTAINERS_TO_CLEAN
- fix_docker_resolvconf:
- on_setup_init:
- exec_in: cat /etc/resolv.conf > /tmp/resolv.conf-docker
- exec_in: umount /etc/resolv.conf
- exec_in: cat /tmp/resolv.conf-docker > /etc/resolv.conf
- fix_docker_etc_hosts:
- on_setup_init:
- exec_in: cat /etc/hosts > /tmp/hosts-docker
- exec_in: umount /etc/hosts
- exec_in: cat /tmp/hosts-docker > /etc/hosts
- start_docker_container:
- on_checkpoint: redo
- exec_out: |
CID=$(docker run -d -i \
-h $${docker_hostname} \
--privileged "$${docker_image}:base" \
cat)
sleep 2
- exec_out: echo "$CID" >> CONTAINERS_TO_CLEAN
- exec_out: echo $CID > MAIN_CONTAINER_ID
- on_export_clean:
- exec_out: rm -f MAIN_CONTAINER_ID
environments/default/steps/checkpoints/docker.yaml 0 → 100644
View file @ fa4c5db7
enabled?:
- exec_local: test -f $KAMELEON_WORKDIR/MAIN_CONTAINER_ID
create:
- exec_local: docker commit $(cat $KAMELEON_WORKDIR/MAIN_CONTAINER_ID) $${docker_image}:@microstep_id
apply:
- exec_local: docker tag -f $${docker_image}:@microstep_id $${docker_image}:latest
list:
- exec_local: |
docker images | grep $${docker_image} | awk '{print $2}' || true
clear:
- exec_local: |
docker images | grep $${docker_image} | awk '{print $3}' \
| xargs -I {} bash -c "docker rmi {} | cat || true"
environments/default/steps/data/helpers/create_appliance.py
View file @ fa4c5db7
......@@ -15,6 +15,7 @@ import contextlib
logger = logging.getLogger(__name__)
tar_options = ["--selinux", "--xattrs", "--xattrs-include='*'", "--numeric-owner", "--one-file-system"]
@contextlib.contextmanager
def temporary_directory():
......@@ -227,11 +228,9 @@ mkfs %s /dev/sda1
if "directory" in input_type:
excludes = ['dev/*', 'proc/*', 'sys/*', 'tmp/*', 'run/*',
'/mnt/*']
tar_options_list = ['--numeric-owner', '--one-file-system',
' '.join(('--exclude="%s"' % s for s in excludes))]
tar_options = ' '.join(tar_options_list)
tar_options_str = ' '.join(tar_options + ['--exclude="%s"' % s for s in excludes])
make_tar_cmd = '%s -cf - %s -C %s $(cd %s; ls -A)' % \
(which("tar"), tar_options, input_, input_)
(which("tar"), tar_options_str, input_, input_)
if make_tar_cmd:
cmd = "%s | %s -a %s -m /dev/sda1:/ tar-in - /" % \
......
environments/default/steps/data/helpers/export_appliance.py
View file @ fa4c5db7
......@@ -17,6 +17,8 @@ logger = logging.getLogger(__name__)
tar_formats = ('tar', 'tar.gz', 'tgz', 'tar.bz2', 'tbz', 'tar.xz', 'txz',
'tar.lzo', 'tzo')
tar_options = ["--selinux", "--xattrs", "--xattrs-include='*'", "--numeric-owner", "--one-file-system"]
disk_formats = ('qcow', 'qcow2', 'qed', 'vdi', 'raw', 'vmdk')
......@@ -62,32 +64,45 @@ def tar_convert(disk, output, excludes, compression_level):
elif output.endswith(('tar.lzo', 'tzo')):
compr = "| %s %s -c -" % (which("lzop"), compression_level_opt)
tar_options_list = ["--selinux", "--acls", "--xattrs",
"--numeric-owner", "--one-file-system"] + \
['--exclude="%s"' % s for s in excludes]
tar_options = ' '.join(tar_options_list)
directory = dir_path = os.path.dirname(os.path.realpath(disk))
cmds = [
which("mkdir") + " %s/.mnt" % directory,
which("guestmount") + " --ro -i -a %s %s/.mnt" % (disk, directory),
which("tar") + " -c %s -C %s/.mnt . %s > %s" % (tar_options, directory, compr, output),
which("guestunmount") + " %s/.mnt" % directory,
which("rmdir") + " %s/.mnt" % directory
]
cmd = " && ".join(cmds)
# NB: guestfish version >= 1.32 supports the special tar options, but not available in Debian stable (jessie): do not use for now
#tar_options_list = ["selinux:true", "acls:true", "xattrs:true",
# "numericowner:true",
# "excludes:\"%s\"" % ' '.join(excludes)]
#tar_options = ' '.join(tar_options_list)
#tar_options_str = ' '.join(tar_options_list)
#cmd = which("guestfish") + \
# " --ro -i tar-out -a %s / - %s %s > %s"
#cmd = cmd % (disk, tar_options, compr, output)
#cmd = cmd % (disk, tar_options_str, compr, output)
#proc = subprocess.Popen(cmd_mount_tar, env=os.environ.copy(), shell=True)
#proc.communicate()
#if proc.returncode:
# raise subprocess.CalledProcessError(proc.returncode, cmd)
proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=True)
tar_options_str = ' '.join(tar_options + ['--exclude="%s"' % s for s in excludes])
directory = dir_path = os.path.dirname(os.path.realpath(disk))
cmds = [
which("mkdir") + " %s/.mnt" % directory,
which("guestmount") + " --ro -i -a %s %s/.mnt" % (disk, directory),
which("tar") + " -c %s -C %s/.mnt . %s > %s" % (tar_options_str, directory, compr, output)
]
cmd_mount_tar = " && ".join(cmds)
proc = subprocess.Popen(cmd_mount_tar, env=os.environ.copy(), shell=True)
proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, cmd)
returncode_mount_tar = proc.returncode
# try to umount even if the previous command failed
cmds = [
which("guestunmount") + " %s/.mnt" % directory,
which("rmdir") + " %s/.mnt" % directory
]
cmd_umount = " && ".join(cmds)
proc = subprocess.Popen(cmd_umount, env=os.environ.copy(), shell=True)
proc.communicate()
returncode_umount = proc.returncode
if returncode_mount_tar:
raise subprocess.CalledProcessError(returncode_mount_tar, cmd_mount_tar)
elif returncode_umount:
raise subprocess.CalledProcessError(returncode_umount, cmd_umount)
def qemu_convert(disk, output_fmt, output_filename):
......
environments/default/steps/export/save_appliance_docker.yaml 0 → 100644
View file @ fa4c5db7
- export_docker_rootfs:
- exec_local: echo "Exporting the docker image to $${rootfs}"
- exec_local: mkdir -p $${rootfs}
- exec_local: docker export $(cat MAIN_CONTAINER_ID) | tar --touch -xf - -C $${rootfs}
- exec_local: chmod 755 $${rootfs}
- on_clean:
- exec_local: rm -f docker_export.tar.gz
environments/steps/NAS_benchmark.yaml
View file @ fa4c5db7
......@@ -4,18 +4,18 @@
- exec_in: apt-get -y install openmpi-bin libopenmpi-dev make gfortran gcc
- download_file_in:
- https://www.nas.nasa.gov/assets/npb/NPB3.3.1.tar.gz
- $$NAS_home/NPB3.3.1.tar.gz
- exec_in: cd $$NAS_home && tar xf NPB3.3.1.tar.gz
- $${NAS_home}/NPB3.3.1.tar.gz
- exec_in: cd $${NAS_home} && tar xf NPB3.3.1.tar.gz
- configure_make_def:
- exec_in: |
cd $$NAS_home/NPB3.3.1/NPB3.3-MPI/
cd $${NAS_home}/NPB3.3.1/NPB3.3-MPI/
cp config/make.def{.template,}
sed -i 's/^MPIF77.*/MPIF77 = mpif77/' config/make.def
sed -i 's/^MPICC.*/MPICC = mpicc/' config/make.def
sed -i 's/^FFLAGS.*/FFLAGS = -O -mcmodel=medium/' config/make.def
- compile_different_MPI_bench:
- exec_in: |
cd $$NAS_home/NPB3.3.1/NPB3.3-MPI/
cd $${NAS_home}/NPB3.3.1/NPB3.3-MPI/
for nbproc in 1 2 4 8 16 32
do
for class in B C D
......
environments/steps/checkpoints/btrfs.yaml
View file @ fa4c5db7
# Use BTRFS as checkpoint mechanism
#
# Assume that the $$btrfs_vol is a btrfs volume and the $$rootfs is in a
# Assume that the $${btrfs_vol} is a btrfs volume and the $${rootfs} is in a
# subpath of this volume.
enabled?:
- exec_local: |
export rootfs="$${rootfs}"
test -f $$kameleon_cwd/checkpoint_enabled
btrfs subvolume create $$rootfs 2> /dev/null || true
btrfs subvolume list $$btrfs_volume | grep -q ${rootfs#"/"}
test -f $${kameleon_cwd}/checkpoint_enabled
btrfs subvolume create $${rootfs} 2> /dev/null || true
btrfs subvolume list $${btrfs_volume} | grep -q ${rootfs#"/"}
create:
- exec_local: btrfs subvolume snapshot $$rootfs $${rootfs}-@microstep_id
- exec_local: btrfs subvolume snapshot $${rootfs} $${rootfs}-@microstep_id
# To apply the snapshot simply remove current rootfs, mv the snapshot to
# rootfs and make a backup
apply:
- exec_local: btrfs subvolume delete $$rootfs
- exec_local: mv $${rootfs}-@microstep_id $$rootfs
- exec_local: btrfs subvolume snapshot $$rootfs $${rootfs}-@microstep_id
- exec_local: btrfs subvolume delete $${rootfs}
- exec_local: mv $${rootfs}-@microstep_id $${rootfs}
- exec_local: btrfs subvolume snapshot $${rootfs} $${rootfs}-@microstep_id
clear:
- exec_local: |
export rootfs="$${rootfs}"
snap_list=$(btrfs subvolume list $$btrfs_volume | grep ${rootfs#"/"} | awk '{print "/"$9 }')
snap_list=$(btrfs subvolume list $${btrfs_volume} | grep ${rootfs#"/"} | awk '{print "/"$9 }')
for snap in $snap_list
do
btrfs subvolume delete $snap
......@@ -32,4 +32,4 @@ clear:
list:
- exec_local: |
export rootfs="$${rootfs}"
btrfs subvolume list $$btrfs_volume | grep ${rootfs#"/"} | awk '{print "/"$9 }' | grep "${rootfs}-" | sed "s#${rootfs}-##" || true
btrfs subvolume list $${btrfs_volume} | grep ${rootfs#"/"} | awk '{print "/"$9 }' | grep "${rootfs}-" | sed "s#${rootfs}-##" || true
environments/steps/docker_export.yaml 0 → 100644
View file @ fa4c5db7
- do_export:
- exec_local: |
docker import $${appliance_filename}.tar.gz oarteam/$${kameleon_recipe_name}:latest
- exec_local: |
docker tag oarteam/$${kameleon_recipe_name}:latest oarteam/$${kameleon_recipe_name}:$(date --iso-8601)
- exec_local: |
docker push oarteam/$${kameleon_recipe_name}:latest
docker push oarteam/$${kameleon_recipe_name}:$(date --iso-8601)
environments/steps/evalys.yaml
View file @ fa4c5db7
......@@ -3,18 +3,17 @@
#
- install dependencies:
- exec_in: |
apt-get -y --force-yes install \
apt install -y \
python3-pip \
python3-pyqt4 \
python3-matplotlib \
python3-pandas \
ipython3-notebook \
python3-mpld3
python3-jupyter-core
- install_evalys:
- exec_in: |
cd $$simctn_dir
cd $${simctn_dir}
git clone https://github.com/oar-team/evalys.git
cd evalys
git checkout $${evalys_commit}
cd $$simctn_dir/evalys
cd $${simctn_dir}/evalys
pip3 install -e .
environments/steps/extrae.yaml
View file @ fa4c5db7
......@@ -10,14 +10,14 @@
build-essential openmpi-bin libopenmpi-dev gfortran libxml2-dev binutils-dev libiberty-dev
# get source and extract
- local2in:
- $$kameleon_data_dir/extrae-3.2.1.tar.bz2
- $$extrae_home/extrae-3.2.1.tar.bz2
- $${kameleon_data_dir}/extrae-3.2.1.tar.bz2
- $${extrae_home}/extrae-3.2.1.tar.bz2
- exec_in: |
cd $$extrae_home
cd $${extrae_home}
tar xf extrae-3.2.1.tar.bz2
- exec_in: |
cd $$extrae_home/extrae-3.2.1
MPICC=/usr/bin/mpicc ./configure --prefix=$$extrae_install \