Update recipe to debian 9

environments/batsim_ci.yaml

@@ -10,9 +10,7 @@
 ---
 ## Choose one of them the following extends
 
-# Checkpoint only with btrfs
-checkpoint: btrfs.yaml
-extend: default/from_image/debian8.yaml
+extend: default/from_upstream_build/debian-stretch
 
 global:
   backend: qemu
@@ -48,16 +46,16 @@ setup:
   #
   # Simulator and schedulers install
   #
-  - configure_apt_repo
+  #- configure_apt_repo
 
-  - install_common_deps:
-
-    - install_build_tools:
-      - exec_in: |
-          apt-get -y install cmake gcc clang g++
-          update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-6 100
-
-  - simgrid:
+  #  - install_common_deps:
+  #
+  #    - install_build_tools:
+  #      - exec_in: |
+  #          apt-get -y install cmake gcc clang g++
+  #          update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-6 100
+  #
+  - simgrid
 
   - batsim_dependencies
   - result_analysis

environments/default/from_upstream_build/base.yaml

+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Base template for Kameleon appliance.
+#
+#==============================================================================
+---
+extend: ../steps/backend/$${backend}.yaml
+
+# Loads some helpful aliases (this files are located in steps/aliases/ directory)
+aliases: defaults.yaml
+
+# Custom shell environement (this files are located in steps/env/ directory)
+env:
+  - bashrc
+  - functions.sh
+
+# Global variables use by Kameleon engine and the steps
+global:
+  ## Select backend for in context isolation
+  backend: qemu
+
+  # Architecture for the target system
+  arch: x86_64
+  # Default hostname
+  hostname: kameleon-$${distrib}
+  user_name: kameleon
+  user_password: $${user_name}
+  root_password: $${user_name}
+  user_groups: sudo
+  default_keyboard_layout: "us,fr,de"
+  default_locales: POSIX C en_US fr_FR de_DE
+  default_lang: en_US.UTF-8
+  default_timezone: UTC
+  appliance_tar_compression_level: "9"
+
+  ## System variables. Required by kameleon engine
+  # Include specific steps
+  include_steps:
+    - $${distrib}/$${release}
+    - $${distrib}
+
+  ## Location of an upstream recipe build to base this recipe on (build from image)
+  upstream_recipe: $${distrib}-$${release}
+  upstream_version: latest
+  upstream_checksum: true
+  upstream_checksign: false
+  upstream_cache: false
+  upstream_url: http://kameleon.imag.fr/builds
+  upstream_store_dir: $${kameleon_cwd}/upstream_build/
+  # rootfs options
+  rootfs: $${kameleon_cwd}/rootfs
+  
+  # output appliance options
+  filesystem_type: ext4
+  image_size: 10G
+  image_disk: $${kameleon_cwd}/base_$${kameleon_recipe_name}
+  # Allowed formats are: tar.gz, tar.bz2, tar.xz, tar.lzo, qcow, qcow2, qed, vdi, raw, vmdk
+  appliance_formats: qcow2 tar.gz
+  appliance_filename: "$${kameleon_cwd}/$${kameleon_recipe_name}"
+  appliance_tar_excludes: >-
+    ./etc/fstab ./root/.bash_history ./root/kameleon_workdir ./root/.ssh
+    ./var/tmp/* ./tmp/* ./var/log/* ./dev/* ./proc/* ./run/*
+    ./sys/*
+
+  ssh_config_file: $${kameleon_cwd}/ssh_config
+  local_ip: 10.0.2.2
+
+
+  # E.g: net.ifnames=0 biosdevname=0 console=tty0 console=ttyS0,115200n8
+  kernel_args: quiet
+
+bootstrap:
+  - "@base"
+
+# Install and configuration steps
+setup:
+  # Install
+  - upgrade_system
+  - install_software:
+    - packages: $${setup_packages}
+  # Configuration
+  - configure_system:
+    - locales: $${default_locales}
+    - lang: $${default_lang}
+    - timezone: $${default_timezone}
+  - configure_keyboard:
+    # set to english keyboard use 'localectl list-keymaps' to see available list
+    - layout: $${default_keyboard_layout}
+  - configure_network
+  - kameleon_customization
+  - create_user:
+    - name: $${user_name}
+    - groups: $${user_groups}
+    - password: $${user_name}
+  - clean_system
+
+export:
+  - "@base"

environments/default/from_upstream_build/debian-base.yaml

+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Debian 7 (Wheezy) appliance.
+#
+#==============================================================================
+---
+extend: base.yaml
+
+# Global variables use by Kameleon engine and the steps
+global:
+  # Distribution
+  distrib: debian
+  release: wheezy
+  release_number: 7
+  kernel_arch: amd64
+
+  # Apt recipe options
+  apt_repository: http://ftp.debian.org/debian/
+  apt_enable_contrib: true
+  apt_enable_nonfree: true
+
+  setup_packages: >-
+    bsdutils ca-certificates locales man-db less libui-dialog-perl dialog
+    isc-dhcp-client ifupdown iptables iputils-ping iproute netbase net-tools
+    psmisc openssh-server acpid acpi-support-base sysvinit bzip2 wget
+    apt-utils pciutils curl sudo vim bash-completion curl resolvconf
+
+# Bootstrap the new system and create the 'in_context'
+bootstrap:
+  - "@base"
+
+# Install and configuration steps
+setup:
+  - configure_apt:
+    - repository: $${apt_repository}
+    - enable_contrib_repo: $${apt_enable_contrib}
+    - enable_nonfree_repo: $${apt_enable_nonfree}
+  - "@base"
+
+# Export the generated appliance in the format of your choice
+export:
+  - "@base"

environments/default/from_upstream_build/debian-stretch.yaml

+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Debian 9 (Stretch) base system.
+#
+#==============================================================================
+---
+extend: debian-base.yaml
+
+global:
+  # Distribution
+  release: stretch
+  release_number: 9
+
+  # Apt recipe options
+  apt_repository: http://deb.debian.org/debian/
+
+  setup_packages: >-
+    sudo vim bash-completion curl resolvconf bzip2 bsdutils ca-certificates
+    locales man-db less dialog isc-dhcp-client ifupdown
+    iptables iputils-ping iproute2 netbase net-tools psmisc openssh-server
+
+bootstrap:
+  - "@base"
+
+setup:
+  - "@base"
+
+export:
+  - "@base"

environments/default/steps/aliases/defaults.yaml

 write_local:
   - exec_local: |
       mkdir -p $(dirname @1);
-      cat >@1 <<EOF
+      cat >@1 <<EOF_KAMELEON_INTERNAL
       @2
-      EOF
+      EOF_KAMELEON_INTERNAL
 
 write_in:
   - exec_in: |
       mkdir -p $(dirname @1);
-      cat >@1 <<EOF
+      cat >@1 <<EOF_KAMELEON_INTERNAL
       @2
-      EOF
+      EOF_KAMELEON_INTERNAL
 
 write_out:
   - exec_out: |
       mkdir -p $(dirname @1);
-      cat >@1 <<EOF
+      cat >@1 <<EOF_KAMELEON_INTERNAL
       @2
-      EOF
+      EOF_KAMELEON_INTERNAL
 
 append_local:
   - exec_local: |
       mkdir -p $(dirname @1);
-      cat >>@1 <<EOF
+      cat >>@1 <<EOF_KAMELEON_INTERNAL
       @2
-      EOF
+      EOF_KAMELEON_INTERNAL
 
 append_in:
   - exec_in: |
       mkdir -p $(dirname @1);
-      cat >>@1 <<EOF
+      cat >>@1 <<EOF_KAMELEON_INTERNAL
       @2
-      EOF
+      EOF_KAMELEON_INTERNAL
 
 append_out:
   - exec_out: |
       mkdir -p $(dirname @1);
-      cat >>@1 <<EOF
+      cat >>@1 <<EOF_KAMELEON_INTERNAL
       @2
-      EOF
+      EOF_KAMELEON_INTERNAL
 
 local2out:
   - exec_out: |
@@ -112,3 +112,6 @@ download_file_out:
 
 download_file_local:
   - exec_local: __download "@1" "@2"
+
+download_recipe_build:
+  - exec_local: __download_recipe_build "@1" "@2" "@3" "@4" "@5" "@6" "@7"

environments/default/steps/backend/VM.yaml

@@ -5,12 +5,12 @@ global:
   out_context:
     cmd: ssh -F $${ssh_config_file} $${kameleon_recipe_name} -t /bin/bash
     workdir: /root/kameleon_workdir
-    proxy_cache: 10.0.2.2
+    proxy_cache: $${local_ip}
 
   in_context:
     cmd: ssh -F $${ssh_config_file} $${kameleon_recipe_name} -t /bin/bash
     workdir: /root/kameleon_workdir
-    proxy_cache: 10.0.2.2
+    proxy_cache: $${local_ip}
 
 bootstrap:
   - create_appliance

environments/default/steps/backend/qemu.yaml

@@ -13,9 +13,9 @@ global:
   qemu_monitor_socket: $${kameleon_cwd}/qemu_monitor.socket
   qemu_arch: $${arch}
   qemu_pidfile: $${kameleon_cwd}/qemu.pid
-  qemu_image_disk: $${image_disk}.$${image_format}
 
 bootstrap:
+  - prepare_ssh_to_out_context
   - "@base"
   - start_qemu
 

environments/default/steps/bootstrap/create_appliance.yaml

 - create_appliance_script: $${kameleon_data_dir}/helpers/create_appliance.py
 
-- download_rootfs_archive:
-  - download_file_local:
-    - $${rootfs_archive_url}
-    - $${rootfs_archive_download_path}
+- download_upstream_build:
+  - download_recipe_build:
+    - $${upstream_recipe}
+    - $${upstream_version}
+    - $${upstream_checksum}
+    - $${upstream_checksign}
+    - $${upstream_cache}
+    - $${upstream_url}
+    - $${upstream_store_dir}
 
 - create_appliance:
   - check_cmd_local: virt-make-fs
   - exec_local: |
       rm -f $${image_disk}.$${image_format}
       # Create an appliance from rootfs directory
-      python $${create_appliance_script} $${rootfs_archive_download_path} \
+      python $${create_appliance_script} $${upstream_store_dir}/$${upstream_recipe}.tar.xz \
         --size $${image_size} \
         -o $${image_disk} \
         --filesystem $${filesystem_type} \
         --format $${image_format} \
         --append "$${kernel_args}"
-  - exec_local: rm -f $${rootfs_archive_download_path}
+  - exec_local: rm -rf $${upstream_store_dir}
 
 - delete_initial_image_at_the_end:
   - on_checkpoint: skip

environments/default/steps/bootstrap/prepare_appliance.yaml

@@ -29,26 +29,9 @@
           --run-command 'chmod 600 $${authorized_keys}' \
           --delete /root/.ssh/.kameleon_authorized_keys
 
-- create_ssh_config:
+- add_insecure_key_to_ssh_config:
   - on_checkpoint: redo
   - exec_local: |
-      # Find empty SSH forwarding port
-      SSH_FWD_PORT=$(__find_free_port 50000 60000)
-      echo "SSH forwarding port: $SSH_FWD_PORT"
-
-  - write_local:
-    - $${ssh_config_file}
-    - |
-      Host $${kameleon_recipe_name}
-      HostName 127.0.0.1
-      Port $SSH_FWD_PORT
-      User root
+      cat <<EOF >> $${ssh_config_file}
       IdentityFile $${insecure_ssh_key}
-      UserKnownHostsFile /dev/null
-      StrictHostKeyChecking no
-      PasswordAuthentication no
-      IdentitiesOnly yes
-      LogLevel FATAL
-      ForwardAgent yes
-      Compression yes
-      Protocol 2
+      EOF

environments/default/steps/bootstrap/prepare_ssh_to_out_context.yaml

+- select_empty_port:
+  - on_checkpoint: redo
+  - exec_local: |
+      # Find empty SSH forwarding port
+      SSH_FWD_PORT=$(__find_free_port 50000 60000)
+      echo "SSH forwarding port: $SSH_FWD_PORT"
+- prepare_ssh_config:
+  - on_checkpoint: redo
+  - write_local:
+    - $${ssh_config_file}
+    - |
+      Host $${kameleon_recipe_name}
+      HostName 127.0.0.1
+      Port ${SSH_FWD_PORT}
+      User root
+      UserKnownHostsFile /dev/null
+      StrictHostKeyChecking no
+      PasswordAuthentication no
+      IdentitiesOnly yes
+      LogLevel FATAL
+      ForwardAgent yes
+      Compression yes
+      Protocol 2

environments/default/steps/bootstrap/start_qemu.yaml

 # Require SSH_FWD_PORT bash environment variable to be set
-- boot_timeout: 100
-- shutdown_timeout: 100
 - disk_cache: unsafe
 # This must be set if you want to boot an ISO image instead of a disk
 - iso_path: ""
+- iso_boot: "yes"
+- vm_expected_service: ssh
+- boot_timeout: 100
+- shutdown_timeout: 100
+- debug: false
+- telnet_port: ""
+- no_reboot: true
+- socat_monitor: socat - UNIX-CONNECT:$${qemu_monitor_socket}
+- qemu_sendkeys_script: $${kameleon_data_dir}/qemu-sendkeys.rb
+- qemu_sendkeys_commands: 
+- vm_expected_service: ssh
+- vm_cleanup_section: setup
+- shutdown_vm_immediately: false
+- force_vm_shutdown: true
+- check_cmd_local: socat
+- qemu_arch: x86_64
 
-- start_qemu_vm:
+- start_vm:
   - on_checkpoint: redo
   - check_cmd_local: qemu-system-$${qemu_arch}
-  - exec_local: echo "Starting qemu..."
   - exec_local: |
-      if [ "$${qemu_enable_kvm}" = "true" ]; then
+      echo "Starting qemu..."
+      if [ "$${qemu_enable_kvm}" == "true" ]; then
         ENABLE_KVM="-enable-kvm -cpu host"
         BOOT_TIMEOUT=$${boot_timeout}
       else
         BOOT_TIMEOUT=$(($${boot_timeout}*2))
       fi
-  - exec_local: |
-        if [ -f "vm_state_to_load.txt" ]
-        then
-            SAVED_STATE="$(cat vm_state_to_load.txt)"
-            LOADVM="-loadvm $SAVED_STATE"
-            rm -f vm_state_to_load.txt
-        fi
-  - exec_local: |
+      if [ -f "vm_state_to_load.txt" ]
+      then
+          SAVED_STATE="$(< vm_state_to_load.txt)"
+          LOADVM="-loadvm $SAVED_STATE"
+          rm -f vm_state_to_load.txt
+      fi
+      if [ "$${debug}" == "true" ]; then
+        VNC_OPT=""
+      else
         # Find empty VNC port
         VNC_PORT=$(( $(__find_free_port 5900 5910) - 5900 ))
         echo "VNC port: $VNC_PORT"
-
-  - exec_local: |
-        # Select disk
-        if [ -z "$${iso_path}" ]; then
-          QEMU_DISK=''
-        else
-          QEMU_DISK='-cdrom $${iso_path} -hda $${qemu_image_disk}'
+        VNC_OPT="-vnc :$VNC_PORT"
+      fi
+      if [ -n "$${telnet_port}" ]; then
+        SERIAL_TELNET="telnet:localhost:$${telnet_port},server"
+      fi
+      # Select disk
+      QEMU_DRIVES="-drive file=$${image_disk}.$${image_format},cache=$${disk_cache},media=disk,if=virtio,id=drive0"
+      QENU_BOOT=
+      if [ -n "$${iso_path}" ]; then
+        QEMU_DRIVES="-drive file=$${iso_path},readonly,media=cdrom $QEMU_DRIVES"
+        if [ "$${iso_boot}" == "yes" ]; then
+          QEMU_BOOT="-boot order=d"
         fi
+      fi
+      if [ "$${no_reboot}" == "true" ]; then
+        NO_REBOOT="-no-reboot"
+      fi
+      if [ -n "${SSH_FWD_PORT}" ]; then
+        HOSTFWD=",hostfwd=tcp::${SSH_FWD_PORT}-:22"
+      fi
+      qemu-system-$${qemu_arch} $ENABLE_KVM -smp $${qemu_cpu} -m $${qemu_memory_size} -rtc base=localtime \
+        -net nic,model=virtio -net user${HOSTFWD} \
+        $QEMU_DRIVES \
+        -monitor unix:$${qemu_monitor_socket},server,nowait -pidfile $${qemu_pidfile} -daemonize \
+        $QEMU_BOOT $NO_REBOOT \
+        $VNC_OPT $SERIAL_TELNET\
+        $LOADVM
   - exec_local: |
-        qemu-system-$${qemu_arch} $ENABLE_KVM \
-          $QEMU_DISK -drive file=$${image_disk}.$${image_format},cache=$${disk_cache},media=disk,if=virtio,id=drive0 \
-          -smp $${qemu_cpu} \
-          -m $${qemu_memory_size} \
-          -rtc base=localtime \
-          -net nic,model=virtio \
-          -net user,hostfwd=tcp::${SSH_FWD_PORT}-:22 \
-          -monitor unix:$${qemu_monitor_socket},server,nowait \
-          -pidfile $${qemu_pidfile} \
-          -daemonize -vnc :$VNC_PORT $LOADVM
-  - exec_local: |
-        SSH_AVAILABLE=0
+      VM_AVAILABLE=0
+      if [ "$${vm_expected_service}" == "ssh" ]; then
         TIMEOUT=$(( $(date +%s) + $BOOT_TIMEOUT ))
-        until timeout 5 ssh -q -F $${ssh_config_file} -o ConnectionAttempts=1  $${kameleon_recipe_name} -t true && SSH_AVAILABLE=1 || [ $(date +%s) -gt $TIMEOUT ];
+        until timeout 5 ssh -q -F $${ssh_config_file} -o ConnectionAttempts=1  $${kameleon_recipe_name} -t true && VM_AVAILABLE=1 || [ $(date +%s) -gt $TIMEOUT ];
         do
-          echo -en "\rWaiting for SSH to become available for out_context...($(( TIMEOUT - $(date +%s) ))s)"
+          echo -en "\rWaiting for SSH to become available in VM for out_context...($(( TIMEOUT - $(date +%s) ))s)"
           sleep 1
         done
-        echo ""
+        echo
+      else
+        TIMEOUT=$(( $(date +%s) + $BOOT_TIMEOUT ))
+        until timeout 1 [ $(date +%s) -gt $TIMEOUT ];
+        do
+          echo -en "\rWaiting for VM to become available : ($(( TIMEOUT - $(date +%s) ))s)"
+          sleep 1
+        done
+        echo
+        VM_AVAILABLE=1
+      fi
   - rescue:
-    - exec_local: test $SSH_AVAILABLE -eq 1
-    - breakpoint: Failed to connect to VM via SSH. Please verify the VM successfully booted with a vnc client.
+    - exec_local: test $VM_AVAILABLE -eq 1
+    - breakpoint: |
+        Failed to get VM up and running (expected service: $${vm_expected_service}). Please verify the VM successfully booted with a vnc client.
+  - test:
+    - exec_local: test -e "$${qemu_sendkeys_commands}" -a -s "$${qemu_sendkeys_commands}"
+    - exec_local: |
+        echo "Sending keyboard commands to the VM: $${qemu_sendkeys_commands}"
+        echo "(Local httpd server url: http://$${local_ip}:$HTTP_PORT)"
+        ruby $${qemu_sendkeys_script} -d 0.05 "$(sed -e s/%LOCAL_IP%/$${local_ip}/g -e s/%HTTP_PORT%/$HTTP_PORT/g $${qemu_sendkeys_commands})" | $${socat_monitor} > /dev/null
+    - exec_local: echo "No keyboard commands to send"
 
-- force_shutdown_qemu_vm:
+- shutdown_vm:
   - on_checkpoint: redo
-  - check_cmd_local: socat
-  - on_setup_clean:
-    - exec_out: echo "Sync"
-    - exec_out: sync
-    - exec_out: echo "Shutting down virtual machine"
-    - exec_out: shutdown -h now
-    - exec_local: |
+  - test:
+    - exec_local: test "$${shutdown_vm_immediately}" == "true"
+    - exec_local: &1 |
         if [ -f $${qemu_pidfile} ]; then
-          _QEMU_PID=$(cat $${qemu_pidfile})
+          _QEMU_PID=$(< $${qemu_pidfile})
           if ps -p $_QEMU_PID > /dev/null; then
-            if [ -S $${qemu_monitor_socket} ]; then
-              NEXT_WAIT_TIME=0
-              echo system_powerdown | socat - UNIX-CONNECT:$${qemu_monitor_socket} || true
-              while ps -p $_QEMU_PID > /dev/null && [ $NEXT_WAIT_TIME -lt $${shutdown_timeout} ];
+            if [ "$${force_vm_shutdown}" == "true" ]; then
+              if [ -S $${qemu_monitor_socket} ]; then
+                NEXT_WAIT_TIME=0
+                echo system_powerdown | socat - UNIX-CONNECT:$${qemu_monitor_socket} || true
+                while ps -p $_QEMU_PID > /dev/null && [ $NEXT_WAIT_TIME -lt $${shutdown_timeout} ];
+                do
+                  sleep 1
+                  echo -en "\rWaiting for qemu virtual machine to shutdown...($(( $${shutdown_timeout} - 1 - NEXT_WAIT_TIME++ ))s)"
+                done
+              fi
+            else
+              echo "Waiting for the VM to shutdown"
+              echo "Run 'vncviewer :$VNC_PORT' to see what's happening in the VM"
+              while ps -p $_QEMU_PID > /dev/null;
               do
-                sleep 1
-                echo -en "\rWaiting for qemu virtual machine to shutdown...($(( $${shutdown_timeout} - 1 - NEXT_WAIT_TIME++ ))s)"
+                sleep 2
               done
             fi
+          fi
+        fi
+  - on_clean:
+    - exec_local: echo "QEMU SHUTDOWN VM ON CLEAN $${shutdown_vm_immediately} $${vm_cleanup_section}"
+    - test:
+      - exec_local: test "$${shutdown_vm_immediately}" == "true"
+      - exec_local: &2 |
+          if [ -f $${qemu_pidfile} ]; then
+            _QEMU_PID=$(< $${qemu_pidfile})
             if ps -p $_QEMU_PID > /dev/null; then
               if [ -S $${qemu_monitor_socket} ]; then
                 echo "  ~> Failed"
@@ -93,5 +149,29 @@
             fi
             rm -f $${qemu_pidfile}
           fi
-        fi
-        rm -f $${qemu_monitor_socket}
+          rm -f $${qemu_monitor_socket}
+      - exec_local: echo "NO VM CLEAN HERE"
+  - on_bootstrap_clean:
+    - exec_local: echo "QEMU SHUTDOWN VM ON BOOTSTRAP CLEAN $${shutdown_vm_immediately} $${vm_cleanup_section}"
+    - test:
+      - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "bootstrap"
+      - group: 
+        - exec_local: *1
+        - exec_local: *2
+      - exec_local: echo "NO VM CLEAN HERE"
+  - on_setup_clean:
+    - exec_local: echo "QEMU SHUTDOWN VM ON SETUP CLEAN $${shutdown_vm_immediately} $${vm_cleanup_section}"
+    - test:
+      - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "setup"
+      - group:
+        - exec_local: *1
+        - exec_local: *2
+      - exec_local: echo "NO VM CLEAN HERE"
+  - on_export_clean:
+    - exec_local: echo "QEMU SHUTDOWN VM ON EXPORT CLEAN $${shutdown_vm_imm