Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit e1829c4d authored by Thomas Lavocat's avatar Thomas Lavocat
Browse files

Spark mpi image

parents
# Build the image
```
kameleon build jessie_custom.yaml --enable-cache
```
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: Base template for Archlinux appliance.
#
#==============================================================================
---
# Loads some helpful aliases (this files are located in steps/aliases/ directory)
aliases: defaults.yaml
# Custom shell environement (this files are located in steps/env/ directory)
env:
- bashrc
- functions.sh
# Global variables use by Kameleon engine and the steps
global:
## User varibales : used by the recipe
user_name: kameleon
user_password: $${user_name}
root_password: $${user_name}
user_groups: sudo
default_keyboard_layout: "us,fr,de"
default_locales: POSIX C en_US fr_FR de_DE
default_lang: en_US.UTF-8
default_timezone: UTC
appliance_tar_compression_level: "9"
## Initial rootfs archive built with kameleon
## See https://github.com/oar-team/kameleon-bootstrap-recipes.git
rootfs_archive_url: http://kameleon.imag.fr/rootfs/$${arch}/$${distrib}$${release_number}_$${bootstrap_image_version}.tar.xz
rootfs_archive_download_path: $${kameleon_cwd}/rootfs.tar.xz
# rootfs options
rootfs: $${kameleon_cwd}/rootfs
# Distribution
arch: x86_64
hostname: kameleon-$${distrib}
# appliance options
filesystem_type: ext4
image_size: 10G
image_format: qcow2
image_disk: $${kameleon_cwd}/base_$${kameleon_recipe_name}
# Allowed formats are: tar.gz, tar.bz2, tar.xz, tar.lzo, qcow, qcow2, qed, vdi, raw, vmdk
appliance_formats: qcow2 tar.gz
appliance_filename: "$${kameleon_cwd}/$${kameleon_recipe_name}"
appliance_tar_excludes: >-
./etc/fstab ./root/.bash_history ./root/kameleon_workdir ./root/.ssh
./var/tmp/* ./tmp/* ./var/log/* ./dev/* ./proc/* ./run/*
./sys/*
## System variables. Required by kameleon engine
# Include specific steps
include_steps:
- $${distrib}/$${release}
- $${distrib}
# Shell session from where we launch exec_out commands. There is often a
# local bash session, but it can be a remote shell on other machines or on
# any shell. (eg. bash, chroot, fakechroot, ssh, tmux, lxc...)
out_context:
cmd: bash
workdir: $${kameleon_cwd}
proxy_cache: 127.0.0.1
# Shell session that allows us to connect to the building machine in order to
# configure it and setup additional programs
in_context:
cmd: USER=root HOME=/root LC_ALL=POSIX chroot $${rootfs} bash
workdir: /root/kameleon_workdir
proxy_cache: 127.0.0.1
# E.g: net.ifnames=0 console=tty0 console=ttyS0,115200n8
kernel_args: "quiet net.ifnames=0 biosdevname=0"
# Bootstrap the new system and create the 'in_context'
bootstrap:
- enable_checkpoint
- prepare_chroot
- start_chroot
# Install and configuration steps
setup:
# Install
- upgrade_system
- install_software:
- packages: $${setup_packages}
# Configuration
- configure_system:
- locales: $${default_locales}
- lang: $${default_lang}
- timezone: $${default_timezone}
- configure_keyboard:
# set to english keyboard use 'localectl list-keymaps' to see available list
- layout: $${default_keyboard_layout}
- configure_network
- kameleon_customization
- create_user:
- name: $${user_name}
- groups: $${user_groups}
- password: $${user_name}
- clean_system
# Export the generated appliance in the format of your choice
export:
- disable_checkpoint
- save_appliance:
- output_filename: $${appliance_filename}
- output_formats: $${appliance_formats}
- tar_compression_level: $${appliance_tar_compression_level}
- tar_excludes: $${appliance_tar_excludes}
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: Debian 7 (Wheezy) appliance built with chroot.
#
#==============================================================================
---
extend: base.yaml
# Global variables use by Kameleon engine and the steps
global:
# Distribution
distrib: debian
release: wheezy
release_number: 7
kernel_arch: amd64
# Apt recipe options
apt_repository: http://ftp.debian.org/debian/
apt_enable_contrib: true
apt_enable_nonfree: true
setup_packages: >-
bsdutils ca-certificates locales man-db less libui-dialog-perl dialog
isc-dhcp-client ifupdown iptables iputils-ping iproute netbase net-tools
psmisc openssh-server acpid acpi-support-base sysvinit bzip2 wget
apt-utils pciutils curl sudo vim bash-completion curl resolvconf
# Bootstrap the new system and create the 'in_context'
bootstrap:
- "@base"
# Install and configuration steps
setup:
- configure_apt:
- repository: $${apt_repository}
- enable_contrib_repo: $${apt_enable_contrib}
- enable_nonfree_repo: $${apt_enable_nonfree}
- "@base"
# Export the generated appliance in the format of your choice
export:
- "@base"
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: Debian 8 (Jessie) base system built with chroot.
#
#==============================================================================
---
extend: debian7.yaml
global:
# Distribution
release: jessie
release_number: 8
bootstrap_image_version: "20150915-1"
setup_packages: >-
sudo vim bash-completion curl resolvconf bzip2 bsdutils ca-certificates
locales man-db less libui-dialog-perl dialog isc-dhcp-client ifupdown
iptables iputils-ping iproute2 netbase net-tools psmisc openssh-server
acpid acpi-support-base sysvinit systemd systemd-sysv pciutils
bootstrap:
- "@base"
setup:
- "@base"
export:
- "@base"
write_local:
- exec_local: |
mkdir -p $(dirname @1);
cat >@1 <<EOF
@2
EOF
write_in:
- exec_in: |
mkdir -p $(dirname @1);
cat >@1 <<EOF
@2
EOF
write_out:
- exec_out: |
mkdir -p $(dirname @1);
cat >@1 <<EOF
@2
EOF
append_local:
- exec_local: |
mkdir -p $(dirname @1);
cat >>@1 <<EOF
@2
EOF
append_in:
- exec_in: |
mkdir -p $(dirname @1);
cat >>@1 <<EOF
@2
EOF
append_out:
- exec_out: |
mkdir -p $(dirname @1);
cat >>@1 <<EOF
@2
EOF
local2out:
- exec_out: |
mkdir -p $(dirname @2)
- pipe:
- exec_local: cat @1
- exec_out: cat > @2
local2in:
- exec_in: mkdir -p $(dirname @2)
- pipe:
- exec_local: cat @1
- exec_in: cat > @2
out2local:
- exec_local: mkdir -p $(dirname @2)
- pipe:
- exec_out: cat @1
- exec_local: cat > @2
out2in:
- exec_in: mkdir -p $(dirname @2)
- pipe:
- exec_out: cat @1
- exec_in: cat > @2
in2local:
- exec_local: mkdir -p $(dirname @2)
- pipe:
- exec_in: cat @1
- exec_local: cat > @2
in2out:
- exec_out: mkdir -p $(dirname @2)
- pipe:
- exec_in: cat @1
- exec_out: cat > @2
check_cmd_out:
- rescue:
- exec_out: command -V @1 2> /dev/null
- breakpoint: "@1 is missing from out_context"
check_cmd_local:
- rescue:
- exec_local: command -V @1 2> /dev/null
- breakpoint: "@1 is missing from local_context"
check_cmd_in:
- rescue:
- exec_in: command -V @1 2> /dev/null
- breakpoint: "@1 is missing from in_context"
umount_out:
- exec_out: |
echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
umount_local:
- exec_local: |
echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
umount_in:
- exec_in: |
echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
download_file_in:
- exec_in: __download "@1" "@2"
download_file_out:
- exec_out: __download "@1" "@2"
download_file_local:
- exec_local: __download "@1" "@2"
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""Convert an rootfs archive to a bootable disk image with guestfish."""
import os
import os.path as op
import sys
import subprocess
import argparse
import logging
import tempfile
import shutil
import contextlib
logger = logging.getLogger(__name__)
@contextlib.contextmanager
def temporary_directory():
"""Context manager for tempfile.mkdtemp()."""
name = tempfile.mkdtemp()
try:
yield name
finally:
shutil.rmtree(name)
def which(command):
"""Locate a command.
Snippet from: http://stackoverflow.com/a/377028
"""
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(command)
if fpath:
if is_exe(command):
return command
else:
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, command)
if is_exe(exe_file):
return exe_file
raise ValueError("Command '%s' not found" % command)
def file_type(path):
"""Get file type."""
if not op.exists(path):
raise Exception("cannot open '%s' (No such file or directory)" % path)
cmd = [which("file"), path]
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=os.environ.copy(),
shell=False)
output, _ = proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd))
return output.decode('utf-8').split(':')[1].strip()
def qemu_convert(disk, output_fmt, output_filename):
"""Convert the disk image filename to disk image output_filename."""
binary = which("qemu-img")
cmd = [binary, "convert", "-p", "-O", output_fmt, disk, output_filename]
if output_fmt in ("qcow", "qcow2"):
cmd.insert(2, "-c")
proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=False)
proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd))
def run_guestfish_script(disk, script, mount=True, piped_output=False):
"""Run guestfish script."""
args = [which("guestfish"), '-a', disk]
if mount:
script = "run\nmount /dev/sda1 /\n%s" % script
else:
script = "run\n%s" % script
if piped_output:
proc = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
env=os.environ.copy())
stdout, _ = proc.communicate(input=script.encode('utf-8'))
else:
proc = subprocess.Popen(args,
stdin=subprocess.PIPE,
env=os.environ.copy())
proc.communicate(input=script.encode('utf-8'))
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, ' '.join(args))
if piped_output:
return stdout.decode('utf-8')
def find_mbr():
""" ..."""
search_paths = (
"/usr/share/syslinux/mbr.bin",
"/usr/lib/bios/syslinux/mbr.bin",
"/usr/lib/syslinux/bios/mbr.bin",
"/usr/lib/extlinux/mbr.bin",
"/usr/lib/syslinux/mbr.bin",
"/usr/lib/syslinux/mbr/mbr.bin",
"/usr/lib/EXTLINUX/mbr.bin"
)
for path in search_paths:
if op.exists(path):
return path
raise Exception("syslinux MBR not found")
def get_boot_information(disk):
"""Looking for boot information"""
script_1 = """
blkid /dev/sda1 | grep ^UUID: | awk '{print $2}'
ls /boot/ | grep ^vmlinuz | head -n 1
ls /boot/ | grep ^init | grep -v fallback | head -n 1
ls /boot/ | grep ^init | grep fallback | head -n 1"""
logger.info(get_boot_information.__doc__)
output_1 = run_guestfish_script(disk, script_1, piped_output=True)
try:
infos = output_1.strip().split('\n')
if len(infos) == 4:
uuid, vmlinuz, initrd, initrd_fallback = infos
if initrd:
return uuid, vmlinuz, initrd
else:
return uuid, vmlinuz, initrd_fallback
else:
uuid, vmlinuz, initrd = infos
return uuid, vmlinuz, initrd
except:
raise Exception("Invalid boot information (missing kernel ?)")
def generate_fstab(disk, uuid, filesystem_type):
"""Generate /etc/fstab file"""
logger.info("Generating /etc/fstab")
script = """
write /etc/fstab "# /etc/fstab: static file system information.\\n"
write-append /etc/fstab "# Generated by kameleon-helpers.\\n\\n"
write-append /etc/fstab "UUID=%s\\t/\\t%s\\tdefaults\\t0\\t1\\n"
""" % (uuid, filesystem_type)
run_guestfish_script(disk, script)
def install_bootloader(disk, mbr, append):
"""Install a bootloader"""
mbr_path = mbr or find_mbr()
mbr_path = op.abspath(mbr_path)
uuid, vmlinuz, initrd = get_boot_information(disk)
logger.info("Root partition UUID: %s" % uuid)
logger.info("Kernel image: /boot/%s" % vmlinuz)
logger.info("Initrd image: /boot/%s" % initrd)
script = """
echo "[guestfish] Upload the master boot record"
upload %s /boot/mbr.bin
echo "[guestfish] Generate /boot/syslinux.cfg"
write /boot/syslinux.cfg "DEFAULT linux\\n"
write-append /boot/syslinux.cfg "LABEL linux\\n"
write-append /boot/syslinux.cfg "SAY Booting the kernel\\n"
write-append /boot/syslinux.cfg "KERNEL /boot/%s\\n"
write-append /boot/syslinux.cfg "INITRD /boot/%s\\n"
write-append /boot/syslinux.cfg "APPEND ro root=UUID=%s %s\\n"
echo "[guestfish] Put the MBR into the boot sector"
copy-file-to-device /boot/mbr.bin /dev/sda size:440
echo "[guestfish] Install extlinux on the first partition"
extlinux /boot
echo "[guestfish] Set the first partition as bootable"
part-set-bootable /dev/sda 1 true
echo "[guestfish] Generate empty fstab"
write /etc/fstab "# UNCONFIGURED FSTAB FOR BASE SYSTEM\\n"
echo "[guestfish] Set / permissions to '0755'"
chmod 0755 /
""" % (mbr_path, vmlinuz, initrd, uuid, append)
run_guestfish_script(disk, script)
return uuid, vmlinuz, initrd
def create_disk(input_, output_filename, fmt, size, filesystem, verbose):
"""Make a disk image from a tar archive or files."""
input_type = file_type(input_).lower()
make_tar_cmd = ""
if "xz compressed data" in input_type:
make_tar_cmd = "%s %s" % (which("xzcat"), input_)
elif "bzip2 compressed data" in input_type:
make_tar_cmd = "%s %s" % (which("bzcat"), input_)
elif "gzip compressed data" in input_type:
make_tar_cmd = "%s %s" % (which("zcat"), input_)
# create empty disk
logger.info("Creating an empty disk image")
qemu_img = which("qemu-img")
cmd = [qemu_img, "create", "-f", fmt, output_filename, size, "-q"]
proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=False)
proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd))
# parition disk and create the filesystem
script = """
echo "[guestfish] Create new MBR partition table on /dev/sda"
part-disk /dev/sda mbr
echo "[guestfish] Create %s filesystem on /dev/sda1"
mkfs %s /dev/sda1
""" % (filesystem, filesystem)
run_guestfish_script(output_filename, script, mount=False)
# Fill disk with our data
logger.info("Copying the data into the disk image")
if "directory" in input_type:
excludes = ['dev/*', 'proc/*', 'sys/*', 'tmp/*', 'run/*',
'/mnt/*']
tar_options_list = ['--numeric-owner', '--one-file-system',
' '.join(('--exclude="%s"' % s for s in excludes))]
tar_options = ' '.join(tar_options_list)
make_tar_cmd = '%s -cf - %s -C %s $(cd %s; ls -A)' % \
(which("tar"), tar_options, input_, input_)
if make_tar_cmd:
cmd = "%s | %s -a %s -m /dev/sda1:/ tar-in - /" % \
(make_tar_cmd, which("guestfish"), output_filename)
else:
cmd = "%s -a %s -m /dev/sda1:/ tar-in %s /" % \
(which("guestfish"), output_filename, input_)
proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=True)
proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, cmd)
def create_appliance(args):
"""Convert disk to another format."""
input_ = op.abspath(args.input)
output = op.abspath(args.output)
temp_filename = next(tempfile._get_candidate_names())
temp_file = op.abspath(".%s" % temp_filename)
output_fmt = args.format.lower()
output_filename = "%s.%s" % (output, output_fmt)
os.environ['LIBGUESTFS_CACHEDIR'] = os.getcwd()
if args.verbose:
os.environ['LIBGUESTFS_DEBUG'] = '1'
create_disk(input_,
temp_file,
args.format,
args.size,
args.filesystem,
args.verbose)
logger.info("Installing bootloader")
uuid, _, _ = install_bootloader(temp_file,
args.extlinux_mbr,
args.append)
generate_fstab(temp_file, uuid, args.filesystem)
logger.info("Exporting appliance to %s" % output_filename)
if output_fmt == "qcow2" and args.compress:
qemu_convert(temp_file, output_fmt, output_filename)
os.remove(temp_file) if os.path.exists(temp_file) else None
else:
shutil.move(temp_file, output_filename)