From de33dbc497077ac3c9330a9ebfa2158702a996de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Tue, 22 May 2018 20:34:21 +0200 Subject: [PATCH] Add Vagrant VMs & combine it with gitlab-ci - add Vagrantfile to contrib/vms - add custom provisioning via shell scripts - move gitlab-ci before_script: to ci/install_dependencies.sh which is also used by vagrant - change COPR setup on CentOS 7 (yum-plugin-copr does not seem to work) --- .gitignore | 1 + .gitlab-ci.yml | 28 +---------- ci/debian_build_gtest.sh | 13 ----- ci/install_dependencies.sh | 69 +++++++++++++++++++++++++++ contrib/vms/README.md | 97 ++++++++++++++++++++++++++++++++++++++ contrib/vms/Vagrantfile | 53 +++++++++++++++++++++ contrib/vms/setup.sh | 36 ++++++++++++++ contrib/vms/setup_user.sh | 46 ++++++++++++++++++ contrib/vms/utils.source | 21 +++++++++ 9 files changed, 325 insertions(+), 39 deletions(-) delete mode 100755 ci/debian_build_gtest.sh create mode 100755 ci/install_dependencies.sh create mode 100644 contrib/vms/README.md create mode 100644 contrib/vms/Vagrantfile create mode 100644 contrib/vms/setup.sh create mode 100644 contrib/vms/setup_user.sh create mode 100644 contrib/vms/utils.source diff --git a/.gitignore b/.gitignore index a7b0261e..c82efefc 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,4 @@ website/var/__xmp_xmpMM__ website/var/__xmp_xmpRights__ website/var/__xmp_xmpTPg__ website/var/__xmp_xmp__ +contrib/vms/.vagrant diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 466db950..5594ab37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,8 @@ # only create artifacts of the build directory when something fails (for cmake logs) # cache the ccache/ directory for each job separately .build_template: &distro_build + before_script: + - ci/install_dependencies.sh script: - python3 ci/test_build.py artifacts: @@ -15,50 +17,24 @@ Fedora: image: fedora:28 - before_script: - - dnf -y --refresh install gcc-c++ clang cmake make ccache expat-devel zlib-devel libssh-devel libcurl-devel gtest-devel which dos2unix <<: *distro_build Debian: image: debian:9 - before_script: - - apt-get update - - apt-get install -y cmake g++ clang make ccache python3 libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev libxml2-utils - - ./ci/debian_build_gtest.sh <<: *distro_build Archlinux: image: base/archlinux - before_script: - - pacman --noconfirm -Sy - - pacman --noconfirm -S gcc clang cmake make ccache expat zlib libssh curl gtest python dos2unix <<: *distro_build Ubuntu: image: ubuntu:18.04 - before_script: - - apt-get update - - apt-get install -y cmake g++ clang make ccache python3 libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev google-mock libxml2-utils - - ./ci/debian_build_gtest.sh <<: *distro_build CentOS: image: centos:7 - before_script: - - yum -y install yum-plugin-copr epel-release - # enable copr for gtest - - yum -y copr enable defolos/devel - - yum clean all - - yum -y install gcc-c++ clang cmake3 make ccache expat-devel zlib-devel libssh-devel libcurl-devel gtest-devel which python36 dos2unix - # symlink up to date versions of python & cmake to 'default' names - - ln -s /usr/bin/python36 /usr/bin/python3 - - mv /bin/cmake /bin/.cmake.old - - ln -s /bin/cmake3 /bin/cmake <<: *distro_build OpenSUSE: image: opensuse:tumbleweed - before_script: - - zypper --non-interactive refresh - - zypper --non-interactive install gcc-c++ clang cmake make ccache libexpat-devel zlib-devel libssh-devel libcurl-devel gtest which dos2unix libxml2-tools <<: *distro_build diff --git a/ci/debian_build_gtest.sh b/ci/debian_build_gtest.sh deleted file mode 100755 index a69a6e5b..00000000 --- a/ci/debian_build_gtest.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Debian & derivatives don't provide binary packages of googletest -# => have to build them ourselves -# -# This script builds a shared library of googletest (not googlemock!) and copies -# it to usr/lib/ - -mkdir gtest_build && cd gtest_build -cmake -DBUILD_SHARED_LIBS=1 /usr/src/googletest/googletest -make -cp libgtest* /usr/lib/ -cd .. diff --git a/ci/install_dependencies.sh b/ci/install_dependencies.sh new file mode 100755 index 00000000..c75a8587 --- /dev/null +++ b/ci/install_dependencies.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -e + +# this script expects one parameter, which is the path to utils.source +# if the parameter is omitted, it defaults to the relative path in the +# repository, but it can be provided manually (e.g. for vagrant VMs) + +if [ $# -eq 0 ]; then + source $(dirname "${BASH_SOURCE[0]}")/../contrib/vms/utils.source +elif [ $# -gt 1 ]; then + cat << EOF +usage: install_dependencies.sh [path/to/utils.source] + +Installs the dependencies required to build & test exiv2 on some Linux +distributions. +The optional parameter is the path to the utils.source file, which can be set to +an alternative location (currently used for vagrant builds) +EOF + exit 1 +else + source "$1" +fi + +distro_id=$(get_distro_id) + +case "$distro_id" in + 'fedora') + dnf -y --refresh install gcc-c++ clang cmake make ccache expat-devel zlib-devel libssh-devel libcurl-devel gtest-devel which dos2unix + ;; + + 'debian') + apt-get update + apt-get install -y cmake g++ clang make ccache python3 libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev libxml2-utils + debian_build_gtest + ;; + + 'arch') + pacman --noconfirm -Sy + pacman --noconfirm -S gcc clang cmake make ccache expat zlib libssh curl gtest python dos2unix + ;; + + 'ubuntu') + apt-get update + apt-get install -y cmake g++ clang make ccache python3 libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev google-mock libxml2-utils + debian_build_gtest + ;; + + 'centos'|'rhel') + yum -y install epel-release + # enable copr for gtest + curl https://copr.fedorainfracloud.org/coprs/defolos/devel/repo/epel-7/defolos-devel-epel-7.repo > /etc/yum.repos.d/_copr_defolos-devel.repo + yum clean all + yum -y install gcc-c++ clang cmake3 make ccache expat-devel zlib-devel libssh-devel libcurl-devel gtest-devel which python36 dos2unix + # symlink up to date versions of python & cmake to 'default' names + ln -s /usr/bin/python36 /usr/bin/python3 + mv /bin/cmake /bin/.cmake.old + ln -s /bin/cmake3 /bin/cmake + ;; + + 'opensuse'|'opensuse-tumbleweed') + zypper --non-interactive refresh + zypper --non-interactive install gcc-c++ clang cmake make ccache libexpat-devel zlib-devel libssh-devel libcurl-devel gtest which dos2unix libxml2-tools + ;; + *) + echo "Sorry, no predefined dependencies for your distribution $distro_id exist yet" + exit 1 + ;; +esac diff --git a/contrib/vms/README.md b/contrib/vms/README.md new file mode 100644 index 00000000..d1fad553 --- /dev/null +++ b/contrib/vms/README.md @@ -0,0 +1,97 @@ +# Vagrant development boxes + +This directory contains a `Vagrantfile` which can be used to automatically +create virtual machines for testing purposes. The virtual machines are +automatically provisioned with all required dependencies for building & testing +of exiv2 (the provisioning is shared with the GitLab CI). + +The following Linux distributions are provided (the name in the brackets is the +name of the Vagrant VM): +- Fedora 28 ("Fedora") +- Debian 9 aka Stretch ("Debian") +- Archlinux ("Archlinux") +- Ubuntu 16.04 aka Bionic Beaver ("Ubuntu") +- CentOS 7 ("CentOS") +- OpenSUSE Tumbleweed ("OpenSUSE") + +The Fedora, Archlinux and OpenSUSE boxes are the 'vanilla' distribution with +some additional packages installed. + +For Debian and Ubuntu, we build gtest manually from source and install the +resulting library to /usr/lib/. + +On CentOS, we have to install a `cmake3` and `python36` (the default cmake is +too old and a default python3 does not exist) which we symlink to +`/usr/bin/cmake` & `/usr/bin/python3` to retain a similar workflow to the other +distributions. + +For further details, consult the shell scripts `setup.sh` and +`ci/install_dependencies.sh`. + + +All boxes come with `conan` installed via pip in the `vagrant` user's home +directory and the `exiv2` git repository cloned. + + +Please note that these VMs are not continuously tested and the provisioning can +break. Please open an issue on GitHub if you happen to encounter a problem. + + +## Usage + +Please install [Vagrant](https://www.vagrantup.com/) and a supported provider +(e.g. libvirt, VirtualBox). + +Choose a box from the above list and run in the directory where the +`Vagrantfile` resides: +``` shell +vagrant up $name +``` +where `$name` is the name in the brackets in the above list, e.g. `OpenSUSE` or +`Archlinux`. Depending on your default provider you may have to set the provider +manually via `vagrant up $name --provider $provider_name` (the Ubuntu image does +only support VirtualBox, which is not the default on Linux and will result in an +error unless you explicitly set the provider to `virtualbox`). + +This will download a box from the vagrantcloud and set it up. Once the whole +process is finished, you can ssh into the machine via: + +``` shell +vagrant ssh $name +``` + +Don't forget to turn it off via `vagrant halt $name` or the VM will keep +running! A VM can be discarded when it is no longer required via `vagrant +destroy $name` (Vagrant will keep the base box around in `~/.vagrant.d/boxes` +and libvirt sometimes leaves images around in `/var/lib/libvirt/` or +`/var/libvirt`, so check these folders too). + + +You can also setup & start all VMs at once via `vagrant up`, but keep in mind +that it will start 6 VMs and occupy between 10 and 20 GB of disk space. + + +# Notes for OpenSUSE Tumbleweed + +Unfortunately the OpenSUSE Tumbleweed box cannot be provisioned easily with +Vagrant as it must perform a system upgrade first, which cannot be done +non-interactively. To get the OpenSUSE box up and running, follow these steps: + +``` shell +$ vagrant up OpenSUSE +# you'll get a failure in the first provisioning script +$ vagrant ssh OpenSUSE +vagrant@opensuse-exiv2:~> su - # the root password is vagrant +Password: +opensuse-exiv2:~ # zypper refresh +opensuse-exiv2:~ # zypper dup +# zypper will now perform a system upgrade +# you'll probably get a few file conflicts, confirm the overwrite with 'yes' +# once the upgrade is done, exit the ssh session +$ vagrant halt OpenSUSE +$ vagrant up OpenSUSE +$ vagrant provision OpenSUSE +``` + +Provided the system upgrade went fine, you should now have an OpenSUSE +Tumbleweed virtual machine ready to go. diff --git a/contrib/vms/Vagrantfile b/contrib/vms/Vagrantfile new file mode 100644 index 00000000..9a438824 --- /dev/null +++ b/contrib/vms/Vagrantfile @@ -0,0 +1,53 @@ +Vagrant.configure("2") do |config| + + config.vm.define "Fedora" do |fedora| + fedora.vm.box = "fedora/28-cloud-base" + fedora.vm.hostname = "Fedora-exiv2" + end + + config.vm.define "Debian" do |debian| + debian.vm.box = "generic/debian9" + debian.vm.hostname = "debian-exiv2" + end + + config.vm.define "Archlinux" do |archlinux| + archlinux.vm.box = "archlinux/archlinux" + archlinux.vm.hostname = "archlinux-exiv2" + end + + config.vm.define "Ubuntu" do |ubuntu| + ubuntu.vm.box = "ubuntu/bionic64" + ubuntu.vm.hostname = "ubuntu-exiv2" + end + + config.vm.define "CentOS" do |centos| + centos.vm.box = "centos/7" + centos.vm.hostname = "centos-exiv2" + end + + config.vm.define "OpenSUSE" do |opensuse| + opensuse.vm.box = "opensuse/openSUSE-Tumbleweed-x86_64" + opensuse.vm.hostname = "opensuse-exiv2" + end + + config.vm.synced_folder ".", "/vagrant", owner: "vagrant", group: "vagrant", + disabled: false, type: "rsync" + + # use the CI script from gitlab to setup all dependencies + config.vm.provision "install_dependencies", type: "shell" do |shell| + shell.path = "../../ci/install_dependencies.sh" + shell.args = "/vagrant/utils.source" + end + + # install additional dependencies for development + config.vm.provision "install_devel_dependencies", type: "shell" do |shell| + shell.path = "setup.sh" + end + + # install conan & clone the exiv2 repo + config.vm.provision "setup_repository", type: "shell" do |shell| + shell.path = "setup_user.sh" + shell.privileged = false + end + +end diff --git a/contrib/vms/setup.sh b/contrib/vms/setup.sh new file mode 100644 index 00000000..6ba07ed2 --- /dev/null +++ b/contrib/vms/setup.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +source /vagrant/utils.source + +distro_id=$(get_distro_id) + +case "$distro_id" in + 'fedora') + dnf -y --refresh install python3-pip git + ;; + + 'debian' | 'ubuntu') + apt-get install -y python3-pip git + ;; + + 'arch') + pacman --noconfirm -S python-pip git + ;; + + 'centos' | 'rhel') + yum -y install centos-release-scl-rh + yum clean all + yum -y install rh-python36-python-pip git + ;; + + 'opensuse' | 'opensuse-tumbleweed') + zypper --non-interactive install python3-pip git + ;; + + *) + echo "Sorry, no predefined dependencies for your distribution exist yet" + exit 1 + ;; +esac diff --git a/contrib/vms/setup_user.sh b/contrib/vms/setup_user.sh new file mode 100644 index 00000000..f8af8630 --- /dev/null +++ b/contrib/vms/setup_user.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e + +source /vagrant/utils.source + +function clone_exiv2() { + git clone https://github.com/Exiv2/exiv2.git + + cd exiv2 + sed -i '/fetch = +refs\/heads\/\*:refs\/remotes\/origin\//a \ \ \ \ \ \ \ \ fetch = +refs\/pull\/\*\/head:refs\/remotes\/origin\/pr\/*' .git/config + cd .. +} + +distro_id=$(get_distro_id) + +case "$distro_id" in + 'debian' | 'ubuntu' | 'fedora' | 'opensuse' | 'opensuse-tumbleweed') + PIP=pip3 + ;; + + 'arch') + PIP=pip + ;; + + 'centos' | 'rhel') + PIP=/opt/rh/rh-python36/root/usr/bin/pip3 + ;; + + *) + echo "Sorry, no predefined dependencies for your distribution exist yet" + exit 1 + ;; +esac + +$PIP install conan --user --upgrade + +CONAN_PROFILE=~/.conan/profiles/default + +# create a new conan profile & set the used libstdc++ to use the C++11 ABI +[ -e $CONAN_PROFILE ] || ~/.local/bin/conan profile new --detect default +sed -i 's/compiler.libcxx=libstdc++/compiler.libcxx=libstdc++11/' $CONAN_PROFILE + +[ -d exiv2 ] || clone_exiv2 + +cd exiv2 && git fetch && cd .. diff --git a/contrib/vms/utils.source b/contrib/vms/utils.source new file mode 100644 index 00000000..ddeef60d --- /dev/null +++ b/contrib/vms/utils.source @@ -0,0 +1,21 @@ +#!/bin/sh + +function get_distro_id() { + local distro_id=$(grep '^ID=' /etc/os-release|awk -F = '{print $2}'|sed 's/\"//g') + echo "$distro_id" +} + +# Debian & derivatives don't provide binary packages of googletest +# => have to build them ourselves +# +# This script builds a shared library of googletest (not googlemock!) inside +# gtest_build and copies it to /usr/lib/ +function debian_build_gtest() { + pushd . + [ -d gtest_build ] || mkdir gtest_build + cd gtest_build + cmake -DBUILD_SHARED_LIBS=1 /usr/src/googletest/googletest + make + cp libgtest* /usr/lib/ + popd +}