From e5311de35216cc541978c10a8f2cf8204acc7b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Wed, 18 Apr 2018 00:27:06 +0200 Subject: Initial commit Sets up LXC containers for CI for Alpine, Arch Linux, CentOS, Debian and Ubuntu. Currently only tested against Cowlib, it's likely that more packages need to be installed for the other projects. --- alpine.sh | 23 +++++++++++++ archlinux.sh | 25 ++++++++++++++ centos.sh | 25 ++++++++++++++ ci.d/apk | 11 ++++++ ci.d/apt-get | 13 +++++++ ci.d/archlinux | 33 ++++++++++++++++++ ci.d/buildkite | 74 ++++++++++++++++++++++++++++++++++++++++ ci.d/lxc | 44 ++++++++++++++++++++++++ ci.d/ninenines | 8 +++++ ci.d/openssh | 22 ++++++++++++ ci.d/openssl | 12 +++++++ ci.d/pacman | 11 ++++++ ci.d/yum | 12 +++++++ debian.sh | 24 +++++++++++++ priv/buildkite-pre-artifact-hook | 21 ++++++++++++ ubuntu.sh | 23 +++++++++++++ 16 files changed, 381 insertions(+) create mode 100755 alpine.sh create mode 100755 archlinux.sh create mode 100755 centos.sh create mode 100644 ci.d/apk create mode 100644 ci.d/apt-get create mode 100644 ci.d/archlinux create mode 100644 ci.d/buildkite create mode 100644 ci.d/lxc create mode 100644 ci.d/ninenines create mode 100644 ci.d/openssh create mode 100644 ci.d/openssl create mode 100644 ci.d/pacman create mode 100644 ci.d/yum create mode 100755 debian.sh create mode 100644 priv/buildkite-pre-artifact-hook create mode 100755 ubuntu.sh diff --git a/alpine.sh b/alpine.sh new file mode 100755 index 0000000..cdc4976 --- /dev/null +++ b/alpine.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=alpine +RELEASE=edge +ARCH=amd64 +PACKAGES="pcre ca-certificates openssl-dev ncurses-dev zlib-dev \ + openssh bash curl zsh vim sudo erlang git build-base autoconf gawk" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +apk_upgrade +apk_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +lxc_restart diff --git a/archlinux.sh b/archlinux.sh new file mode 100755 index 0000000..2a39155 --- /dev/null +++ b/archlinux.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=archlinux +RELEASE=current +ARCH=amd64 +PACKAGES="cronie openssh openssl-1.0 zsh vim sudo erlang-nox git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +archlinux_fix_network +lxc_wait_for_ip +pacman_upgrade +pacman_install $PACKAGES +openssl_fix_10 +archlinux_enable_cron +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +lxc_restart diff --git a/centos.sh b/centos.sh new file mode 100755 index 0000000..c2ff9a0 --- /dev/null +++ b/centos.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=centos +RELEASE=7 +ARCH=amd64 +# We don't install Erlang because it's complicated on CentOS. +# We will instead use the Erlang versions built with kerl. +PACKAGES="curl ncurses-devel openssl-devel zsh vim sudo git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +yum_upgrade +yum_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +ssh_copy_host_key buildkite-agent /var/lib/buildkite-agent +lxc_restart diff --git a/ci.d/apk b/ci.d/apk new file mode 100644 index 0000000..e622754 --- /dev/null +++ b/ci.d/apk @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +# apk_upgrade +apk_upgrade() { + lxc_do apk upgrade +} + +# apk_install $PACKAGES +apk_install() { + lxc_do apk add "$@" +} diff --git a/ci.d/apt-get b/ci.d/apt-get new file mode 100644 index 0000000..5394f63 --- /dev/null +++ b/ci.d/apt-get @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +# apt_get_upgrade +apt_get_upgrade() { + lxc_do apt-get update + lxc_do apt-get upgrade -y + lxc_do apt-get dist-upgrade -y +} + +# apt_get_install $PACKAGES +apt_get_install() { + lxc_do apt-get install -y "$@" +} diff --git a/ci.d/archlinux b/ci.d/archlinux new file mode 100644 index 0000000..100bc60 --- /dev/null +++ b/ci.d/archlinux @@ -0,0 +1,33 @@ +#!/usr/bin/env sh + +# archlinux_enable_cron +archlinux_enable_cron() { + lxc_do systemctl enable cronie +} + +# archlinux_fix_network +archlinux_fix_network() { + # We ForceConnect because the interface is up at container startup. + lxc_do bash -c \ + 'printf "%s\n" \ + "Interface=eth0" \ + "Connection=ethernet" \ + "IP=dhcp" \ + "ForceConnect=yes" > /etc/netctl/eth0' + sleep 1 + lxc_do netctl enable eth0 + + # Fix a bug in Netctl that prevents starting the service. + lxc_do bash -c ' \ + echo ".include /usr/lib/systemd/system/netctl@.service" \ + > /etc/systemd/system/netctl@eth0.service' + lxc_do systemctl daemon-reload + sleep 1 + + # There's probably a better way to do this. + lxc_do rm /etc/resolv.conf + lxc_do bash -c 'echo "nameserver 10.0.3.1" > /etc/resolv.conf' + + # Should be good! + lxc_do systemctl start netctl@eth0 +} diff --git a/ci.d/buildkite b/ci.d/buildkite new file mode 100644 index 0000000..21988f8 --- /dev/null +++ b/ci.d/buildkite @@ -0,0 +1,74 @@ +#!/usr/bin/env sh + +_buildkite_config() { + BUILDKITE_PATH=/root/.buildkite-agent + BUILDKITE_USER=root + if [ "$1" = 'ubuntu' -o "$1" = 'debian' -o "$1" = 'centos' ] + then + BUILDKITE_PATH=/etc/buildkite-agent + BUILDKITE_USER=buildkite-agent + fi + # Update the configuration. + lxc_do bash -c "echo name=\"$1-%n\" >> $BUILDKITE_PATH/buildkite-agent.cfg" + lxc_do bash -c "echo tags=\"os=$1\" >> $BUILDKITE_PATH/buildkite-agent.cfg" + # Install hooks. + lxc_do bash -c " \ + echo \"export BUILDKITE_API_TOKEN=$BUILDKITE_API_TOKEN\" >> /etc/buildkite-env" + lxc_do bash -c " \ + echo \"export BUILDKITE_LOGS_SERVER=$BUILDKITE_LOGS_SERVER\" >> /etc/buildkite-env" + <$( cd "$( dirname "$0" )" && pwd )/priv/buildkite-pre-artifact-hook \ + lxc_do bash -c "cat > $BUILDKITE_PATH/hooks/pre-artifact" + lxc_do chmod +x $BUILDKITE_PATH/hooks/pre-artifact + lxc_do chown $BUILDKITE_USER:$BUILDKITE_USER $BUILDKITE_PATH/hooks/pre-artifact +} + +_buildkite_install_centos() { + lxc_do bash -c 'echo -e "[buildkite-agent]\nname = Buildkite Pty Ltd\nbaseurl = https://yum.buildkite.com/buildkite-agent/stable/x86_64/\nenabled=1\ngpgcheck=0\npriority=1" > /etc/yum.repos.d/buildkite-agent.repo' + lxc_do yum install -y buildkite-agent + lxc_do sed -i "s/xxx/$2/g" /etc/buildkite-agent/buildkite-agent.cfg + lxc_do systemctl enable buildkite-agent +} + +_buildkite_install_generic() { + lxc_do curl -sLO https://raw.githubusercontent.com/buildkite/agent/master/install.sh + lxc_do bash -c "TOKEN=\"$2\" bash install.sh" + lxc_do rm install.sh + lxc_do bash -c 'echo "@reboot /root/.buildkite-agent/bin/buildkite-agent start &" >> tmpcron' + lxc_do crontab tmpcron + lxc_do rm tmpcron +} + +_buildkite_install_ubuntu() { + lxc_do bash -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main \ + > /etc/apt/sources.list.d/buildkite-agent.list' + lxc_do apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \ + --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 + lxc_do apt-get update + lxc_do apt-get install -y buildkite-agent + lxc_do sed -i "s/xxx/$2/g" /etc/buildkite-agent/buildkite-agent.cfg + lxc_do systemctl enable buildkite-agent +} + +# buildkite_install dist token +buildkite_install() { + if [ "$1" = 'ubuntu' -o "$1" = 'debian' ] + then + _buildkite_install_ubuntu "$@" + elif [ "$1" = 'centos' ] + then + _buildkite_install_centos "$@" + else + _buildkite_install_generic "$@" + fi + _buildkite_config $1 +} + +# buildkite_stop dist +buildkite_stop() { + if [ "$1" = 'ubuntu' -o "$1" = 'debian' ] + then + lxc_do systemctl stop buildkite-agent || true + else + lxc_do killall -q buildkite-agent || true + fi +} diff --git a/ci.d/lxc b/ci.d/lxc new file mode 100644 index 0000000..c0c5ed2 --- /dev/null +++ b/ci.d/lxc @@ -0,0 +1,44 @@ +#!/usr/bin/env sh + +# $NAME must be set to the name of the container. +if [ -z "$NAME" ]; then + echo 'Error: $NAME is empty or not set.' 1>&2 + exit 1 +fi + +# lxc_create dist release arch +lxc_create() { + lxc-create -n $NAME -t download -B btrfs -- --dist $1 --release $2 --arch $3 + echo "lxc.start.auto = 1" >> /home/lxc/$NAME/config + echo "lxc.environment = HOME=/root" >> /home/lxc/$NAME/config + lxc-start -n $NAME + lxc-wait -n $NAME -s RUNNING +} + +# lxc_destroy +lxc_destroy() { + lxc-stop -n $NAME -k || true + lxc-destroy -n $NAME || true +} + +# lxc_do +lxc_do() { + lxc-attach -n $NAME --clear-env -- "$@" +} + +# lxc_restart +lxc_restart() { + lxc-stop -n $NAME + lxc-start -n $NAME + lxc_wait_for_ip + lxc-ls -f $NAME +} + +# lxc_wait_for_ip +lxc_wait_for_ip() { + until lxc-info -n $NAME -i | grep IP: + do + echo -n . + sleep 1 + done +} diff --git a/ci.d/ninenines b/ci.d/ninenines new file mode 100644 index 0000000..f78bce3 --- /dev/null +++ b/ci.d/ninenines @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +# ci_cowlib +ci_cowlib() { + lxc_do mkdir ninenines + lxc_do git clone https://git.ninenines.eu/cowlib.git ninenines/cowlib + lxc_do make -C ninenines/cowlib ci CI_ERLANG_MK=1 +} diff --git a/ci.d/openssh b/ci.d/openssh new file mode 100644 index 0000000..4abeb02 --- /dev/null +++ b/ci.d/openssh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh + +# ssh_copy_host_key +# ssh_copy_host_key user home +ssh_copy_host_key() { + USER=root + USER_HOME=/root + if [ -n "$2" ] + then + USER=$1 + USER_HOME=$2 + fi + lxc_do mkdir -p $USER_HOME/.ssh + lxc_do chmod 700 $USER_HOME/.ssh + lxc_do chown $USER:$USER $USER_HOME/.ssh + $USER_HOME/.ssh/id_ecdsa" + lxc_do chmod 600 $USER_HOME/.ssh/id_ecdsa + lxc_do chown $USER:$USER $USER_HOME/.ssh/id_ecdsa + $USER_HOME/.ssh/known_hosts" + lxc_do chmod 600 $USER_HOME/.ssh/known_hosts + lxc_do chown $USER:$USER $USER_HOME/.ssh/known_hosts +} diff --git a/ci.d/openssl b/ci.d/openssl new file mode 100644 index 0000000..0165cc1 --- /dev/null +++ b/ci.d/openssl @@ -0,0 +1,12 @@ +#!/usr/bin/env sh + +# openssl_fix_10 +# +# Setup OpenSSL 1.0 for older Erlang versions. +openssl_fix_10() { + lxc_do mkdir /root/openssl-1.0 + lxc_do ln -s /usr/include/openssl-1.0 /root/openssl-1.0/include + lxc_do ln -s /usr/lib/openssl-1.0 /root/openssl-1.0/lib + echo "lxc.environment = KERL_CONFIGURE_OPTIONS=--with-ssl=/root/openssl-1.0" \ + >> /home/lxc/$NAME/config +} diff --git a/ci.d/pacman b/ci.d/pacman new file mode 100644 index 0000000..6e76699 --- /dev/null +++ b/ci.d/pacman @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +# pacman_upgrade +pacman_upgrade() { + lxc_do pacman --noconfirm -Syu +} + +# pacman_install $PACKAGES +pacman_install() { + lxc_do pacman --noconfirm -S "$@" +} diff --git a/ci.d/yum b/ci.d/yum new file mode 100644 index 0000000..27607c7 --- /dev/null +++ b/ci.d/yum @@ -0,0 +1,12 @@ +#!/usr/bin/env sh + +# yum_upgrade +yum_upgrade() { + lxc_do yum update +} + +# yum_install $PACKAGES +yum_install() { + lxc_do yum install -y "$@" +} + diff --git a/debian.sh b/debian.sh new file mode 100755 index 0000000..6c920de --- /dev/null +++ b/debian.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=debian +RELEASE=stretch +ARCH=amd64 +PACKAGES="gnupg apt-transport-https curl libssl1.0-dev libncurses5-dev \ + zsh vim sudo erlang-nox git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +apt_get_upgrade +apt_get_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +ssh_copy_host_key buildkite-agent /var/lib/buildkite-agent +lxc_restart diff --git a/priv/buildkite-pre-artifact-hook b/priv/buildkite-pre-artifact-hook new file mode 100644 index 0000000..ff76fdd --- /dev/null +++ b/priv/buildkite-pre-artifact-hook @@ -0,0 +1,21 @@ +#!/bin/bash + +source /etc/buildkite-env + +LOGS_PATH=logs/$BUILDKITE_PIPELINE_SLUG/$BUILDKITE_BUILD_NUMBER/$BUILDKITE_AGENT_META_DATA_OS/ +ssh buildkite@$BUILDKITE_LOGS_SERVER mkdir -p $LOGS_PATH + +# Terminal output. +curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/jobs/$BUILDKITE_JOB_ID/log" -H "Accept: text/plain" -o output.txt +scp -p output.txt buildkite@$BUILDKITE_LOGS_SERVER:$LOGS_PATH +rm -f output.txt + +# Common Test logs. +test -d logs/ && \ + scp -rp logs/* buildkite@$BUILDKITE_LOGS_SERVER:$LOGS_PATH + +# Erlang.mk packages. +test -f test/packages/errors.log && \ + scp -p test/packages/errors.log buildkite@$BUILDKITE_LOGS_SERVER:$LOGS_PATH + +echo "" > logs.html diff --git a/ubuntu.sh b/ubuntu.sh new file mode 100755 index 0000000..5328113 --- /dev/null +++ b/ubuntu.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=ubuntu +RELEASE=artful +ARCH=amd64 +PACKAGES="curl libssl-dev libncurses5-dev zsh vim sudo erlang-nox git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +apt_get_upgrade +apt_get_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +ssh_copy_host_key buildkite-agent /var/lib/buildkite-agent +lxc_restart -- cgit v1.2.3