diff options
-rwxr-xr-x | alpine.sh | 23 | ||||
-rwxr-xr-x | archlinux.sh | 25 | ||||
-rwxr-xr-x | centos.sh | 25 | ||||
-rw-r--r-- | ci.d/apk | 11 | ||||
-rw-r--r-- | ci.d/apt-get | 13 | ||||
-rw-r--r-- | ci.d/archlinux | 33 | ||||
-rw-r--r-- | ci.d/buildkite | 74 | ||||
-rw-r--r-- | ci.d/lxc | 44 | ||||
-rw-r--r-- | ci.d/ninenines | 8 | ||||
-rw-r--r-- | ci.d/openssh | 22 | ||||
-rw-r--r-- | ci.d/openssl | 12 | ||||
-rw-r--r-- | ci.d/pacman | 11 | ||||
-rw-r--r-- | ci.d/yum | 12 | ||||
-rwxr-xr-x | debian.sh | 24 | ||||
-rw-r--r-- | priv/buildkite-pre-artifact-hook | 21 | ||||
-rwxr-xr-x | ubuntu.sh | 23 |
16 files changed, 381 insertions, 0 deletions
diff --git a/alpine.sh b/alpine.sh new file mode 100755 index 0000000..cdc4976 --- /dev/null +++ b/alpine.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=alpine +RELEASE=edge +ARCH=amd64 +PACKAGES="pcre ca-certificates openssl-dev ncurses-dev zlib-dev \ + openssh bash curl zsh vim sudo erlang git build-base autoconf gawk" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +apk_upgrade +apk_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +lxc_restart diff --git a/archlinux.sh b/archlinux.sh new file mode 100755 index 0000000..2a39155 --- /dev/null +++ b/archlinux.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=archlinux +RELEASE=current +ARCH=amd64 +PACKAGES="cronie openssh openssl-1.0 zsh vim sudo erlang-nox git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +archlinux_fix_network +lxc_wait_for_ip +pacman_upgrade +pacman_install $PACKAGES +openssl_fix_10 +archlinux_enable_cron +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +lxc_restart diff --git a/centos.sh b/centos.sh new file mode 100755 index 0000000..c2ff9a0 --- /dev/null +++ b/centos.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=centos +RELEASE=7 +ARCH=amd64 +# We don't install Erlang because it's complicated on CentOS. +# We will instead use the Erlang versions built with kerl. +PACKAGES="curl ncurses-devel openssl-devel zsh vim sudo git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +yum_upgrade +yum_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +ssh_copy_host_key buildkite-agent /var/lib/buildkite-agent +lxc_restart diff --git a/ci.d/apk b/ci.d/apk new file mode 100644 index 0000000..e622754 --- /dev/null +++ b/ci.d/apk @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +# apk_upgrade +apk_upgrade() { + lxc_do apk upgrade +} + +# apk_install $PACKAGES +apk_install() { + lxc_do apk add "$@" +} diff --git a/ci.d/apt-get b/ci.d/apt-get new file mode 100644 index 0000000..5394f63 --- /dev/null +++ b/ci.d/apt-get @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +# apt_get_upgrade +apt_get_upgrade() { + lxc_do apt-get update + lxc_do apt-get upgrade -y + lxc_do apt-get dist-upgrade -y +} + +# apt_get_install $PACKAGES +apt_get_install() { + lxc_do apt-get install -y "$@" +} diff --git a/ci.d/archlinux b/ci.d/archlinux new file mode 100644 index 0000000..100bc60 --- /dev/null +++ b/ci.d/archlinux @@ -0,0 +1,33 @@ +#!/usr/bin/env sh + +# archlinux_enable_cron +archlinux_enable_cron() { + lxc_do systemctl enable cronie +} + +# archlinux_fix_network +archlinux_fix_network() { + # We ForceConnect because the interface is up at container startup. + lxc_do bash -c \ + 'printf "%s\n" \ + "Interface=eth0" \ + "Connection=ethernet" \ + "IP=dhcp" \ + "ForceConnect=yes" > /etc/netctl/eth0' + sleep 1 + lxc_do netctl enable eth0 + + # Fix a bug in Netctl that prevents starting the service. + lxc_do bash -c ' \ + echo ".include /usr/lib/systemd/system/[email protected]" \ + > /etc/systemd/system/[email protected]' + lxc_do systemctl daemon-reload + sleep 1 + + # There's probably a better way to do this. + lxc_do rm /etc/resolv.conf + lxc_do bash -c 'echo "nameserver 10.0.3.1" > /etc/resolv.conf' + + # Should be good! + lxc_do systemctl start netctl@eth0 +} diff --git a/ci.d/buildkite b/ci.d/buildkite new file mode 100644 index 0000000..21988f8 --- /dev/null +++ b/ci.d/buildkite @@ -0,0 +1,74 @@ +#!/usr/bin/env sh + +_buildkite_config() { + BUILDKITE_PATH=/root/.buildkite-agent + BUILDKITE_USER=root + if [ "$1" = 'ubuntu' -o "$1" = 'debian' -o "$1" = 'centos' ] + then + BUILDKITE_PATH=/etc/buildkite-agent + BUILDKITE_USER=buildkite-agent + fi + # Update the configuration. + lxc_do bash -c "echo name=\"$1-%n\" >> $BUILDKITE_PATH/buildkite-agent.cfg" + lxc_do bash -c "echo tags=\"os=$1\" >> $BUILDKITE_PATH/buildkite-agent.cfg" + # Install hooks. + lxc_do bash -c " \ + echo \"export BUILDKITE_API_TOKEN=$BUILDKITE_API_TOKEN\" >> /etc/buildkite-env" + lxc_do bash -c " \ + echo \"export BUILDKITE_LOGS_SERVER=$BUILDKITE_LOGS_SERVER\" >> /etc/buildkite-env" + <$( cd "$( dirname "$0" )" && pwd )/priv/buildkite-pre-artifact-hook \ + lxc_do bash -c "cat > $BUILDKITE_PATH/hooks/pre-artifact" + lxc_do chmod +x $BUILDKITE_PATH/hooks/pre-artifact + lxc_do chown $BUILDKITE_USER:$BUILDKITE_USER $BUILDKITE_PATH/hooks/pre-artifact +} + +_buildkite_install_centos() { + lxc_do bash -c 'echo -e "[buildkite-agent]\nname = Buildkite Pty Ltd\nbaseurl = https://yum.buildkite.com/buildkite-agent/stable/x86_64/\nenabled=1\ngpgcheck=0\npriority=1" > /etc/yum.repos.d/buildkite-agent.repo' + lxc_do yum install -y buildkite-agent + lxc_do sed -i "s/xxx/$2/g" /etc/buildkite-agent/buildkite-agent.cfg + lxc_do systemctl enable buildkite-agent +} + +_buildkite_install_generic() { + lxc_do curl -sLO https://raw.githubusercontent.com/buildkite/agent/master/install.sh + lxc_do bash -c "TOKEN=\"$2\" bash install.sh" + lxc_do rm install.sh + lxc_do bash -c 'echo "@reboot /root/.buildkite-agent/bin/buildkite-agent start &" >> tmpcron' + lxc_do crontab tmpcron + lxc_do rm tmpcron +} + +_buildkite_install_ubuntu() { + lxc_do bash -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main \ + > /etc/apt/sources.list.d/buildkite-agent.list' + lxc_do apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \ + --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 + lxc_do apt-get update + lxc_do apt-get install -y buildkite-agent + lxc_do sed -i "s/xxx/$2/g" /etc/buildkite-agent/buildkite-agent.cfg + lxc_do systemctl enable buildkite-agent +} + +# buildkite_install dist token +buildkite_install() { + if [ "$1" = 'ubuntu' -o "$1" = 'debian' ] + then + _buildkite_install_ubuntu "$@" + elif [ "$1" = 'centos' ] + then + _buildkite_install_centos "$@" + else + _buildkite_install_generic "$@" + fi + _buildkite_config $1 +} + +# buildkite_stop dist +buildkite_stop() { + if [ "$1" = 'ubuntu' -o "$1" = 'debian' ] + then + lxc_do systemctl stop buildkite-agent || true + else + lxc_do killall -q buildkite-agent || true + fi +} diff --git a/ci.d/lxc b/ci.d/lxc new file mode 100644 index 0000000..c0c5ed2 --- /dev/null +++ b/ci.d/lxc @@ -0,0 +1,44 @@ +#!/usr/bin/env sh + +# $NAME must be set to the name of the container. +if [ -z "$NAME" ]; then + echo 'Error: $NAME is empty or not set.' 1>&2 + exit 1 +fi + +# lxc_create dist release arch +lxc_create() { + lxc-create -n $NAME -t download -B btrfs -- --dist $1 --release $2 --arch $3 + echo "lxc.start.auto = 1" >> /home/lxc/$NAME/config + echo "lxc.environment = HOME=/root" >> /home/lxc/$NAME/config + lxc-start -n $NAME + lxc-wait -n $NAME -s RUNNING +} + +# lxc_destroy +lxc_destroy() { + lxc-stop -n $NAME -k || true + lxc-destroy -n $NAME || true +} + +# lxc_do +lxc_do() { + lxc-attach -n $NAME --clear-env -- "$@" +} + +# lxc_restart +lxc_restart() { + lxc-stop -n $NAME + lxc-start -n $NAME + lxc_wait_for_ip + lxc-ls -f $NAME +} + +# lxc_wait_for_ip +lxc_wait_for_ip() { + until lxc-info -n $NAME -i | grep IP: + do + echo -n . + sleep 1 + done +} diff --git a/ci.d/ninenines b/ci.d/ninenines new file mode 100644 index 0000000..f78bce3 --- /dev/null +++ b/ci.d/ninenines @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +# ci_cowlib +ci_cowlib() { + lxc_do mkdir ninenines + lxc_do git clone https://git.ninenines.eu/cowlib.git ninenines/cowlib + lxc_do make -C ninenines/cowlib ci CI_ERLANG_MK=1 +} diff --git a/ci.d/openssh b/ci.d/openssh new file mode 100644 index 0000000..4abeb02 --- /dev/null +++ b/ci.d/openssh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh + +# ssh_copy_host_key +# ssh_copy_host_key user home +ssh_copy_host_key() { + USER=root + USER_HOME=/root + if [ -n "$2" ] + then + USER=$1 + USER_HOME=$2 + fi + lxc_do mkdir -p $USER_HOME/.ssh + lxc_do chmod 700 $USER_HOME/.ssh + lxc_do chown $USER:$USER $USER_HOME/.ssh + </root/.ssh/id_ecdsa lxc_do bash -c "cat > $USER_HOME/.ssh/id_ecdsa" + lxc_do chmod 600 $USER_HOME/.ssh/id_ecdsa + lxc_do chown $USER:$USER $USER_HOME/.ssh/id_ecdsa + </root/.ssh/known_hosts lxc_do bash -c "cat > $USER_HOME/.ssh/known_hosts" + lxc_do chmod 600 $USER_HOME/.ssh/known_hosts + lxc_do chown $USER:$USER $USER_HOME/.ssh/known_hosts +} diff --git a/ci.d/openssl b/ci.d/openssl new file mode 100644 index 0000000..0165cc1 --- /dev/null +++ b/ci.d/openssl @@ -0,0 +1,12 @@ +#!/usr/bin/env sh + +# openssl_fix_10 +# +# Setup OpenSSL 1.0 for older Erlang versions. +openssl_fix_10() { + lxc_do mkdir /root/openssl-1.0 + lxc_do ln -s /usr/include/openssl-1.0 /root/openssl-1.0/include + lxc_do ln -s /usr/lib/openssl-1.0 /root/openssl-1.0/lib + echo "lxc.environment = KERL_CONFIGURE_OPTIONS=--with-ssl=/root/openssl-1.0" \ + >> /home/lxc/$NAME/config +} diff --git a/ci.d/pacman b/ci.d/pacman new file mode 100644 index 0000000..6e76699 --- /dev/null +++ b/ci.d/pacman @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +# pacman_upgrade +pacman_upgrade() { + lxc_do pacman --noconfirm -Syu +} + +# pacman_install $PACKAGES +pacman_install() { + lxc_do pacman --noconfirm -S "$@" +} diff --git a/ci.d/yum b/ci.d/yum new file mode 100644 index 0000000..27607c7 --- /dev/null +++ b/ci.d/yum @@ -0,0 +1,12 @@ +#!/usr/bin/env sh + +# yum_upgrade +yum_upgrade() { + lxc_do yum update +} + +# yum_install $PACKAGES +yum_install() { + lxc_do yum install -y "$@" +} + diff --git a/debian.sh b/debian.sh new file mode 100755 index 0000000..6c920de --- /dev/null +++ b/debian.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=debian +RELEASE=stretch +ARCH=amd64 +PACKAGES="gnupg apt-transport-https curl libssl1.0-dev libncurses5-dev \ + zsh vim sudo erlang-nox git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +apt_get_upgrade +apt_get_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +ssh_copy_host_key buildkite-agent /var/lib/buildkite-agent +lxc_restart diff --git a/priv/buildkite-pre-artifact-hook b/priv/buildkite-pre-artifact-hook new file mode 100644 index 0000000..ff76fdd --- /dev/null +++ b/priv/buildkite-pre-artifact-hook @@ -0,0 +1,21 @@ +#!/bin/bash + +source /etc/buildkite-env + +LOGS_PATH=logs/$BUILDKITE_PIPELINE_SLUG/$BUILDKITE_BUILD_NUMBER/$BUILDKITE_AGENT_META_DATA_OS/ +ssh buildkite@$BUILDKITE_LOGS_SERVER mkdir -p $LOGS_PATH + +# Terminal output. +curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/jobs/$BUILDKITE_JOB_ID/log" -H "Accept: text/plain" -o output.txt +scp -p output.txt buildkite@$BUILDKITE_LOGS_SERVER:$LOGS_PATH +rm -f output.txt + +# Common Test logs. +test -d logs/ && \ + scp -rp logs/* buildkite@$BUILDKITE_LOGS_SERVER:$LOGS_PATH + +# Erlang.mk packages. +test -f test/packages/errors.log && \ + scp -p test/packages/errors.log buildkite@$BUILDKITE_LOGS_SERVER:$LOGS_PATH + +echo "<html><head><meta http-equiv='refresh' content='0;URL=https://builds.ninenines.eu/$LOGS_PATH'/></head></html>" > logs.html diff --git a/ubuntu.sh b/ubuntu.sh new file mode 100755 index 0000000..5328113 --- /dev/null +++ b/ubuntu.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +set -e +#set -x + +NAME=$1 +DIST=ubuntu +RELEASE=artful +ARCH=amd64 +PACKAGES="curl libssl-dev libncurses5-dev zsh vim sudo erlang-nox git make autoconf automake gcc" + +for f in ci.d/*; do source "$f"; done + +buildkite_stop $DIST +lxc_destroy +lxc_create $DIST $RELEASE $ARCH +lxc_wait_for_ip +apt_get_upgrade +apt_get_install $PACKAGES +buildkite_install $DIST $BUILDKITE_TOKEN +ssh_copy_host_key +ssh_copy_host_key buildkite-agent /var/lib/buildkite-agent +lxc_restart |