From dfa5569f9b60554077fd6f32e1b0b57f0e8e1e06 Mon Sep 17 00:00:00 2001
From: pierreozoux <pierre@ozoux.net>
Date: Wed, 5 Nov 2014 12:43:16 +0000
Subject: [PATCH] Adds backup vm in Vagrant
---
Vagrantfile | 23 ++++++++++++
cloud-config | 6 +++
deploy/deploy.sh | 32 +++++++---------
deploy/onServer.sh | 3 ++
doc/deploying-a-server.md | 20 +++-------
importers/backup-snapshot.sh | 48 ------------------------
scripts/backup-init.sh | 50 +++++++++----------------
scripts/setup.sh | 4 --
unit-files/backup@.service | 28 +++++++++++++-
unit-files/static-git-importer@.service | 2 +
unit-files/static-importer@.service | 2 +
unit-files/wordpress-importer@.service | 1 +
12 files changed, 98 insertions(+), 121 deletions(-)
delete mode 100755 importers/backup-snapshot.sh
diff --git a/Vagrantfile b/Vagrantfile
index 4bcc60a..b19b3c0 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -24,6 +24,21 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box_version = ">= 308.0.1"
config.vm.box_url = "http://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json" % $update_channel
+ config.vm.define "backup" do |backup|
+ backup.vm.provider :virtualbox do |vb|
+ vb.memory = 512
+ vb.cpus = 1
+ vb.check_guest_additions = false
+ vb.functional_vboxsf = false
+ end
+ # plugin conflict
+ if Vagrant.has_plugin?("vagrant-vbguest") then
+ backup.vbguest.auto_update = false
+ end
+ backup.vm.hostname = "backup.dev"
+ backup.vm.network :private_network, ip: "192.168.65.100"
+ end
+
(1..$num_instances).each do |i|
config.vm.define "core-#{i}" do |core|
core.vm.provider :virtualbox do |vb|
@@ -44,8 +59,16 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
core.vm.network :private_network, ip: "#{BASE_IP_ADDR}.#{i+1}"
core.vm.synced_folder ".", "/data/indiehosters", id: "coreos-indiehosters", :nfs => true, :mount_options => ['nolock,vers=3,udp']
core.vm.provision :file, source: "./cloud-config", destination: "/tmp/vagrantfile-user-data"
+ $install_insecure_keys = <<SCRIPT
+mkdir ~/.ssh
+wget https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub -O ~/.ssh/id_rsa.pub
+wget https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant -O ~/.ssh/id_rsa
+chmod 600 ~/.ssh/id_rsa
+SCRIPT
+ core.vm.provision :shell, inline: $install_insecure_keys
core.vm.provision :shell, inline: "mkdir -p /data/runtime/haproxy/approved-certs; cp /data/indiehosters/scripts/unsecure-certs/*.pem /data/runtime/haproxy/approved-certs"
core.vm.provision :shell, path: "./scripts/setup.sh", args: [HOSTNAME]
end
+
end
end
diff --git a/cloud-config b/cloud-config
index 75a4a5c..a24fe09 100644
--- a/cloud-config
+++ b/cloud-config
@@ -9,3 +9,9 @@ coreos:
units:
- name: etcd.service
command: start
+write_files:
+ - path: /data/BACKUP_DESTINATION
+ permissions: 0644
+ owner: root
+ content: |
+ core@backup.dev
diff --git a/deploy/deploy.sh b/deploy/deploy.sh
index 195fb6d..eb32501 100755
--- a/deploy/deploy.sh
+++ b/deploy/deploy.sh
@@ -1,15 +1,11 @@
#!/bin/sh
-if [ $# -ge 1 ]; then
+if [ $# -ge 2 ]; then
SERVER=$1
+ BACKUP_DEST=$2
else
- echo "Usage: sh ./deploy/deploy.sh server [folder [branch [user]]]"
+ echo "Usage: sh ./deploy/deploy.sh server backup_dest [branch [user]]]"
exit 1
fi
-if [ $# -ge 2 ]; then
- FOLDER=$2
-else
- FOLDER="./data/"
-fi
if [ $# -ge 3 ]; then
BRANCH=$3
else
@@ -21,20 +17,18 @@ else
USER="core"
fi
-if [ -e "${FOLDER}runtime/haproxy/approved-certs/${SERVER}.pem" ]; then
- DEFAULTSITE=$SERVER
-else
- echo "Please make sure ${FOLDER}runtime/haproxy/approved-certs/${SERVER}.pem exists, then retry"
- exit 1
-fi
-
-echo "Hoster data folder is $FOLDER"
-echo "Infrastructure branch is $BRANCH"
+echo "Server to deploy is $SERVER"
+echo "Backups will live under $BACKUP_DEST"
+echo "IndieHosters repo branch is $BRANCH"
echo "Remote user is $USER"
-echo "Default site is $DEFAULTSITE"
-scp -r $FOLDER $USER@$SERVER:/data
scp ./deploy/onServer.sh $USER@$SERVER:
+
ssh $USER@$SERVER sudo mkdir -p /var/lib/coreos-install/
scp cloud-config $USER@$SERVER:/var/lib/coreos-install/user_data
-ssh $USER@$SERVER sudo sh ./onServer.sh $BRANCH $DEFAULTSITE
+ssh $USER@$SERVER sudo sh ./onServer.sh $BRANCH $SERVER
+
+# overrides BACKUP_DESTINATION from cloud-config
+echo $BACKUP_DEST > ./deploy/tmp.txt
+scp ./deploy/tmp.txt $USER@SERVER:/data/BACKUP_DESTINATION
+rm ./deploy/tmp.txt
diff --git a/deploy/onServer.sh b/deploy/onServer.sh
index 95f0283..61e819a 100755
--- a/deploy/onServer.sh
+++ b/deploy/onServer.sh
@@ -1,5 +1,8 @@
#!/bin/sh
+#Usage from deploy/deploy.sh:
+#ssh $USER@$SERVER sudo sh ./onServer.sh $BRANCH $SERVER
+
echo Starting etcd:
/usr/bin/coreos-cloudinit --from-file=/var/lib/coreos-install/user_data
diff --git a/doc/deploying-a-server.md b/doc/deploying-a-server.md
index eafe5f6..6fef377 100644
--- a/doc/deploying-a-server.md
+++ b/doc/deploying-a-server.md
@@ -11,18 +11,12 @@ Make sure you read [getting started](getting-started-as-a-hoster.md) first.
* Give the new server a name (in this example, we call the server 'k3')
* Add k3 to your /etc/hosts with the right IP address
* If you have used this name before, run `./deploy/forget-server-fingerprint.sh k3`
-* From the root folder of this repository, run `sh ./deploy/deploy.sh k3 ./data/ master root` (where `./data/` should contain
- `runtime/postfix/`
- and `runtime/haproxy/approved-certs/k3.pem`; see the existing folder `data/` in this repo for an example of what the email forwards and
- TLS certificate files should look like).
-* Add the default site by following the 'Adding a website to your server' instructions below with domain name k3 instead of example.com
-* The rest should be automatic!
-
-### Preparing backups
-
-* ssh into your server, and run `ssh-keygen -t rsa`
+* ssh into your server, and run `ssh-keygen -t rsa` (use all the default settings, empty passphrase)
* set up a backups server at an independent location (at least a different data center, but preferably also a different IaaS provider, the bu25 plan of https://securedragon.net/ is a good option at 3 dollars per month).
-* set up a git server with one private git repo per domain by following http://www.git-scm.com/book/en/v2/Git-on-the-Server-Setting-Up-the-Server (instead of 'project.git' you can use 'domainname.com.git')
+* set up a git server by following http://www.git-scm.com/book/en/v2/Git-on-the-Server-Setting-Up-the-Server (no need to set up any repos like 'project.git' yet). Let's call the backup server 'bu25' (add this to /etc/hosts on k3).
+* add the ssh key from k3 to the authorized_keys for the git user (not the root user) on bu25.
+* Exit from the double ssh back to your laptop, and from the root folder of this repository, run `sh ./deploy/deploy.sh k3 master root`
+* The rest should be automatic!
### Adding a website to your server
* For each site you want to deploy on the server, e.g. example.com, do the following:
@@ -47,9 +41,5 @@ Make sure you read [getting started](getting-started-as-a-hoster.md) first.
* Now run `deploy/add-site.sh k3 example.com ../hoster-data/TLS/example.com.pem nginx https://github.com/someone/example.com.git root`.
It will make sure the server is in the correct state, and git pull and scp the user data and the
approved cert into place, start a container running the image requested, update haproxy config, and restart the haproxy container.
- * set up a git repo for the new site on the backup server (see http://www.git-scm.com/book/en/v2/Git-on-the-Server-Setting-Up-the-Server again), and for instance if you called the backup repo example.com.git and your backup server is in /etc/hosts on k3 as 'bu25', ssh into k3 and run:
-
- sh scripts/backup-init.sh example.com git@bu25:/opt/git/example.com.git
-
* Test the site using your /etc/hosts. You should see the data from the git repo on both http and https.
* Switch DNS and monitoring.
diff --git a/importers/backup-snapshot.sh b/importers/backup-snapshot.sh
deleted file mode 100755
index ec69584..0000000
--- a/importers/backup-snapshot.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash -eux
-
-if [ -e /data/domains/$DOMAIN/mysql ]; then
- echo backing up mysql databases for $DOMAIN
- mkdir -p /data/domains/$DOMAIN/backup/mysql/
- cp /data/domains/$DOMAIN/mysql/.env /data/domains/$DOMAIN/backup/mysql/.env
- /usr/bin/docker run --link mysql-$DOMAIN:db\
- --env-file /data/domains/$DOMAIN/mysql/.env \
- indiehosters/mysql mysqldump --all-databases --events -u admin \
- -p$(cat /data/domains/$DOMAIN/mysql/.env | cut -d'=' -f2) \
- -h db > /data/domains/$DOMAIN/backup/mysql/dump.sql
-fi
-
-if [ -e /data/domains/$DOMAIN/wordpress ]; then
- echo backing up www from wordpress for $DOMAIN
- mkdir -p /data/domains/$DOMAIN/backup/www/
- rsync -r /data/domains/$DOMAIN/wordpress /data/domains/$DOMAIN/backup/www/wordpress
-fi
-
-if [ -e /data/domains/$DOMAIN/nginx ]; then
- echo backing up www from nginx for $DOMAIN
- mkdir -p /data/domains/$DOMAIN/backup/www/nginx/
- if [ -e /data/domains/$DOMAIN/nginx/data/GITURL ]; then
- cp /data/domains/$DOMAIN/nginx/data/GITURL /data/domains/$DOMAIN/backup/www/nginx/GITURL
- else
- rsync -r /data/domains/$DOMAIN/nginx/data/www-content /data/domains/$DOMAIN/backup/www/nginx/www-content
- fi
-fi
-
-echo copying TLS cert
-mkdir -p /data/domains/$DOMAIN/backup/TLS/
-cp /data/runtime/haproxy/approved-certs/$DOMAIN.pem /data/domains/$DOMAIN/backup/TLS/$DOMAIN.pem
-
-echo committing everything
-cd /data/domains/$DOMAIN/backup/
-pwd
-git add *
-git status
-
-git config --local user.email "backups@`hostname`"
-git config --local user.name "`hostname` hourly backups"
-git config --local push.default simple
-
-git commit -m"backup $DOMAIN @ `hostname` - `date`"
-if [ -e /data/domains/$DOMAIN/backup/BACKUPDEST ]; then
- git pull --rebase
- git push
-fi
diff --git a/scripts/backup-init.sh b/scripts/backup-init.sh
index c10e764..a4b6625 100755
--- a/scripts/backup-init.sh
+++ b/scripts/backup-init.sh
@@ -1,36 +1,20 @@
-#!/bin/sh
-if [ $# -ge 2 ]; then
- DOMAIN=$1
- BACKUPDEST=$2
-else
- echo "Usage: sh ./scripts/backups-init.sh domain gitrepo"
- exit 1
-fi
-echo "Adding backup job for $DOMAIN to $BACKUPDEST"
-
-echo "First, trying to clone latest master from $BACKUPDEST"
-git clone $BACKUPDEST /data/domains/$DOMAIN/backup
+#!/bin/bash -eux
-sudo mkdir -p /data/domains/$DOMAIN/backup
-sudo echo "$BACKUPDEST" > /data/domains/$DOMAIN/backup/BACKUPDEST
+BACKUP_DESTINATION=`cat /data/BACKUP_DESTINATION`
-echo initializing backups for $DOMAIN
-mkdir -p /data/domains/$DOMAIN/backup/mysql
-mkdir -p /data/domains/$DOMAIN/backup/www
-mkdir -p /data/domains/$DOMAIN/backup/TLS
-cd /data/domains/$DOMAIN/backup/
-git config --local user.email "backups@`hostname`"
-git config --local user.name "`hostname` hourly backups"
-git config --local push.default simple
-
-if [ -e /data/domains/$DOMAIN/backup/.git ]; then
- git init
+echo "Intitializing backups with $BACKUP_DESTINATION"
+if [ ! -d /data/domains/$DOMAIN ]; then
+ ssh $BACKUP_DESTINATION " \
+ if [ ! -d $DOMAIN ]; then \
+ mkdir -p $DOMAIN; \
+ cd $DOMAIN; \
+ git init --bare; \
+ else
+ echo \"Git folder already present\"
+ fi"
+ git clone $BACKUP_DESTINATION:$DOMAIN /data/domains/$DOMAIN
+ cd /data/domains/$DOMAIN
+ git config --local user.email "backups@`hostname`"
+ git config --local user.name "`hostname` hourly backups"
+ git config --local push.default simple
fi
-echo "backups of $DOMAIN at IndieHosters server `hostname`" > README.md
-git add README.md
-git commit -m"initial commit"
-
-echo "Pushing initial commit to $BACKUPDEST master branch"
-cd /data/domains/$DOMAIN/backup/
-git remote add destination $BACKUPDEST
-git push -u destination master
diff --git a/scripts/setup.sh b/scripts/setup.sh
index 7a944d7..b4c6fb1 100755
--- a/scripts/setup.sh
+++ b/scripts/setup.sh
@@ -24,10 +24,6 @@ docker pull indiehosters/wordpress
cp /data/indiehosters/unit-files/* /etc/systemd/system
systemctl daemon-reload
-# Activate default domain
-sh /data/indiehosters/scripts/activate-user.sh $HOSTNAME static
-etcdctl set /services/default '{"app":"static", "hostname":"'$HOSTNAME'"}'
-
# Configure and start HAproxy
mkdir -p /data/runtime/haproxy/approved-certs
systemctl enable haproxy-confd.service
diff --git a/unit-files/backup@.service b/unit-files/backup@.service
index 134e803..14d3408 100644
--- a/unit-files/backup@.service
+++ b/unit-files/backup@.service
@@ -1,7 +1,31 @@
[Unit]
-Description= Back up mysql and www data to a git repo and optionally push it out
+Description=Back up domain data to a git repo and push it out
[Service]
Type=oneshot
Environment=DOMAIN=%i
-ExecStart=/data/indiehosters/importers/backup-snapshot.sh
+ExecStartPre=/bin/bash -euxc ' \
+ if [ -e /data/domains/$DOMAIN/mysql ]; then \
+ echo "Backing up mysql databases for $DOMAIN"; \
+ /usr/bin/docker run \
+ --link mysql-$DOMAIN:db \
+ --env-file /data/domains/$DOMAIN/mysql/.env \
+ indiehosters/mysql \
+ mysqldump \
+ --all-databases \
+ --events \
+ -u admin \
+ -p$(cat /data/domains/$DOMAIN/mysql/.env | cut -d'=' -f2) \
+ -h db \
+ > /data/domains/$DOMAIN/mysql/dump.sql; \
+ fi'
+
+ExecStart=/bin/bash -euxc ' \
+ echo "Committing everything"; \
+ cd /data/domains/$DOMAIN/; \
+ git add *; \
+ git status; \
+ git commit -m"backup $DOMAIN @ `hostname` - `date`"; \
+ # be careful: hidden sync functionnality; \
+ git pull --rebase; \
+ git push'
diff --git a/unit-files/static-git-importer@.service b/unit-files/static-git-importer@.service
index 255bb80..5951dbd 100644
--- a/unit-files/static-git-importer@.service
+++ b/unit-files/static-git-importer@.service
@@ -7,6 +7,8 @@ Before=static-git@%i.service
[Service]
Type=oneshot
RemainAfterExit=yes
+Environment=DOMAIN=%i
+ExecStartPre=/data/indiehosters/scripts/backup-init.sh
ExecStart=/bin/bash -euxc ' \
domain_folder=/data/domains/%i; \
cp $domain_folder/TLS/%i.pem /data/runtime/haproxy/approved-certs/%i.pem; \
diff --git a/unit-files/static-importer@.service b/unit-files/static-importer@.service
index b1ec540..ebd5fe2 100644
--- a/unit-files/static-importer@.service
+++ b/unit-files/static-importer@.service
@@ -7,6 +7,8 @@ Before=static@%i.service
[Service]
Type=oneshot
RemainAfterExit=yes
+Environment=DOMAIN=%i
+ExecStartPre=/data/indiehosters/scripts/backup-init.sh
ExecStart=/bin/bash -euxc ' \
domain_folder=/data/domains/%i; \
cp $domain_folder/TLS/%i.pem /data/runtime/haproxy/approved-certs/%i.pem; \
diff --git a/unit-files/wordpress-importer@.service b/unit-files/wordpress-importer@.service
index 2d92374..8a47b2e 100644
--- a/unit-files/wordpress-importer@.service
+++ b/unit-files/wordpress-importer@.service
@@ -12,6 +12,7 @@ BindsTo=wordpress@%i.service
Type=oneshot
RemainAfterExit=yes
Environment=DOMAIN=%i
+ExecStartPre=/data/indiehosters/scripts/backup-init.sh
ExecStart=/bin/bash -euxc ' \
domain_folder=/data/domains/%i; \
cp $domain_folder/TLS/%i.pem /data/runtime/haproxy/approved-certs/%i.pem; \
--
GitLab