diff --git a/unit-files/backup@.service b/unit-files/backup@.service
index b0590e86ca185243d93f5ad2eb09887a8dbb7ee0..85145bcb0a8d0a35586b920999a8f57086690d22 100644
--- a/unit-files/backup@.service
+++ b/unit-files/backup@.service
@@ -3,11 +3,15 @@ Description=Back up domain data to a git repo and push it out
 
 [Service]
 Type=oneshot
+ExecStartPre=-/usr/bin/docker kill mysqldump-%i
+ExecStartPre=-/usr/bin/docker rm mysqldump-%i
 ExecStartPre=/bin/bash -euxc ' \
   if [ -d /data/domains/%i/mysql ]; then \
     echo "Backing up mysql databases for %i"; \
     mysql_passwd=`cat /data/domains/%i/mysql/.env | cut -d= -f2`; \
     /usr/bin/docker run \
+      --rm \
+      --name mysqldump-%i \
       --link mysql-%i:db \
       --env-file /data/domains/%i/mysql/.env \
       indiehosters/mysql \