diff options
Diffstat (limited to 'src/ceph/mirroring')
| -rw-r--r-- | src/ceph/mirroring/MIRRORS | 11 | ||||
| -rw-r--r-- | src/ceph/mirroring/README.md | 64 | ||||
| -rw-r--r-- | src/ceph/mirroring/apache2.vhost.conf | 18 | ||||
| -rwxr-xr-x | src/ceph/mirroring/mirror-ceph.sh | 102 | ||||
| -rw-r--r-- | src/ceph/mirroring/rsyncd.conf | 9 | ||||
| -rwxr-xr-x | src/ceph/mirroring/test-mirrors.sh | 69 |
6 files changed, 0 insertions, 273 deletions
diff --git a/src/ceph/mirroring/MIRRORS b/src/ceph/mirroring/MIRRORS deleted file mode 100644 index 408a2bb..0000000 --- a/src/ceph/mirroring/MIRRORS +++ /dev/null @@ -1,11 +0,0 @@ -download.ceph.com: Red Hat <ceph-users@lists.ceph.com> -eu.ceph.com: Wido den Hollander <wido@42on.com> -au.ceph.com: Matthew Taylor <matthew.taylor@digitalpacific.com.au> -de.ceph.com: Oliver Dzombic <info@ip-interactive.de> -se.ceph.com: Josef Johansson <se-ceph-com@oderland.se> -cz.ceph.com: Tomáš Kukrál <kukratom@fit.cvut.cz> -us-east.ceph.com: Tyler Bishop <tyler.bishop@beyondhosting.net> -hk.ceph.com: Mart van Santen <mart@greenhost.nl> -fr.ceph.com: Adrien Gillard <gillard.adrien@gmail.com> -uk.ceph.com: Tim Bishop <T.D.Bishop@kent.ac.uk> -cn.ceph.com: USTC LUG <lug@ustc.edu.cn> diff --git a/src/ceph/mirroring/README.md b/src/ceph/mirroring/README.md deleted file mode 100644 index 8a295d8..0000000 --- a/src/ceph/mirroring/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# Mirroring Ceph -Ceph is primarily distributed from download.ceph.com which is based in the US. - -However, globally there are multiple mirrors which offer the same content. Often -faster than downloading from the primary source. - -Using the script found in this directory you can easily mirror Ceph to your local -datacenter and serve packages from there to your servers. - -## Guidelines -If you want to mirror Ceph please follow these guidelines: -* Please use a mirror close to you -* Do not sync in a shorter interval than 3 hours -* Avoid syncing at minute 0 of the hour, use something between 0 and 59. - -## Mirror script -The 'mirror-ceph.sh' script is written in Bash and will use rsync to mirror -all the contents to a local directory. - -Usage is simple: - -<pre> -./mirror-ceph.sh -q -s eu -t /srv/mirrors/ceph -</pre> - -This example will mirror all contents from the source 'eu' which is *eu.ceph.com*. - -### Running with CRON -The script can easily be run with CRON: - -<pre> -13 1,5,9,13,17,21 * * * /home/ceph/mirror-ceph.sh -q -s eu -t /srv/mirrors/ceph -</pre> - -This will sync from *eu.ceph.com* on 01:13, 05:13, 09:13, 13:13, 17:13 and 21:13. - -## Becoming a mirror source -If you have spare hardware and resources available you can opt for becoming a mirror -source for others. - -A few things which are required: -* 1Gbit connection or more -* Native IPv4 **and** IPv6 -* HTTP access -* rsync access -* 2TB of storage or more -* Monitoring of the mirror/source - -You can then run the *mirror-ceph.sh* script and mirror all the contents. - -### Logs -The project wants to analyze the downloads of Ceph a few times a year. From mirrors -we expect that they store HTTP access logs for at least 6 months so they can be -used for analysis. - -### DNS -Using a DNS CNAME record a XX.ceph.com entry can be forwarded to the server and -added to the mirror script. - -You can request such a DNS entry on the ceph mailinglists. - -### Apache configuration -A Apache 2.4 VirtualHost example configuration can be found the Git repository -with the name *apache2.vhost.conf* diff --git a/src/ceph/mirroring/apache2.vhost.conf b/src/ceph/mirroring/apache2.vhost.conf deleted file mode 100644 index b7fce7e..0000000 --- a/src/ceph/mirroring/apache2.vhost.conf +++ /dev/null @@ -1,18 +0,0 @@ -# -# This is a example Apache 2 VirtualHost being used -# on eu.ceph.com which runs on Ubuntu 14.04 -# - -<VirtualHost *:80> - ServerName eu.ceph.com - ServerAdmin webmaster@localhost - DocumentRoot /srv/mirror/ceph/download - <Directory /srv/mirror/ceph/download> - Options FollowSymLinks Indexes - AllowOverride none - Require all granted - </Directory> - - ErrorLog ${APACHE_LOG_DIR}/error.log - CustomLog ${APACHE_LOG_DIR}/access.log combined -</VirtualHost> diff --git a/src/ceph/mirroring/mirror-ceph.sh b/src/ceph/mirroring/mirror-ceph.sh deleted file mode 100755 index ef80a63..0000000 --- a/src/ceph/mirroring/mirror-ceph.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -set -e -# -# Script to mirror Ceph locally -# -# Please, choose a local source and do not sync in a shorter interval than -# 3 hours. -# -SILENT=0 - -# All available source mirrors -declare -A SOURCES -SOURCES[eu]="eu.ceph.com" -SOURCES[de]="de.ceph.com" -SOURCES[se]="se.ceph.com" -SOURCES[cz]="cz.ceph.com" -SOURCES[au]="au.ceph.com" -SOURCES[us]="download.ceph.com" -SOURCES[hk]="hk.ceph.com" -SOURCES[fr]="fr.ceph.com" -SOURCES[us-east]="us-east.ceph.com" -SOURCES[us-west]="us-west.ceph.com" -SOURCES[global]="download.ceph.com" - -function print_usage() { - echo "$0 [-q ] -s <source mirror> -t <target directory>" -} - -while getopts ":qhs:t:" opt; do - case $opt in - q) - SILENT=1 - ;; - s) - SOURCE=$OPTARG - ;; - t) - TARGET=$OPTARG - ;; - h) - HELP=1 - ;; - \?) - print_usage - exit 1 - ;; - esac -done - -if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then - print_usage - exit 1 -fi - -if [ ! -d "$TARGET" ]; then - echo "$TARGET is not a valid target directory" - exit 1 -fi - -for i in "${!SOURCES[@]}"; do - if [ "$i" == "$SOURCE" ]; then - SOURCE_HOST=${SOURCES[$i]} - fi -done - -if [ -z "$SOURCE_HOST" ]; then - echo -n "Please select one of the following sources:" - for i in "${!SOURCES[@]}"; do - echo -n " $i" - done - echo "" - exit 1 -fi - -RSYNC_OPTS="--stats --progress" -if [ $SILENT -eq 1 ]; then - RSYNC_OPTS="--quiet" -fi - -# We start a two-stage sync here for DEB and RPM -# Based on: https://www.debian.org/mirror/ftpmirror -# -# The idea is to prevent temporary situations where metadata points to files -# which do not exist -# - -# Exclude all metadata files -rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \ - --hard-links \ - --exclude Packages* \ - --exclude Sources* \ - --exclude Release* \ - --exclude InRelease \ - --exclude i18n/* \ - --exclude ls-lR* \ - --exclude repodata/* \ - ${TARGET} - -# Now also transfer the metadata and delete afterwards -rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \ - --hard-links --delete-after \ - ${TARGET} diff --git a/src/ceph/mirroring/rsyncd.conf b/src/ceph/mirroring/rsyncd.conf deleted file mode 100644 index fa45595..0000000 --- a/src/ceph/mirroring/rsyncd.conf +++ /dev/null @@ -1,9 +0,0 @@ -uid = nobody -gid = nogroup -max connections = 15 -socket options = SO_KEEPALIVE - -[ceph] - path = /path/to/ceph/mirror/data - comment = Ceph mirror - read only = true diff --git a/src/ceph/mirroring/test-mirrors.sh b/src/ceph/mirroring/test-mirrors.sh deleted file mode 100755 index 4b6392b..0000000 --- a/src/ceph/mirroring/test-mirrors.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# -# Simple script which performs a HTTP and rsync check on -# all Ceph mirrors over IPv4 and IPv6 to see if they are online -# -# Requires IPv4, IPv6, rsync and curl -# -# Example usage: -# - ./test-mirrors.sh eu.ceph.com,de.ceph.com,hk.ceph.com -# - cat MIRRORS |cut -d ':' -f 1|xargs -n 1 ./test-mirrors.sh -# - -function print_usage { - echo "Usage: $0 mirror1,mirror2,mirror3,mirror4,etc" -} - -function test_http { - HOST=$1 - - echo -n "$HOST HTTP IPv4: " - curl -s -I -4 -o /dev/null http://$HOST - if [ "$?" -ne 0 ]; then - echo "FAIL" - else - echo "OK" - fi - - echo -n "$HOST HTTP IPv6: " - curl -s -I -6 -o /dev/null http://$HOST - if [ "$?" -ne 0 ]; then - echo "FAIL" - else - echo "OK" - fi -} - -function test_rsync { - HOST=$1 - - echo -n "$HOST RSYNC IPv4: " - rsync -4 -avrqn ${HOST}::ceph /tmp 2>/dev/null - if [ "$?" -ne 0 ]; then - echo "FAIL" - else - echo "OK" - fi - - echo -n "$HOST RSYNC IPv6: " - rsync -6 -avrqn ${HOST}::ceph /tmp 2>/dev/null - if [ "$?" -ne 0 ]; then - echo "FAIL" - else - echo "OK" - fi -} - -MIRRORS=$1 - -if [ -z "$MIRRORS" ]; then - print_usage - exit 1 -fi - -IFS=', ' read -r -a array <<< "$MIRRORS" - -for MIRROR in "${array[@]}"; do - test_http $MIRROR - test_rsync $MIRROR -done |