initial code repo

This patch creates initial code repo.

For ceph, luminous stable release will be used for base code,
and next changes and optimization for ceph will be added to it.

For opensds, currently any changes can be upstreamed into original
opensds repo (https://github.com/opensds/opensds), and so stor4nfv
will directly clone opensds code to deploy stor4nfv environment.
And the scripts for deployment based on ceph and opensds will be
put into 'ci' directory.

Change-Id: I46a32218884c75dda2936337604ff03c554648e4
Signed-off-by: Qiaowei Ren <qiaowei.ren@intel.com>

16 files changed, 364 insertions, 0 deletions

diff --git a/src/ceph/qa/workunits/suites/blogbench.sh b/src/ceph/qa/workunits/suites/blogbench.sh
new file mode 100755
index 0000000..17c91c8
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/blogbench.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+echo "getting blogbench"
+wget http://download.ceph.com/qa/blogbench-1.0.tar.bz2
+#cp /home/gregf/src/blogbench-1.0.tar.bz2 .
+tar -xvf blogbench-1.0.tar.bz2
+cd blogbench*
+echo "making blogbench"
+./configure
+make
+cd src
+mkdir blogtest_in
+echo "running blogbench"
+./blogbench -d blogtest_in
diff --git a/src/ceph/qa/workunits/suites/bonnie.sh b/src/ceph/qa/workunits/suites/bonnie.sh
new file mode 100755
index 0000000..698ba9c
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/bonnie.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+bonnie_bin=`which bonnie++`
+[ $? -eq 1 ] && bonnie_bin=/usr/sbin/bonnie++
+
+uid_flags=""
+[ "`id -u`" == "0" ] && uid_flags="-u root"
+
+$bonnie_bin $uid_flags -n 100
diff --git a/src/ceph/qa/workunits/suites/cephfs_journal_tool_smoke.sh b/src/ceph/qa/workunits/suites/cephfs_journal_tool_smoke.sh
new file mode 100755
index 0000000..60e9149
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/cephfs_journal_tool_smoke.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -e
+set -x
+
+export BIN="${BIN:-cephfs-journal-tool}"
+export JOURNAL_FILE=/tmp/journal.bin
+export JSON_OUTPUT=/tmp/json.tmp
+export BINARY_OUTPUT=/tmp/binary.tmp
+
+if [ -d $BINARY_OUTPUT ] ; then
+    rm -rf $BINARY_OUTPUT
+fi
+
+# Check that the import/export stuff really works as expected
+# first because it's used as the reset method between
+# following checks.
+echo "Testing that export/import cycle preserves state"
+HEADER_STATE=`$BIN header get`
+EVENT_LIST=`$BIN event get list`
+$BIN journal export $JOURNAL_FILE
+$BIN journal import $JOURNAL_FILE
+NEW_HEADER_STATE=`$BIN header get`
+NEW_EVENT_LIST=`$BIN event get list`
+
+if [ ! "$HEADER_STATE" = "$NEW_HEADER_STATE" ] ; then
+    echo "Import failed to preserve header state"
+    echo $HEADER_STATE
+    echo $NEW_HEADER_STATE
+    exit -1
+fi
+
+if [ ! "$EVENT_LIST" = "$NEW_EVENT_LIST" ] ; then
+    echo "Import failed to preserve event state"
+    echo $EVENT_LIST
+    echo $NEW_EVENT_LIST
+    exit -1
+fi
+
+echo "Testing 'journal' commands..."
+
+# Simplest thing: print the vital statistics of the journal
+$BIN journal inspect
+$BIN header get
+
+# Make a copy of the journal in its original state
+$BIN journal export $JOURNAL_FILE
+if [ ! -s $JOURNAL_FILE ] ; then
+    echo "Export to $JOURNAL_FILE failed"
+    exit -1
+fi
+
+# Can we execute a journal reset?
+$BIN journal reset
+$BIN journal inspect
+$BIN header get
+
+echo "Rolling back journal to original state..."
+$BIN journal import $JOURNAL_FILE
+
+echo "Testing 'header' commands..."
+$BIN header get
+$BIN header set write_pos 123
+$BIN header set expire_pos 123
+$BIN header set trimmed_pos 123
+
+echo "Rolling back journal to original state..."
+$BIN journal import $JOURNAL_FILE
+
+echo "Testing 'event' commands..."
+$BIN event get summary
+$BIN event get --type=UPDATE --path=/ --inode=0 --frag=0x100 summary
+$BIN event get json --path $JSON_OUTPUT
+if [ ! -s $JSON_OUTPUT ] ; then
+    echo "Export to $JSON_OUTPUT failed"
+    exit -1
+fi
+$BIN event get binary --path $BINARY_OUTPUT
+if [ ! -s $BINARY_OUTPUT ] ; then
+    echo "Export to $BINARY_OUTPUT failed"
+    exit -1
+fi
+$BIN event recover_dentries summary
+$BIN event splice summary
+
+# Tests finish.
+# Metadata objects have been modified by the 'event recover_dentries' command.
+# Journal is no long consistent with respect to metadata objects (especially inotable).
+# To ensure mds successfully replays its journal, we need to do journal reset.
+$BIN journal reset
+cephfs-table-tool all reset session
+
diff --git a/src/ceph/qa/workunits/suites/dbench-short.sh b/src/ceph/qa/workunits/suites/dbench-short.sh
new file mode 100755
index 0000000..7297d83
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/dbench-short.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+dbench 1
diff --git a/src/ceph/qa/workunits/suites/dbench.sh b/src/ceph/qa/workunits/suites/dbench.sh
new file mode 100755
index 0000000..ea2be1c
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/dbench.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+dbench 1
+dbench 10
diff --git a/src/ceph/qa/workunits/suites/ffsb.sh b/src/ceph/qa/workunits/suites/ffsb.sh
new file mode 100755
index 0000000..9ed66ab
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/ffsb.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+mydir=`dirname $0`
+
+wget http://download.ceph.com/qa/ffsb.tar.bz2
+tar jxvf ffsb.tar.bz2
+cd ffsb-*
+./configure
+make
+cd ..
+mkdir tmp
+cd tmp
+
+for f in $mydir/*.ffsb
+do
+    ../ffsb-*/ffsb $f
+done
+cd ..
+rm -r tmp ffsb*
+
diff --git a/src/ceph/qa/workunits/suites/fio.sh b/src/ceph/qa/workunits/suites/fio.sh
new file mode 100755
index 0000000..04e0645
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/fio.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -x
+
+gen_fio_file() {
+  iter=$1
+  f=$2
+  cat > randio-$$-${iter}.fio <<EOF
+[randio]
+blocksize_range=32m:128m
+blocksize_unaligned=1
+filesize=10G:20G
+readwrite=randrw
+runtime=300
+size=20G
+filename=${f}
+EOF
+}
+
+sudo apt-get -y install fio
+for i in $(seq 1 20); do
+  fcount=$(ls donetestfile* 2>/dev/null | wc -l)
+  donef="foo"
+  fiof="bar"
+  if test ${fcount} -gt 0; then
+     # choose random file
+     r=$[ ${RANDOM} % ${fcount} ]
+     testfiles=( $(ls donetestfile*) )
+     donef=${testfiles[${r}]}
+     fiof=$(echo ${donef} | sed -e "s|done|fio|")
+     gen_fio_file $i ${fiof}
+  else
+     fiof=fiotestfile.$$.$i
+     donef=donetestfile.$$.$i
+     gen_fio_file $i ${fiof}
+  fi
+
+  sudo rm -f ${donef}
+  sudo fio randio-$$-$i.fio
+  sudo ln ${fiof} ${donef}
+  ls -la
+done
diff --git a/src/ceph/qa/workunits/suites/fsstress.sh b/src/ceph/qa/workunits/suites/fsstress.sh
new file mode 100755
index 0000000..92e123b
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/fsstress.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+BIN_PATH=${TESTDIR}/fsstress/ltp-full-20091231/testcases/kernel/fs/fsstress/fsstress
+
+path=`pwd`
+trap "rm -rf ${TESTDIR}/fsstress" EXIT
+mkdir -p ${TESTDIR}/fsstress
+cd ${TESTDIR}/fsstress
+wget -q -O ${TESTDIR}/fsstress/ltp-full.tgz http://download.ceph.com/qa/ltp-full-20091231.tgz
+tar xzf ${TESTDIR}/fsstress/ltp-full.tgz
+rm ${TESTDIR}/fsstress/ltp-full.tgz
+cd ${TESTDIR}/fsstress/ltp-full-20091231/testcases/kernel/fs/fsstress
+make
+cd $path
+
+command="${BIN_PATH} -d fsstress-`hostname`$$ -l 1 -n 1000 -p 10 -v"
+
+echo "Starting fsstress $command"
+mkdir fsstress`hostname`-$$
+$command
diff --git a/src/ceph/qa/workunits/suites/fsx.sh b/src/ceph/qa/workunits/suites/fsx.sh
new file mode 100755
index 0000000..8a34806
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/fsx.sh
@@ -0,0 +1,16 @@
+#!/bin/sh -x
+
+set -e
+
+git clone git://git.ceph.com/xfstests.git
+cd xfstests
+git checkout b7fd3f05d6a7a320d13ff507eda2e5b183cae180
+make
+cd ..
+cp xfstests/ltp/fsx .
+
+OPTIONS="-z"  # don't use zero range calls; not supported by cephfs
+
+./fsx $OPTIONS  1MB -N 50000 -p 10000 -l 1048576
+./fsx $OPTIONS  10MB -N 50000 -p 10000 -l 10485760
+./fsx $OPTIONS 100MB -N 50000 -p 10000 -l 104857600
diff --git a/src/ceph/qa/workunits/suites/fsync-tester.sh b/src/ceph/qa/workunits/suites/fsync-tester.sh
new file mode 100755
index 0000000..345fbde
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/fsync-tester.sh
@@ -0,0 +1,12 @@
+#!/bin/sh -x
+
+set -e
+
+wget http://download.ceph.com/qa/fsync-tester.c
+gcc fsync-tester.c -o fsync-tester
+
+./fsync-tester
+
+echo $PATH
+whereis lsof
+lsof
diff --git a/src/ceph/qa/workunits/suites/iogen.sh b/src/ceph/qa/workunits/suites/iogen.sh
new file mode 100755
index 0000000..d159bde
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/iogen.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+echo "getting iogen"
+wget http://download.ceph.com/qa/iogen_3.1p0.tar
+tar -xvzf iogen_3.1p0.tar
+cd iogen*
+echo "making iogen"
+make
+echo "running iogen"
+./iogen -n 5 -s 2g
+echo "sleep for 10 min"
+sleep 600
+echo "stopping iogen"
+./iogen -k
+
+echo "OK"
diff --git a/src/ceph/qa/workunits/suites/iozone-sync.sh b/src/ceph/qa/workunits/suites/iozone-sync.sh
new file mode 100755
index 0000000..c094952
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/iozone-sync.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+# basic tests of O_SYNC, O_DSYNC, O_RSYNC
+# test O_SYNC
+iozone -c -e -s 512M -r 1M -t 1 -F osync1 -i 0 -i 1 -o
+# test O_DSYNC
+iozone -c -e -s 512M -r 1M -t 1 -F odsync1 -i 0 -i 1 -+D
+# test O_RSYNC
+iozone -c -e -s 512M -r 1M -t 1 -F orsync1 -i 0 -i 1 -+r
+
+# test same file with O_SYNC in one process, buffered in the other
+# the sync test starts first, so the buffered test should blow
+# past it and 
+iozone -c -e -s 512M -r 1M -t 1 -F osync2 -i 0 -i 1 -o &
+sleep 1
+iozone -c -e -s 512M -r 256K -t 1 -F osync2 -i 0
+wait $!
+
+# test same file with O_SYNC from different threads
+iozone -c -e -s 512M -r 1M -t 2 -F osync3 -i 2 -o
diff --git a/src/ceph/qa/workunits/suites/iozone.sh b/src/ceph/qa/workunits/suites/iozone.sh
new file mode 100755
index 0000000..4fcf8f1
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/iozone.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+iozone -c -e -s 1024M -r 16K -t 1 -F f1 -i 0 -i 1
+iozone -c -e -s 1024M -r 1M -t 1 -F f2 -i 0 -i 1
+iozone -c -e -s 10240M -r 1M -t 1 -F f3 -i 0 -i 1
diff --git a/src/ceph/qa/workunits/suites/pjd.sh b/src/ceph/qa/workunits/suites/pjd.sh
new file mode 100755
index 0000000..e6df309
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/pjd.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+wget http://download.ceph.com/qa/pjd-fstest-20090130-RC-aclfixes.tgz
+tar zxvf pjd*.tgz
+cd pjd*
+make clean
+make
+cd ..
+mkdir tmp
+cd tmp
+# must be root!
+sudo prove -r -v --exec 'bash -x' ../pjd*/tests
+cd ..
+rm -rf tmp pjd*
+
diff --git a/src/ceph/qa/workunits/suites/random_write.32.ffsb b/src/ceph/qa/workunits/suites/random_write.32.ffsb
new file mode 100644
index 0000000..ba83e47
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/random_write.32.ffsb
@@ -0,0 +1,48 @@
+# Large file random writes.
+# 1024 files, 100MB per file.
+
+time=300  # 5 min
+alignio=1
+
+[filesystem0]
+        location=.
+        num_files=128
+        min_filesize=104857600  # 100 MB
+        max_filesize=104857600
+        reuse=1
+[end0]
+
+[threadgroup0]
+        num_threads=32
+
+        write_random=1
+        write_weight=1
+
+        write_size=5242880  # 5 MB
+        write_blocksize=4096
+
+        [stats]
+                enable_stats=1
+                enable_range=1
+
+                msec_range    0.00      0.01
+                msec_range    0.01      0.02
+                msec_range    0.02      0.05
+                msec_range    0.05      0.10
+                msec_range    0.10      0.20
+                msec_range    0.20      0.50
+                msec_range    0.50      1.00
+                msec_range    1.00      2.00
+                msec_range    2.00      5.00
+                msec_range    5.00     10.00
+                msec_range   10.00     20.00
+                msec_range   20.00     50.00
+                msec_range   50.00    100.00
+                msec_range  100.00    200.00
+                msec_range  200.00    500.00
+                msec_range  500.00   1000.00
+                msec_range 1000.00   2000.00
+                msec_range 2000.00   5000.00
+                msec_range 5000.00  10000.00
+        [end]
+[end0]
diff --git a/src/ceph/qa/workunits/suites/wac.sh b/src/ceph/qa/workunits/suites/wac.sh
new file mode 100755
index 0000000..49b4f14
--- /dev/null
+++ b/src/ceph/qa/workunits/suites/wac.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+wget http://download.ceph.com/qa/wac.c
+gcc -o wac wac.c
+set +e
+timeout 5m ./wac -l 65536 -n 64 -r wac-test
+RET=$?
+set -e
+[[ $RET -eq 124 ]]
+echo OK