initial code repo

This patch creates initial code repo.

For ceph, luminous stable release will be used for base code,
and next changes and optimization for ceph will be added to it.

For opensds, currently any changes can be upstreamed into original
opensds repo (https://github.com/opensds/opensds), and so stor4nfv
will directly clone opensds code to deploy stor4nfv environment.
And the scripts for deployment based on ceph and opensds will be
put into 'ci' directory.

Change-Id: I46a32218884c75dda2936337604ff03c554648e4
Signed-off-by: Qiaowei Ren <qiaowei.ren@intel.com>

18 files changed, 524 insertions, 0 deletions

diff --git a/src/ceph/qa/workunits/fs/snaps/snap-rm-diff.sh b/src/ceph/qa/workunits/fs/snaps/snap-rm-diff.sh
new file mode 100755
index 0000000..c1b6c24
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snap-rm-diff.sh
@@ -0,0 +1,11 @@
+#!/bin/sh -ex
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+wget -q http://download.ceph.com/qa/linux-2.6.33.tar.bz2
+mkdir foo
+cp linux* foo
+mkdir foo/.snap/barsnap
+rm foo/linux*
+diff -q foo/.snap/barsnap/linux* linux* && echo "passed: files are identical"
+rmdir foo/.snap/barsnap
+echo OK
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-0.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-0.sh
new file mode 100755
index 0000000..b57763a
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-0.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -x
+
+expect_failure() {
+	if "$@"; then return 1; else return 0; fi
+}
+set -e
+
+ceph mds set allow_new_snaps false
+expect_failure mkdir .snap/foo
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+echo asdf > foo
+mkdir .snap/foo
+grep asdf .snap/foo/foo
+rmdir .snap/foo
+
+echo asdf > bar
+mkdir .snap/bar
+rm bar
+grep asdf .snap/bar/bar
+rmdir .snap/bar
+rm foo
+
+ceph mds set allow_new_snaps false
+expect_failure mkdir .snap/baz
+
+echo OK
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-1.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-1.sh
new file mode 100755
index 0000000..f8fb614
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-1.sh
@@ -0,0 +1,31 @@
+#!/bin/bash -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+echo 1 > file1
+echo 2 > file2
+echo 3 > file3
+[ -e file4 ] && rm file4
+mkdir .snap/snap1
+echo 4 > file4
+now=`ls`
+then=`ls .snap/snap1`
+rmdir .snap/snap1
+if [ "$now" = "$then" ]; then
+    echo live and snap contents are identical?
+    false
+fi
+
+# do it again
+echo 1 > file1
+echo 2 > file2
+echo 3 > file3
+mkdir .snap/snap1
+echo 4 > file4
+rmdir .snap/snap1
+
+rm file?
+
+echo OK
\ No newline at end of file
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-2.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-2.sh
new file mode 100755
index 0000000..b2458d9
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-2.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+echo "Create dir 100 to 199 ..."
+for i in $(seq 100 199); do
+	echo "    create dir $i"
+	mkdir "$i"
+	for y in $(seq 10 20); do
+		echo "This is a test file before any snapshot was taken." >"$i/$y"
+	done
+done
+
+echo "Take first snapshot .snap/test1"
+mkdir .snap/test1
+
+echo "Create dir 200 to 299 ..."
+for i in $(seq 200 299); do
+	echo "    create dir $i"
+        mkdir $i
+        for y in $(seq 20 29); do
+                echo "This is a test file. Created after .snap/test1" >"$i/$y"
+        done
+done
+
+echo "Create a snapshot in every first level dir ..."
+for dir in $(ls); do
+	echo "    create $dir/.snap/snap-subdir-test"
+	mkdir "$dir/.snap/snap-subdir-test"
+	for y in $(seq 30 39); do
+		echo "        create $dir/$y file after the snapshot"
+                echo "This is a test file. Created after $dir/.snap/snap-subdir-test" >"$dir/$y"
+        done
+done
+
+echo "Take second snapshot .snap/test2"
+mkdir .snap/test2
+
+echo "Copy content of .snap/test1 to copyofsnap1 ..."
+mkdir copyofsnap1
+cp -Rv .snap/test1 copyofsnap1/
+
+
+echo "Take third snapshot .snap/test3"
+mkdir .snap/test3
+
+echo "Delete the snapshots..."
+
+find ./ -type d -print | \
+        xargs -I% -n1 find %/.snap -mindepth 1 -maxdepth 1 \
+                         \( ! -name "_*" \) -print 2>/dev/null
+
+find ./ -type d -print | \
+	xargs -I% -n1 find %/.snap -mindepth 1 -maxdepth 1 \
+                         \( ! -name "_*" \) -print 2>/dev/null | \
+	xargs -n1 rmdir
+
+echo "Delete all the files and directories ..."
+rm -Rfv ./*
+
+echo OK
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-authwb.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-authwb.sh
new file mode 100755
index 0000000..9dd9845
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-authwb.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+touch foo
+chmod +x foo
+mkdir .snap/s
+find .snap/s/foo -executable | grep foo
+rmdir .snap/s
+rm foo
+
+echo OK
\ No newline at end of file
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-capwb.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-capwb.sh
new file mode 100755
index 0000000..3b6a01a
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-capwb.sh
@@ -0,0 +1,35 @@
+#!/bin/sh -x
+
+set -e
+
+mkdir foo
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+# make sure mds handles it when the client does not send flushsnap
+echo x > foo/x
+sync
+mkdir foo/.snap/ss
+ln foo/x foo/xx
+cat foo/.snap/ss/x
+rmdir foo/.snap/ss
+
+#
+echo a > foo/a
+echo b > foo/b
+mkdir foo/.snap/s
+r=`cat foo/.snap/s/a`
+[ -z "$r" ] && echo "a appears empty in snapshot" && false
+
+ln foo/b foo/b2
+cat foo/.snap/s/b
+
+echo "this used to hang:"
+echo more >> foo/b2
+echo "oh, it didn't hang! good job."
+cat foo/b
+rmdir foo/.snap/s
+
+rm -r foo
+
+echo OK
\ No newline at end of file
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-dir-rename.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-dir-rename.sh
new file mode 100755
index 0000000..b98358a
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-dir-rename.sh
@@ -0,0 +1,19 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+#
+# make sure we keep an existing dn's seq
+#
+
+mkdir a
+mkdir .snap/bar
+mkdir a/.snap/foo
+rmdir a/.snap/foo
+rmdir a
+stat .snap/bar/a
+rmdir .snap/bar
+
+echo OK
\ No newline at end of file
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-double-null.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-double-null.sh
new file mode 100755
index 0000000..b547213
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-double-null.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+# multiple intervening snapshots with no modifications, and thus no
+# snapflush client_caps messages.  make sure the mds can handle this.
+
+for f in `seq 1 20` ; do
+
+mkdir a
+cat > a/foo &
+mkdir a/.snap/one
+mkdir a/.snap/two
+chmod 777 a/foo
+sync   # this might crash the mds
+ps
+rmdir a/.snap/*
+rm a/foo
+rmdir a
+
+done
+
+echo OK
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-estale.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-estale.sh
new file mode 100755
index 0000000..1465a35
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-estale.sh
@@ -0,0 +1,15 @@
+#!/bin/sh -x
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+mkdir .snap/foo
+
+echo "We want ENOENT, not ESTALE, here."
+for f in `seq 1 100`
+do
+    stat .snap/foo/$f 2>&1 | grep 'No such file'
+done
+
+rmdir .snap/foo
+
+echo "OK"
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-git-ceph.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-git-ceph.sh
new file mode 100755
index 0000000..1769fe8
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-git-ceph.sh
@@ -0,0 +1,35 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+git clone git://git.ceph.com/ceph.git
+cd ceph
+
+versions=`seq 1 21`
+
+for v in $versions
+do
+    ver="v0.$v"
+    echo $ver
+    git reset --hard $ver
+    mkdir .snap/$ver
+done
+
+for v in $versions
+do
+    ver="v0.$v"
+    echo checking $ver
+    cd .snap/$ver
+    git diff --exit-code
+    cd ../..
+done
+
+for v in $versions
+do
+    ver="v0.$v"
+    rmdir .snap/$ver
+done
+
+echo OK
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-intodir.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-intodir.sh
new file mode 100755
index 0000000..729baa1
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-intodir.sh
@@ -0,0 +1,24 @@
+#!/bin/sh -ex
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+# this tests fix for #1399
+mkdir foo
+mkdir foo/.snap/one
+touch bar
+mv bar foo
+sync
+# should not crash :)
+
+mkdir baz
+mkdir baz/.snap/two
+mv baz foo
+sync
+# should not crash :)
+
+# clean up.
+rmdir foo/baz/.snap/two
+rmdir foo/.snap/one
+rm -r foo
+
+echo OK
\ No newline at end of file
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh
new file mode 100755
index 0000000..bc58bac
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh
@@ -0,0 +1,44 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+echo asdf > a
+mkdir .snap/1
+chmod 777 a
+mkdir .snap/2
+echo qwer > a
+mkdir .snap/3
+chmod 666 a
+mkdir .snap/4
+echo zxcv > a
+mkdir .snap/5
+
+ls -al .snap/?/a
+
+grep asdf .snap/1/a
+stat .snap/1/a | grep 'Size: 5'
+
+grep asdf .snap/2/a
+stat .snap/2/a | grep 'Size: 5'
+stat .snap/2/a | grep -- '-rwxrwxrwx'
+
+grep qwer .snap/3/a
+stat .snap/3/a | grep 'Size: 5'
+stat .snap/3/a | grep -- '-rwxrwxrwx'
+
+grep qwer .snap/4/a
+stat .snap/4/a | grep 'Size: 5'
+stat .snap/4/a | grep -- '-rw-rw-rw-'
+
+grep zxcv .snap/5/a
+stat .snap/5/a | grep 'Size: 5'
+stat .snap/5/a | grep -- '-rw-rw-rw-'
+
+rmdir .snap/[12345]
+
+echo "OK"
+
+
+
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-parents.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-parents.sh
new file mode 100755
index 0000000..6b76fdb
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-parents.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+echo "making directory tree and files"
+mkdir -p 1/a/b/c/
+echo "i'm file1" > 1/a/file1
+echo "i'm file2" > 1/a/b/file2
+echo "i'm file3" > 1/a/b/c/file3
+echo "snapshotting"
+mkdir 1/.snap/foosnap1
+mkdir 2
+echo "moving tree"
+mv 1/a 2
+echo "checking snapshot contains tree..."
+dir1=`find 1/.snap/foosnap1 | wc -w`
+dir2=`find 2/ | wc -w`
+#diff $dir1 $dir2 && echo "Success!"
+test $dir1==$dir2 && echo "Success!"
+echo "adding folder and file to tree..."
+mkdir 2/a/b/c/d
+echo "i'm file 4!" > 2/a/b/c/d/file4
+echo "snapshotting tree 2"
+mkdir 2/.snap/barsnap2
+echo "comparing snapshots"
+dir1=`find 1/.snap/foosnap1/ -maxdepth 2 | wc -w`
+dir2=`find 2/.snap/barsnap2/ -maxdepth 2 | wc -w`
+#diff $dir1 $dir2 && echo "Success!"
+test $dir1==$dir2 && echo "Success!"
+echo "moving subtree to first folder"
+mv 2/a/b/c 1
+echo "comparing snapshots and new tree"
+dir1=`find 1/ | wc -w`
+dir2=`find 2/.snap/barsnap2/a/b/c | wc -w`
+#diff $dir1 $dir2 && echo "Success!"
+test $dir1==$dir2 && echo "Sucess!"
+rmdir 1/.snap/*
+rmdir 2/.snap/*
+echo "OK"
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-snap-rename.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-snap-rename.sh
new file mode 100755
index 0000000..e48b10b
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-snap-rename.sh
@@ -0,0 +1,35 @@
+#!/bin/sh -x
+
+expect_failure() {
+	if "$@"; then return 1; else return 0; fi
+}
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+mkdir -p d1/d2
+mkdir -p d1/d3
+mkdir d1/.snap/foo
+mkdir d1/d2/.snap/foo
+mkdir d1/d3/.snap/foo
+mkdir d1/d3/.snap/bar
+mv d1/d2/.snap/foo d1/d2/.snap/bar
+# snapshot name can't start with _
+expect_failure mv d1/d2/.snap/bar d1/d2/.snap/_bar
+# can't rename parent snapshot
+expect_failure mv d1/d2/.snap/_foo_* d1/d2/.snap/foo
+expect_failure mv d1/d2/.snap/_foo_* d1/d2/.snap/_foo_1
+# can't rename snapshot to different directroy
+expect_failure mv d1/d2/.snap/bar d1/.snap/
+# can't overwrite existing snapshot
+expect_failure python -c "import os; os.rename('d1/d3/.snap/foo', 'd1/d3/.snap/bar')"
+# can't move snaphost out of snapdir
+expect_failure python -c "import os; os.rename('d1/.snap/foo', 'd1/foo')"
+
+rmdir d1/.snap/foo
+rmdir d1/d2/.snap/bar
+rmdir d1/d3/.snap/foo
+rmdir d1/d3/.snap/bar
+rm -rf d1
+
+echo OK
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh
new file mode 100755
index 0000000..8b1ca5b
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh
@@ -0,0 +1,26 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+file=linux-2.6.33.tar.bz2
+wget -q http://download.ceph.com/qa/$file
+
+real=`md5sum $file | awk '{print $1}'`
+
+for f in `seq 1 20`
+do
+    echo $f
+    cp $file a
+    mkdir .snap/s
+    rm a
+    cp .snap/s/a /tmp/a
+    cur=`md5sum /tmp/a | awk '{print $1}'`
+    if [ "$cur" != "$real" ]; then
+	echo "FAIL: bad match, /tmp/a $cur != real $real"
+	false
+    fi
+    rmdir .snap/s
+done
+rm $file
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-upchildrealms.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-upchildrealms.sh
new file mode 100755
index 0000000..64a99ea
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-upchildrealms.sh
@@ -0,0 +1,30 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+#
+# verify that a snap update on a parent realm will induce
+# snap cap writeback for inodes child realms
+#
+
+mkdir a
+mkdir a/b
+mkdir a/.snap/a1
+mkdir a/b/.snap/b1
+echo asdf > a/b/foo
+mkdir a/.snap/a2
+# client _should_ have just queued a capsnap for writeback
+ln a/b/foo a/b/bar       # make the server cow the inode
+
+echo "this should not hang..."
+cat a/b/.snap/_a2_*/foo
+echo "good, it did not hang."
+
+rmdir a/b/.snap/b1
+rmdir a/.snap/a1
+rmdir a/.snap/a2
+rm -r a
+
+echo "OK"
\ No newline at end of file
diff --git a/src/ceph/qa/workunits/fs/snaps/snaptest-xattrwb.sh b/src/ceph/qa/workunits/fs/snaps/snaptest-xattrwb.sh
new file mode 100755
index 0000000..af28b63
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/snaptest-xattrwb.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -x
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+echo "testing simple xattr wb"
+touch x
+setfattr -n user.foo x
+mkdir .snap/s1
+getfattr -n user.foo .snap/s1/x | grep user.foo
+rm x
+rmdir .snap/s1
+
+echo "testing wb with pre-wb server cow"
+mkdir a
+mkdir a/b
+mkdir a/b/c
+# b now has As but not Ax
+setfattr -n user.foo a/b
+mkdir a/.snap/s
+mkdir a/b/cc
+# b now has been cowed on the server, but we still have dirty xattr caps
+getfattr -n user.foo a/b          # there they are...
+getfattr -n user.foo a/.snap/s/b | grep user.foo  # should be there, too!
+
+# ok, clean up
+rmdir a/.snap/s
+rm -r a
+
+echo OK
\ No newline at end of file
diff --git a/src/ceph/qa/workunits/fs/snaps/untar_snap_rm.sh b/src/ceph/qa/workunits/fs/snaps/untar_snap_rm.sh
new file mode 100755
index 0000000..b337aea
--- /dev/null
+++ b/src/ceph/qa/workunits/fs/snaps/untar_snap_rm.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+
+ceph mds set allow_new_snaps true --yes-i-really-mean-it
+
+do_tarball() {
+    wget http://download.ceph.com/qa/$1
+    tar xvf$2 $1
+    mkdir .snap/k
+    sync
+    rm -rv $3
+    cp -av .snap/k .
+    rmdir .snap/k
+    rm -rv k
+    rm $1
+}
+
+do_tarball coreutils_8.5.orig.tar.gz z coreutils-8.5
+do_tarball linux-2.6.33.tar.bz2 j linux-2.6.33