From 812ff6ca9fcd3e629e49d4328905f33eee8ca3f5 Mon Sep 17 00:00:00 2001 From: Qiaowei Ren Date: Thu, 4 Jan 2018 13:43:33 +0800 Subject: initial code repo This patch creates initial code repo. For ceph, luminous stable release will be used for base code, and next changes and optimization for ceph will be added to it. For opensds, currently any changes can be upstreamed into original opensds repo (https://github.com/opensds/opensds), and so stor4nfv will directly clone opensds code to deploy stor4nfv environment. And the scripts for deployment based on ceph and opensds will be put into 'ci' directory. Change-Id: I46a32218884c75dda2936337604ff03c554648e4 Signed-off-by: Qiaowei Ren --- src/ceph/qa/suites/upgrade/luminous-x/parallel/% | 0 .../suites/upgrade/luminous-x/parallel/0-cluster/+ | 0 .../luminous-x/parallel/0-cluster/openstack.yaml | 4 ++ .../luminous-x/parallel/0-cluster/start.yaml | 40 ++++++++++++++++++++ .../parallel/1-ceph-install/luminous.yaml | 43 ++++++++++++++++++++++ .../upgrade/luminous-x/parallel/2-workload/+ | 0 .../luminous-x/parallel/2-workload/blogbench.yaml | 14 +++++++ .../parallel/2-workload/ec-rados-default.yaml | 24 ++++++++++++ .../luminous-x/parallel/2-workload/rados_api.yaml | 11 ++++++ .../parallel/2-workload/rados_loadgenbig.yaml | 11 ++++++ .../parallel/2-workload/test_rbd_api.yaml | 11 ++++++ .../parallel/2-workload/test_rbd_python.yaml | 11 ++++++ .../parallel/3-upgrade-sequence/upgrade-all.yaml | 16 ++++++++ .../3-upgrade-sequence/upgrade-mon-osd-mds.yaml | 35 ++++++++++++++++++ .../upgrade/luminous-x/parallel/5-final-workload/+ | 0 .../parallel/5-final-workload/blogbench.yaml | 13 +++++++ .../5-final-workload/rados-snaps-few-objects.yaml | 17 +++++++++ .../5-final-workload/rados_loadgenmix.yaml | 9 +++++ .../5-final-workload/rados_mon_thrash.yaml | 18 +++++++++ .../parallel/5-final-workload/rbd_cls.yaml | 9 +++++ .../rbd_import_export_no_upgrated.yaml | 13 +++++++ .../rbd_import_export_upgrated.yaml | 12 ++++++ .../parallel/5-final-workload/rgw_swift.yaml | 13 +++++++ .../qa/suites/upgrade/luminous-x/parallel/distros | 1 + .../suites/upgrade/luminous-x/parallel/objectstore | 1 + 25 files changed, 326 insertions(+) create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/% create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/+ create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/openstack.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/1-ceph-install/luminous.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/+ create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/blogbench.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/ec-rados-default.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_api.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_loadgenbig.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_api.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_python.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-all.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-mon-osd-mds.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/+ create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/blogbench.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados-snaps-few-objects.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_loadgenmix.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_mon_thrash.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_cls.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_upgrated.yaml create mode 100644 src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rgw_swift.yaml create mode 120000 src/ceph/qa/suites/upgrade/luminous-x/parallel/distros create mode 120000 src/ceph/qa/suites/upgrade/luminous-x/parallel/objectstore (limited to 'src/ceph/qa/suites/upgrade/luminous-x/parallel') diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/% b/src/ceph/qa/suites/upgrade/luminous-x/parallel/% new file mode 100644 index 0000000..e69de29 diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/+ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/+ new file mode 100644 index 0000000..e69de29 diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/openstack.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/openstack.yaml new file mode 100644 index 0000000..f4d1349 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/openstack.yaml @@ -0,0 +1,4 @@ +openstack: + - volumes: # attached to each instance + count: 3 + size: 30 # GB diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml new file mode 100644 index 0000000..3684b1e --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml @@ -0,0 +1,40 @@ +meta: +- desc: | + Run ceph on two nodes, + with a separate client 0,1,2 third node. + Use xfs beneath the osds. + CephFS tests running on client 2,3 +roles: +- - mon.a + - mgr.x + - mds.a + - osd.0 + - osd.1 +- - mon.b + - mon.c + - osd.2 + - osd.3 +- - client.0 + - client.1 + - client.2 + - client.3 +- - client.4 +overrides: + ceph: + log-whitelist: + - scrub mismatch + - ScrubResult + - wrongly marked + - (POOL_APP_NOT_ENABLED) + - overall HEALTH_ + conf: + global: + enable experimental unrecoverable data corrupting features: "*" + mon: + mon warn on osd down out interval zero: false + osd: + osd_class_load_list: "cephfs hello journal lock log numops rbd refcount + replica_log rgw sdk statelog timeindex user version" + osd_class_default_list: "cephfs hello journal lock log numops rbd refcount + replica_log rgw sdk statelog timeindex user version" + fs: xfs diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/1-ceph-install/luminous.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/1-ceph-install/luminous.yaml new file mode 100644 index 0000000..3d57f79 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/1-ceph-install/luminous.yaml @@ -0,0 +1,43 @@ +meta: +- desc: | + install ceph/luminous latest + run workload and upgrade-sequence in parallel + upgrade the client node +tasks: +- install: + branch: luminous +- print: "**** done installing luminous" +- ceph: + log-whitelist: + - overall HEALTH_ + - \(FS_ + - \(MDS_ + - \(OSD_ + - \(MON_DOWN\) + - \(CACHE_POOL_ + - \(POOL_ + - \(MGR_DOWN\) + - \(PG_ + - \(SMALLER_PGP_NUM\) + - Monitor daemon marked osd + - Behind on trimming + - Manager daemon + conf: + global: + mon warn on pool no app: false +- exec: + osd.0: + - ceph osd require-osd-release luminous + - ceph osd set-require-min-compat-client luminous +- print: "**** done ceph" +- install.upgrade: + mon.a: + mon.b: +- print: "**** done install.upgrade both hosts" +- parallel: + - workload + - upgrade-sequence +- print: "**** done parallel" +- install.upgrade: + client.0: +- print: "**** done install.upgrade on client.0" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/+ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/+ new file mode 100644 index 0000000..e69de29 diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/blogbench.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/blogbench.yaml new file mode 100644 index 0000000..021fcc6 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/blogbench.yaml @@ -0,0 +1,14 @@ +meta: +- desc: | + run a cephfs stress test + mount ceph-fuse on client.2 before running workunit +workload: + full_sequential: + - sequential: + - ceph-fuse: + - print: "**** done ceph-fuse 2-workload" + - workunit: + clients: + client.2: + - suites/blogbench.sh + - print: "**** done suites/blogbench.sh 2-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/ec-rados-default.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/ec-rados-default.yaml new file mode 100644 index 0000000..5c5a958 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/ec-rados-default.yaml @@ -0,0 +1,24 @@ +meta: +- desc: | + run run randomized correctness test for rados operations + on an erasure-coded pool +workload: + full_sequential: + - rados: + clients: [client.0] + ops: 4000 + objects: 50 + ec_pool: true + write_append_excl: false + op_weights: + read: 100 + write: 0 + append: 100 + delete: 50 + snap_create: 50 + snap_remove: 50 + rollback: 50 + copy_from: 50 + setattr: 25 + rmattr: 25 + - print: "**** done rados ec task" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_api.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_api.yaml new file mode 100644 index 0000000..e4cc9f9 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_api.yaml @@ -0,0 +1,11 @@ +meta: +- desc: | + object class functional tests +workload: + full_sequential: + - workunit: + branch: luminous + clients: + client.0: + - cls + - print: "**** done cls 2-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_loadgenbig.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_loadgenbig.yaml new file mode 100644 index 0000000..874a8c5 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/rados_loadgenbig.yaml @@ -0,0 +1,11 @@ +meta: +- desc: | + generate read/write load with rados objects ranging from 1MB to 25MB +workload: + full_sequential: + - workunit: + branch: luminous + clients: + client.0: + - rados/load-gen-big.sh + - print: "**** done rados/load-gen-big.sh 2-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_api.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_api.yaml new file mode 100644 index 0000000..81563c9 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_api.yaml @@ -0,0 +1,11 @@ +meta: +- desc: | + librbd C and C++ api tests +workload: + full_sequential: + - workunit: + branch: luminous + clients: + client.0: + - rbd/test_librbd.sh + - print: "**** done rbd/test_librbd.sh 2-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_python.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_python.yaml new file mode 100644 index 0000000..e17207d --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/2-workload/test_rbd_python.yaml @@ -0,0 +1,11 @@ +meta: +- desc: | + librbd python api tests +workload: + full_sequential: + - workunit: + branch: luminous + clients: + client.0: + - rbd/test_librbd_python.sh + - print: "**** done rbd/test_librbd_python.sh 2-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-all.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-all.yaml new file mode 100644 index 0000000..cff3a68 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-all.yaml @@ -0,0 +1,16 @@ +meta: +- desc: | + upgrade the ceph cluster +upgrade-sequence: + sequential: + - ceph.restart: + daemons: [mon.a, mon.b, mon.c, mgr.x] + - ceph.restart: + daemons: [osd.0, osd.1, osd.2, osd.3] + wait-for-healthy: false + wait-for-osds-up: true + - ceph.restart: + daemons: [mds.a] + wait-for-healthy: false + wait-for-osds-up: true + - print: "**** done ceph.restart all" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-mon-osd-mds.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-mon-osd-mds.yaml new file mode 100644 index 0000000..f197de6 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/3-upgrade-sequence/upgrade-mon-osd-mds.yaml @@ -0,0 +1,35 @@ +meta: +- desc: | + upgrade the ceph cluster, + upgrate in two steps + step one ordering: mon.a, osd.0, osd.1, mds.a + step two ordering: mon.b, mon.c, osd.2, osd.3 + ceph expected to be healthy state after each step +upgrade-sequence: + sequential: + - ceph.restart: + daemons: [mon.a] + wait-for-healthy: true + - sleep: + duration: 60 + - ceph.restart: + daemons: [mon.b, mon.c, mgr.x] + wait-for-healthy: true + - sleep: + duration: 60 + - ceph.restart: + daemons: [osd.0, osd.1] + wait-for-healthy: true + - sleep: + duration: 60 + - ceph.restart: [mds.a] + - sleep: + duration: 60 + - sleep: + duration: 60 + - ceph.restart: + daemons: [osd.2, osd.3] + wait-for-healthy: false + wait-for-osds-up: true + - sleep: + duration: 60 diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/+ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/+ new file mode 100644 index 0000000..e69de29 diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/blogbench.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/blogbench.yaml new file mode 100644 index 0000000..d2629c0 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/blogbench.yaml @@ -0,0 +1,13 @@ +meta: +- desc: | + run a cephfs stress test + mount ceph-fuse on client.3 before running workunit +tasks: +- sequential: + - ceph-fuse: + - print: "**** done ceph-fuse 5-final-workload" + - workunit: + clients: + client.3: + - suites/blogbench.sh + - print: "**** done suites/blogbench.sh 5-final-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados-snaps-few-objects.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados-snaps-few-objects.yaml new file mode 100644 index 0000000..d8b3dcb --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados-snaps-few-objects.yaml @@ -0,0 +1,17 @@ +meta: +- desc: | + randomized correctness test for rados operations on a replicated pool with snapshots +tasks: + - rados: + clients: [client.1] + ops: 4000 + objects: 50 + write_append_excl: false + op_weights: + read: 100 + write: 100 + delete: 50 + snap_create: 50 + snap_remove: 50 + rollback: 50 + - print: "**** done rados 4-final-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_loadgenmix.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_loadgenmix.yaml new file mode 100644 index 0000000..922a9da --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_loadgenmix.yaml @@ -0,0 +1,9 @@ +meta: +- desc: | + generate read/write load with rados objects ranging from 1 byte to 1MB +tasks: + - workunit: + clients: + client.1: + - rados/load-gen-mix.sh + - print: "**** done rados/load-gen-mix.sh 4-final-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_mon_thrash.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_mon_thrash.yaml new file mode 100644 index 0000000..a42b7d2 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rados_mon_thrash.yaml @@ -0,0 +1,18 @@ +meta: +- desc: | + librados C and C++ api tests +overrides: + ceph: + log-whitelist: + - reached quota +tasks: + - mon_thrash: + revive_delay: 20 + thrash_delay: 1 + - print: "**** done mon_thrash 4-final-workload" + - workunit: + branch: luminous + clients: + client.1: + - rados/test.sh + - print: "**** done rados/test.sh 4-final-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_cls.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_cls.yaml new file mode 100644 index 0000000..aaf0a37 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_cls.yaml @@ -0,0 +1,9 @@ +meta: +- desc: | + rbd object class functional tests +tasks: + - workunit: + clients: + client.1: + - cls/test_cls_rbd.sh + - print: "**** done cls/test_cls_rbd.sh 4-final-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml new file mode 100644 index 0000000..5de8a23 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml @@ -0,0 +1,13 @@ +meta: +- desc: | + run basic import/export cli tests for rbd + on NO upgrated client +tasks: + - workunit: + branch: luminous + clients: + client.4: + - rbd/import_export.sh + env: + RBD_CREATE_ARGS: --new-format + - print: "**** done rbd/import_export.sh 4-final-workload on NO upgrated client" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_upgrated.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_upgrated.yaml new file mode 100644 index 0000000..2c7c484 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_upgrated.yaml @@ -0,0 +1,12 @@ +meta: +- desc: | + run basic import/export cli tests for rbd + on upgrated client +tasks: + - workunit: + clients: + client.1: + - rbd/import_export.sh + env: + RBD_CREATE_ARGS: --new-format + - print: "**** done rbd/import_export.sh 4-final-workload on upgrated client" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rgw_swift.yaml b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rgw_swift.yaml new file mode 100644 index 0000000..7a7659f --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rgw_swift.yaml @@ -0,0 +1,13 @@ +meta: +- desc: | + swift api tests for rgw +overrides: + rgw: + frontend: civetweb +tasks: + - rgw: [client.1] + - print: "**** done rgw 4-final-workload" + - swift: + client.1: + rgw_server: client.1 + - print: "**** done swift 4-final-workload" diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/distros b/src/ceph/qa/suites/upgrade/luminous-x/parallel/distros new file mode 120000 index 0000000..ca99fee --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/distros @@ -0,0 +1 @@ +../../../../distros/supported/ \ No newline at end of file diff --git a/src/ceph/qa/suites/upgrade/luminous-x/parallel/objectstore b/src/ceph/qa/suites/upgrade/luminous-x/parallel/objectstore new file mode 120000 index 0000000..016cbf9 --- /dev/null +++ b/src/ceph/qa/suites/upgrade/luminous-x/parallel/objectstore @@ -0,0 +1 @@ +../stress-split/objectstore/ \ No newline at end of file -- cgit 1.2.3-korg