From 4faa7f927149a5c4ef7a03523f7bc14523cb9baa Mon Sep 17 00:00:00 2001 From: Stuart Mackie Date: Fri, 7 Oct 2016 12:24:58 -0700 Subject: Charms for Contrail 3.1 with Mitaka Change-Id: Id37f3b9743d1974e31fcd7cd9c54be41bb0c47fb Signed-off-by: Stuart Mackie --- charms/trusty/kafka/.bzr/README | 3 + charms/trusty/kafka/.bzr/branch-format | 1 + charms/trusty/kafka/.bzr/branch/format | 1 + charms/trusty/kafka/.bzr/branch/location | 1 + charms/trusty/kafka/.bzr/checkout/conflicts | 1 + charms/trusty/kafka/.bzr/checkout/dirstate | Bin 0 -> 10605 bytes charms/trusty/kafka/.bzr/checkout/format | 1 + charms/trusty/kafka/.bzr/checkout/views | 0 charms/trusty/kafka/LICENSE | 177 ++++++++++++++++++++ charms/trusty/kafka/README.md | 84 ++++++++++ charms/trusty/kafka/actions.yaml | 48 ++++++ charms/trusty/kafka/actions/create-topic | 40 +++++ charms/trusty/kafka/actions/delete-topic | 36 ++++ charms/trusty/kafka/actions/list-topics | 31 ++++ charms/trusty/kafka/actions/list-zks | 28 ++++ charms/trusty/kafka/actions/read-topic | 35 ++++ charms/trusty/kafka/actions/write-topic | 36 ++++ charms/trusty/kafka/config.yaml | 7 + charms/trusty/kafka/copyright | 16 ++ charms/trusty/kafka/dist.yaml | 30 ++++ charms/trusty/kafka/hooks/callbacks.py | 181 +++++++++++++++++++++ charms/trusty/kafka/hooks/common.py | 90 ++++++++++ charms/trusty/kafka/hooks/config-changed | 15 ++ charms/trusty/kafka/hooks/install | 17 ++ charms/trusty/kafka/hooks/kafka-relation-changed | 15 ++ charms/trusty/kafka/hooks/setup.py | 33 ++++ charms/trusty/kafka/hooks/start | 15 ++ charms/trusty/kafka/hooks/stop | 15 ++ .../trusty/kafka/hooks/zookeeper-relation-changed | 15 ++ .../trusty/kafka/hooks/zookeeper-relation-departed | 15 ++ charms/trusty/kafka/icon.svg | 90 ++++++++++ charms/trusty/kafka/metadata.yaml | 30 ++++ charms/trusty/kafka/resources.yaml | 12 ++ .../kafka/resources/python/PyYAML-3.11.tar.gz | Bin 0 -> 248685 bytes .../resources/python/charmhelpers-0.3.1.tar.gz | Bin 0 -> 62031 bytes .../resources/python/jujuresources-0.2.11.tar.gz | Bin 0 -> 12679 bytes .../kafka/resources/python/pyaml-15.5.7.tar.gz | Bin 0 -> 14374 bytes .../python/six-1.9.0-py2.py3-none-any.whl | Bin 0 -> 10222 bytes charms/trusty/kafka/templates/upstart.conf | 14 ++ charms/trusty/kafka/tests/00-setup | 5 + charms/trusty/kafka/tests/100-deploy-kafka | 29 ++++ .../trusty/kafka/tests/remote/test_dist_config.py | 71 ++++++++ charms/trusty/kafka/tests/tests.yaml | 10 ++ 43 files changed, 1248 insertions(+) create mode 100644 charms/trusty/kafka/.bzr/README create mode 100644 charms/trusty/kafka/.bzr/branch-format create mode 100644 charms/trusty/kafka/.bzr/branch/format create mode 100644 charms/trusty/kafka/.bzr/branch/location create mode 100644 charms/trusty/kafka/.bzr/checkout/conflicts create mode 100644 charms/trusty/kafka/.bzr/checkout/dirstate create mode 100644 charms/trusty/kafka/.bzr/checkout/format create mode 100644 charms/trusty/kafka/.bzr/checkout/views create mode 100644 charms/trusty/kafka/LICENSE create mode 100644 charms/trusty/kafka/README.md create mode 100644 charms/trusty/kafka/actions.yaml create mode 100755 charms/trusty/kafka/actions/create-topic create mode 100755 charms/trusty/kafka/actions/delete-topic create mode 100755 charms/trusty/kafka/actions/list-topics create mode 100755 charms/trusty/kafka/actions/list-zks create mode 100755 charms/trusty/kafka/actions/read-topic create mode 100755 charms/trusty/kafka/actions/write-topic create mode 100644 charms/trusty/kafka/config.yaml create mode 100644 charms/trusty/kafka/copyright create mode 100644 charms/trusty/kafka/dist.yaml create mode 100644 charms/trusty/kafka/hooks/callbacks.py create mode 100755 charms/trusty/kafka/hooks/common.py create mode 100755 charms/trusty/kafka/hooks/config-changed create mode 100755 charms/trusty/kafka/hooks/install create mode 100755 charms/trusty/kafka/hooks/kafka-relation-changed create mode 100644 charms/trusty/kafka/hooks/setup.py create mode 100755 charms/trusty/kafka/hooks/start create mode 100755 charms/trusty/kafka/hooks/stop create mode 100755 charms/trusty/kafka/hooks/zookeeper-relation-changed create mode 100755 charms/trusty/kafka/hooks/zookeeper-relation-departed create mode 100644 charms/trusty/kafka/icon.svg create mode 100644 charms/trusty/kafka/metadata.yaml create mode 100644 charms/trusty/kafka/resources.yaml create mode 100644 charms/trusty/kafka/resources/python/PyYAML-3.11.tar.gz create mode 100644 charms/trusty/kafka/resources/python/charmhelpers-0.3.1.tar.gz create mode 100644 charms/trusty/kafka/resources/python/jujuresources-0.2.11.tar.gz create mode 100644 charms/trusty/kafka/resources/python/pyaml-15.5.7.tar.gz create mode 100644 charms/trusty/kafka/resources/python/six-1.9.0-py2.py3-none-any.whl create mode 100644 charms/trusty/kafka/templates/upstart.conf create mode 100755 charms/trusty/kafka/tests/00-setup create mode 100755 charms/trusty/kafka/tests/100-deploy-kafka create mode 100755 charms/trusty/kafka/tests/remote/test_dist_config.py create mode 100644 charms/trusty/kafka/tests/tests.yaml (limited to 'charms/trusty/kafka') diff --git a/charms/trusty/kafka/.bzr/README b/charms/trusty/kafka/.bzr/README new file mode 100644 index 0000000..f82dc1c --- /dev/null +++ b/charms/trusty/kafka/.bzr/README @@ -0,0 +1,3 @@ +This is a Bazaar control directory. +Do not change any files in this directory. +See http://bazaar.canonical.com/ for more information about Bazaar. diff --git a/charms/trusty/kafka/.bzr/branch-format b/charms/trusty/kafka/.bzr/branch-format new file mode 100644 index 0000000..9eb09b7 --- /dev/null +++ b/charms/trusty/kafka/.bzr/branch-format @@ -0,0 +1 @@ +Bazaar-NG meta directory, format 1 diff --git a/charms/trusty/kafka/.bzr/branch/format b/charms/trusty/kafka/.bzr/branch/format new file mode 100644 index 0000000..b391ffd --- /dev/null +++ b/charms/trusty/kafka/.bzr/branch/format @@ -0,0 +1 @@ +Bazaar-NG Branch Reference Format 1 diff --git a/charms/trusty/kafka/.bzr/branch/location b/charms/trusty/kafka/.bzr/branch/location new file mode 100644 index 0000000..2f40eef --- /dev/null +++ b/charms/trusty/kafka/.bzr/branch/location @@ -0,0 +1 @@ +http://bazaar.launchpad.net/~sdn-charmers/charms/trusty/apache-kafka/trunk/ \ No newline at end of file diff --git a/charms/trusty/kafka/.bzr/checkout/conflicts b/charms/trusty/kafka/.bzr/checkout/conflicts new file mode 100644 index 0000000..0dc2d3a --- /dev/null +++ b/charms/trusty/kafka/.bzr/checkout/conflicts @@ -0,0 +1 @@ +BZR conflict list format 1 diff --git a/charms/trusty/kafka/.bzr/checkout/dirstate b/charms/trusty/kafka/.bzr/checkout/dirstate new file mode 100644 index 0000000..9d2ef8a Binary files /dev/null and b/charms/trusty/kafka/.bzr/checkout/dirstate differ diff --git a/charms/trusty/kafka/.bzr/checkout/format b/charms/trusty/kafka/.bzr/checkout/format new file mode 100644 index 0000000..e0261c7 --- /dev/null +++ b/charms/trusty/kafka/.bzr/checkout/format @@ -0,0 +1 @@ +Bazaar Working Tree Format 6 (bzr 1.14) diff --git a/charms/trusty/kafka/.bzr/checkout/views b/charms/trusty/kafka/.bzr/checkout/views new file mode 100644 index 0000000..e69de29 diff --git a/charms/trusty/kafka/LICENSE b/charms/trusty/kafka/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/charms/trusty/kafka/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/charms/trusty/kafka/README.md b/charms/trusty/kafka/README.md new file mode 100644 index 0000000..31e8e4f --- /dev/null +++ b/charms/trusty/kafka/README.md @@ -0,0 +1,84 @@ +## Overview +Apache Kafka is an open-source message broker project developed by the Apache +Software Foundation written in Scala. The project aims to provide a unified, +high-throughput, low-latency platform for handling real-time data feeds. Learn +more at [kafka.apache.org](http://kafka.apache.org/). + + +## Usage +Kafka requires the Zookeeper distributed coordination service. Deploy and +relate them as follows: + + juju deploy apache-zookeeper zookeeper + juju deploy apache-kafka kafka + juju add-relation kafka zookeeper + +Once deployed, we can list the zookeeper servers that our kafka brokers +are connected to. The following will list `:` information for each +zookeeper unit in the environment (e.g.: `10.0.3.221:2181`). + + juju action do kafka/0 list-zks + juju action fetch # <-- id from above command + +We can create a Kafka topic with: + + juju action do kafka/0 create-topic topic= \ + partitions=<#> replication=<#> + juju action fetch # <-- id from above command + +We can list topics with: + + juju action do kafka/0 list-topics + juju action fetch # <-- id from above command + +We can write to a topic with: + + juju action do kafka/0 write-topic topic= data= + juju action fetch # <-- id from above command + +We can read from a topic with: + + juju action do kafka/0 read-topic topic= partition=<#> + juju action fetch # <-- id from above command + +And finally, we can delete a topic with: + + juju action do kafka/0 delete-topic topic= + juju action fetch # <-- id from above command + +## Deploying in Network-Restricted Environments +This charm can be deployed in environments with limited network access. To +deploy in this environment, you will need a local mirror to serve the packages +and resources required by this charm. + +### Mirroring Packages +You can setup a local mirror for apt packages using squid-deb-proxy. +For instructions on configuring juju to use this, see the +[Juju Proxy Documentation](https://juju.ubuntu.com/docs/howto-proxies.html). + +### Mirroring Resources +In addition to apt packages, this charm requires a few binary resources +which are normally hosted on Launchpad. If access to Launchpad is not +available, the `jujuresources` library makes it easy to create a mirror +of these resources: + + sudo pip install jujuresources + juju-resources fetch --all /path/to/resources.yaml -d /tmp/resources + juju-resources serve -d /tmp/resources + +This will fetch all of the resources needed by this charm and serve them via a +simple HTTP server. The output from `juju-resources serve` will give you a +URL that you can set as the `resources_mirror` config option for this charm. +Setting this option will cause all resources required by this charm to be +downloaded from the configured URL. + + +## Contact Information +- + + +## Help +- [Apache Kafka home page](http://kafka.apache.org/) +- [Apache Kafka issue tracker](https://issues.apache.org/jira/browse/KAFKA) +- [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) +- [Juju community](https://jujucharms.com/community) diff --git a/charms/trusty/kafka/actions.yaml b/charms/trusty/kafka/actions.yaml new file mode 100644 index 0000000..6026d0b --- /dev/null +++ b/charms/trusty/kafka/actions.yaml @@ -0,0 +1,48 @@ +create-topic: + description: Create a new Kafka topic + params: + topic: + type: string + description: Topic name + partitions: + type: integer + description: Number of partitions for the topic being created + replication: + type: integer + description: Replication factor for each partition in the topic + required: [topic, partitions, replication] + additionalProperties: false +delete-topic: + description: Delete a Kafka topic + params: + topic: + type: string + description: Topic name + required: [topic] + additionalProperties: false +list-topics: + description: List all Kafka topics +list-zks: + description: List ip:port info for connected Zookeeper servers +read-topic: + description: Consume an existing kafka topic + params: + topic: + type: string + description: Topic name + partition: + type: integer + description: Partition to consume + required: [topic, partition] + additionalProperties: false +write-topic: + description: Write to a kafka topic + params: + topic: + type: string + description: Topic name + data: + type: string + description: Data to write to topic + required: [topic, data] + additionalProperties: false diff --git a/charms/trusty/kafka/actions/create-topic b/charms/trusty/kafka/actions/create-topic new file mode 100755 index 0000000..4910430 --- /dev/null +++ b/charms/trusty/kafka/actions/create-topic @@ -0,0 +1,40 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +# Grab the business +topic_name = hookenv.action_get('topic') +topic_partitions = hookenv.action_get('partitions') +topic_replication = hookenv.action_get('replication') + +# Create the topic if we've got zookeepers; otherwise fail. +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + output = utils.run_as('kafka', 'kafka-topics.sh', + '--zookeeper', zookeepers, '--create', + '--topic', topic_name, + '--partitions', topic_partitions, + '--replication-factor', topic_replication, + capture_output=True) + hookenv.action_set({'output': output}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/delete-topic b/charms/trusty/kafka/actions/delete-topic new file mode 100755 index 0000000..b56f004 --- /dev/null +++ b/charms/trusty/kafka/actions/delete-topic @@ -0,0 +1,36 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +# Grab the business +topic_name = hookenv.action_get('topic') + +# Delete the topic if we've got zookeepers; otherwise fail. +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + output = utils.run_as('kafka', 'kafka-topics.sh', + '--zookeeper', zookeepers, '--delete', + '--topic', topic_name, + capture_output=True) + hookenv.action_set({'output': output}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/list-topics b/charms/trusty/kafka/actions/list-topics new file mode 100755 index 0000000..629d2b4 --- /dev/null +++ b/charms/trusty/kafka/actions/list-topics @@ -0,0 +1,31 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + output = utils.run_as('kafka', '/usr/lib/kafka/bin/kafka-topics.sh', + '--zookeeper', zookeepers, '--list', + capture_output=True) + hookenv.action_set({'topics': output}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/list-zks b/charms/trusty/kafka/actions/list-zks new file mode 100755 index 0000000..9de9e9a --- /dev/null +++ b/charms/trusty/kafka/actions/list-zks @@ -0,0 +1,28 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + hookenv.action_set({'zookeepers': zookeepers}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/read-topic b/charms/trusty/kafka/actions/read-topic new file mode 100755 index 0000000..9f59396 --- /dev/null +++ b/charms/trusty/kafka/actions/read-topic @@ -0,0 +1,35 @@ +#!/usr/bin/env python +#pylint: disable=C0103 +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + import jujubigdata + from jujubigdata import utils + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +kafka_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports'] +dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml', + required_keys=kafka_reqs) + +# Grab the business +topic_name = hookenv.action_get('topic') +topic_partition = hookenv.action_get('partition') + +output = utils.run_as( + 'kafka', 'kafka-simple-consumer-shell.sh', + '--broker-list', '{}:{}'.format( + hookenv.unit_private_ip(), + dist_config.port('kafka'), + ), + '--topic', topic_name, + '--partition', topic_partition, + '--no-wait-at-logend', + capture_output=True) +hookenv.action_set({'output': output}) diff --git a/charms/trusty/kafka/actions/write-topic b/charms/trusty/kafka/actions/write-topic new file mode 100755 index 0000000..fce8e1b --- /dev/null +++ b/charms/trusty/kafka/actions/write-topic @@ -0,0 +1,36 @@ +#!/usr/bin/env python +#pylint: disable=C0103 + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + import jujubigdata + from jujubigdata import utils + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + + +kafka_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports'] +dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml', + required_keys=kafka_reqs) + +# Grab the business +topic_name = hookenv.action_get('topic') +data = hookenv.action_get('data') + +output = utils.run_as( + 'kafka', 'kafka-console-producer.sh', + '--broker-list', '{}:{}'.format( + hookenv.unit_private_ip(), + dist_config.port('kafka'), + ), + '--topic', topic_name, + capture_output=True, + input=data) +hookenv.action_set({'output': output}) diff --git a/charms/trusty/kafka/config.yaml b/charms/trusty/kafka/config.yaml new file mode 100644 index 0000000..f2483b0 --- /dev/null +++ b/charms/trusty/kafka/config.yaml @@ -0,0 +1,7 @@ +options: + resources_mirror: + type: string + default: '' + description: | + URL from which to fetch resources (e.g., Hadoop binaries) instead of Launchpad. + diff --git a/charms/trusty/kafka/copyright b/charms/trusty/kafka/copyright new file mode 100644 index 0000000..e900b97 --- /dev/null +++ b/charms/trusty/kafka/copyright @@ -0,0 +1,16 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2015, Canonical Ltd., All Rights Reserved. +License: Apache License 2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/charms/trusty/kafka/dist.yaml b/charms/trusty/kafka/dist.yaml new file mode 100644 index 0000000..e8f1a37 --- /dev/null +++ b/charms/trusty/kafka/dist.yaml @@ -0,0 +1,30 @@ +# This file contains values that are likely to change per distribution. +# The aim is to make it easier to update / extend the charms with +# minimal changes to the shared code in the jujubigdata library. +vendor: 'apache' +packages: + - 'openjdk-7-jdk' +groups: + - 'hadoop' +users: + kafka: + groups: ['hadoop'] +dirs: + kafka: + path: '/usr/lib/kafka' + kafka_conf: + path: '/etc/kafka/conf' + kafka_app_logs: + path: '/var/log/kafka' + owner: 'kafka' + kafka_data_logs: + path: '/var/lib/kafka' + owner: 'kafka' +ports: + # Ports that need to be exposed, overridden, or manually specified. + # Only expose ports serving a UI or external API (i.e., namenode and + # resourcemanager). Communication among units within the cluster does + # not need ports to be explicitly opened. + kafka: + port: 9092 + exposed_on: 'kafka' diff --git a/charms/trusty/kafka/hooks/callbacks.py b/charms/trusty/kafka/hooks/callbacks.py new file mode 100644 index 0000000..bac68cb --- /dev/null +++ b/charms/trusty/kafka/hooks/callbacks.py @@ -0,0 +1,181 @@ +import os +from socket import gaierror, gethostbyname, gethostname +from subprocess import Popen, check_call + +import jujuresources +from charmhelpers.core import hookenv, templating +from charmhelpers.core import host +from charmhelpers.core import unitdata +from jujubigdata import utils +from jujubigdata.relations import Zookeeper + + +# Extended status support +# We call update_blocked_status from the "requires" section of our service +# block, so be sure to return True. Otherwise, we'll block the "requires" +# and never move on to callbacks. The other status update methods are called +# from the "callbacks" section and therefore don't need to return True. +def update_blocked_status(): + if unitdata.kv().get('charm.active', False): + return True + if not Zookeeper().connected_units(): + hookenv.status_set('blocked', 'Waiting for relation to apache-zookeeper') + elif not Zookeeper().is_ready(): + hookenv.status_set('waiting', 'Waiting for Zookeeper to become ready') + return True + + +def update_working_status(): + if unitdata.kv().get('charm.active', False): + hookenv.status_set('maintenance', 'Updating configuration') + return + hookenv.status_set('maintenance', 'Setting up Apache Kafka') + + +def update_active_status(): + unitdata.kv().set('charm.active', True) + hookenv.status_set('active', 'Ready') + + +def clear_active_flag(): + unitdata.kv().set('charm.active', False) + + +# Main Kafka class for callbacks +class Kafka(object): + def __init__(self, dist_config): + self.dist_config = dist_config + self.resources = { + 'kafka': 'kafka-%s' % host.cpu_arch(), + } + self.verify_resources = utils.verify_resources(*self.resources.values()) + + def fix_hostname(self): + # ensure hostname is resolvable + hostname = gethostname() + try: + gethostbyname(hostname) + except gaierror: + check_call(['sed', '-E', '-i', '-e', + '/127.0.0.1[[:blank:]]+/a \\\n127.0.1.1 ' + hostname, + '/etc/hosts']) + + def is_installed(self): + return unitdata.kv().get('kafka.installed') + + def install(self, force=False): + if not force and self.is_installed(): + return + self.fix_hostname() + self.dist_config.add_users() + self.dist_config.add_dirs() + self.dist_config.add_packages() + jujuresources.install(self.resources['kafka'], + destination=self.dist_config.path('kafka'), + skip_top_level=True) + self.setup_kafka_config() + unitdata.kv().set('kafka.installed', True) + + def setup_kafka_config(self): + ''' + copy the default configuration files to kafka_conf property + defined in dist.yaml + ''' + default_conf = self.dist_config.path('kafka') / 'config' + kafka_conf = self.dist_config.path('kafka_conf') + kafka_conf.rmtree_p() + default_conf.copytree(kafka_conf) + # Now remove the conf included in the tarball and symlink our real conf + # dir. we've seen issues where kafka still looks for config in + # KAFKA_HOME/config. + default_conf.rmtree_p() + kafka_conf.symlink(default_conf) + + # Configure immutable bits + kafka_bin = self.dist_config.path('kafka') / 'bin' + with utils.environment_edit_in_place('/etc/environment') as env: + if kafka_bin not in env['PATH']: + env['PATH'] = ':'.join([env['PATH'], kafka_bin]) + env['LOG_DIR'] = self.dist_config.path('kafka_app_logs') + + # note: we set the advertised.host.name below to the public_address + # to ensure that external (non-Juju) clients can connect to Kafka + public_address = hookenv.unit_get('public-address') + private_ip = utils.resolve_private_address(hookenv.unit_get('private-address')) + kafka_server_conf = self.dist_config.path('kafka_conf') / 'server.properties' + service, unit_num = os.environ['JUJU_UNIT_NAME'].split('/', 1) + utils.re_edit_in_place(kafka_server_conf, { + r'^broker.id=.*': 'broker.id=%s' % unit_num, + r'^port=.*': 'port=%s' % self.dist_config.port('kafka'), + r'^log.dirs=.*': 'log.dirs=%s' % self.dist_config.path('kafka_data_logs'), + r'^#?advertised.host.name=.*': 'advertised.host.name=%s' % public_address, + }) + + kafka_log4j = self.dist_config.path('kafka_conf') / 'log4j.properties' + utils.re_edit_in_place(kafka_log4j, { + r'^kafka.logs.dir=.*': 'kafka.logs.dir=%s' % self.dist_config.path('kafka_app_logs'), + }) + + # fix for lxc containers and some corner cases in manual provider + # ensure that public_address is resolvable internally by mapping it to the private IP + utils.update_etc_hosts({private_ip: public_address}) + + templating.render( + 'upstart.conf', + '/etc/init/kafka.conf', + context={ + 'kafka_conf': self.dist_config.path('kafka_conf'), + 'kafka_bin': '{}/bin'.format(self.dist_config.path('kafka')) + }, + ) + + def configure_kafka(self): + # Get ip:port data from our connected zookeepers + if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zk_connect = ",".join(zks) + + # update consumer props + cfg = self.dist_config.path('kafka_conf') / 'consumer.properties' + utils.re_edit_in_place(cfg, { + r'^zookeeper.connect=.*': 'zookeeper.connect=%s' % zk_connect, + }) + + # update server props + cfg = self.dist_config.path('kafka_conf') / 'server.properties' + utils.re_edit_in_place(cfg, { + r'^zookeeper.connect=.*': 'zookeeper.connect=%s' % zk_connect, + }) + else: + # if we have no zookeepers, make sure kafka is stopped + self.stop() + + def run_bg(self, user, command, *args): + """ + Run a Kafka command as the `kafka` user in the background. + + :param str command: Command to run + :param list args: Additional args to pass to the command + """ + parts = [command] + list(args) + quoted = ' '.join("'%s'" % p for p in parts) + e = utils.read_etc_env() + Popen(['su', user, '-c', quoted], env=e) + + def restart(self): + self.stop() + self.start() + + def start(self): + host.service_start('kafka') + + def stop(self): + host.service_stop('kafka') + + def cleanup(self): + self.dist_config.remove_users() + self.dist_config.remove_dirs() diff --git a/charms/trusty/kafka/hooks/common.py b/charms/trusty/kafka/hooks/common.py new file mode 100755 index 0000000..1ca080b --- /dev/null +++ b/charms/trusty/kafka/hooks/common.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Common implementation for all hooks. +""" + +import jujuresources +from charmhelpers.core import hookenv +from charmhelpers.core import unitdata +from charmhelpers.core import charmframework + + +def bootstrap_resources(): + """ + Install required resources defined in resources.yaml + """ + if unitdata.kv().get('charm.bootstrapped', False): + return True + hookenv.status_set('maintenance', 'Installing base resources') + mirror_url = jujuresources.config_get('resources_mirror') + if not jujuresources.fetch(mirror_url=mirror_url): + missing = jujuresources.invalid() + hookenv.status_set('blocked', 'Unable to fetch required resource%s: %s' % ( + 's' if len(missing) > 1 else '', + ', '.join(missing), + )) + return False + jujuresources.install(['pathlib', 'jujubigdata']) + unitdata.kv().set('charm.bootstrapped', True) + return True + + +def manage(): + if not bootstrap_resources(): + # defer until resources are available, since charmhelpers, and thus + # the framework, are required (will require manual intervention) + return + + import jujubigdata + import callbacks + + kafka_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports'] + dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml', + required_keys=kafka_reqs) + kafka = callbacks.Kafka(dist_config) + manager = charmframework.Manager([ + { + 'name': 'kafka', + 'provides': [ + jujubigdata.relations.Kafka(port=dist_config.port('kafka')) + ], + 'requires': [ + kafka.verify_resources, + jujubigdata.relations.Zookeeper(), + callbacks.update_blocked_status, # not really a requirement, but best way to fit into framework + ], + 'callbacks': [ + callbacks.update_working_status, + kafka.install, + kafka.configure_kafka, + kafka.restart, + charmframework.helpers.open_ports( + dist_config.exposed_ports('kafka')), + callbacks.update_active_status, + ], + 'cleanup': [ + callbacks.clear_active_flag, + charmframework.helpers.close_ports( + dist_config.exposed_ports('kafka')), + kafka.stop, + kafka.cleanup, + callbacks.update_blocked_status, + ], + }, + ]) + manager.manage() + + +if __name__ == '__main__': + manage() diff --git a/charms/trusty/kafka/hooks/config-changed b/charms/trusty/kafka/hooks/config-changed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/config-changed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/install b/charms/trusty/kafka/hooks/install new file mode 100755 index 0000000..7ea6d0f --- /dev/null +++ b/charms/trusty/kafka/hooks/install @@ -0,0 +1,17 @@ +#!/usr/bin/python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import setup +setup.pre_install() + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/kafka-relation-changed b/charms/trusty/kafka/hooks/kafka-relation-changed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/kafka-relation-changed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/setup.py b/charms/trusty/kafka/hooks/setup.py new file mode 100644 index 0000000..496edc6 --- /dev/null +++ b/charms/trusty/kafka/hooks/setup.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import subprocess +from glob import glob + + +def pre_install(): + """ + Do any setup required before the install hook. + """ + install_pip() + install_bundled_resources() + + +def install_pip(): + subprocess.check_call(['apt-get', 'install', '-yq', 'python-pip', 'bzr']) + + +def install_bundled_resources(): + """ + Install the bundled resources libraries. + """ + archives = glob('resources/python/*') + subprocess.check_call(['pip', 'install'] + archives) diff --git a/charms/trusty/kafka/hooks/start b/charms/trusty/kafka/hooks/start new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/start @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/stop b/charms/trusty/kafka/hooks/stop new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/stop @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/zookeeper-relation-changed b/charms/trusty/kafka/hooks/zookeeper-relation-changed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/zookeeper-relation-changed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/zookeeper-relation-departed b/charms/trusty/kafka/hooks/zookeeper-relation-departed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/zookeeper-relation-departed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/icon.svg b/charms/trusty/kafka/icon.svg new file mode 100644 index 0000000..1564f99 --- /dev/null +++ b/charms/trusty/kafka/icon.svg @@ -0,0 +1,90 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + diff --git a/charms/trusty/kafka/metadata.yaml b/charms/trusty/kafka/metadata.yaml new file mode 100644 index 0000000..735c20e --- /dev/null +++ b/charms/trusty/kafka/metadata.yaml @@ -0,0 +1,30 @@ +name: apache-kafka +maintainer: Kevin Monroe +summary: High-performance pub/sub as distributed commit log. +description: | + Fast + A single Kafka broker can handle hundreds of megabytes of reads and writes per + second from thousands of clients. + + Scalable + Kafka is designed to allow a single cluster to serve as the central data + backbone for a large organization. It can be elastically and transparently + expanded without downtime. Data streams are partitioned and spread over a + cluster of machines to allow data streams larger than the capability of any + single machine and to allow clusters of co-ordinated consumers. + + Durable + Messages are persisted on disk and replicated within the cluster to prevent + data loss. Each broker can handle terabytes of messages without performance + impact. + + Distributed by Design + Kafka has a modern cluster-centric design that offers strong durability and + fault-tolerance guarantees. +tags: ["bigdata", "apache"] +provides: + kafka: + interface: kafka +requires: + zookeeper: + interface: zookeeper diff --git a/charms/trusty/kafka/resources.yaml b/charms/trusty/kafka/resources.yaml new file mode 100644 index 0000000..a3b5c74 --- /dev/null +++ b/charms/trusty/kafka/resources.yaml @@ -0,0 +1,12 @@ +options: + output_dir: /home/ubuntu/resources +resources: + pathlib: + pypi: path.py>=7.0 + jujubigdata: + pypi: jujubigdata>=4.1.0,<5.0.0 +optional_resources: + kafka-x86_64: + url: http://mirrors.ibiblio.org/apache/kafka/0.9.0.0/kafka_2.10-0.9.0.0.tgz + hash: 4bd0a264b84e8d88445b2712dd9e28ac + hash_type: md5 diff --git a/charms/trusty/kafka/resources/python/PyYAML-3.11.tar.gz b/charms/trusty/kafka/resources/python/PyYAML-3.11.tar.gz new file mode 100644 index 0000000..2a5d431 Binary files /dev/null and b/charms/trusty/kafka/resources/python/PyYAML-3.11.tar.gz differ diff --git a/charms/trusty/kafka/resources/python/charmhelpers-0.3.1.tar.gz b/charms/trusty/kafka/resources/python/charmhelpers-0.3.1.tar.gz new file mode 100644 index 0000000..0ccdbd9 Binary files /dev/null and b/charms/trusty/kafka/resources/python/charmhelpers-0.3.1.tar.gz differ diff --git a/charms/trusty/kafka/resources/python/jujuresources-0.2.11.tar.gz b/charms/trusty/kafka/resources/python/jujuresources-0.2.11.tar.gz new file mode 100644 index 0000000..c491086 Binary files /dev/null and b/charms/trusty/kafka/resources/python/jujuresources-0.2.11.tar.gz differ diff --git a/charms/trusty/kafka/resources/python/pyaml-15.5.7.tar.gz b/charms/trusty/kafka/resources/python/pyaml-15.5.7.tar.gz new file mode 100644 index 0000000..c51f6d1 Binary files /dev/null and b/charms/trusty/kafka/resources/python/pyaml-15.5.7.tar.gz differ diff --git a/charms/trusty/kafka/resources/python/six-1.9.0-py2.py3-none-any.whl b/charms/trusty/kafka/resources/python/six-1.9.0-py2.py3-none-any.whl new file mode 100644 index 0000000..743ee12 Binary files /dev/null and b/charms/trusty/kafka/resources/python/six-1.9.0-py2.py3-none-any.whl differ diff --git a/charms/trusty/kafka/templates/upstart.conf b/charms/trusty/kafka/templates/upstart.conf new file mode 100644 index 0000000..d24fe8c --- /dev/null +++ b/charms/trusty/kafka/templates/upstart.conf @@ -0,0 +1,14 @@ +#!upstart +description "kafka" + +start on startup +stop on shutdown + +setuid kafka +setgid hadoop + +respawn + +script +"{{kafka_bin}}/kafka-server-start.sh" "{{kafka_conf}}/server.properties" +end script diff --git a/charms/trusty/kafka/tests/00-setup b/charms/trusty/kafka/tests/00-setup new file mode 100755 index 0000000..36549ea --- /dev/null +++ b/charms/trusty/kafka/tests/00-setup @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install python3 amulet -y diff --git a/charms/trusty/kafka/tests/100-deploy-kafka b/charms/trusty/kafka/tests/100-deploy-kafka new file mode 100755 index 0000000..713a4b4 --- /dev/null +++ b/charms/trusty/kafka/tests/100-deploy-kafka @@ -0,0 +1,29 @@ +#!/usr/bin/python3 +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Deployment test for Apache Kafka + """ + + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='trusty') + # Deploy Kafka Service + cls.d.add('kafka', charm='cs:~bigdata-dev/trusty/apache-kafka') + cls.d.add('zookeeper', charm='cs:~bigdata-dev/trusty/apache-zookeeper') + cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper') + + cls.d.setup(timeout=1800) + cls.d.sentry.wait(timeout=1800) + cls.unit = cls.d.sentry['kafka'][0] + + def test_deploy(self): + output, retcode = self.unit.run("pgrep -a java") + assert 'Kafka' in output, "Kafka daemon is not started" + + +if __name__ == '__main__': + unittest.main() diff --git a/charms/trusty/kafka/tests/remote/test_dist_config.py b/charms/trusty/kafka/tests/remote/test_dist_config.py new file mode 100755 index 0000000..eb2c3aa --- /dev/null +++ b/charms/trusty/kafka/tests/remote/test_dist_config.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import grp +import os +import pwd +import unittest + +import jujubigdata + + +class TestDistConfig(unittest.TestCase): + """ + Test that the ``dist.yaml`` settings were applied properly, such as users, groups, and dirs. + + This is done as a remote test on the deployed unit rather than a regular + test under ``tests/`` because filling in the ``dist.yaml`` requires Juju + context (e.g., config). + """ + @classmethod + def setUpClass(cls): + config = None + config_dir = os.environ['JUJU_CHARM_DIR'] + config_file = 'dist.yaml' + if os.path.isfile(os.path.join(config_dir, config_file)): + config = os.path.join(config_dir, config_file) + if not config: + raise IOError('Could not find {} in {}'.format(config_file, config_dir)) + reqs = ['vendor', 'hadoop_version', 'groups', 'users', 'dirs'] + cls.dist_config = jujubigdata.utils.DistConfig(config, reqs) + + def test_groups(self): + for name in self.dist_config.groups: + try: + grp.getgrnam(name) + except KeyError: + self.fail('Group {} is missing'.format(name)) + + def test_users(self): + for username, details in self.dist_config.users.items(): + try: + user = pwd.getpwnam(username) + except KeyError: + self.fail('User {} is missing'.format(username)) + for groupname in details['groups']: + try: + group = grp.getgrnam(groupname) + except KeyError: + self.fail('Group {} referenced by user {} does not exist'.format( + groupname, username)) + if group.gr_gid != user.pw_gid: + self.assertIn(username, group.gr_mem, 'User {} not in group {}'.format( + username, groupname)) + + def test_dirs(self): + for name, details in self.dist_config.dirs.items(): + dirpath = self.dist_config.path(name) + self.assertTrue(dirpath.isdir(), 'Dir {} is missing'.format(name)) + stat = dirpath.stat() + owner = pwd.getpwuid(stat.st_uid).pw_name + group = grp.getgrgid(stat.st_gid).gr_name + perms = stat.st_mode & ~0o40000 + self.assertEqual(owner, details.get('owner', 'root'), + 'Dir {} ({}) has wrong owner: {}'.format(name, dirpath, owner)) + self.assertEqual(group, details.get('group', 'root'), + 'Dir {} ({}) has wrong group: {}'.format(name, dirpath, group)) + self.assertEqual(perms, details.get('perms', 0o755), + 'Dir {} ({}) has wrong perms: 0o{:o}'.format(name, dirpath, perms)) + + +if __name__ == '__main__': + unittest.main() diff --git a/charms/trusty/kafka/tests/tests.yaml b/charms/trusty/kafka/tests/tests.yaml new file mode 100644 index 0000000..771f3fd --- /dev/null +++ b/charms/trusty/kafka/tests/tests.yaml @@ -0,0 +1,10 @@ +# Driver for bundletester: https://github.com/juju-solutions/bundletester +# +# It may be useful to alter the defaults during manual testing. For example, +# set 'reset: false' to reuse existing charms instead of redeploying them. + +# Allow bootstrap of current env, default: true +bootstrap: true + +# Use juju-deployer to reset env between test, default: true +reset: true -- cgit 1.2.3-korg