diff options
Diffstat (limited to 'charms/trusty/kafka')
43 files changed, 1248 insertions, 0 deletions
diff --git a/charms/trusty/kafka/.bzr/README b/charms/trusty/kafka/.bzr/README new file mode 100644 index 0000000..f82dc1c --- /dev/null +++ b/charms/trusty/kafka/.bzr/README @@ -0,0 +1,3 @@ +This is a Bazaar control directory. +Do not change any files in this directory. +See http://bazaar.canonical.com/ for more information about Bazaar. diff --git a/charms/trusty/kafka/.bzr/branch-format b/charms/trusty/kafka/.bzr/branch-format new file mode 100644 index 0000000..9eb09b7 --- /dev/null +++ b/charms/trusty/kafka/.bzr/branch-format @@ -0,0 +1 @@ +Bazaar-NG meta directory, format 1 diff --git a/charms/trusty/kafka/.bzr/branch/format b/charms/trusty/kafka/.bzr/branch/format new file mode 100644 index 0000000..b391ffd --- /dev/null +++ b/charms/trusty/kafka/.bzr/branch/format @@ -0,0 +1 @@ +Bazaar-NG Branch Reference Format 1 diff --git a/charms/trusty/kafka/.bzr/branch/location b/charms/trusty/kafka/.bzr/branch/location new file mode 100644 index 0000000..2f40eef --- /dev/null +++ b/charms/trusty/kafka/.bzr/branch/location @@ -0,0 +1 @@ +http://bazaar.launchpad.net/~sdn-charmers/charms/trusty/apache-kafka/trunk/
\ No newline at end of file diff --git a/charms/trusty/kafka/.bzr/checkout/conflicts b/charms/trusty/kafka/.bzr/checkout/conflicts new file mode 100644 index 0000000..0dc2d3a --- /dev/null +++ b/charms/trusty/kafka/.bzr/checkout/conflicts @@ -0,0 +1 @@ +BZR conflict list format 1 diff --git a/charms/trusty/kafka/.bzr/checkout/dirstate b/charms/trusty/kafka/.bzr/checkout/dirstate Binary files differnew file mode 100644 index 0000000..9d2ef8a --- /dev/null +++ b/charms/trusty/kafka/.bzr/checkout/dirstate diff --git a/charms/trusty/kafka/.bzr/checkout/format b/charms/trusty/kafka/.bzr/checkout/format new file mode 100644 index 0000000..e0261c7 --- /dev/null +++ b/charms/trusty/kafka/.bzr/checkout/format @@ -0,0 +1 @@ +Bazaar Working Tree Format 6 (bzr 1.14) diff --git a/charms/trusty/kafka/.bzr/checkout/views b/charms/trusty/kafka/.bzr/checkout/views new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/charms/trusty/kafka/.bzr/checkout/views diff --git a/charms/trusty/kafka/LICENSE b/charms/trusty/kafka/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/charms/trusty/kafka/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/charms/trusty/kafka/README.md b/charms/trusty/kafka/README.md new file mode 100644 index 0000000..31e8e4f --- /dev/null +++ b/charms/trusty/kafka/README.md @@ -0,0 +1,84 @@ +## Overview +Apache Kafka is an open-source message broker project developed by the Apache +Software Foundation written in Scala. The project aims to provide a unified, +high-throughput, low-latency platform for handling real-time data feeds. Learn +more at [kafka.apache.org](http://kafka.apache.org/). + + +## Usage +Kafka requires the Zookeeper distributed coordination service. Deploy and +relate them as follows: + + juju deploy apache-zookeeper zookeeper + juju deploy apache-kafka kafka + juju add-relation kafka zookeeper + +Once deployed, we can list the zookeeper servers that our kafka brokers +are connected to. The following will list `<ip>:<port>` information for each +zookeeper unit in the environment (e.g.: `10.0.3.221:2181`). + + juju action do kafka/0 list-zks + juju action fetch <id> # <-- id from above command + +We can create a Kafka topic with: + + juju action do kafka/0 create-topic topic=<topic_name> \ + partitions=<#> replication=<#> + juju action fetch <id> # <-- id from above command + +We can list topics with: + + juju action do kafka/0 list-topics + juju action fetch <id> # <-- id from above command + +We can write to a topic with: + + juju action do kafka/0 write-topic topic=<topic_name> data=<data> + juju action fetch <id> # <-- id from above command + +We can read from a topic with: + + juju action do kafka/0 read-topic topic=<topic_name> partition=<#> + juju action fetch <id> # <-- id from above command + +And finally, we can delete a topic with: + + juju action do kafka/0 delete-topic topic=<topic_name> + juju action fetch <id> # <-- id from above command + +## Deploying in Network-Restricted Environments +This charm can be deployed in environments with limited network access. To +deploy in this environment, you will need a local mirror to serve the packages +and resources required by this charm. + +### Mirroring Packages +You can setup a local mirror for apt packages using squid-deb-proxy. +For instructions on configuring juju to use this, see the +[Juju Proxy Documentation](https://juju.ubuntu.com/docs/howto-proxies.html). + +### Mirroring Resources +In addition to apt packages, this charm requires a few binary resources +which are normally hosted on Launchpad. If access to Launchpad is not +available, the `jujuresources` library makes it easy to create a mirror +of these resources: + + sudo pip install jujuresources + juju-resources fetch --all /path/to/resources.yaml -d /tmp/resources + juju-resources serve -d /tmp/resources + +This will fetch all of the resources needed by this charm and serve them via a +simple HTTP server. The output from `juju-resources serve` will give you a +URL that you can set as the `resources_mirror` config option for this charm. +Setting this option will cause all resources required by this charm to be +downloaded from the configured URL. + + +## Contact Information +- <bigdata-dev@lists.launchpad.net> + + +## Help +- [Apache Kafka home page](http://kafka.apache.org/) +- [Apache Kafka issue tracker](https://issues.apache.org/jira/browse/KAFKA) +- [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) +- [Juju community](https://jujucharms.com/community) diff --git a/charms/trusty/kafka/actions.yaml b/charms/trusty/kafka/actions.yaml new file mode 100644 index 0000000..6026d0b --- /dev/null +++ b/charms/trusty/kafka/actions.yaml @@ -0,0 +1,48 @@ +create-topic: + description: Create a new Kafka topic + params: + topic: + type: string + description: Topic name + partitions: + type: integer + description: Number of partitions for the topic being created + replication: + type: integer + description: Replication factor for each partition in the topic + required: [topic, partitions, replication] + additionalProperties: false +delete-topic: + description: Delete a Kafka topic + params: + topic: + type: string + description: Topic name + required: [topic] + additionalProperties: false +list-topics: + description: List all Kafka topics +list-zks: + description: List ip:port info for connected Zookeeper servers +read-topic: + description: Consume an existing kafka topic + params: + topic: + type: string + description: Topic name + partition: + type: integer + description: Partition to consume + required: [topic, partition] + additionalProperties: false +write-topic: + description: Write to a kafka topic + params: + topic: + type: string + description: Topic name + data: + type: string + description: Data to write to topic + required: [topic, data] + additionalProperties: false diff --git a/charms/trusty/kafka/actions/create-topic b/charms/trusty/kafka/actions/create-topic new file mode 100755 index 0000000..4910430 --- /dev/null +++ b/charms/trusty/kafka/actions/create-topic @@ -0,0 +1,40 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +# Grab the business +topic_name = hookenv.action_get('topic') +topic_partitions = hookenv.action_get('partitions') +topic_replication = hookenv.action_get('replication') + +# Create the topic if we've got zookeepers; otherwise fail. +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + output = utils.run_as('kafka', 'kafka-topics.sh', + '--zookeeper', zookeepers, '--create', + '--topic', topic_name, + '--partitions', topic_partitions, + '--replication-factor', topic_replication, + capture_output=True) + hookenv.action_set({'output': output}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/delete-topic b/charms/trusty/kafka/actions/delete-topic new file mode 100755 index 0000000..b56f004 --- /dev/null +++ b/charms/trusty/kafka/actions/delete-topic @@ -0,0 +1,36 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +# Grab the business +topic_name = hookenv.action_get('topic') + +# Delete the topic if we've got zookeepers; otherwise fail. +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + output = utils.run_as('kafka', 'kafka-topics.sh', + '--zookeeper', zookeepers, '--delete', + '--topic', topic_name, + capture_output=True) + hookenv.action_set({'output': output}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/list-topics b/charms/trusty/kafka/actions/list-topics new file mode 100755 index 0000000..629d2b4 --- /dev/null +++ b/charms/trusty/kafka/actions/list-topics @@ -0,0 +1,31 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + output = utils.run_as('kafka', '/usr/lib/kafka/bin/kafka-topics.sh', + '--zookeeper', zookeepers, '--list', + capture_output=True) + hookenv.action_set({'topics': output}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/list-zks b/charms/trusty/kafka/actions/list-zks new file mode 100755 index 0000000..9de9e9a --- /dev/null +++ b/charms/trusty/kafka/actions/list-zks @@ -0,0 +1,28 @@ +#!/usr/bin/env python +import sys + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + from jujubigdata import utils + from jujubigdata.relations import Zookeeper + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zookeepers = ",".join(zks) + hookenv.action_set({'zookeepers': zookeepers}) +else: + hookenv.action_fail('Zookeeper relation is not present/ready') + sys.exit() diff --git a/charms/trusty/kafka/actions/read-topic b/charms/trusty/kafka/actions/read-topic new file mode 100755 index 0000000..9f59396 --- /dev/null +++ b/charms/trusty/kafka/actions/read-topic @@ -0,0 +1,35 @@ +#!/usr/bin/env python +#pylint: disable=C0103 +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + import jujubigdata + from jujubigdata import utils + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + +kafka_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports'] +dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml', + required_keys=kafka_reqs) + +# Grab the business +topic_name = hookenv.action_get('topic') +topic_partition = hookenv.action_get('partition') + +output = utils.run_as( + 'kafka', 'kafka-simple-consumer-shell.sh', + '--broker-list', '{}:{}'.format( + hookenv.unit_private_ip(), + dist_config.port('kafka'), + ), + '--topic', topic_name, + '--partition', topic_partition, + '--no-wait-at-logend', + capture_output=True) +hookenv.action_set({'output': output}) diff --git a/charms/trusty/kafka/actions/write-topic b/charms/trusty/kafka/actions/write-topic new file mode 100755 index 0000000..fce8e1b --- /dev/null +++ b/charms/trusty/kafka/actions/write-topic @@ -0,0 +1,36 @@ +#!/usr/bin/env python +#pylint: disable=C0103 + +try: + from charmhelpers.core import hookenv + from charmhelpers.core import unitdata + import jujubigdata + from jujubigdata import utils + charm_ready = unitdata.kv().get('charm.active', False) +except ImportError: + charm_ready = False + +if not charm_ready: + # might not have hookenv.action_fail available yet + from subprocess import call + call(['action-fail', 'Kafka service not yet ready']) + + +kafka_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports'] +dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml', + required_keys=kafka_reqs) + +# Grab the business +topic_name = hookenv.action_get('topic') +data = hookenv.action_get('data') + +output = utils.run_as( + 'kafka', 'kafka-console-producer.sh', + '--broker-list', '{}:{}'.format( + hookenv.unit_private_ip(), + dist_config.port('kafka'), + ), + '--topic', topic_name, + capture_output=True, + input=data) +hookenv.action_set({'output': output}) diff --git a/charms/trusty/kafka/config.yaml b/charms/trusty/kafka/config.yaml new file mode 100644 index 0000000..f2483b0 --- /dev/null +++ b/charms/trusty/kafka/config.yaml @@ -0,0 +1,7 @@ +options: + resources_mirror: + type: string + default: '' + description: | + URL from which to fetch resources (e.g., Hadoop binaries) instead of Launchpad. + diff --git a/charms/trusty/kafka/copyright b/charms/trusty/kafka/copyright new file mode 100644 index 0000000..e900b97 --- /dev/null +++ b/charms/trusty/kafka/copyright @@ -0,0 +1,16 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2015, Canonical Ltd., All Rights Reserved. +License: Apache License 2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/charms/trusty/kafka/dist.yaml b/charms/trusty/kafka/dist.yaml new file mode 100644 index 0000000..e8f1a37 --- /dev/null +++ b/charms/trusty/kafka/dist.yaml @@ -0,0 +1,30 @@ +# This file contains values that are likely to change per distribution. +# The aim is to make it easier to update / extend the charms with +# minimal changes to the shared code in the jujubigdata library. +vendor: 'apache' +packages: + - 'openjdk-7-jdk' +groups: + - 'hadoop' +users: + kafka: + groups: ['hadoop'] +dirs: + kafka: + path: '/usr/lib/kafka' + kafka_conf: + path: '/etc/kafka/conf' + kafka_app_logs: + path: '/var/log/kafka' + owner: 'kafka' + kafka_data_logs: + path: '/var/lib/kafka' + owner: 'kafka' +ports: + # Ports that need to be exposed, overridden, or manually specified. + # Only expose ports serving a UI or external API (i.e., namenode and + # resourcemanager). Communication among units within the cluster does + # not need ports to be explicitly opened. + kafka: + port: 9092 + exposed_on: 'kafka' diff --git a/charms/trusty/kafka/hooks/callbacks.py b/charms/trusty/kafka/hooks/callbacks.py new file mode 100644 index 0000000..bac68cb --- /dev/null +++ b/charms/trusty/kafka/hooks/callbacks.py @@ -0,0 +1,181 @@ +import os +from socket import gaierror, gethostbyname, gethostname +from subprocess import Popen, check_call + +import jujuresources +from charmhelpers.core import hookenv, templating +from charmhelpers.core import host +from charmhelpers.core import unitdata +from jujubigdata import utils +from jujubigdata.relations import Zookeeper + + +# Extended status support +# We call update_blocked_status from the "requires" section of our service +# block, so be sure to return True. Otherwise, we'll block the "requires" +# and never move on to callbacks. The other status update methods are called +# from the "callbacks" section and therefore don't need to return True. +def update_blocked_status(): + if unitdata.kv().get('charm.active', False): + return True + if not Zookeeper().connected_units(): + hookenv.status_set('blocked', 'Waiting for relation to apache-zookeeper') + elif not Zookeeper().is_ready(): + hookenv.status_set('waiting', 'Waiting for Zookeeper to become ready') + return True + + +def update_working_status(): + if unitdata.kv().get('charm.active', False): + hookenv.status_set('maintenance', 'Updating configuration') + return + hookenv.status_set('maintenance', 'Setting up Apache Kafka') + + +def update_active_status(): + unitdata.kv().set('charm.active', True) + hookenv.status_set('active', 'Ready') + + +def clear_active_flag(): + unitdata.kv().set('charm.active', False) + + +# Main Kafka class for callbacks +class Kafka(object): + def __init__(self, dist_config): + self.dist_config = dist_config + self.resources = { + 'kafka': 'kafka-%s' % host.cpu_arch(), + } + self.verify_resources = utils.verify_resources(*self.resources.values()) + + def fix_hostname(self): + # ensure hostname is resolvable + hostname = gethostname() + try: + gethostbyname(hostname) + except gaierror: + check_call(['sed', '-E', '-i', '-e', + '/127.0.0.1[[:blank:]]+/a \\\n127.0.1.1 ' + hostname, + '/etc/hosts']) + + def is_installed(self): + return unitdata.kv().get('kafka.installed') + + def install(self, force=False): + if not force and self.is_installed(): + return + self.fix_hostname() + self.dist_config.add_users() + self.dist_config.add_dirs() + self.dist_config.add_packages() + jujuresources.install(self.resources['kafka'], + destination=self.dist_config.path('kafka'), + skip_top_level=True) + self.setup_kafka_config() + unitdata.kv().set('kafka.installed', True) + + def setup_kafka_config(self): + ''' + copy the default configuration files to kafka_conf property + defined in dist.yaml + ''' + default_conf = self.dist_config.path('kafka') / 'config' + kafka_conf = self.dist_config.path('kafka_conf') + kafka_conf.rmtree_p() + default_conf.copytree(kafka_conf) + # Now remove the conf included in the tarball and symlink our real conf + # dir. we've seen issues where kafka still looks for config in + # KAFKA_HOME/config. + default_conf.rmtree_p() + kafka_conf.symlink(default_conf) + + # Configure immutable bits + kafka_bin = self.dist_config.path('kafka') / 'bin' + with utils.environment_edit_in_place('/etc/environment') as env: + if kafka_bin not in env['PATH']: + env['PATH'] = ':'.join([env['PATH'], kafka_bin]) + env['LOG_DIR'] = self.dist_config.path('kafka_app_logs') + + # note: we set the advertised.host.name below to the public_address + # to ensure that external (non-Juju) clients can connect to Kafka + public_address = hookenv.unit_get('public-address') + private_ip = utils.resolve_private_address(hookenv.unit_get('private-address')) + kafka_server_conf = self.dist_config.path('kafka_conf') / 'server.properties' + service, unit_num = os.environ['JUJU_UNIT_NAME'].split('/', 1) + utils.re_edit_in_place(kafka_server_conf, { + r'^broker.id=.*': 'broker.id=%s' % unit_num, + r'^port=.*': 'port=%s' % self.dist_config.port('kafka'), + r'^log.dirs=.*': 'log.dirs=%s' % self.dist_config.path('kafka_data_logs'), + r'^#?advertised.host.name=.*': 'advertised.host.name=%s' % public_address, + }) + + kafka_log4j = self.dist_config.path('kafka_conf') / 'log4j.properties' + utils.re_edit_in_place(kafka_log4j, { + r'^kafka.logs.dir=.*': 'kafka.logs.dir=%s' % self.dist_config.path('kafka_app_logs'), + }) + + # fix for lxc containers and some corner cases in manual provider + # ensure that public_address is resolvable internally by mapping it to the private IP + utils.update_etc_hosts({private_ip: public_address}) + + templating.render( + 'upstart.conf', + '/etc/init/kafka.conf', + context={ + 'kafka_conf': self.dist_config.path('kafka_conf'), + 'kafka_bin': '{}/bin'.format(self.dist_config.path('kafka')) + }, + ) + + def configure_kafka(self): + # Get ip:port data from our connected zookeepers + if Zookeeper().connected_units() and Zookeeper().is_ready(): + zks = [] + for unit, data in Zookeeper().filtered_data().items(): + ip = utils.resolve_private_address(data['private-address']) + zks.append("%s:%s" % (ip, data['port'])) + zks.sort() + zk_connect = ",".join(zks) + + # update consumer props + cfg = self.dist_config.path('kafka_conf') / 'consumer.properties' + utils.re_edit_in_place(cfg, { + r'^zookeeper.connect=.*': 'zookeeper.connect=%s' % zk_connect, + }) + + # update server props + cfg = self.dist_config.path('kafka_conf') / 'server.properties' + utils.re_edit_in_place(cfg, { + r'^zookeeper.connect=.*': 'zookeeper.connect=%s' % zk_connect, + }) + else: + # if we have no zookeepers, make sure kafka is stopped + self.stop() + + def run_bg(self, user, command, *args): + """ + Run a Kafka command as the `kafka` user in the background. + + :param str command: Command to run + :param list args: Additional args to pass to the command + """ + parts = [command] + list(args) + quoted = ' '.join("'%s'" % p for p in parts) + e = utils.read_etc_env() + Popen(['su', user, '-c', quoted], env=e) + + def restart(self): + self.stop() + self.start() + + def start(self): + host.service_start('kafka') + + def stop(self): + host.service_stop('kafka') + + def cleanup(self): + self.dist_config.remove_users() + self.dist_config.remove_dirs() diff --git a/charms/trusty/kafka/hooks/common.py b/charms/trusty/kafka/hooks/common.py new file mode 100755 index 0000000..1ca080b --- /dev/null +++ b/charms/trusty/kafka/hooks/common.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Common implementation for all hooks. +""" + +import jujuresources +from charmhelpers.core import hookenv +from charmhelpers.core import unitdata +from charmhelpers.core import charmframework + + +def bootstrap_resources(): + """ + Install required resources defined in resources.yaml + """ + if unitdata.kv().get('charm.bootstrapped', False): + return True + hookenv.status_set('maintenance', 'Installing base resources') + mirror_url = jujuresources.config_get('resources_mirror') + if not jujuresources.fetch(mirror_url=mirror_url): + missing = jujuresources.invalid() + hookenv.status_set('blocked', 'Unable to fetch required resource%s: %s' % ( + 's' if len(missing) > 1 else '', + ', '.join(missing), + )) + return False + jujuresources.install(['pathlib', 'jujubigdata']) + unitdata.kv().set('charm.bootstrapped', True) + return True + + +def manage(): + if not bootstrap_resources(): + # defer until resources are available, since charmhelpers, and thus + # the framework, are required (will require manual intervention) + return + + import jujubigdata + import callbacks + + kafka_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports'] + dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml', + required_keys=kafka_reqs) + kafka = callbacks.Kafka(dist_config) + manager = charmframework.Manager([ + { + 'name': 'kafka', + 'provides': [ + jujubigdata.relations.Kafka(port=dist_config.port('kafka')) + ], + 'requires': [ + kafka.verify_resources, + jujubigdata.relations.Zookeeper(), + callbacks.update_blocked_status, # not really a requirement, but best way to fit into framework + ], + 'callbacks': [ + callbacks.update_working_status, + kafka.install, + kafka.configure_kafka, + kafka.restart, + charmframework.helpers.open_ports( + dist_config.exposed_ports('kafka')), + callbacks.update_active_status, + ], + 'cleanup': [ + callbacks.clear_active_flag, + charmframework.helpers.close_ports( + dist_config.exposed_ports('kafka')), + kafka.stop, + kafka.cleanup, + callbacks.update_blocked_status, + ], + }, + ]) + manager.manage() + + +if __name__ == '__main__': + manage() diff --git a/charms/trusty/kafka/hooks/config-changed b/charms/trusty/kafka/hooks/config-changed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/config-changed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/install b/charms/trusty/kafka/hooks/install new file mode 100755 index 0000000..7ea6d0f --- /dev/null +++ b/charms/trusty/kafka/hooks/install @@ -0,0 +1,17 @@ +#!/usr/bin/python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import setup +setup.pre_install() + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/kafka-relation-changed b/charms/trusty/kafka/hooks/kafka-relation-changed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/kafka-relation-changed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/setup.py b/charms/trusty/kafka/hooks/setup.py new file mode 100644 index 0000000..496edc6 --- /dev/null +++ b/charms/trusty/kafka/hooks/setup.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import subprocess +from glob import glob + + +def pre_install(): + """ + Do any setup required before the install hook. + """ + install_pip() + install_bundled_resources() + + +def install_pip(): + subprocess.check_call(['apt-get', 'install', '-yq', 'python-pip', 'bzr']) + + +def install_bundled_resources(): + """ + Install the bundled resources libraries. + """ + archives = glob('resources/python/*') + subprocess.check_call(['pip', 'install'] + archives) diff --git a/charms/trusty/kafka/hooks/start b/charms/trusty/kafka/hooks/start new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/start @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/stop b/charms/trusty/kafka/hooks/stop new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/stop @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/zookeeper-relation-changed b/charms/trusty/kafka/hooks/zookeeper-relation-changed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/zookeeper-relation-changed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/hooks/zookeeper-relation-departed b/charms/trusty/kafka/hooks/zookeeper-relation-departed new file mode 100755 index 0000000..1b4e92c --- /dev/null +++ b/charms/trusty/kafka/hooks/zookeeper-relation-departed @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import common +common.manage() diff --git a/charms/trusty/kafka/icon.svg b/charms/trusty/kafka/icon.svg new file mode 100644 index 0000000..1564f99 --- /dev/null +++ b/charms/trusty/kafka/icon.svg @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="75pt" + height="117pt" + viewBox="0 0 75 117" + version="1.1" + id="svg3201" + inkscape:version="0.48.4 r9939" + sodipodi:docname="icon.svg"> + <metadata + id="metadata3221"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <defs + id="defs3219" /> + <sodipodi:namedview + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1" + objecttolerance="10" + gridtolerance="10" + guidetolerance="10" + inkscape:pageopacity="0" + inkscape:pageshadow="2" + inkscape:window-width="1236" + inkscape:window-height="847" + id="namedview3217" + showgrid="false" + inkscape:zoom="4.5641627" + inkscape:cx="35.369926" + inkscape:cy="80.60281" + inkscape:window-x="397" + inkscape:window-y="78" + inkscape:window-maximized="0" + inkscape:current-layer="layer1" /> + <path + d="m 0,0.04322 76.8,0 0,76.8 -76.8,0 0,-76.8 z" + id="path3203" + style="opacity:0;fill:#fffffe;fill-opacity:0.94977172" + inkscape:connector-curvature="0" /> + <path + d="M 13.76256,3.6403482 C 20.992,-0.05524154 33.024,1.9665021 36.42368,7.4934765 40.37632,12.889169 34.816,19.735528 25.77408,20.917066 c 0,1.732924 0.01024,3.459282 0,5.185642 4.6592,0.708923 8.87808,2.258051 12.05248,4.56205 2.21184,-0.892717 4.526079,-1.700102 6.72768,-2.605948 -0.59392,-3.209847 -0.59392,-6.833231 3.10272,-9.340718 5.77536,-4.719589 18.51392,-4.476718 23.72608,0.531692 4.751359,4.076308 3.266559,10.450051 -3.21536,13.403898 -6.03136,3.012923 -15.28832,2.271179 -20.41856,-1.266872 -2.16064,0.833641 -4.43392,1.549128 -6.52288,2.454975 -0.307201,1.122461 0.45056,2.251486 0.6144,3.373948 0.70656,1.956103 -0.78848,3.859693 -0.68608,5.809231 2.11968,0.951795 4.47488,1.68041 6.79936,2.415589 6.36928,-4.653948 19.445759,-3.918768 24.10496,1.575386 5.12,5.087179 0.62464,12.668717 -8.52992,14.211281 -6.85056,1.273436 -14.7968,-0.794256 -17.84832,-4.988717 -1.81248,-2.23836 -1.69984,-4.857437 -0.8704,-7.259898 -2.29376,-0.951795 -4.67968,-1.805128 -6.97344,-2.756923 -3.16416,2.284307 -7.34208,3.885949 -12.01152,4.516102 -0.06144,1.746051 -0.04096,3.498667 -0.0512,5.251283 4.78208,0.728615 8.98048,2.921025 10.81344,5.868307 2.75456,4.299487 0.07168,9.872411 -6.51264,12.025436 -6.2464,2.422154 -15.06304,1.234052 -19.33312,-2.546872 -4.89472,-3.846564 -3.84,-10.095589 2.17088,-13.213539 1.91488,-1.102768 4.41344,-1.634461 6.78912,-2.19241 0.03072,-1.68041 0.04096,-3.367384 0.07168,-5.047794 C 14.98112,50.009169 10.21952,48.597887 7.26016,45.919734 1.44384,41.180451 2.2528,33.671118 9.20576,29.581682 c 2.78528,-1.93641 6.77888,-2.776616 10.567679,-3.577436 -0.03072,-1.68041 -0.04096,-3.360821 -0.07168,-5.034667 C 15.59552,20.181887 11.60192,18.678708 9.5641599,16.164655 5.82656,12.081784 7.69024,6.3644508 13.76256,3.6403482 z" + id="path3205" + inkscape:connector-curvature="0" + style="fill:#201f1f" /> + <path + d="M 18.95424,7.4869124 C 23.58272,5.3338867 30.53568,8.2614765 29.85984,11.799528 29.48096,14.924041 23.5008,17.182092 19.2,15.462297 14.42816,13.926297 14.27456,9.1410662 18.95424,7.4869124 z" + id="path3207" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 55.76704,20.844861 c 4.51584,-1.673846 10.91584,0.761436 10.56768,4.135384 0.235519,3.649642 -7.33184,5.96677 -11.64288,3.557744 -4.106241,-1.897025 -3.38944,-6.255589 1.0752,-7.693128 z" + id="path3209" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 18.78016,32.305784 c 7.311359,-2.303999 16.35328,2.829129 13.8752,7.75877 -1.44384,4.555487 -11.17184,6.721641 -16.5376,3.472409 -6.05184,-2.894768 -4.352,-9.570461 2.6624,-11.231179 z" + id="path3211" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 54.69184,48.348451 c 4.66944,-2.619077 12.759039,0.347897 11.601919,4.253538 -0.409599,3.629949 -8.345599,5.218462 -12.103679,2.651898 -3.2256,-1.772308 -2.8672,-5.303795 0.50176,-6.905436 z" + id="path3213" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 20.67456,61.030297 c 5.10976,-1.13559 10.78272,2.566565 8.82688,5.809231 -1.269761,3.190153 -8.48896,4.483282 -11.84768,1.857641 -4.06528,-2.19241 -2.03776,-6.872615 3.0208,-7.666872 z" + id="path3215" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <g + inkscape:groupmode="layer" + id="layer1" + inkscape:label="Alpha" + style="opacity:1" /> +</svg> diff --git a/charms/trusty/kafka/metadata.yaml b/charms/trusty/kafka/metadata.yaml new file mode 100644 index 0000000..735c20e --- /dev/null +++ b/charms/trusty/kafka/metadata.yaml @@ -0,0 +1,30 @@ +name: apache-kafka +maintainer: Kevin Monroe <kevin.monroe@canonical.com> +summary: High-performance pub/sub as distributed commit log. +description: | + Fast + A single Kafka broker can handle hundreds of megabytes of reads and writes per + second from thousands of clients. + + Scalable + Kafka is designed to allow a single cluster to serve as the central data + backbone for a large organization. It can be elastically and transparently + expanded without downtime. Data streams are partitioned and spread over a + cluster of machines to allow data streams larger than the capability of any + single machine and to allow clusters of co-ordinated consumers. + + Durable + Messages are persisted on disk and replicated within the cluster to prevent + data loss. Each broker can handle terabytes of messages without performance + impact. + + Distributed by Design + Kafka has a modern cluster-centric design that offers strong durability and + fault-tolerance guarantees. +tags: ["bigdata", "apache"] +provides: + kafka: + interface: kafka +requires: + zookeeper: + interface: zookeeper diff --git a/charms/trusty/kafka/resources.yaml b/charms/trusty/kafka/resources.yaml new file mode 100644 index 0000000..a3b5c74 --- /dev/null +++ b/charms/trusty/kafka/resources.yaml @@ -0,0 +1,12 @@ +options: + output_dir: /home/ubuntu/resources +resources: + pathlib: + pypi: path.py>=7.0 + jujubigdata: + pypi: jujubigdata>=4.1.0,<5.0.0 +optional_resources: + kafka-x86_64: + url: http://mirrors.ibiblio.org/apache/kafka/0.9.0.0/kafka_2.10-0.9.0.0.tgz + hash: 4bd0a264b84e8d88445b2712dd9e28ac + hash_type: md5 diff --git a/charms/trusty/kafka/resources/python/PyYAML-3.11.tar.gz b/charms/trusty/kafka/resources/python/PyYAML-3.11.tar.gz Binary files differnew file mode 100644 index 0000000..2a5d431 --- /dev/null +++ b/charms/trusty/kafka/resources/python/PyYAML-3.11.tar.gz diff --git a/charms/trusty/kafka/resources/python/charmhelpers-0.3.1.tar.gz b/charms/trusty/kafka/resources/python/charmhelpers-0.3.1.tar.gz Binary files differnew file mode 100644 index 0000000..0ccdbd9 --- /dev/null +++ b/charms/trusty/kafka/resources/python/charmhelpers-0.3.1.tar.gz diff --git a/charms/trusty/kafka/resources/python/jujuresources-0.2.11.tar.gz b/charms/trusty/kafka/resources/python/jujuresources-0.2.11.tar.gz Binary files differnew file mode 100644 index 0000000..c491086 --- /dev/null +++ b/charms/trusty/kafka/resources/python/jujuresources-0.2.11.tar.gz diff --git a/charms/trusty/kafka/resources/python/pyaml-15.5.7.tar.gz b/charms/trusty/kafka/resources/python/pyaml-15.5.7.tar.gz Binary files differnew file mode 100644 index 0000000..c51f6d1 --- /dev/null +++ b/charms/trusty/kafka/resources/python/pyaml-15.5.7.tar.gz diff --git a/charms/trusty/kafka/resources/python/six-1.9.0-py2.py3-none-any.whl b/charms/trusty/kafka/resources/python/six-1.9.0-py2.py3-none-any.whl Binary files differnew file mode 100644 index 0000000..743ee12 --- /dev/null +++ b/charms/trusty/kafka/resources/python/six-1.9.0-py2.py3-none-any.whl diff --git a/charms/trusty/kafka/templates/upstart.conf b/charms/trusty/kafka/templates/upstart.conf new file mode 100644 index 0000000..d24fe8c --- /dev/null +++ b/charms/trusty/kafka/templates/upstart.conf @@ -0,0 +1,14 @@ +#!upstart +description "kafka" + +start on startup +stop on shutdown + +setuid kafka +setgid hadoop + +respawn + +script +"{{kafka_bin}}/kafka-server-start.sh" "{{kafka_conf}}/server.properties" +end script diff --git a/charms/trusty/kafka/tests/00-setup b/charms/trusty/kafka/tests/00-setup new file mode 100755 index 0000000..36549ea --- /dev/null +++ b/charms/trusty/kafka/tests/00-setup @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install python3 amulet -y diff --git a/charms/trusty/kafka/tests/100-deploy-kafka b/charms/trusty/kafka/tests/100-deploy-kafka new file mode 100755 index 0000000..713a4b4 --- /dev/null +++ b/charms/trusty/kafka/tests/100-deploy-kafka @@ -0,0 +1,29 @@ +#!/usr/bin/python3 +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Deployment test for Apache Kafka + """ + + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='trusty') + # Deploy Kafka Service + cls.d.add('kafka', charm='cs:~bigdata-dev/trusty/apache-kafka') + cls.d.add('zookeeper', charm='cs:~bigdata-dev/trusty/apache-zookeeper') + cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper') + + cls.d.setup(timeout=1800) + cls.d.sentry.wait(timeout=1800) + cls.unit = cls.d.sentry['kafka'][0] + + def test_deploy(self): + output, retcode = self.unit.run("pgrep -a java") + assert 'Kafka' in output, "Kafka daemon is not started" + + +if __name__ == '__main__': + unittest.main() diff --git a/charms/trusty/kafka/tests/remote/test_dist_config.py b/charms/trusty/kafka/tests/remote/test_dist_config.py new file mode 100755 index 0000000..eb2c3aa --- /dev/null +++ b/charms/trusty/kafka/tests/remote/test_dist_config.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import grp +import os +import pwd +import unittest + +import jujubigdata + + +class TestDistConfig(unittest.TestCase): + """ + Test that the ``dist.yaml`` settings were applied properly, such as users, groups, and dirs. + + This is done as a remote test on the deployed unit rather than a regular + test under ``tests/`` because filling in the ``dist.yaml`` requires Juju + context (e.g., config). + """ + @classmethod + def setUpClass(cls): + config = None + config_dir = os.environ['JUJU_CHARM_DIR'] + config_file = 'dist.yaml' + if os.path.isfile(os.path.join(config_dir, config_file)): + config = os.path.join(config_dir, config_file) + if not config: + raise IOError('Could not find {} in {}'.format(config_file, config_dir)) + reqs = ['vendor', 'hadoop_version', 'groups', 'users', 'dirs'] + cls.dist_config = jujubigdata.utils.DistConfig(config, reqs) + + def test_groups(self): + for name in self.dist_config.groups: + try: + grp.getgrnam(name) + except KeyError: + self.fail('Group {} is missing'.format(name)) + + def test_users(self): + for username, details in self.dist_config.users.items(): + try: + user = pwd.getpwnam(username) + except KeyError: + self.fail('User {} is missing'.format(username)) + for groupname in details['groups']: + try: + group = grp.getgrnam(groupname) + except KeyError: + self.fail('Group {} referenced by user {} does not exist'.format( + groupname, username)) + if group.gr_gid != user.pw_gid: + self.assertIn(username, group.gr_mem, 'User {} not in group {}'.format( + username, groupname)) + + def test_dirs(self): + for name, details in self.dist_config.dirs.items(): + dirpath = self.dist_config.path(name) + self.assertTrue(dirpath.isdir(), 'Dir {} is missing'.format(name)) + stat = dirpath.stat() + owner = pwd.getpwuid(stat.st_uid).pw_name + group = grp.getgrgid(stat.st_gid).gr_name + perms = stat.st_mode & ~0o40000 + self.assertEqual(owner, details.get('owner', 'root'), + 'Dir {} ({}) has wrong owner: {}'.format(name, dirpath, owner)) + self.assertEqual(group, details.get('group', 'root'), + 'Dir {} ({}) has wrong group: {}'.format(name, dirpath, group)) + self.assertEqual(perms, details.get('perms', 0o755), + 'Dir {} ({}) has wrong perms: 0o{:o}'.format(name, dirpath, perms)) + + +if __name__ == '__main__': + unittest.main() diff --git a/charms/trusty/kafka/tests/tests.yaml b/charms/trusty/kafka/tests/tests.yaml new file mode 100644 index 0000000..771f3fd --- /dev/null +++ b/charms/trusty/kafka/tests/tests.yaml @@ -0,0 +1,10 @@ +# Driver for bundletester: https://github.com/juju-solutions/bundletester +# +# It may be useful to alter the defaults during manual testing. For example, +# set 'reset: false' to reuse existing charms instead of redeploying them. + +# Allow bootstrap of current env, default: true +bootstrap: true + +# Use juju-deployer to reset env between test, default: true +reset: true |
