diff options
Diffstat (limited to 'jjb/cperf')
| -rw-r--r-- | jjb/cperf/cirros-upload.yaml.ansible | 39 | ||||
| -rw-r--r-- | jjb/cperf/cperf-ci-jobs.yaml | 143 | ||||
| -rwxr-xr-x | jjb/cperf/cperf-prepare-robot.sh | 32 | ||||
| -rwxr-xr-x | jjb/cperf/cperf-robot-netvirt-csit.sh | 174 | ||||
| -rw-r--r-- | jjb/cperf/cperf-upload-logs-csit.sh | 12 | ||||
| -rw-r--r-- | jjb/cperf/csit-clean.yaml.ansible | 11 | ||||
| -rw-r--r-- | jjb/cperf/parse-node-yaml.py | 71 |
7 files changed, 426 insertions, 56 deletions
diff --git a/jjb/cperf/cirros-upload.yaml.ansible b/jjb/cperf/cirros-upload.yaml.ansible new file mode 100644 index 000000000..855bb1f3d --- /dev/null +++ b/jjb/cperf/cirros-upload.yaml.ansible @@ -0,0 +1,39 @@ +--- +- hosts: all + tasks: + - copy: + src: "{{ lookup('env', 'WORKSPACE') }}/{{ item }}" + dest: "/home/heat-admin/{{ item }}" + owner: heat-admin + group: heat-admin + mode: 0775 + with_items: + - cirros-0.3.5-x86_64-disk.img + - overcloudrc + - name: Upload cirros glance image + shell: > + source /home/heat-admin/overcloudrc && openstack image create + cirros-0.3.5-x86_64-disk --public + --file /home/heat-admin/cirros-0.3.5-x86_64-disk.img + --disk-format qcow2 --container-format bare + - name: Create nano flavor + shell: > + source /home/heat-admin/overcloudrc && openstack flavor create + --id 42 --ram 64 --disk 0 --vcpus 1 m1.nano + - name: Open CSIT TCP port for netcat + iptables: + chain: INPUT + action: insert + protocol: tcp + destination_port: 12345 + jump: ACCEPT + become: yes + - name: Open CSIT UDP port for netcat + iptables: + chain: INPUT + action: insert + protocol: udp + destination_port: 12345 + jump: ACCEPT + become: yes + diff --git a/jjb/cperf/cperf-ci-jobs.yaml b/jjb/cperf/cperf-ci-jobs.yaml index fdd3509d1..efcf51faa 100644 --- a/jjb/cperf/cperf-ci-jobs.yaml +++ b/jjb/cperf/cperf-ci-jobs.yaml @@ -9,47 +9,30 @@ # ------------------------------- # BRANCH ANCHORS # ------------------------------- - master: &master - stream: master - branch: '{stream}' - gs-pathname: '' - docker-tag: 'latest' - danube: &danube - stream: danube - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - docker-tag: 'stable' + stream: master + branch: '{stream}' + gs-pathname: '' + docker-tag: 'latest' - # ------------------------------- - # POD, INSTALLER, AND BRANCH MAPPING - # ------------------------------- - pod: - # ------------------------------- - # master - # ------------------------------- - - intel-pod2: - installer: apex - <<: *master - - intel-pod2: - installer: apex - <<: *danube + installer: apex testsuite: - - 'daily' + - csit + - cbench jobs: - - 'cperf-{installer}-{pod}-{testsuite}-{stream}' + - 'cperf-{installer}-{testsuite}-{stream}' + - 'cperf-upload-logs-csit' ################################ # job template ################################ - job-template: - name: 'cperf-{installer}-{pod}-{testsuite}-{stream}' + name: 'cperf-{installer}-{testsuite}-{stream}' concurrent: true properties: - - logrotate-default - throttle: enabled: true max-per-node: 1 @@ -57,28 +40,46 @@ wrappers: - build-name: - name: '$BUILD_NUMBER Suite: $CPERF_SUITE_NAME Scenario: $DEPLOY_SCENARIO' + name: '$BUILD_NUMBER Suite: $CPERF_SUITE_NAME ODL BRANCH: $ODL_BRANCH' - timeout: timeout: 400 abort: true parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - '{pod}-defaults' - - '{installer}-defaults' - cperf-parameter: testsuite: '{testsuite}' gs-pathname: '{gs-pathname}' docker-tag: '{docker-tag}' - - scm: - - git-scm + stream: '{stream}' builders: - 'cperf-{testsuite}-builder' +- job-template: + name: 'cperf-upload-logs-csit' + + concurrent: true + + disabled: false + + parameters: + - cperf-parameter: + testsuite: 'csit' + gs-pathname: '{gs-pathname}' + docker-tag: '{docker-tag}' + stream: '{stream}' + + # yamllint enable rule:line-length + properties: + - logrotate-default + - throttle: + max-per-node: 1 + max-total: 10 + option: 'project' + + builders: + - 'cperf-upload-logs-csit' + ######################## # parameter macros ######################## @@ -90,6 +91,14 @@ default: '{testsuite}' description: "Suite name to run" - string: + name: ODL_BRANCH + default: 'master' + description: "Branch that OpenDaylight is running" + - string: + name: OS_VERSION + default: 'master' + description: "OpenStack version (short name, no stable/ prefix)" + - string: name: GS_PATHNAME default: '{gs-pathname}' description: "Version directory where the opnfv documents will be stored in gs repository" @@ -101,6 +110,22 @@ name: DOCKER_TAG default: '{docker-tag}' description: 'Tag to pull docker image' + - string: + name: RC_FILE_PATH + default: '' + description: "Path to the OS credentials file if given" + - string: + name: SSH_KEY_PATH + default: '' + description: "Path to the private SSH key to access OPNFV nodes" + - string: + name: NODE_FILE_PATH + default: '' + description: "Path to the yaml file describing overcloud nodes" + - string: + name: ODL_CONTAINERIZED + default: 'true' + description: "boolean set true if ODL on overcloud is a container" ######################## # trigger macros @@ -110,12 +135,26 @@ # builder macros ######################## - builder: - name: cperf-daily-builder + name: cperf-csit-builder + builders: + - 'cperf-cleanup' + - 'cperf-prepare-robot' + - 'cperf-robot-netvirt-csit' + +- builder: + name: cperf-cbench-builder builders: - 'cperf-cleanup' + - 'cperf-prepare-robot' - 'cperf-robot-cbench' - builder: + name: cperf-prepare-robot + builders: + - shell: + !include-raw: ./cperf-prepare-robot.sh + +- builder: name: cperf-robot-cbench builders: - shell: | @@ -123,23 +162,6 @@ set -o errexit set -o nounset set -o pipefail - undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ - grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") - INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) - - sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/overcloudrc /tmp/overcloudrc - sudo chmod 755 /tmp/overcloudrc - source /tmp/overcloudrc - - # robot suites need the ssh key to log in to controller nodes, so throwing it - # in tmp, and mounting /tmp as $HOME as far as robot is concerned - sudo rm -rf /tmp/.ssh - sudo mkdir /tmp/.ssh - sudo chmod 0700 /tmp/.ssh - sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/.ssh/id_rsa /tmp/.ssh/ - sudo chown -R jenkins-ci:jenkins-ci /tmp/.ssh - # done with sudo. jenkins-ci is the user from this point - chmod 0600 /tmp/.ssh/id_rsa # cbench requires the openflow drop test feature to be installed. sshpass -p karaf ssh -o StrictHostKeyChecking=no \ @@ -148,8 +170,6 @@ -p 8101 karaf@$SDN_CONTROLLER_IP \ feature:install odl-openflowplugin-flow-services-ui odl-openflowplugin-drop-test - docker pull opnfv/cperf:$DOCKER_TAG - robot_cmd="pybot -e exclude -L TRACE -d /tmp \ -v ODL_SYSTEM_1_IP:${SDN_CONTROLLER_IP} \ -v ODL_SYSTEM_IP:${SDN_CONTROLLER_IP} \ @@ -165,6 +185,12 @@ docker run -i -v /tmp:/tmp opnfv/cperf:$DOCKER_TAG ${robot_cmd} ${robot_suite} - builder: + name: cperf-robot-netvirt-csit + builders: + - shell: + !include-raw: ./cperf-robot-netvirt-csit.sh + +- builder: name: cperf-cleanup builders: - shell: | @@ -188,3 +214,8 @@ docker rmi opnfv/cperf:$tag >/dev/null done fi + +- builder: + name: cperf-upload-logs-csit + builders: + - shell: !include-raw: ./cperf-upload-logs-csit.sh diff --git a/jjb/cperf/cperf-prepare-robot.sh b/jjb/cperf/cperf-prepare-robot.sh new file mode 100755 index 000000000..d88c6d510 --- /dev/null +++ b/jjb/cperf/cperf-prepare-robot.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if [ -z ${RC_FILE_PATH+x} ]; then + undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ + grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") + INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) + sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/overcloudrc /tmp/overcloudrc +else + cp -f $RC_FILE_PATH ${WORKSPACE}/overcloudrc +fi + +sudo chmod 755 ${WORKSPACE}/overcloudrc +source ${WORKSPACE}/overcloudrc + +# copy ssh key for robot + +if [ -z ${SSH_KEY_PATH+x} ]; then + sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/.ssh/id_rsa ${WORKSPACE}/ + sudo chown -R jenkins-ci:jenkins-ci ${WORKSPACE}/ + # done with sudo. jenkins-ci is the user from this point + chmod 0600 ${WORKSPACE}/id_rsa +else + cp -f ${SSH_KEY_PATH} ${WORKSPACE}/ +fi + +docker pull opnfv/cperf:$DOCKER_TAG + +sudo mkdir -p /tmp/robot_results diff --git a/jjb/cperf/cperf-robot-netvirt-csit.sh b/jjb/cperf/cperf-robot-netvirt-csit.sh new file mode 100755 index 000000000..4fdcd102b --- /dev/null +++ b/jjb/cperf/cperf-robot-netvirt-csit.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if [ "$OS_VERSION" == 'master' ]; then + FULL_OS_VER='master' +else + FULL_OS_VER="stable/${OS_VERSION}" +fi + +if [ "$ODL_BRANCH" == 'master' ]; then + ODL_STREAM='neon' +else + ODL_STREAM=${ODL_BRANCH#"stable/"} +fi + +echo "ODL Stream set: ${ODL_STREAM} and OS Version is ${FULL_OS_VER}" + +sudo rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng.git +REL_PATH='releng/jjb/cperf' + +# NOTE: sourcing overcloudrc unsets any variable with OS_ prefix +source ${WORKSPACE}/overcloudrc +# note SDN_CONTROLLER_IP is set in overcloudrc, which is the VIP +# for admin/public network (since we are running single network deployment) + +NUM_CONTROL_NODES=$(python ${REL_PATH}/parse-node-yaml.py num_nodes --file $NODE_FILE_PATH) +NUM_COMPUTE_NODES=$(python ${REL_PATH}/parse-node-yaml.py num_nodes --node-type compute --file $NODE_FILE_PATH) + +echo "Number of Control nodes found: ${NUM_CONTROL_NODES}" +echo "Number of Compute nodes found: ${NUM_COMPUTE_NODES}" + +# Only 1 combo or ctrl node is specified, even for OS HA deployments +# Currently supported combinations are: +# 0cmb-1ctl-2cmp +# 1cmb-0ctl-0cmp +# 1cmb-0ctl-1cmp +if [ "$NUM_COMPUTE_NODES" -eq 0 ]; then + OPENSTACK_TOPO="1cmb-0ctl-0cmp" +else + OPENSTACK_TOPO="0cmb-1ctl-2cmp" +fi + +idx=1 +EXTRA_ROBOT_ARGS="" +for idx in `seq 1 $NUM_CONTROL_NODES`; do + CONTROLLER_IP=$(python ${REL_PATH}/parse-node-yaml.py get_value -k address --node-number ${idx} --file $NODE_FILE_PATH) + EXTRA_ROBOT_ARGS+=" -v ODL_SYSTEM_${idx}_IP:${CONTROLLER_IP} \ + -v OS_CONTROL_NODE_${idx}_IP:${CONTROLLER_IP} \ + -v ODL_SYSTEM_${idx}_IP:${CONTROLLER_IP} \ + -v HA_PROXY_${idx}_IP:${SDN_CONTROLLER_IP}" +done + +# In all-in-one these Compute IPs still need to be passed to robot +if [ "$NUM_COMPUTE_NODES" -eq 0 ]; then + EXTRA_ROBOT_ARGS+=" -v OS_COMPUTE_1_IP:'' -v OS_COMPUTE_2_IP:''" +else + idx=1 + for idx in `seq 1 $NUM_COMPUTE_NODES`; do + COMPUTE_IP=$(python ${REL_PATH}/parse-node-yaml.py get_value -k address --node-type compute --node-number ${idx} --file $NODE_FILE_PATH) + EXTRA_ROBOT_ARGS+=" -v OS_COMPUTE_${idx}_IP:${COMPUTE_IP}" + done +fi + +CONTROLLER_1_IP=$(python ${REL_PATH}/parse-node-yaml.py get_value -k address --node-number 1 --file $NODE_FILE_PATH) + +if [ "$ODL_CONTAINERIZED" == 'false' ]; then + EXTRA_ROBOT_ARGS+=" -v NODE_KARAF_COUNT_COMMAND:'ps axf | grep org.apache.karaf | grep -v grep | wc -l || echo 0' \ + -v NODE_START_COMMAND:'sudo systemctl start opendaylight_api' \ + -v NODE_KILL_COMMAND:'sudo systemctl stop opendaylight_api' \ + -v NODE_STOP_COMMAND:'sudo systemctl stop opendaylight_api' \ + -v NODE_FREEZE_COMMAND:'sudo systemctl stop opendaylight_api' " +else + EXTRA_ROBOT_ARGS+=" -v NODE_KARAF_COUNT_COMMAND:'sudo docker ps | grep opendaylight_api | wc -l || echo 0' \ + -v NODE_START_COMMAND:'sudo docker start opendaylight_api' \ + -v NODE_KILL_COMMAND:'sudo docker stop opendaylight_api' \ + -v NODE_STOP_COMMAND:'sudo docker stop opendaylight_api' \ + -v NODE_FREEZE_COMMAND:'sudo docker stop opendaylight_api' " +fi + +# FIXME(trozet) remove this once it is fixed in csit +# Upload glance image into openstack +wget -O ${WORKSPACE}/cirros-0.3.5-x86_64-disk.img http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img +export ANSIBLE_HOST_KEY_CHECKING=False +ansible-playbook -i ${CONTROLLER_1_IP}, -u heat-admin --key-file ${WORKSPACE}/id_rsa ${REL_PATH}/cirros-upload.yaml.ansible -vvv + +LOGS_LOCATION=/tmp/robot_results + +robot_cmd="pybot \ + --removekeywords wuks \ + --xunit robotxunit.xml \ + --name 'CSIT' \ + -e exclude \ + -d $LOGS_LOCATION \ + -v BUNDLEFOLDER:/opt/opendaylight \ + -v CONTROLLER_USER:heat-admin \ + -v DEFAULT_LINUX_PROMPT:\$ \ + -v DEFAULT_LINUX_PROMPT_STRICT:]\$ \ + -v DEFAULT_USER:heat-admin \ + -v DEVSTACK_DEPLOY_PATH:/tmp \ + -v EXTERNAL_GATEWAY:$CONTROLLER_1_IP \ + -v EXTERNAL_PNF:$CONTROLLER_1_IP \ + -v EXTERNAL_SUBNET:192.0.2.0/24 \ + -v EXTERNAL_SUBNET_ALLOCATION_POOL:start=192.0.2.100,end=192.0.2.200 \ + -v EXTERNAL_INTERNET_ADDR:$CONTROLLER_1_IP \ + -v HA_PROXY_IP:$SDN_CONTROLLER_IP \ + -v NUM_ODL_SYSTEM:$NUM_CONTROL_NODES \ + -v NUM_OS_SYSTEM:$(($NUM_CONTROL_NODES + $NUM_COMPUTE_NODES)) \ + -v NUM_TOOLS_SYSTEM:0 \ + -v ODL_SNAT_MODE:conntrack \ + -v ODL_STREAM:$ODL_STREAM \ + -v ODL_SYSTEM_IP:$CONTROLLER_1_IP \ + -v OS_CONTROL_NODE_IP:$CONTROLLER_1_IP \ + -v OPENSTACK_BRANCH:$FULL_OS_VER \ + -v OPENSTACK_TOPO:$OPENSTACK_TOPO \ + -v OS_USER:heat-admin \ + -v ODL_ENABLE_L3_FWD:yes \ + -v ODL_SYSTEM_USER:heat-admin \ + -v ODL_SYSTEM_PROMPT:\$ \ + -v PRE_CLEAN_OPENSTACK_ALL:True \ + -v PUBLIC_PHYSICAL_NETWORK:datacentre \ + -v RESTCONFPORT:8081 \ + -v ODL_RESTCONF_USER:admin \ + -v ODL_RESTCONF_PASSWORD:$SDN_CONTROLLER_PASSWORD \ + -v KARAF_PROMPT_LOGIN:'opendaylight-user' \ + -v KARAF_PROMPT:'opendaylight-user.*root.*>' \ + -v SECURITY_GROUP_MODE:stateful \ + -v USER:heat-admin \ + -v USER_HOME:\$HOME \ + -v TOOLS_SYSTEM_IP:'' \ + -v NODE_ROLE_INDEX_START:0 \ + -v WORKSPACE:/tmp \ + $EXTRA_ROBOT_ARGS \ + -v of_port:6653 " + +SUITE_HOME='/home/opnfv/repos/odl_test/csit/suites' + +# Disabled suites +# +# ${SUITE_HOME}/openstack/connectivity/live_migration.robot +# Live migration will not work unless we use a shared storage backend like +# Ceph which we do not currently use with CSIT images +# + +suites="${SUITE_HOME}/openstack/connectivity/l2.robot \ + ${SUITE_HOME}/openstack/connectivity/l3.robot \ + ${SUITE_HOME}/openstack/connectivity/external_network.robot \ + ${SUITE_HOME}/openstack/connectivity/security_group.robot \ + ${SUITE_HOME}/openstack/securitygroup/neutron_security_group.robot \ + ${SUITE_HOME}/openstack/securitygroup/security_group_l3bcast.robot \ + ${SUITE_HOME}/netvirt/vpnservice/vpn_basic.robot \ + ${SUITE_HOME}/netvirt/vpnservice/vpn_basic_ipv6.robot \ + ${SUITE_HOME}/netvirt/elan/elan.robot \ + ${SUITE_HOME}/netvirt/vpnservice/arp_learning.robot \ + ${SUITE_HOME}/netvirt/l2l3_gatewaymac_arp.robot \ + ${SUITE_HOME}/integration/Create_JVM_Plots.robot" + +echo "Robot command set: ${robot_cmd}" +echo "Running robot..." +docker run -i --net=host \ + -v ${LOGS_LOCATION}:${LOGS_LOCATION} \ + -v ${WORKSPACE}/id_rsa:/tmp/id_rsa \ + -v ${WORKSPACE}/overcloudrc:/tmp/overcloudrc \ + opnfv/cperf:$DOCKER_TAG \ + /bin/bash -c "source /tmp/overcloudrc; mkdir -p \$HOME/.ssh; cp /tmp/id_rsa \$HOME/.ssh; \ + cd /home/opnfv/repos/odl_test/ && git pull origin master; \ + pip install odltools; \ + ${robot_cmd} ${suites};" + +echo "Running post CSIT clean" +ansible-playbook -i ${CONTROLLER_1_IP}, -u heat-admin --key-file ${WORKSPACE}/id_rsa ${REL_PATH}/csit-clean.yaml.ansible -vvv diff --git a/jjb/cperf/cperf-upload-logs-csit.sh b/jjb/cperf/cperf-upload-logs-csit.sh new file mode 100644 index 000000000..bd8680406 --- /dev/null +++ b/jjb/cperf/cperf-upload-logs-csit.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +ODL_STREAM=${ODL_BRANCH#"stable/"} + +LOGS_LOCATION=/tmp/robot_results +UPLOAD_LOCATION=artifacts.opnfv.org/cperf/cperf-apex-csit-${ODL_STREAM}/${BUILD_NUMBER}/ +echo "Uploading robot logs to ${UPLOAD_LOCATION}" +gsutil -m cp -r -v ${LOGS_LOCATION} gs://${UPLOAD_LOCATION} > gsutil.latest_logs.log diff --git a/jjb/cperf/csit-clean.yaml.ansible b/jjb/cperf/csit-clean.yaml.ansible new file mode 100644 index 000000000..0151dd824 --- /dev/null +++ b/jjb/cperf/csit-clean.yaml.ansible @@ -0,0 +1,11 @@ +--- +- hosts: all + tasks: + - name: Delete cirros glance image + shell: > + source /home/heat-admin/overcloudrc && openstack image delete + cirros-0.3.5-x86_64-disk + - name: Delete nano flavor + shell: > + source /home/heat-admin/overcloudrc && openstack flavor delete + m1.nano diff --git a/jjb/cperf/parse-node-yaml.py b/jjb/cperf/parse-node-yaml.py new file mode 100644 index 000000000..5a7575540 --- /dev/null +++ b/jjb/cperf/parse-node-yaml.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import argparse +import sys +import yaml + + +def get_node_data_by_number(node_type, node_number): + node_idx = 1 + for node_name, node_data in data['servers'].items(): + if node_type == node_data['type']: + if node_idx == node_number: + return node_name, node_data + else: + node_idx += 1 + + +def get_node_value(node_type, node_number, key): + node_name, node_data = get_node_data_by_number(node_type, node_number) + if not key and node_name is not None: + return node_name + elif node_data and isinstance(node_data, dict) and key in node_data: + return node_data[key] + + +def get_number_of_nodes(node_type): + nodes = data['servers'] + num_nodes = 0 + for node_name, node_data in nodes.items(): + if node_data['type'] == node_type: + num_nodes += 1 + return num_nodes + + +FUNCTION_MAP = {'num_nodes': + {'func': get_number_of_nodes, + 'args': ['node_type']}, + 'get_value': + {'func': get_node_value, + 'args': ['node_type', 'node_number', 'key']}, + } + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=FUNCTION_MAP.keys()) + parser.add_argument('-f', '--file', + dest='node_file', + required=True) + parser.add_argument('--node-type', + default='controller', + required=False) + parser.add_argument('--node-number', + default=1, + type=int, + required=False) + parser.add_argument('-k', '--key', + required=False) + args = parser.parse_args(sys.argv[1:]) + with open(args.node_file, 'r') as fh: + data = yaml.safe_load(fh) + assert 'servers' in data + func = FUNCTION_MAP[args.command]['func'] + args = [getattr(args, x) for x in FUNCTION_MAP[args.command]['args']] + print(func(*args)) |