From bb38e087e70dd00575e441e375b1be2a41d4c657 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Thu, 28 Feb 2019 15:46:19 +0100 Subject: [virtual] Parameterize scenarios based on PDF/IDF NOTE: only os-nosdn-nofeature-noha is parameterized for now. - move config drive & disk creation from prepare_vms to create_vms; - make default disk size(s) configurable based on scenario defaults and vPDF; * compute nodes require 2 disks to be defined in vPDF, since the pillar reclass model assumes /dev/vdb is reserved for cinder; * if multiple disks are defined in vPDF, they are created and attached accordinly (only ctl01 and cmp nodes are parameterized in this change; only for the os-nosdn-nofeature-noha scenario); - vCPU specifications are deduced based on vPDF (sockets, cores); * threads/core is hard set to 2 since vPDF does not have a key for it; * NUMA resources are distributed evenly based on the number of sockets configured in PDF; * no less than the mininum requirement for a scenario is allocated (e.g. if PDF specifies 2 cores, but the scenario requires at least 4 cores, the larger value will be used); - RAM is deduced based on PDF (but no less than the mininum req is allocated, e.g. if PDF specifies 2GB RAM for computes, but the scenario requires at least 8GB, the larger value will be used); Change-Id: I97188aa2a1006865b8429eb6483e10c76795f7d2 Signed-off-by: Alexandru Avadanii (cherry picked from commit 9e5b19768d6eb9b32564240065ecba6c791cac12) --- ci/deploy.sh | 3 +- mcp/config/scenario/.gitignore | 1 + mcp/config/scenario/defaults.yaml.j2 | 2 + mcp/config/scenario/os-nosdn-fdio-ha.yaml | 2 + mcp/config/scenario/os-nosdn-fdio-noha.yaml | 2 + mcp/config/scenario/os-nosdn-nofeature-ha.yaml | 2 + mcp/config/scenario/os-nosdn-nofeature-noha.yaml | 54 ---------------------- .../scenario/os-nosdn-nofeature-noha.yaml.j2 | 52 +++++++++++++++++++++ mcp/config/scenario/os-nosdn-onap-ha.yaml | 2 + mcp/config/scenario/os-nosdn-onap-noha.yaml | 2 + mcp/config/scenario/os-nosdn-ovs-ha.yaml | 2 + mcp/config/scenario/os-nosdn-ovs-noha.yaml | 2 + mcp/config/scenario/os-odl-bgpvpn-noha.yaml | 2 + mcp/config/scenario/os-odl-nofeature-ha.yaml | 2 + mcp/config/scenario/os-odl-nofeature-noha.yaml | 4 ++ mcp/config/scenario/os-odl-ovs-noha.yaml | 2 + mcp/config/scenario/os-odl-sfc-noha.yaml | 4 ++ mcp/config/scenario/os-ovn-nofeature-ha.yaml | 2 + mcp/config/scenario/os-ovn-nofeature-noha.yaml | 2 + mcp/scripts/lib_jump_deploy.sh | 48 +++++++++---------- mcp/scripts/lib_template.sh | 6 +-- mcp/scripts/xdf_data.sh.j2 | 3 +- 22 files changed, 116 insertions(+), 85 deletions(-) delete mode 100644 mcp/config/scenario/os-nosdn-nofeature-noha.yaml create mode 100644 mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 diff --git a/ci/deploy.sh b/ci/deploy.sh index 8515d5c21..75a1a8864 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -306,8 +306,7 @@ elif [ ${USE_EXISTING_INFRA} -gt 0 ]; then notify "[NOTE] Use existing infra: skip first ${USE_EXISTING_INFRA} states" 2 notify "[STATE] Skipping: ${cluster_states[*]::${USE_EXISTING_INFRA}}" 2 else - prepare_vms "${base_image}" "${MCP_STORAGE_DIR}" "${virtual_repos_pkgs}" \ - "${virtual_nodes[@]}" + prepare_vms "${base_image}" "${MCP_STORAGE_DIR}" "${virtual_repos_pkgs}" create_networks "${OPNFV_BRIDGES[@]}" do_sysctl_cfg do_udev_cfg diff --git a/mcp/config/scenario/.gitignore b/mcp/config/scenario/.gitignore index 13346b908..0e5ba3d09 100644 --- a/mcp/config/scenario/.gitignore +++ b/mcp/config/scenario/.gitignore @@ -1 +1,2 @@ defaults.yaml +os-nosdn-nofeature-noha.yaml diff --git a/mcp/config/scenario/defaults.yaml.j2 b/mcp/config/scenario/defaults.yaml.j2 index ddcd0a039..3ea639bc3 100644 --- a/mcp/config/scenario/defaults.yaml.j2 +++ b/mcp/config/scenario/defaults.yaml.j2 @@ -10,6 +10,7 @@ x86_64: base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img default: + disks: 100G # ';'-separated list of disk drives to create vcpus: 4 ram: 8192 cluster: &arch_default_cluster_states @@ -41,6 +42,7 @@ x86_64: aarch64: base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-uefi1.img default: + disks: 100G # ';'-separated list of disk drives to create vcpus: 6 ram: 4096 cluster: *arch_default_cluster_states diff --git a/mcp/config/scenario/os-nosdn-fdio-ha.yaml b/mcp/config/scenario/os-nosdn-fdio-ha.yaml index 3ff4e5c5d..384771298 100644 --- a/mcp/config/scenario/os-nosdn-fdio-ha.yaml +++ b/mcp/config/scenario/os-nosdn-fdio-ha.yaml @@ -38,8 +38,10 @@ virtual: vcpus: 4 ram: 14336 cmp001: + disks: 100G;100G vcpus: 4 ram: 8192 cmp002: + disks: 100G;100G vcpus: 4 ram: 8192 diff --git a/mcp/config/scenario/os-nosdn-fdio-noha.yaml b/mcp/config/scenario/os-nosdn-fdio-noha.yaml index 747adbee2..187b5bf48 100644 --- a/mcp/config/scenario/os-nosdn-fdio-noha.yaml +++ b/mcp/config/scenario/os-nosdn-fdio-noha.yaml @@ -35,6 +35,7 @@ virtual: memory: 8388608 cpus: 0-7 cmp001: + disks: 100G;100G vcpus: 8 ram: 8192 cpu_topology: @@ -46,6 +47,7 @@ virtual: memory: 8388608 cpus: 0-7 cmp002: + disks: 100G;100G vcpus: 8 ram: 8192 cpu_topology: diff --git a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml index ce79b838e..d06d2ae0b 100644 --- a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml +++ b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml @@ -35,8 +35,10 @@ virtual: vcpus: 4 ram: 14336 cmp001: + disks: 100G;100G vcpus: 4 ram: 8192 cmp002: + disks: 100G;100G vcpus: 4 ram: 8192 diff --git a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml deleted file mode 100644 index 76fed8c12..000000000 --- a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml +++ /dev/null @@ -1,54 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## ---- -cluster: - domain: mcp-ovs-noha.local - states: - - openstack_noha - - neutron_gateway - - networks -virtual: - nodes: - control: - - ctl01 - - gtw01 - compute: - - cmp001 - - cmp002 - ctl01: - ram: 14336 - gtw01: - ram: 4096 - cmp001: - vcpus: 8 - ram: 16384 - cpu_topology: - sockets: 2 - cores: 2 - threads: 2 - numa: - cell0: - memory: 8388608 - cpus: 0-3 - cell1: - memory: 8388608 - cpus: 4-7 - cmp002: - vcpus: 8 - ram: 16384 - cpu_topology: - sockets: 2 - cores: 2 - threads: 2 - numa: - cell0: - memory: 8388608 - cpus: 0-3 - cell1: - memory: 8388608 - cpus: 4-7 diff --git a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 new file mode 100644 index 000000000..a1f0ddaa6 --- /dev/null +++ b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2017 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- import 'net_map.j2' as nm with context %} +--- +cluster: + domain: mcp-ovs-noha.local + states: + - openstack_noha + - neutron_gateway + - networks +virtual: + nodes: + control: + - ctl01 + - gtw01 + compute: + - cmp001 + - cmp002 + ctl01: + disks: {{ conf.nodes[nm.ctl01.idx].disks | map(attribute='disk_capacity') | join(';') }} + vcpus: 4 + ram: 14336 + gtw01: + ram: 2048 +{%- if nm.cmp001.idx < conf.nodes | length %} + {%- set cmp_node = conf.nodes[nm.cmp001.idx].node %} + {%- set cmp_ram = ([16384, ('%.0f' | format((cmp_node.memory | storage_size_num | float) * 1024 / 1000000000) | int)] | sort)[-1] %} + {%- set cmp_sockets = ([2, cmp_node.cpus | int] | sort)[-1] %} + {%- set cmp_threads = 2 %} + {%- set cmp_cores = ([2, ('%.0f' | format((cmp_node.cores | float) / cmp_threads)) | int] | sort)[-1] %} + {%- for cmpi in range(1, 3) %} + cmp00{{ cmpi }}: + disks: {{ conf.nodes[nm.cmp001.idx].disks | map(attribute='disk_capacity') | join(';') }} + vcpus: {{ cmp_sockets * cmp_cores * cmp_threads }} + ram: {{ cmp_ram }} + cpu_topology: + sockets: {{ cmp_sockets }} + cores: {{ cmp_cores }} + threads: {{ cmp_threads }} + numa: + {%- for ci in range(0, cmp_sockets) %} + cell{{ ci }}: + memory: {{ '%.0f' | format((cmp_ram | float) * 1024 / cmp_sockets) }} + cpus: {{ cmp_cores * cmp_threads * (ci | int) }}-{{ cmp_cores * cmp_threads * (ci | int + 1) - 1 }} + {%- endfor %} + {%- endfor %} +{%- endif %} diff --git a/mcp/config/scenario/os-nosdn-onap-ha.yaml b/mcp/config/scenario/os-nosdn-onap-ha.yaml index d0e9c40c0..653599f6c 100644 --- a/mcp/config/scenario/os-nosdn-onap-ha.yaml +++ b/mcp/config/scenario/os-nosdn-onap-ha.yaml @@ -39,8 +39,10 @@ virtual: vcpus: 4 ram: 14336 cmp001: + disks: 100G;100G vcpus: 32 ram: 102400 cmp002: + disks: 100G;100G vcpus: 32 ram: 102400 diff --git a/mcp/config/scenario/os-nosdn-onap-noha.yaml b/mcp/config/scenario/os-nosdn-onap-noha.yaml index ae6255436..5a08a2cd6 100644 --- a/mcp/config/scenario/os-nosdn-onap-noha.yaml +++ b/mcp/config/scenario/os-nosdn-onap-noha.yaml @@ -32,8 +32,10 @@ virtual: gtw01: ram: 2048 cmp001: + disks: 100G;100G vcpus: 36 ram: 122880 cmp002: + disks: 100G;100G vcpus: 36 ram: 122880 diff --git a/mcp/config/scenario/os-nosdn-ovs-ha.yaml b/mcp/config/scenario/os-nosdn-ovs-ha.yaml index 9be6540c7..72a55ea6f 100644 --- a/mcp/config/scenario/os-nosdn-ovs-ha.yaml +++ b/mcp/config/scenario/os-nosdn-ovs-ha.yaml @@ -35,8 +35,10 @@ virtual: vcpus: 4 ram: 14336 cmp001: + disks: 100G;100G vcpus: 4 ram: 10240 cmp002: + disks: 100G;100G vcpus: 4 ram: 10240 diff --git a/mcp/config/scenario/os-nosdn-ovs-noha.yaml b/mcp/config/scenario/os-nosdn-ovs-noha.yaml index 30a55fe35..b109200e1 100644 --- a/mcp/config/scenario/os-nosdn-ovs-noha.yaml +++ b/mcp/config/scenario/os-nosdn-ovs-noha.yaml @@ -25,8 +25,10 @@ virtual: gtw01: ram: 4096 cmp001: + disks: 100G;100G vcpus: 6 ram: 14336 cmp002: + disks: 100G;100G vcpus: 6 ram: 14336 diff --git a/mcp/config/scenario/os-odl-bgpvpn-noha.yaml b/mcp/config/scenario/os-odl-bgpvpn-noha.yaml index b4db80986..a083e154e 100644 --- a/mcp/config/scenario/os-odl-bgpvpn-noha.yaml +++ b/mcp/config/scenario/os-odl-bgpvpn-noha.yaml @@ -31,6 +31,8 @@ virtual: odl01: ram: 6144 cmp001: + disks: 100G;100G ram: 12288 cmp002: + disks: 100G;100G ram: 12288 diff --git a/mcp/config/scenario/os-odl-nofeature-ha.yaml b/mcp/config/scenario/os-odl-nofeature-ha.yaml index 51d476de5..f295df2ec 100644 --- a/mcp/config/scenario/os-odl-nofeature-ha.yaml +++ b/mcp/config/scenario/os-odl-nofeature-ha.yaml @@ -36,8 +36,10 @@ virtual: vcpus: 4 ram: 14336 cmp001: + disks: 100G;100G vcpus: 4 ram: 8192 cmp002: + disks: 100G;100G vcpus: 4 ram: 8192 diff --git a/mcp/config/scenario/os-odl-nofeature-noha.yaml b/mcp/config/scenario/os-odl-nofeature-noha.yaml index c9847b9ba..1bb47bbc7 100644 --- a/mcp/config/scenario/os-odl-nofeature-noha.yaml +++ b/mcp/config/scenario/os-odl-nofeature-noha.yaml @@ -29,3 +29,7 @@ virtual: ram: 2048 odl01: ram: 6144 + cmp001: + disks: 100G;100G + cmp002: + disks: 100G;100G diff --git a/mcp/config/scenario/os-odl-ovs-noha.yaml b/mcp/config/scenario/os-odl-ovs-noha.yaml index b24538754..d47ad9668 100644 --- a/mcp/config/scenario/os-odl-ovs-noha.yaml +++ b/mcp/config/scenario/os-odl-ovs-noha.yaml @@ -30,8 +30,10 @@ virtual: odl01: ram: 6144 cmp001: + disks: 100G;100G vcpus: 6 ram: 14336 cmp002: + disks: 100G;100G vcpus: 6 ram: 14336 diff --git a/mcp/config/scenario/os-odl-sfc-noha.yaml b/mcp/config/scenario/os-odl-sfc-noha.yaml index f2932c653..735855433 100644 --- a/mcp/config/scenario/os-odl-sfc-noha.yaml +++ b/mcp/config/scenario/os-odl-sfc-noha.yaml @@ -30,3 +30,7 @@ virtual: ram: 2048 odl01: ram: 6144 + cmp001: + disks: 100G;100G + cmp002: + disks: 100G;100G diff --git a/mcp/config/scenario/os-ovn-nofeature-ha.yaml b/mcp/config/scenario/os-ovn-nofeature-ha.yaml index f76b0dce4..69c5b9495 100644 --- a/mcp/config/scenario/os-ovn-nofeature-ha.yaml +++ b/mcp/config/scenario/os-ovn-nofeature-ha.yaml @@ -35,8 +35,10 @@ virtual: vcpus: 4 ram: 14336 cmp001: + disks: 100G;100G vcpus: 4 ram: 8192 cmp002: + disks: 100G;100G vcpus: 4 ram: 8192 diff --git a/mcp/config/scenario/os-ovn-nofeature-noha.yaml b/mcp/config/scenario/os-ovn-nofeature-noha.yaml index 8dd682c39..e7ff2a7d8 100644 --- a/mcp/config/scenario/os-ovn-nofeature-noha.yaml +++ b/mcp/config/scenario/os-ovn-nofeature-noha.yaml @@ -21,8 +21,10 @@ virtual: ctl01: ram: 14336 cmp001: + disks: 100G;100G vcpus: 6 ram: 10240 cmp002: + disks: 100G;100G vcpus: 6 ram: 10240 diff --git a/mcp/scripts/lib_jump_deploy.sh b/mcp/scripts/lib_jump_deploy.sh index 5d442eb0b..aad6ce586 100644 --- a/mcp/scripts/lib_jump_deploy.sh +++ b/mcp/scripts/lib_jump_deploy.sh @@ -215,7 +215,6 @@ function prepare_vms { local base_image=$1; shift local image_dir=$1; shift local repos_pkgs_str=$1; shift # ^-sep list of repos, pkgs to install/rm - local vnodes=("$@") local image=base_image_opnfv_fuel.img local vcp_image=${image%.*}_vcp.img local _o=${base_image/*\/} @@ -249,18 +248,6 @@ function prepare_vms { ln -sf "${image_dir}/${_tmp}" "${image_dir}/${image}" fi - # Create config ISO and resize OS disk image for each foundation node VM - for node in "${vnodes[@]}"; do - ./create-config-drive.sh -k "$(basename "${SSH_KEY}").pub" \ - -u 'user-data.sh' -h "${node}" "${image_dir}/mcp_${node}.iso" - cp "${image_dir}/${image}" "${image_dir}/mcp_${node}.qcow2" - qemu-img resize "${image_dir}/mcp_${node}.qcow2" 100G - # Prepare dedicated drive for cinder on cmp nodes - if [[ "${node}" =~ ^(cmp) ]]; then - qemu-img create "${image_dir}/mcp_${node}_storage.qcow2" 100G - fi - done - # VCP VMs base image specific changes if [[ ! "${repos_pkgs_str}" =~ \^{3}$ ]] && [ -n "${repos_pkgs[*]:4}" ]; then echo "[INFO] Lookup cache / build patched VCP image for md5sum: ${_h}" @@ -342,8 +329,9 @@ function create_networks { function create_vms { local image_dir=$1; shift + local image=base_image_opnfv_fuel.img # vnode data should be serialized with the following format: - # ,,[,,,[,,, + # ,,,[,,,[,,, # ,,,]]|,...' IFS='|' read -r -a vnodes <<< "$1"; shift @@ -358,18 +346,34 @@ function create_vms { for serialized_vnode_data in "${vnodes[@]}"; do if [ -z "${serialized_vnode_data}" ]; then continue; fi IFS=',' read -r -a vnode_data <<< "${serialized_vnode_data}" + IFS=';' read -r -a disks_data <<< "${vnode_data[1]}" + + # Create config ISO and resize OS disk image for each foundation node VM + ./create-config-drive.sh -k "$(basename "${SSH_KEY}").pub" \ + -u 'user-data.sh' -h "${vnode_data[0]}" "${image_dir}/mcp_${vnode_data[0]}.iso" + cp "${image_dir}/${image}" "${image_dir}/mcp_${vnode_data[0]}.qcow2" + qemu-img resize "${image_dir}/mcp_${vnode_data[0]}.qcow2" "${disks_data[0]}" + # Prepare additional drives if present + idx=0 + virt_extra_storage= + for dsize in "${disks_data[@]:1}"; do + ((idx+=1)) + qcow_file="${image_dir}/mcp_${vnode_data[0]}_${idx}.qcow2" + qemu-img create "${qcow_file}" "${dsize}" + virt_extra_storage+=" --disk path=${qcow_file},format=qcow2,bus=virtio,cache=none,io=native" + done # prepare VM CPU model, count, topology (optional), NUMA cells (optional, requires topo) local virt_cpu_args=' --cpu host-passthrough' - local idx=6 # cell0.name index in serialized data + local idx=7 # cell0.name index in serialized data while [ -n "${vnode_data[${idx}]}" ]; do virt_cpu_args+=",${vnode_data[${idx}]}.memory=${vnode_data[$((idx + 1))]}" virt_cpu_args+=",${vnode_data[${idx}]}.cpus=${vnode_data[$((idx + 2))]}" idx=$((idx + 3)) done - virt_cpu_args+=" --vcpus vcpus=${vnode_data[2]}" - if [ -n "${vnode_data[5]}" ]; then - virt_cpu_args+=",sockets=${vnode_data[3]},cores=${vnode_data[4]},threads=${vnode_data[5]}" + virt_cpu_args+=" --vcpus vcpus=${vnode_data[3]}" + if [ -n "${vnode_data[6]}" ]; then + virt_cpu_args+=",sockets=${vnode_data[4]},cores=${vnode_data[5]},threads=${vnode_data[6]}" fi # prepare network args @@ -379,18 +383,12 @@ function create_vms { net_args="${net_args} --network bridge=${net},model=virtio" done - # dedicated storage drive for cinder on cmp nodes - virt_extra_storage= - if [[ "${vnode_data[0]}" =~ ^(cmp) ]]; then - virt_extra_storage="--disk path=${image_dir}/mcp_${vnode_data[0]}_storage.qcow2,format=qcow2,bus=virtio,cache=none,io=native" - fi - [ ! -e "${image_dir}/virt-manager" ] || VIRT_PREFIX="${image_dir}/virt-manager/" # shellcheck disable=SC2086 ${VIRT_PREFIX}${VIRSH/virsh/virt-install} --name "${vnode_data[0]}" \ ${virt_cpu_args} --accelerate \ ${net_args} \ - --ram "${vnode_data[1]}" \ + --ram "${vnode_data[2]}" \ --disk path="${image_dir}/mcp_${vnode_data[0]}.qcow2",format=qcow2,bus=virtio,cache=none,io=native \ ${virt_extra_storage} \ --os-type linux --os-variant none \ diff --git a/mcp/scripts/lib_template.sh b/mcp/scripts/lib_template.sh index 980827c7a..b44702235 100644 --- a/mcp/scripts/lib_template.sh +++ b/mcp/scripts/lib_template.sh @@ -47,9 +47,6 @@ function do_templates_scenario { notify_e "[ERROR] IDF does not match yaml schema!" fi fi - for _yaml in "${extra_yaml[@]}"; do - awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}" - done if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")" \ -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then @@ -63,6 +60,9 @@ function do_templates_scenario { -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then notify_e '[ERROR] Could not convert j2 scenario definitions!' fi + for _yaml in "${extra_yaml[@]}"; do + awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}" + done } # Expand reclass and virsh network templates based on PDF + IDF + others diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2 index 187f09296..346a9e53e 100644 --- a/mcp/scripts/xdf_data.sh.j2 +++ b/mcp/scripts/xdf_data.sh.j2 @@ -44,7 +44,8 @@ {%- do V[n].update({'s_numa': c if 's_numa' not in V[n] else pack([c, V[n].s_numa])}) -%} {%- endfor -%} {%- endif -%} - {%- do arr.append(pack([n, V[n].ram or arch.default.ram, + {%- do arr.append(pack([n, V[n].disks or arch.default.disks, + V[n].ram or arch.default.ram, V[n].vcpus or arch.default.vcpus, '' if not cpu_topo else pack([ V[n].cpu_topology.sockets, -- cgit 1.2.3-korg