summaryrefslogtreecommitdiffstats
path: root/VNFs/DPPD-PROX/tools/flow_extract
diff options
context:
space:
mode:
authorDeepak S <deepak.s@linux.intel.com>2017-07-13 21:26:50 -0700
committerDeepak S <deepak.s@linux.intel.com>2017-07-14 04:58:47 -0700
commit7286b2518ec8e4398b512ce95def9166a7af2e4a (patch)
treec93ef65d9e73e8893ccecb720152e16aae96a8b6 /VNFs/DPPD-PROX/tools/flow_extract
parentadcb79da90176b27224eeb1d00aa0e611ef85a9b (diff)
Adding PROX(Packet pROcessing eXecution engine) VNF to sampleVNF
JIRA: SAMPLEVNF-55 PROX is a DPDK-based application implementing Telco use-cases such as a simplified BRAS/BNG, light-weight AFTR... It also allows configuring finer grained network functions like QoS, Routing, load-balancing... (We are moving PROX version v039 to sampleVNF https://01.org/intel-data-plane-performance-demonstrators/prox-overview) Change-Id: Ia3cb02cf0e49ac5596e922c197ff7e010293d033 Signed-off-by: Deepak S <deepak.s@linux.intel.com>
Diffstat (limited to 'VNFs/DPPD-PROX/tools/flow_extract')
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/Makefile59
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/README20
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/allocator.cpp84
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/allocator.hpp38
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/bundle.cpp28
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/bundle.hpp38
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/crc.hpp51
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.cpp67
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.hpp35
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/flowtable.hpp174
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/halfstream.cpp101
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/halfstream.hpp63
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/main.cpp37
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/mappedfile.cpp109
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/mappedfile.hpp40
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/memreader.cpp106
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/memreader.hpp45
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/netsocket.cpp33
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/netsocket.hpp31
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/path.cpp97
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/path.hpp42
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcappkt.cpp266
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcappkt.hpp104
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcappktref.cpp32
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcappktref.hpp40
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcapreader.cpp76
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcapreader.hpp48
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.cpp46
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.hpp33
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/programconfig.cpp119
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/programconfig.hpp47
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/progress.cpp96
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/progress.hpp50
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/stream.cpp171
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/stream.hpp94
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/stream2.cpp151
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/stream2.hpp54
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/stream3.cpp95
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/stream3.hpp55
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/streamextract.cpp406
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/streamextract.hpp55
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/streamsorter.cpp203
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/streamsorter.hpp47
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/timestamp.cpp65
-rw-r--r--VNFs/DPPD-PROX/tools/flow_extract/timestamp.hpp45
45 files changed, 3696 insertions, 0 deletions
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/Makefile b/VNFs/DPPD-PROX/tools/flow_extract/Makefile
new file mode 100644
index 00000000..a772b8c5
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/Makefile
@@ -0,0 +1,59 @@
+##
+## Copyright (c) 2010-2017 Intel Corporation
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+
+SOURCES = main.cpp
+SOURCES += streamextract.cpp
+SOURCES += pcapreader.cpp
+SOURCES += pcapwriter.cpp
+SOURCES += timestamp.cpp
+SOURCES += pcappkt.cpp
+SOURCES += netsocket.cpp
+SOURCES += stream3.cpp
+SOURCES += stream2.cpp
+SOURCES += stream.cpp
+SOURCES += path.cpp
+SOURCES += allocator.cpp
+SOURCES += halfstream.cpp
+SOURCES += bundle.cpp
+SOURCES += progress.cpp
+SOURCES += mappedfile.cpp
+SOURCES += streamsorter.cpp
+SOURCES += memreader.cpp
+SOURCES += programconfig.cpp
+
+BUILD_DIR = build
+OBJECTS = $(SOURCES:%.cpp=$(BUILD_DIR)/%.o)
+PROG = flowextract
+
+CXXFLAGS += -D__STDC_LIMIT_MACROS -g -O2 -Wall -ansi -pedantic -Wno-unused -msse4.2
+LDFLAGS = -lpcap
+
+$(BUILD_DIR)/$(PROG): $(OBJECTS)
+ @echo -e "LD\t$<"
+ @$(CXX) $(CXXFLAGS) $(LDFLAGS) $(OBJECTS) -o $@
+
+-include $(SOURCES:%.cpp=$(BUILD_DIR)/%.d)
+
+$(BUILD_DIR)/%.o: %.cpp
+ @mkdir -p $(BUILD_DIR)
+ @echo -e "CXX\t $<"
+ @$(CXX) -c $(CXXFLAGS) $*.cpp -o $@
+ @$(CXX) -MM $(CXXFLAGS) $*.cpp -MT $(BUILD_DIR)/$*.o > $(BUILD_DIR)/$*.d
+ @cp -f $(BUILD_DIR)/$*.d $(BUILD_DIR)/$*.d.tmp
+ @sed -e 's/.*://' -e 's/\\$$//' < $(BUILD_DIR)/$*.d.tmp | fmt -1 | sed -e 's/^ *//' -e 's/$$/:/' >> $(BUILD_DIR)/$*.d
+ @rm -f $(BUILD_DIR)/$*.d.tmp
+clean:
+ @rm -f $(BUILD_DIR)/$(PROG) $(BUILD_DIR)/*.o $(BUILD_DIR)/*.d
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/README b/VNFs/DPPD-PROX/tools/flow_extract/README
new file mode 100644
index 00000000..fb8754b3
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/README
@@ -0,0 +1,20 @@
+##
+## Copyright (c) 2010-2017 Intel Corporation
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+
+The flow extract tool is meant a to be run as a first pass on a pcap
+file. The output is a lua config file describing the relations between
+flows together with a binary file that contains all the packet headers
+and payload.
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/allocator.cpp b/VNFs/DPPD-PROX/tools/flow_extract/allocator.cpp
new file mode 100644
index 00000000..c861ebfe
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/allocator.cpp
@@ -0,0 +1,84 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <iostream>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <cerrno>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#define USEHP
+
+using namespace std;
+
+#include "allocator.hpp"
+
+Allocator::Allocator(size_t size, size_t threshold)
+ : m_size(size), m_threshold(threshold), m_alloc_offset(0)
+{
+#ifdef USEHP
+ int fd = open("/mnt/huge/hp", O_CREAT | O_RDWR, 0755);
+ if (fd < 0) {
+ cerr << "Allocator failed to open huge page file descriptor: " << strerror(errno) << endl;
+ exit(EXIT_FAILURE);
+ }
+ m_mem = (uint8_t *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (m_mem == MAP_FAILED) {
+ perror("mmap");
+ unlink("/mnt/huge");
+ cerr << "Allocator mmap failed: " << strerror(errno) << endl;
+ exit (EXIT_FAILURE);
+ }
+#else
+ m_mem = new uint8_t[size];
+#endif
+}
+
+Allocator::~Allocator()
+{
+#ifdef USEHP
+ munmap((void *)m_mem, m_size);
+#else
+ delete[] m_mem;
+#endif
+}
+
+void *Allocator::alloc(size_t size)
+{
+ void *ret = &m_mem[m_alloc_offset];
+
+ m_alloc_offset += size;
+ return ret;
+}
+
+void Allocator::reset()
+{
+ m_alloc_offset = 0;
+}
+
+size_t Allocator::getFreeSize() const
+{
+ return m_size - m_alloc_offset;
+}
+
+bool Allocator::lowThresholdReached() const
+{
+ return (m_size - m_alloc_offset) < m_threshold;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/allocator.hpp b/VNFs/DPPD-PROX/tools/flow_extract/allocator.hpp
new file mode 100644
index 00000000..d3f1537e
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/allocator.hpp
@@ -0,0 +1,38 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _ALLOCATOR_H_
+#define _ALLOCATOR_H_
+
+#include <cstddef>
+#include <inttypes.h>
+
+class Allocator {
+public:
+ Allocator(size_t size, size_t lowThreshold);
+ ~Allocator();
+ bool lowThresholdReached() const;
+ void *alloc(size_t size);
+ void reset();
+ size_t getFreeSize() const;
+private:
+ size_t m_size;
+ size_t m_threshold;
+ size_t m_alloc_offset;
+ uint8_t *m_mem;
+};
+
+#endif /* _ALLOCATOR_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/bundle.cpp b/VNFs/DPPD-PROX/tools/flow_extract/bundle.cpp
new file mode 100644
index 00000000..abeaf14e
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/bundle.cpp
@@ -0,0 +1,28 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include "bundle.hpp"
+
+void Bundle::toLua(ofstream *f, const string& streamTableName, uint32_t idx) const
+{
+ (*f) << "bundles[" << idx << "] = {";
+
+ for(vector<uint32_t>::const_iterator i = streams.begin(); i != streams.end(); ++i) {
+ (*f) << streamTableName << "[" << (*i) << "]," ;
+ }
+
+ (*f) << "}" << endl;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/bundle.hpp b/VNFs/DPPD-PROX/tools/flow_extract/bundle.hpp
new file mode 100644
index 00000000..cb5d81b6
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/bundle.hpp
@@ -0,0 +1,38 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _BUNDLE_H_
+#define _BUNDLE_H_
+
+#include <vector>
+#include <inttypes.h>
+#include <fstream>
+
+using namespace std;
+
+class Bundle
+{
+public:
+ void addStream(uint32_t streamId, uint32_t port) {streams.push_back(streamId); ports.push_back(port);}
+ const vector<uint32_t>& getStream() const {return streams;}
+ const vector<uint32_t>& getPorts() const {return ports;}
+ void toLua(ofstream *f, const string& streamTableName, uint32_t idx) const;
+private:
+ vector<uint32_t> streams;
+ vector<uint32_t> ports;
+};
+
+#endif /* _BUNDLE_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/crc.hpp b/VNFs/DPPD-PROX/tools/flow_extract/crc.hpp
new file mode 100644
index 00000000..713b4abd
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/crc.hpp
@@ -0,0 +1,51 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _CRC_H_
+#define _CRC_H_
+
+static uint32_t crc32(const uint8_t *buf, size_t len, int init)
+{
+ uint32_t ret = init;
+
+ while (len/8) {
+ ret = __builtin_ia32_crc32di(ret, *((uint64_t*)buf));
+ len -= 8;
+ buf += 8;
+ }
+
+ while (len/4) {
+ ret = __builtin_ia32_crc32si(ret, *((uint32_t*)buf));
+ len -= 4;
+ buf += 4;
+ }
+
+ while (len/2) {
+ ret = __builtin_ia32_crc32hi(ret, *((uint16_t*)buf));
+ len -= 2;
+ buf += 2;
+ }
+
+ while (len) {
+ ret = __builtin_ia32_crc32qi(ret, *((uint8_t*)buf));
+ len -= 1;
+ buf += 1;
+ }
+
+ return ret;
+}
+
+#endif /* _CRC_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.cpp b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.cpp
new file mode 100644
index 00000000..909fc94d
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.cpp
@@ -0,0 +1,67 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <iostream>
+#include <cstdlib>
+#include <cstring>
+#include <stdint.h>
+
+#include "csvfilereader.hpp"
+
+int CsvFileReader::open(const string& str)
+{
+ char *resolved_path = new char[1024];
+
+ memset(resolved_path, 0, 1024);
+ realpath(str.c_str(), resolved_path);
+ file.open(resolved_path);
+
+ delete []resolved_path;
+ return file.is_open();
+}
+
+vector<string> CsvFileReader::read()
+{
+ vector<string> ret;
+ size_t prev = 0, cur = 0;
+ string line;
+
+ if (file.eof())
+ return vector<string>();
+
+ std::getline(file, line);
+ if (line.empty())
+ return vector<string>();
+
+ while (true) {
+ cur = line.find_first_of(',', prev);
+
+ if (cur != SIZE_MAX) {
+ ret.push_back(line.substr(prev, cur - prev));
+ prev = cur + 1;
+ }
+ else {
+ ret.push_back(line.substr(prev, line.size() - prev));
+ break;
+ }
+ }
+ return ret;
+}
+
+void CsvFileReader::close()
+{
+ file.close();
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.hpp b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.hpp
new file mode 100644
index 00000000..21f397a7
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.hpp
@@ -0,0 +1,35 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _CSVFILE_H_
+#define _CSVFILE_H_
+
+#include <fstream>
+#include <string>
+#include <vector>
+
+using namespace std;
+
+class CsvFileReader {
+public:
+ int open(const string& str);
+ vector<string> read();
+ void close();
+private:
+ ifstream file;
+};
+
+#endif /* _CSVFILE_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/flowtable.hpp b/VNFs/DPPD-PROX/tools/flow_extract/flowtable.hpp
new file mode 100644
index 00000000..ebb4d927
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/flowtable.hpp
@@ -0,0 +1,174 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _FLOWTABLE_H_
+#define _FLOWTABLE_H_
+
+#include <inttypes.h>
+#include <sys/time.h>
+#include <stdio.h>
+#include <cstring>
+
+#include <vector>
+#include <list>
+#include <cstddef>
+#include <utility>
+
+#include "crc.hpp"
+#include "timestamp.hpp"
+
+using namespace std;
+
+template <typename K, typename T>
+class FlowTable {
+public:
+ struct entry {
+ entry(K key, T value, const struct timeval& tv, list<struct entry> *parent) :
+ key(key), value(value), tv(tv), parent(parent) {}
+ bool expired(const Timestamp &now, const Timestamp &maxDiff) const
+ {
+ return now - Timestamp(tv) > maxDiff;
+ }
+ K key;
+ T value;
+ struct timeval tv; /* List time entry has been hit */
+ list<struct entry> *parent;
+ };
+ class Iterator {
+ friend class FlowTable;
+ public:
+ bool operator!=(const Iterator& other) {
+ return m_v != other.m_v ||
+ m_vec_pos != other.m_vec_pos ||
+ m_a != other.m_a;
+
+ }
+ Iterator& operator++() {
+ m_a++;
+ while (m_vec_pos != m_v->size() - 1 && m_a == (*m_v)[m_vec_pos].end()) {
+ m_vec_pos++;
+ m_a = (*m_v)[m_vec_pos].begin();
+ }
+
+ return *this;
+ }
+ struct entry &operator*() {
+ return *m_a;
+ }
+ private:
+ Iterator(uint32_t vec_pos, vector<list<struct entry> > *v)
+ : m_vec_pos(vec_pos), m_v(v)
+ {
+ m_a = (*m_v)[vec_pos].begin();
+ while (m_vec_pos != m_v->size() - 1 && m_a == (*m_v)[m_vec_pos].end()) {
+ m_vec_pos++;
+ m_a = (*m_v)[m_vec_pos].begin();
+ }
+ }
+ Iterator(uint32_t vec_pos, vector<list<struct entry> > *v, const typename list< struct entry>::iterator& a)
+ : m_vec_pos(vec_pos), m_v(v), m_a(a)
+ { }
+ uint32_t m_vec_pos;
+ vector<list<struct entry> > *m_v;
+ typename list<struct entry>::iterator m_a;
+ };
+ uint32_t getEntryCount() const {return m_entryCount;}
+ FlowTable(uint32_t size);
+ void expire(const struct timeval& tv);
+ struct entry* lookup(const K& key);
+ void remove(struct FlowTable<K,T>::entry* entry);
+ struct entry* insert(const K& key, const T& value, const struct timeval& tv);
+ Iterator begin() {return Iterator(0, &m_elems);}
+ Iterator end() {return Iterator(m_elems.size() - 1, &m_elems, m_elems.back().end());}
+ void clear();
+private:
+ void clearBucket(list<struct entry> *l);
+ vector<list<struct entry> > m_elems;
+ uint32_t m_entryCount;
+};
+
+template <typename K, typename T>
+FlowTable<K, T>::FlowTable(uint32_t size)
+ : m_elems(), m_entryCount(0)
+
+{
+ m_elems.resize(size);
+}
+
+template <typename K, typename T>
+struct FlowTable<K, T>::entry* FlowTable<K, T>::lookup(const K& key)
+{
+ uint32_t ret = crc32((uint8_t*)&key, sizeof(K), 0);
+
+ list<struct entry> &l = m_elems[ret % m_elems.size()];
+
+ if (l.empty())
+ return NULL;
+
+ for (typename list<struct entry>::iterator it = l.begin(); it != l.end(); ++it) {
+ if (memcmp(&((*it).key), &key, sizeof(key)) == 0)
+ return &(*it);
+ }
+ return NULL;
+}
+
+template <typename K, typename T>
+struct FlowTable<K, T>::entry *FlowTable<K, T>::insert(const K& key, const T& value, const struct timeval& tv)
+{
+ uint32_t ret = crc32((uint8_t*)&key, sizeof(K), 0);
+ list<struct entry> &l = m_elems[ret % m_elems.size()];
+
+ l.push_back(entry(key, value, tv, &l));
+
+ struct entry &n = l.back();
+ m_entryCount++;
+ n.key = key;
+ n.value = value;
+ return &n;
+}
+
+template <typename K, typename T>
+void FlowTable<K, T>::remove(struct FlowTable<K,T>::entry* entry)
+{
+ list<struct entry> &l = *entry->parent;
+
+ for (typename list<struct entry>::iterator it = l.begin(); it != l.end(); ++it) {
+ if (memcmp(&((*it).key), &entry->key, sizeof(entry->key)) == 0) {
+ l.erase(it);
+ m_entryCount--;
+ return ;
+ }
+ }
+}
+
+template <typename K, typename T>
+void FlowTable<K, T>::clearBucket(list<struct entry> *l)
+{
+ while (!l->empty()) {
+ m_entryCount--;
+ l->erase(l->begin());
+ }
+}
+
+template <typename K, typename T>
+void FlowTable<K, T>::clear()
+{
+ for (size_t i = 0; i < m_elems.size(); ++i) {
+ clearBucket(&m_elems[i]);
+ }
+}
+
+#endif /* _FLOWTABLE_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/halfstream.cpp b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.cpp
new file mode 100644
index 00000000..7d8f1fe2
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.cpp
@@ -0,0 +1,101 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <fstream>
+#include <arpa/inet.h>
+
+#include "halfstream.hpp"
+
+HalfStream::Action::Part HalfStream::addPkt(const PcapPkt &pkt)
+{
+ const uint32_t pktId = pkts.size();
+ const uint8_t *l5;
+ uint32_t l5Len;
+ uint16_t tmpHdrLen;
+
+ const struct PcapPkt::tcp_hdr *tcp;
+
+ struct pkt_tuple pt = pkt.parsePkt((const uint8_t **)&tcp, &tmpHdrLen, &l5, &l5Len);
+
+ if (pt.proto_id == IPPROTO_TCP) {
+ if (tcp->tcp_flags & 0x02)
+ tcpOpen = true;
+ if (tcp->tcp_flags & 0x01)
+ tcpClose = true;
+ }
+
+ if (pkts.empty()) {
+ first = pkt.ts();
+ hdrLen = tmpHdrLen;
+ memcpy(hdr, pkt.payload(), hdrLen);
+ }
+ last = pkt.ts();
+ totLen += pkt.len();
+ contentLen += l5Len;
+
+ pkts.push_back(pkt);
+
+ return Action::Part(pktId, l5 - pkt.payload(), l5Len);
+}
+
+double HalfStream::getRate() const
+{
+ if (pkts.empty())
+ return 0;
+ if (first == last)
+ return 1250000000;
+
+ return totLen / (last - first);
+}
+
+HalfStream::Action::Action(HalfStream* stream, const Part &p, bool isClient)
+ : halfStream(stream), m_isClient(isClient)
+{
+ addPart(p);
+}
+
+void HalfStream::Action::addPart(const Part &p)
+{
+ parts.push_back(p);
+}
+
+uint32_t HalfStream::Action::totLen() const
+{
+ uint32_t ret = 0;
+
+ for (list<Part>::const_iterator i = parts.begin(); i != parts.end(); ++i) {
+ ret += (*i).len;
+ }
+
+ return ret;
+}
+
+void HalfStream::Action::toFile(ofstream *f) const
+{
+ for (list<Part>::const_iterator i = parts.begin(); i != parts.end(); ++i) {
+ const PcapPkt &pkt = halfStream->pkts[i->pktId];
+ const uint8_t *payload = &pkt.payload()[i->offset];
+ const uint16_t len = i->len;
+
+ f->write((const char *)payload, len);
+ }
+}
+
+HalfStream::HalfStream()
+ : first(0, 0), last(0, 0), totLen(0), hdrLen(0), contentLen(0), tcpOpen(false), tcpClose(false)
+{
+
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/halfstream.hpp b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.hpp
new file mode 100644
index 00000000..6216979d
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.hpp
@@ -0,0 +1,63 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <inttypes.h>
+#include <list>
+#include <vector>
+
+#include "timestamp.hpp"
+#include "pcappkt.hpp"
+
+struct HalfStream {
+ struct Action {
+ public:
+ struct Part {
+ Part(uint32_t pktId, uint32_t offset, uint32_t len)
+ : pktId(pktId), offset(offset), len(len) {}
+ uint32_t pktId;
+ uint32_t offset;
+ uint32_t len;
+ };
+
+ Action(HalfStream* stream, const Part &p, bool isClient);
+ void addPart(const Part& p);
+ bool isClient() const {return m_isClient;}
+ /* An action can consist of multiple
+ packets. The data is not stored in the
+ action. Instead, a packet id together with
+ an offset into the packet and a length is
+ kept to save space */
+ void toFile(ofstream* f) const;
+ uint32_t totLen() const;
+ private:
+ HalfStream *halfStream;
+ bool m_isClient;
+ list<Part> parts;
+ };
+
+ HalfStream();
+ Timestamp first;
+ Timestamp last;
+ uint64_t totLen;
+ uint64_t hdrLen;
+ uint8_t hdr[64];
+ vector<PcapPkt> pkts;
+ uint64_t contentLen;
+ bool tcpOpen;
+ bool tcpClose;
+ Action::Part addPkt(const PcapPkt &pkt);
+ double getRate() const;
+};
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/main.cpp b/VNFs/DPPD-PROX/tools/flow_extract/main.cpp
new file mode 100644
index 00000000..d1476c5f
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/main.cpp
@@ -0,0 +1,37 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <inttypes.h>
+#include <cstdlib>
+
+#include "streamextract.hpp"
+
+using namespace std;
+
+int main(int argc, char *argv[])
+{
+ ProgramConfig programConfig;
+
+ if (programConfig.parseOptions(argc, argv)) {
+ cerr << programConfig.getError() << endl;
+ cerr << programConfig.getUsage() << endl;
+ return EXIT_FAILURE;
+ }
+
+ StreamExtract se(programConfig);
+
+ return se.run();
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.cpp b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.cpp
new file mode 100644
index 00000000..b2d1a9da
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.cpp
@@ -0,0 +1,109 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <cstdlib>
+#include <cstdio>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <iostream>
+#include <cerrno>
+#include <sys/mman.h>
+#include <cstring>
+
+#include "mappedfile.hpp"
+
+static void zeroOutFile(int fd, size_t size)
+{
+ void *empty = calloc(1, 4096);
+
+ while (size > 4096) {
+ write(fd, empty, 4096);
+ size -= 4096;
+ }
+ write(fd, empty, size);
+ free(empty);
+}
+
+int MappedFile::open(const string& filePath, size_t size)
+{
+ mappedFileSize = size;
+
+ fd = ::open(filePath.c_str(), O_RDWR | O_TRUNC | O_CREAT, S_IRUSR | S_IWUSR);
+ if (fd < 0) {
+ cerr << "Failed to open file " << filePath << ":" << strerror(errno) << endl;
+ return -1;
+ }
+
+ zeroOutFile(fd, size);
+ data = mmap(NULL, mappedFileSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
+
+ if (data == MAP_FAILED) {
+ cerr << "Failed to map file: " << strerror(errno) << endl;
+ return -1;
+ }
+ return 0;
+}
+
+static size_t getFileSize(const string& filePath)
+{
+ struct stat s;
+ if (stat(filePath.c_str(), &s))
+ return -1;
+
+ return s.st_size;
+}
+
+int MappedFile::open(const string& filePath)
+{
+ mappedFileSize = getFileSize(filePath);
+
+ fd = ::open(filePath.c_str(), O_RDONLY);
+ if (fd < 0) {
+ cerr << "Failed to open file " << filePath << ":" << strerror(errno) << endl;
+ return -1;
+ }
+
+ data = mmap(NULL, mappedFileSize, PROT_READ, MAP_SHARED, fd, 0);
+
+ if (data == MAP_FAILED) {
+ cerr << "Failed to map file: " << strerror(errno) << endl;
+ return -1;
+ }
+ return 0;
+}
+
+int MappedFile::sync()
+{
+ if (msync(data, mappedFileSize, MS_SYNC) == -1) {
+ cerr << "Failed to sync: " << strerror(errno) << endl;
+ return -1;
+ }
+ return 0;
+}
+
+
+void MappedFile::close()
+{
+ sync();
+ munmap(data, mappedFileSize);
+ ::close(fd);
+}
+
+size_t MappedFile::size() const
+{
+ return mappedFileSize;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.hpp b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.hpp
new file mode 100644
index 00000000..7bf79df5
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.hpp
@@ -0,0 +1,40 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _MAPPEDFILE_H_
+#define _MAPPEDFILE_H_
+
+#include <inttypes.h>
+#include <string>
+
+using namespace std;
+
+class MappedFile {
+public:
+ int open(const string& filePath, size_t size);
+ int open(const string& filePath);
+ void close();
+ int sync();
+ uint8_t* getMapBeg() {return (uint8_t *)data;}
+ uint8_t* getMapEnd() {return (uint8_t *)data + mappedFileSize;}
+ size_t size() const;
+private:
+ int fd;
+ size_t mappedFileSize;
+ void *data;
+};
+
+#endif /* _MAPPEDFILE_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/memreader.cpp b/VNFs/DPPD-PROX/tools/flow_extract/memreader.cpp
new file mode 100644
index 00000000..df77631c
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/memreader.cpp
@@ -0,0 +1,106 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <cstdlib>
+
+#include "memreader.hpp"
+#include "mappedfile.hpp"
+#include "stream3.hpp"
+
+MemReader::MemReader(MappedFile *file, const vector<size_t> &offsets)
+{
+ initRanges(file->getMapBeg(), file->getMapEnd(), offsets);
+}
+
+bool MemReader::read(Stream3 *stream)
+{
+ if (ranges.empty())
+ return false;
+
+ readStream(stream, getLowestID());
+ removeEmptyRanges();
+ return true;
+}
+
+uint32_t MemReader::getLowestID() const
+{
+ uint32_t lowestID = UINT32_MAX;
+ uint32_t rangeID;
+
+ for (size_t i = 0; i < ranges.size(); ++i) {
+ rangeID = Stream3::getIDFromMem(ranges[i].first);
+ if (rangeID < lowestID)
+ lowestID = rangeID;
+ }
+ return lowestID;
+}
+
+void MemReader::readStream(Stream3 *stream, uint32_t id)
+{
+ stream->removeAllPackets();
+ stream->setID(id);
+
+ size_t len = 0;
+ for (size_t i = 0; i < ranges.size(); ++i) {
+ if (Stream3::getIDFromMem(ranges[i].first) == id) {
+ stream->addFromMemory(ranges[i].first, &len);
+ ranges[i].first += len;
+ }
+ }
+}
+
+void MemReader::removeEmptyRanges()
+{
+ vector<pair <uint8_t *, uint8_t *> > original = ranges;
+ size_t destinationIdx = 0;
+
+ for (size_t i = 0; i < original.size(); ++i) {
+ if (original[i].first < original[i].second)
+ ranges[destinationIdx++] = original[i];
+ }
+ ranges.resize(destinationIdx);
+}
+
+void MemReader::initRanges(uint8_t *begin, uint8_t *end, const vector<size_t> &offsets)
+{
+ ranges.resize(offsets.size());
+
+ totalLength = 0;
+ for (size_t i = 0; i < offsets.size(); ++i) {
+ ranges[i].first = begin + offsets[i];
+ if (i != offsets.size() - 1)
+ ranges[i].second = begin + offsets[i + 1];
+ else
+ ranges[i].second = end;
+ totalLength += ranges[i].second - ranges[i].first;
+ }
+ removeEmptyRanges();
+}
+
+size_t MemReader::getRangeLengths() const
+{
+ size_t total = 0;
+
+ for (size_t i = 0; i < ranges.size(); ++i) {
+ total += ranges[i].second - ranges[i].first;
+ }
+ return total;
+}
+
+size_t MemReader::consumed() const
+{
+ return totalLength - getRangeLengths();
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/memreader.hpp b/VNFs/DPPD-PROX/tools/flow_extract/memreader.hpp
new file mode 100644
index 00000000..31be4c31
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/memreader.hpp
@@ -0,0 +1,45 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _MEMREADER_H_
+#define _MEMREADER_H_
+
+#include <vector>
+#include <inttypes.h>
+
+using namespace std;
+
+class Stream3;
+class MappedFile;
+
+class MemReader {
+public:
+ MemReader(MappedFile *file, const vector<size_t> &offsets);
+ bool read(Stream3 *stream);
+ size_t getTotalLength() const {return totalLength;}
+ size_t consumed() const;
+private:
+ size_t getRangeLengths() const;
+ uint32_t getLowestID() const;
+ void removeEmptyRanges();
+ void readStream(Stream3 *stream, uint32_t id);
+ void initRanges(uint8_t *begin, uint8_t *end, const vector<size_t> &offsets);
+
+ size_t totalLength;
+ vector<pair <uint8_t *, uint8_t *> > ranges;
+};
+
+#endif /* _MEMREADER_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/netsocket.cpp b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.cpp
new file mode 100644
index 00000000..8c61ba7d
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.cpp
@@ -0,0 +1,33 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include "netsocket.hpp"
+
+NetSocket::NetSocket(uint32_t host, uint16_t port)
+ : host(host), port(port)
+{
+
+}
+
+bool NetSocket::operator>(const NetSocket& other) const
+{
+ return host > other.host || (host == other.host && port > other.port);
+}
+
+bool NetSocket::operator<(const NetSocket& other) const
+{
+ return host < other.host || (host == other.host && port < other.port);
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/netsocket.hpp b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.hpp
new file mode 100644
index 00000000..bfd6bec9
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.hpp
@@ -0,0 +1,31 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _NETSOCKET_H_
+#define _NETSOCKET_H_
+
+#include <inttypes.h>
+
+struct NetSocket {
+ NetSocket() {}
+ NetSocket(uint32_t host, uint16_t port);
+ bool operator>(const NetSocket& other) const;
+ bool operator<(const NetSocket& other) const;
+ uint32_t host;
+ uint16_t port;
+};
+
+#endif /* _NETSOCKET_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/path.cpp b/VNFs/DPPD-PROX/tools/flow_extract/path.cpp
new file mode 100644
index 00000000..7d94aae6
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/path.cpp
@@ -0,0 +1,97 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <iostream>
+#include <iomanip>
+#include <sys/stat.h>
+#include <sstream>
+#include <fstream>
+
+#include "path.hpp"
+
+bool Path::isDir() const
+{
+ struct stat s = { 0 };
+
+ if (stat(path.c_str(), &s)) {
+ return false;
+ }
+
+ return s.st_mode & S_IFDIR;
+}
+
+bool Path::isFile() const
+{
+ struct stat s = { 0 };
+
+ if (stat(path.c_str(), &s)) {
+ return false;
+ }
+
+ return s.st_mode & S_IFREG;
+}
+
+Path Path::add(const string& str) const
+{
+ stringstream ss;
+
+ ss << path << str;
+
+ return Path(ss.str());
+}
+
+Path Path::add(int number) const
+{
+ stringstream ss;
+
+ ss << path << number;
+
+ return Path(ss.str());
+}
+
+Path &Path::concat(const string &add)
+{
+ stringstream ss;
+
+ ss << path << add;
+ path = ss.str();
+
+ return *this;
+}
+
+int Path::mkdir() const
+{
+ if (!isDir())
+ return ::mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+ return 0;
+}
+
+std::ostream& operator<<(std::ofstream &stream, const Path &p)
+{
+ stream << p.path.c_str();
+
+ return stream;
+}
+
+string Path::getFileName() const
+{
+ for (size_t i = path.size() - 1; i >= 0; --i) {
+ if (path[i] == '/') {
+ return path.substr(i + 1);
+ }
+ }
+ return path;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/path.hpp b/VNFs/DPPD-PROX/tools/flow_extract/path.hpp
new file mode 100644
index 00000000..e56c9050
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/path.hpp
@@ -0,0 +1,42 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _PATH_H_
+#define _PATH_H_
+
+#include <string>
+
+using namespace std;
+
+class Path {
+public:
+ Path();
+ Path(const Path& other) : path(other.path) {}
+ Path(const string& str) : path(str) {}
+ Path add(const string& str) const;
+ Path add(int number) const;
+ Path &concat(const string &str);
+ const string& str() const {return path;}
+ bool isDir() const;
+ bool isFile() const;
+ string getFileName() const;
+ int mkdir() const;
+ friend std::ostream& operator<<(std::ofstream &stream, const Path &path);
+private:
+ string path;
+};
+
+#endif /* _PATH_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.cpp
new file mode 100644
index 00000000..91708bb1
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.cpp
@@ -0,0 +1,266 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <pcap.h>
+#include <inttypes.h>
+#include <cstring>
+#include <arpa/inet.h>
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include "allocator.hpp"
+#include "pcappkt.hpp"
+
+Allocator *PcapPkt::allocator = NULL;
+
+void* PcapPkt::operator new(size_t size)
+{
+ if (allocator)
+ return allocator->alloc(size);
+ else
+ return ::operator new(size);
+}
+
+void PcapPkt::operator delete(void *pointer)
+{
+ if (!allocator)
+ :: operator delete(pointer);
+}
+
+PcapPkt::PcapPkt(uint8_t *mem)
+{
+ header = *(struct pcap_pkthdr *)mem;
+ mem += sizeof(header);
+ buf = new uint8_t[header.len];
+ memcpy(buf, mem, header.len);
+}
+
+PcapPkt::PcapPkt()
+{
+ buf = new uint8_t[1514];
+ memset(&header, 0, sizeof(header));
+}
+
+PcapPkt::PcapPkt(const PcapPkt& other)
+{
+ if (!allocator) {
+ buf = new uint8_t[other.len()];
+ }
+ else {
+ buf = (uint8_t *)allocator->alloc(other.len());
+ }
+
+ memcpy(buf, other.buf, other.len());
+ header = other.header;
+}
+
+PcapPkt::~PcapPkt()
+{
+ if (!allocator)
+ delete[] buf;
+}
+
+#define ETYPE_IPv4 0x0008 /* IPv4 in little endian */
+#define ETYPE_IPv6 0xDD86 /* IPv6 in little endian */
+#define ETYPE_ARP 0x0608 /* ARP in little endian */
+#define ETYPE_VLAN 0x0081 /* 802-1aq - VLAN */
+#define ETYPE_MPLSU 0x4788 /* MPLS unicast */
+#define ETYPE_MPLSM 0x4888 /* MPLS multicast */
+#define ETYPE_8021ad 0xA888 /* Q-in-Q */
+#define ETYPE_LLDP 0xCC88 /* Link Layer Discovery Protocol (LLDP) */
+#define ETYPE_EoGRE 0x5865 /* EoGRE in little endian */
+
+struct ipv4_hdr {
+ uint8_t version_ihl; /**< version and header length */
+ uint8_t type_of_service; /**< type of service */
+ uint16_t total_length; /**< length of packet */
+ uint16_t packet_id; /**< packet ID */
+ uint16_t fragment_offset; /**< fragmentation offset */
+ uint8_t time_to_live; /**< time to live */
+ uint8_t next_proto_id; /**< protocol ID */
+ uint16_t hdr_checksum; /**< header checksum */
+ uint32_t src_addr; /**< source address */
+ uint32_t dst_addr; /**< destination address */
+} __attribute__((__packed__));
+
+struct ether_addr {
+ uint8_t addr_bytes[6]; /**< Address bytes in transmission order */
+} __attribute__((__packed__));
+
+struct ether_hdr {
+ struct ether_addr d_addr; /**< Destination address. */
+ struct ether_addr s_addr; /**< Source address. */
+ uint16_t ether_type; /**< Frame type. */
+} __attribute__((__packed__));
+
+struct vlan_hdr {
+ uint16_t vlan_tci; /**< Priority (3) + CFI (1) + Identifier Code (12) */
+ uint16_t eth_proto;/**< Ethernet type of encapsulated frame. */
+} __attribute__((__packed__));
+
+struct udp_hdr {
+ uint16_t src_port; /**< UDP source port. */
+ uint16_t dst_port; /**< UDP destination port. */
+ uint16_t dgram_len; /**< UDP datagram length */
+ uint16_t dgram_cksum; /**< UDP datagram checksum */
+} __attribute__((__packed__));
+
+struct pkt_tuple PcapPkt::parsePkt(const uint8_t **l4_hdr, uint16_t *hdr_len, const uint8_t **l5, uint32_t *l5_len) const
+{
+ struct pkt_tuple pt = {0};
+
+ const struct ether_hdr *peth = (struct ether_hdr *)buf;
+ int l2_types_count = 0;
+ const struct ipv4_hdr* pip = 0;
+
+ switch (peth->ether_type) {
+ case ETYPE_IPv4:
+ pip = (const struct ipv4_hdr *)(peth + 1);
+ break;
+ case ETYPE_VLAN: {
+ const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1);
+ if (vlan->eth_proto == ETYPE_IPv4) {
+ pip = (const struct ipv4_hdr *)(peth + 1);
+ }
+ else if (vlan->eth_proto == ETYPE_VLAN) {
+ const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1);
+ if (vlan->eth_proto == ETYPE_IPv4) {
+ pip = (const struct ipv4_hdr *)(peth + 1);
+ }
+ else if (vlan->eth_proto == ETYPE_IPv6) {
+ throw 0;
+ }
+ else {
+ /* TODO: handle BAD PACKET */
+ throw 0;
+ }
+ }
+ }
+ break;
+ case ETYPE_8021ad: {
+ const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1);
+ if (vlan->eth_proto == ETYPE_VLAN) {
+ const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1);
+ if (vlan->eth_proto == ETYPE_IPv4) {
+ pip = (const struct ipv4_hdr *)(peth + 1);
+ }
+ else {
+ throw 0;
+ }
+ }
+ else {
+ throw 0;
+ }
+ }
+ break;
+ case ETYPE_MPLSU:
+ break;
+ default:
+ break;
+ }
+
+ /* L3 */
+ if ((pip->version_ihl >> 4) == 4) {
+
+ if ((pip->version_ihl & 0x0f) != 0x05) {
+ /* TODO: optional fields */
+ throw 0;
+ }
+
+ pt.proto_id = pip->next_proto_id;
+ pt.src_addr = pip->src_addr;
+ pt.dst_addr = pip->dst_addr;
+ }
+ else {
+ /* TODO: IPv6 and bad packets */
+ throw 0;
+ }
+
+ /* L4 parser */
+ if (pt.proto_id == IPPROTO_UDP) {
+ const struct udp_hdr *udp = (const struct udp_hdr*)(pip + 1);
+ if (l4_hdr)
+ *l4_hdr = (const uint8_t*)udp;
+ if (hdr_len)
+ *hdr_len = (const uint8_t*)udp - buf;
+ pt.src_port = udp->src_port;
+ pt.dst_port = udp->dst_port;
+ if (l5)
+ *l5 = ((const uint8_t*)udp) + sizeof(struct udp_hdr);
+ if (l5_len)
+ *l5_len = ntohs(udp->dgram_len) - sizeof(struct udp_hdr);
+ }
+ else if (pt.proto_id == IPPROTO_TCP) {
+ const struct tcp_hdr *tcp = (const struct tcp_hdr *)(pip + 1);
+ if (l4_hdr)
+ *l4_hdr = (const uint8_t*)tcp;
+ if (hdr_len)
+ *hdr_len = (const uint8_t*)tcp - buf;
+ pt.src_port = tcp->src_port;
+ pt.dst_port = tcp->dst_port;
+
+ if (l5)
+ *l5 = ((const uint8_t*)tcp) + ((tcp->data_off >> 4)*4);
+ if (l5_len)
+ *l5_len = ntohs(pip->total_length) - sizeof(struct ipv4_hdr) - ((tcp->data_off >> 4)*4);
+ }
+ else {
+ fprintf(stderr, "unsupported protocol %d\n", pt.proto_id);
+ throw 0;
+ }
+
+ return pt;
+}
+
+void PcapPkt::toMem(uint8_t *mem) const
+{
+ memcpy(mem, &header, sizeof(header));
+ mem += sizeof(header);
+ memcpy(mem, buf, header.len);
+}
+
+void PcapPkt::fromMem(uint8_t *mem)
+{
+ memcpy(&header, mem, sizeof(header));
+ mem += sizeof(header);
+ memcpy(buf, mem, header.len);
+}
+
+void PcapPkt::toFile(ofstream *file) const
+{
+ file->write(reinterpret_cast<const char *>(&header), sizeof(header));
+ file->write(reinterpret_cast<const char *>(buf), header.len);
+}
+size_t PcapPkt::memSize() const
+{
+ return sizeof(header) + header.len;
+}
+
+PcapPkt::L4Proto PcapPkt::getProto() const
+{
+ struct pkt_tuple pt = parsePkt();
+ return pt.proto_id == IPPROTO_TCP? PROTO_TCP : PROTO_UDP;
+}
+
+ostream& operator<<(ostream& stream, const pkt_tuple &other)
+{
+ stream << other.src_addr << ","
+ << other.dst_addr << ","
+ << (int)other.proto_id << ","
+ << other.src_port << ","
+ << other.dst_port;
+ return stream;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.hpp
new file mode 100644
index 00000000..e437c790
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.hpp
@@ -0,0 +1,104 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _PCAPPKT_H_
+#define _PCAPPKT_H_
+
+#include <inttypes.h>
+#include <pcap.h>
+#include <string>
+#include <cstring>
+
+using namespace std;
+
+struct pkt_tuple {
+ uint32_t src_addr;
+ uint32_t dst_addr;
+ uint8_t proto_id;
+ uint16_t src_port;
+ uint16_t dst_port;
+ bool operator!=(const pkt_tuple& other) const
+ {
+ return src_addr != other.src_addr ||
+ dst_addr != other.dst_addr ||
+ proto_id != other.proto_id ||
+ src_port != other.src_port ||
+ dst_port != other.dst_port;
+ }
+ bool operator==(const pkt_tuple& other) const
+ {
+ return src_addr == other.src_addr &&
+ dst_addr == other.dst_addr &&
+ proto_id == other.proto_id &&
+ src_port == other.src_port &&
+ dst_port == other.dst_port;
+ }
+ friend ostream& operator<<(ostream& stream, const pkt_tuple &other);
+ struct pkt_tuple flip() const
+ {
+ struct pkt_tuple ret;
+
+ ret = *this;
+ ret.src_addr = dst_addr;
+ ret.src_port = dst_port;
+ ret.dst_addr = src_addr;
+ ret.dst_port = src_port;
+ return ret;
+ }
+
+} __attribute__((packed));
+
+class Allocator;
+
+class PcapPkt {
+ friend class PcapReader;
+public:
+ struct tcp_hdr {
+ uint16_t src_port; /**< TCP source port. */
+ uint16_t dst_port; /**< TCP destination port. */
+ uint32_t sent_seq; /**< TX data sequence number. */
+ uint32_t recv_ack; /**< RX data acknowledgement sequence number. */
+ uint8_t data_off; /**< Data offset. */
+ uint8_t tcp_flags; /**< TCP flags */
+ uint16_t rx_win; /**< RX flow control window. */
+ uint16_t cksum; /**< TCP checksum. */
+ uint16_t tcp_urp; /**< TCP urgent pointer, if any. */
+ } __attribute__((__packed__));
+
+ static Allocator *allocator;
+ enum L4Proto {PROTO_TCP, PROTO_UDP};
+ PcapPkt();
+ void* operator new(size_t size);
+ static void operator delete(void *pointer);
+ PcapPkt(const PcapPkt& other);
+ PcapPkt(uint8_t *mem);
+ void toMem(uint8_t *mem) const;
+ void fromMem(uint8_t *mem);
+ void toFile(ofstream *file) const;
+ size_t memSize() const;
+ const struct timeval &ts() const {return header.ts;}
+ const uint16_t len() const {return header.len;}
+ pkt_tuple parsePkt(const uint8_t **l4_hdr = NULL, uint16_t *hdr_len = NULL, const uint8_t **l5 = NULL, uint32_t *l5_len = NULL) const;
+ const struct pcap_pkthdr &hdr() const {return header;}
+ const uint8_t *payload() const {return buf;}
+ enum L4Proto getProto() const;
+ ~PcapPkt();
+private:
+ struct pcap_pkthdr header;
+ uint8_t *buf;
+};
+
+#endif /* _PCAPPKT_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.cpp
new file mode 100644
index 00000000..2a0f2f05
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.cpp
@@ -0,0 +1,32 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include "pcappktref.hpp"
+
+PcapPktRef::PcapPktRef(const PcapPktRef &other)
+ : pos(other.pos), pr(other.pr)
+{
+}
+
+PcapPkt PcapPktRef::getPcapPkt()
+{
+ PcapPkt ret;
+
+ if (!pr->readOnce(&ret, pos)) {
+ cerr << "failed to read pcap from pcap pkt ref" << endl;
+ }
+ return ret;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.hpp
new file mode 100644
index 00000000..1afaf2a5
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.hpp
@@ -0,0 +1,40 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _PCAPPKTREF_H_
+#define _PCAPPKTREF_H_
+
+#include <iostream>
+
+#include "pcapreader.hpp"
+#include "pcappkt.hpp"
+
+using namespace std;
+
+class PcapPktRef
+{
+public:
+ PcapPktRef(uint64_t pos, PcapReader *pr) : pos(pos), pr(pr) {}
+ PcapPktRef(const PcapPktRef &other);
+ PcapPktRef() : pos(0), pr(0) {}
+ bool isValid() const {return pos != 0;}
+ PcapPkt getPcapPkt();
+private:
+ uint64_t pos;
+ PcapReader *pr;
+};
+
+#endif /* _PCAPPKTREF_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.cpp
new file mode 100644
index 00000000..6b5a6734
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.cpp
@@ -0,0 +1,76 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <pcap.h>
+#include <cstring>
+#include <linux/in.h>
+
+#include "pcapreader.hpp"
+
+int PcapReader::open(const string& file_path)
+{
+ char err_str[PCAP_ERRBUF_SIZE];
+
+ if (m_handle) {
+ m_error = "Pcap file already open";
+ return -1;
+ }
+
+ m_handle = pcap_open_offline_with_tstamp_precision(file_path.c_str(),
+ PCAP_TSTAMP_PRECISION_NANO,
+ err_str);
+
+ if (!m_handle) {
+ m_error = "Failed to open pcap file";
+ return -1;
+ }
+
+ m_file_beg = ftell(pcap_file(m_handle));
+ fseek(pcap_file(m_handle), 0, SEEK_END);
+ m_file_end = ftell(pcap_file(m_handle));
+ fseek(pcap_file(m_handle), m_file_beg, SEEK_SET);
+
+ return 0;
+}
+
+int PcapReader::readOnce(PcapPkt *pkt, uint64_t pos)
+{
+ return -1;
+}
+
+int PcapReader::read(PcapPkt *pkt)
+{
+ if (!m_handle) {
+ m_error = "No pcap file opened";
+ }
+
+ const uint8_t *buf = pcap_next(m_handle, &pkt->header);
+
+ if (buf) {
+ memcpy(pkt->buf, buf, pkt->header.len);
+ pktReadCount++;
+ }
+
+ return !!buf;
+}
+
+void PcapReader::close()
+{
+ if (m_handle)
+ pcap_close(m_handle);
+
+ m_handle = NULL;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.hpp
new file mode 100644
index 00000000..3766c67b
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.hpp
@@ -0,0 +1,48 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _PCAPREADER_H_
+#define _PCAPREADER_H_
+
+#include <inttypes.h>
+#include <string>
+
+#include <pcap.h>
+
+#include "pcappkt.hpp"
+
+using namespace std;
+
+class PcapReader {
+public:
+ PcapReader() : m_handle(NULL), pktReadCount(0) {}
+ int open(const string& file_path);
+ size_t pos() {return ftell(pcap_file(m_handle)) - m_file_beg;}
+ size_t end() {return m_file_end;}
+ int read(PcapPkt *pkt);
+ int readOnce(PcapPkt *pkt, uint64_t pos);
+ size_t getPktReadCount() const {return pktReadCount;}
+ void close();
+ const string &getError() const {return m_error;}
+private:
+ pcap_t *m_handle;
+ size_t m_file_beg;
+ size_t m_file_end;
+ size_t pktReadCount;
+ string m_error;
+};
+
+#endif /* _PCAPREADER_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.cpp
new file mode 100644
index 00000000..4c7c4cea
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.cpp
@@ -0,0 +1,46 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include "pcapwriter.hpp"
+
+int PcapWriter::open(const string& file_path)
+{
+ m_handle = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65536, PCAP_TSTAMP_PRECISION_NANO);
+ if (m_handle == NULL)
+ return -1;
+
+ m_pcap_dumper = pcap_dump_open(m_handle, file_path.c_str());
+ if (m_pcap_dumper == NULL) {
+ pcap_close(m_handle);
+ return -1;
+ }
+
+ return 0;
+}
+
+int PcapWriter::write(const PcapPkt& pkt)
+{
+ pcap_dump((unsigned char *)m_pcap_dumper, &pkt.hdr(), pkt.payload());
+ return 0;
+}
+
+void PcapWriter::close()
+{
+ if (m_pcap_dumper)
+ pcap_dump_close(m_pcap_dumper);
+ if (m_handle)
+ pcap_close(m_handle);
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.hpp
new file mode 100644
index 00000000..32f79369
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.hpp
@@ -0,0 +1,33 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _PCAPWRITER_H_
+#define _PCAPWRITER_H_
+
+#include "pcappkt.hpp"
+
+class PcapWriter {
+public:
+ PcapWriter() {}
+ int open(const string& file_path);
+ int write(const PcapPkt& pkt);
+ void close();
+private:
+ pcap_t *m_handle;
+ pcap_dumper_t *m_pcap_dumper;
+};
+
+#endif /* _PCAPWRITER_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/programconfig.cpp b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.cpp
new file mode 100644
index 00000000..7b1e18e1
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.cpp
@@ -0,0 +1,119 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <sstream>
+#include <getopt.h>
+#include <iostream>
+#include <cstdlib>
+#include "programconfig.hpp"
+
+ProgramConfig::ProgramConfig()
+ : path_file_in_pcap(""), path_dir_out("output"),
+ path_file_dest_lua("lua"), max_pkts(UINT32_MAX),
+ max_streams(UINT32_MAX), sampleCount(20000), flowTableSize(8*1024*1024),
+ run_first_step(true), write_pcaps(false)
+{
+}
+
+string ProgramConfig::getUsage() const
+{
+ stringstream ret;
+
+ ret << "Usage example: "<< m_programName << " -i in.pcap\n\n"
+ << "Flow Extract 2.0 analyzes and extracts a traffic profile\n"
+ << "configuration from a pcap file. The output is a lua\n"
+ << "configuration file and a binary file containing all the\n"
+ << "headers and payloads for each stream.\n\n"
+
+ << "The program supports analyzing large pcap file (> 300 GB).\n"
+ << "For this, it uses a multi-pass approach. The output of \n"
+ << "intermediary steps is stored in the working directory. The\n"
+ << "algorithm can be described by the following steps:\n\n"
+ << " 1. The pcap file in read chunks of 16 GB. The packets in\n"
+ << " each chunk are associated with streams. The streams are\n"
+ << " ordered through a global ID. Each stream is stored as a"
+ << " sequence of packets that belong to that stream. The\n"
+ << " resulting file at 'DIR/tmp' where DIR is specified\n"
+ << " through -o options as shown below.\n"
+ << " Each chunk in tmp is merged and the result is written\n"
+ << " to file1. Reading the stream with a given ID from all chunks\n"
+ << " gets all the packets for the stream from the whole pcap in\n"
+ << " memory. This first step forms is implemented by an\n"
+ << " external sorting algorithm.\n"
+ << " 2. File2 is read and the source IP for each stream is used to\n"
+ << " associate each stream with a bundle. SAMPLE_COUNT samples\n"
+ << " are taken from the set of bundles. The set of streams that\n"
+ << " are still referenced by the sampled bundles extracted from\n"
+ << " file2 and written to the final binary file. This binary file\n"
+ << " is referenced from the lua configuration. The lua config file\n"
+ << " is written out as part of this step.\n"
+ << "Arguments:\n"
+ << "-i FILE Input pcap to process\n"
+ << "-o DIR output directory and working directory\n"
+ << "-s SAMPLE_COUNT Number of samples to take (default is 20K)\n"
+ << "-k Skip the first step as described above. Useful to\n"
+ << " adjust the number of samples without having to\n"
+ << " repeat the whole process\n";
+
+
+ return ret.str();
+}
+
+int ProgramConfig::checkConfig()
+{
+ if (path_file_in_pcap.empty()) {
+ m_error = "Missing input pcap file\n";
+ return -1;
+ }
+ return 0;
+}
+
+int ProgramConfig::parseOptions(int argc, char *argv[])
+{
+ char c;
+
+ m_programName = argv[0];
+ while ((c = getopt(argc, argv, "hki:o:s:p")) != -1) {
+ switch (c) {
+ case 'h':
+ return -1;
+ break;
+ case 'k':
+ run_first_step = false;
+ break;
+ case 'i':
+ path_file_in_pcap = optarg;
+ break;
+ case 'o':
+ path_dir_out = optarg;
+ break;
+ case 's':
+ sampleCount = atoi(optarg);
+ break;
+ case 'p':
+ write_pcaps = true;
+ break;
+ case '?':
+ cerr << getUsage() << endl;
+ return 0;
+ default:
+ m_error = "Invalid parameter\n";
+ return -1;
+ }
+ }
+
+ return checkConfig();
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/programconfig.hpp b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.hpp
new file mode 100644
index 00000000..59b7104d
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.hpp
@@ -0,0 +1,47 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _PROGRAMCONFIG_H_
+#define _PROGRAMCONFIG_H_
+
+#include <string>
+#include <inttypes.h>
+
+using namespace std;
+
+class ProgramConfig {
+public:
+ ProgramConfig();
+ string getUsage() const;
+ int parseOptions(int argc, char *argv[]);
+ const string& getError() const {return m_error;}
+
+ string path_file_in_pcap;
+ string path_dir_out;
+ string path_file_dest_lua;
+ uint32_t max_pkts;
+ uint32_t max_streams;
+ uint32_t sampleCount;
+ uint32_t flowTableSize;
+ bool run_first_step;
+ bool write_pcaps;
+private:
+ int checkConfig();
+ string m_error;
+ string m_programName;
+};
+
+#endif /* _PROGRAMCONFIG_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/progress.cpp b/VNFs/DPPD-PROX/tools/flow_extract/progress.cpp
new file mode 100644
index 00000000..2c65960f
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/progress.cpp
@@ -0,0 +1,96 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <sys/time.h>
+#include <iostream>
+#include <cstdio>
+#include <sstream>
+
+#include "progress.hpp"
+
+static uint64_t getSec()
+{
+ struct timeval tv;
+
+ gettimeofday(&tv, NULL);
+ return tv.tv_sec;
+}
+
+Progress::Progress(size_t maxProgress, bool inPlace, bool showElapsedTime)
+ : maxProgress(maxProgress), curProgress(0), inPlace(inPlace), showElapsedTime(showElapsedTime), prevLength(0), title("Progress")
+{
+ lastRefresh = -1;
+ firstRefresh = getSec();
+}
+
+void Progress::setProgress(size_t curProgress)
+{
+ this->curProgress = curProgress;
+}
+
+void Progress::setProgress()
+{
+ this->curProgress = maxProgress;
+}
+
+uint32_t Progress::addDetail(const string& detail)
+{
+ details.push_back(make_pair(detail, 0));
+ return details.size() - 1;
+}
+
+void Progress::setDetail(uint32_t idx, uint32_t val)
+{
+ details[idx].second = val;
+}
+
+bool Progress::couldRefresh()
+{
+ uint32_t cur = getSec();
+
+ return (lastRefresh != cur);
+}
+
+void Progress::refresh(bool withNewLine)
+{
+ lastRefresh = getSec();
+ uint64_t elapsed = lastRefresh - firstRefresh;
+ size_t progress = curProgress * 100 / maxProgress;
+ size_t remainingTime = curProgress? (elapsed * maxProgress - elapsed * curProgress) / curProgress : 0;
+
+ stringstream ss;
+
+ if (inPlace)
+ ss << "\r";
+ ss << title << ": " << progress << "%";
+ ss << ", remaining: " << remainingTime;
+ if (showElapsedTime)
+ ss << ", elapsed: " << elapsed;
+ for (size_t i = 0; i < details.size(); ++i)
+ ss << ", " << details[i].first << ": " << details[i].second;
+
+ size_t prevLength2 = ss.str().size();
+
+ while (ss.str().size() < prevLength)
+ ss << " ";
+ prevLength = prevLength2;
+
+ if (!inPlace || withNewLine)
+ ss << "\n";
+
+ cout << ss.str();
+ cout.flush();
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/progress.hpp b/VNFs/DPPD-PROX/tools/flow_extract/progress.hpp
new file mode 100644
index 00000000..7f55cf98
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/progress.hpp
@@ -0,0 +1,50 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _PROGRESS_H_
+#define _PROGRESS_H_
+
+#include <inttypes.h>
+#include <vector>
+#include <utility>
+#include <string>
+
+using namespace std;
+
+class Progress {
+public:
+ Progress(size_t maxProgress, bool inPlace = true, bool showElapsedTime = true);
+ void setTitle(const string &prefix) {this->title = title;}
+ void setProgress(size_t curProgress);
+ void setProgress();
+ uint32_t addDetail(const string& detail);
+ void clearDetails() {details.clear();}
+ void setDetail(uint32_t idx, uint32_t val);
+ bool couldRefresh();
+ void refresh(bool withNewLine = false);
+private:
+ uint64_t firstRefresh;
+ uint64_t lastRefresh;
+ size_t maxProgress;
+ size_t curProgress;
+ bool inPlace;
+ bool showElapsedTime;
+ size_t prevLength;
+ string title;
+ vector<pair<string, uint32_t> > details;
+};
+
+#endif /* _PROGRESS_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream.cpp b/VNFs/DPPD-PROX/tools/flow_extract/stream.cpp
new file mode 100644
index 00000000..b8056852
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/stream.cpp
@@ -0,0 +1,171 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <iostream>
+#include <iomanip>
+#include <arpa/inet.h>
+
+#include "pcapwriter.hpp"
+#include "stream.hpp"
+
+Stream::Stream(uint32_t id, uint32_t sizeHint)
+ : m_id(id), m_prevPktIsClient(false)
+{
+ m_client.pkts.reserve(sizeHint / 2);
+ m_server.pkts.reserve(sizeHint / 2);
+ m_pkts.reserve(sizeHint);
+}
+
+bool Stream::isClient(const PcapPkt &pkt) const
+{
+ return m_pt == pkt.parsePkt();
+}
+
+size_t Stream::pktCount() const
+{
+ return m_client.pkts.size() + m_server.pkts.size();
+}
+
+void Stream::setTupleFromPkt(const PcapPkt &pkt)
+{
+ m_pt = pkt.parsePkt();
+}
+
+void Stream::addPkt(const PcapPkt &pkt)
+{
+ if (!pktCount())
+ setTupleFromPkt(pkt);
+
+ bool isClientPkt = isClient(pkt);
+ HalfStream *half;
+
+ if (isClientPkt)
+ half = &m_client;
+ else
+ half = &m_server;
+
+ HalfStream::Action::Part p = half->addPkt(pkt);
+
+ if (p.len) {
+ addAction(half, p, isClientPkt);
+ }
+
+ m_pkts.push_back(pkt);
+}
+
+void Stream::addAction(HalfStream *half, HalfStream::Action::Part p, bool isClientPkt)
+{
+ if (m_actions.empty() || m_prevPktIsClient != isClientPkt || m_pt.proto_id == IPPROTO_UDP)
+ m_actions.push_back(HalfStream::Action(half, p, isClientPkt));
+ else
+ m_actions.back().addPart(p);
+ m_prevPktIsClient = isClientPkt;
+}
+
+Stream::Header Stream::getHeader() const
+{
+ Header h;
+
+ h.streamId = m_id;
+ h.clientHdrLen = m_client.hdrLen;
+ h.clientContentLen = m_client.contentLen;
+ h.serverHdrLen = m_server.hdrLen;
+ h.serverContentLen = m_server.contentLen;
+ h.actionCount = m_actions.size();
+ h.clientIP = m_pt.src_addr;
+ h.clientPort = m_pt.src_port;
+ h.serverIP = m_pt.dst_addr;
+ h.serverPort = m_pt.dst_port;
+ h.upRate = m_client.getRate();
+ h.dnRate = m_server.getRate();
+ h.protocol = m_pt.proto_id;
+ h.completedTCP = (m_client.tcpOpen && m_client.tcpClose && m_server.tcpOpen && m_server.tcpClose) ||
+ (!m_client.tcpOpen && !m_client.tcpClose && !m_server.tcpOpen && !m_server.tcpClose);
+
+ return h;
+}
+
+void Stream::Header::toFile(ofstream *f) const
+{
+ f->write((const char *)this, sizeof(*this));
+}
+
+int Stream::Header::fromFile(ifstream *f)
+{
+ const size_t readSize = sizeof(*this);
+
+ f->read((char *)this, readSize);
+ return f->gcount() == readSize? 0 : -1;
+}
+
+size_t Stream::Header::getStreamLen() const
+{
+ return actionCount * sizeof(ActionEntry)
+ + clientHdrLen + clientContentLen
+ + serverHdrLen + serverContentLen;
+}
+
+void Stream::actionsToFile(ofstream *f) const
+{
+ ActionEntry actionEntry;
+ uint32_t runningTotalLen[2] = {0};
+
+ for (size_t i = 0; i < m_actions.size(); ++i) {
+ actionEntry.peer = m_actions[i].isClient()? 0 : 1;
+ actionEntry.beg = runningTotalLen[actionEntry.peer];
+ actionEntry.len = m_actions[i].totLen();
+
+ runningTotalLen[actionEntry.peer] += actionEntry.len;
+ f->write((const char *)&actionEntry, sizeof(actionEntry));
+ }
+}
+
+void Stream::clientHdrToFile(ofstream *f) const
+{
+ f->write((const char *)m_client.hdr, m_client.hdrLen);
+}
+
+void Stream::serverHdrToFile(ofstream *f) const
+{
+ f->write((const char *)m_server.hdr, m_server.hdrLen);
+}
+
+void Stream::contentsToFile(ofstream *f, bool isClient) const
+{
+ for (size_t i = 0; i < m_actions.size(); ++i)
+ if (m_actions[i].isClient() == isClient)
+ m_actions[i].toFile(f);
+}
+
+void Stream::toFile(ofstream *f)
+{
+ getHeader().toFile(f);
+ actionsToFile(f);
+ clientHdrToFile(f);
+ serverHdrToFile(f);
+ contentsToFile(f, true);
+ contentsToFile(f, false);
+}
+
+void Stream::toPcap(const string& outFile)
+{
+ PcapWriter pw;
+
+ pw.open(outFile);
+ for (size_t i = 0; i < m_pkts.size(); ++i)
+ pw.write(m_pkts[i]);
+ pw.close();
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream.hpp b/VNFs/DPPD-PROX/tools/flow_extract/stream.hpp
new file mode 100644
index 00000000..28547d18
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/stream.hpp
@@ -0,0 +1,94 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _STREAM_H_
+#define _STREAM_H_
+
+#include <list>
+#include <string>
+#include <fstream>
+#include <cstring>
+#include <vector>
+#include <cstdlib>
+#include <sys/time.h>
+
+#include "pcappktref.hpp"
+#include "pcappkt.hpp"
+#include "netsocket.hpp"
+#include "timestamp.hpp"
+#include "halfstream.hpp"
+
+using namespace std;
+
+class PcapReader;
+
+class Stream {
+public:
+ struct Header {
+ uint32_t streamId;
+ uint16_t clientHdrLen;
+ uint32_t clientContentLen;
+ uint16_t serverHdrLen;
+ uint32_t serverContentLen;
+ uint32_t actionCount;
+ uint32_t clientIP;
+ uint16_t clientPort;
+ uint32_t serverIP;
+ uint16_t serverPort;
+ double upRate;
+ double dnRate;
+ uint8_t protocol;
+ uint8_t completedTCP;
+ void toFile(ofstream *f) const;
+ int fromFile(ifstream *f);
+ size_t getStreamLen() const;
+ };
+ struct ActionEntry {
+ uint8_t peer;
+ uint32_t beg;
+ uint32_t len;
+ } __attribute__((packed));
+
+ Stream(uint32_t id = -1, uint32_t sizeHint = 0);
+ void addPkt(const PcapPkt &pkt);
+ void toFile(ofstream *f);
+ void toPcap(const string& outFile);
+ double getRate() const;
+ size_t actionCount() const {return m_actions.size();}
+
+private:
+ Header getHeader() const;
+ void actionsToFile(ofstream *f) const;
+ void clientHdrToFile(ofstream *f) const;
+ void serverHdrToFile(ofstream *f) const;
+ void contentsToFile(ofstream *f, bool isClient) const;
+ bool isClient(const PcapPkt &pkt) const;
+ size_t pktCount() const;
+ struct pkt_tuple m_pt;
+ void setTupleFromPkt(const PcapPkt &pkt);
+ void addToClient(const PcapPkt &pkt);
+ void addToServer(const PcapPkt &pkt);
+ void addAction(HalfStream *half, HalfStream::Action::Part p, bool isClientPkt);
+
+ int m_id;
+ vector<PcapPkt> m_pkts;
+ vector<HalfStream::Action> m_actions;
+ HalfStream m_client;
+ HalfStream m_server;
+ bool m_prevPktIsClient;
+};
+
+#endif /* _STREAM_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream2.cpp b/VNFs/DPPD-PROX/tools/flow_extract/stream2.cpp
new file mode 100644
index 00000000..51057e7d
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/stream2.cpp
@@ -0,0 +1,151 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <iomanip>
+#include <arpa/inet.h>
+#include <sstream>
+
+#include "stream.hpp"
+#include "stream2.hpp"
+
+int Stream2::fromFile(ifstream *f)
+{
+ m_actions.clear();
+ if (streamHdr.fromFile(f))
+ return -1;
+ if (actionsFromFile(f, streamHdr.actionCount))
+ return -1;
+ if (setReferences(f))
+ return -1;
+
+ return 0;
+}
+
+int Stream2::actionsFromFile(ifstream *f, size_t actionCount)
+{
+ m_actions.resize(actionCount);
+ for (size_t i = 0; i < actionCount; ++i)
+ f->read((char *)&m_actions[i], sizeof(Stream::ActionEntry));
+
+ return 0;
+}
+
+int Stream2::setReferences(ifstream *f)
+{
+ size_t toRead = streamHdr.clientHdrLen +
+ streamHdr.serverHdrLen +
+ streamHdr.clientContentLen +
+ streamHdr.serverContentLen;
+
+ delete [] clientServerHdrContent;
+ clientServerHdrContent = new uint8_t[toRead];
+ f->read((char *)clientServerHdrContent, toRead);
+ return 0;
+}
+
+void Stream2::calcOffsets(ofstream *out)
+{
+ size_t curPos = out->tellp();
+
+ clientHdrBeg = curPos;
+ serverHdrBeg = clientHdrBeg + streamHdr.clientHdrLen;
+ clientContentBeg = serverHdrBeg + streamHdr.serverHdrLen;
+ serverContentBeg = clientContentBeg + streamHdr.clientContentLen;
+}
+
+void Stream2::toFile(ofstream *out) const
+{
+ size_t len = streamHdr.clientHdrLen +
+ streamHdr.serverHdrLen +
+ streamHdr.clientContentLen +
+ streamHdr.serverContentLen;
+
+ out->write((const char *)clientServerHdrContent, len);
+}
+
+static string ipToString(const uint32_t ip)
+{
+ uint32_t ip_ne = htonl(ip);
+ stringstream ss;
+
+ ss << ((ip_ne >> 24) & 0xff) << "."
+ << ((ip_ne >> 16) & 0xff) << "."
+ << ((ip_ne >> 8) & 0xff) << "."
+ << (ip_ne & 0xff);
+
+ return ss.str();
+}
+
+static string spaces(uint32_t count)
+{
+ stringstream ss;
+
+ while (count--)
+ ss << " ";
+ return ss.str();
+}
+
+NetSocket Stream2::getServerNetSocket() const
+{
+ return NetSocket(streamHdr.serverIP, ntohs(streamHdr.serverPort));
+}
+
+NetSocket Stream2::getClientNetSocket() const
+{
+ return NetSocket(streamHdr.clientIP, ntohs(streamHdr.clientPort));
+}
+void Stream2::setServerNetSocket(const NetSocket& netSocket)
+{
+ streamHdr.serverPort = htons(netSocket.port);
+ streamHdr.serverIP = netSocket.host;
+}
+
+void Stream2::setClientNetSocket(const NetSocket& netSocket)
+{
+ streamHdr.clientPort = htons(netSocket.port);
+ streamHdr.clientIP = netSocket.host;
+}
+void Stream2::toLua(ofstream *f, const string& binFileName, const string& streamTableName) const
+
+{
+ (*f) << std::fixed;
+
+ (*f) << streamTableName << "[" << streamHdr.streamId << "] = {" << endl
+ << spaces(3) << "client_data = {" << endl
+ << spaces(6) << "header = bin_read(" << binFileName << "," << clientHdrBeg << "," << streamHdr.clientHdrLen << "), " << endl
+ << spaces(6) << "content = bin_read(" << binFileName << "," << clientContentBeg << "," << streamHdr.clientContentLen << "), " << endl
+ << spaces(3) << "}," << endl
+ << spaces(3) << "server_data = {" << endl
+ << spaces(6) << "header = bin_read(" << binFileName << "," << serverHdrBeg << "," << streamHdr.serverHdrLen << "), " << endl
+ << spaces(6) << "content = bin_read(" << binFileName << "," << serverContentBeg << "," << streamHdr.serverContentLen << "), " << endl
+ << spaces(3) << "}," << endl
+ << spaces(3) << "actions = {" << endl;
+
+ for (size_t i = 0; i < m_actions.size(); ++i) {
+ const char *peer_str = m_actions[i].peer == 0? "client" : "server";
+
+ (*f) << spaces(6) << peer_str << "_content(" << m_actions[i].beg << "," << m_actions[i].len << ")," << endl;
+ }
+
+ (*f) << spaces(3) << "}," << endl
+ << spaces(3) << "clients = {ip = ip(\"" << ipToString(streamHdr.clientIP) << "\"), port = " << ntohs(streamHdr.clientPort) << "}," << endl
+ << spaces(3) << "servers = {ip = ip(\"" << ipToString(streamHdr.serverIP) << "\"), port = " << ntohs(streamHdr.serverPort) << "}," << endl
+ << spaces(3) << "l4_proto = \"" << (streamHdr.protocol == 0x06? "tcp" : "udp") << "\"," << endl
+ << spaces(3) << "up_bps = " << setprecision(4) << streamHdr.upRate << "," << endl
+ << spaces(3) << "dn_bps = " << setprecision(4) << streamHdr.dnRate << "," << endl;
+
+ (*f) << "}" << endl;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream2.hpp b/VNFs/DPPD-PROX/tools/flow_extract/stream2.hpp
new file mode 100644
index 00000000..fd9d9c8c
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/stream2.hpp
@@ -0,0 +1,54 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _STREAM2_H_
+#define _STREAM2_H_
+
+#include <inttypes.h>
+#include <fstream>
+
+#include "netsocket.hpp"
+
+using namespace std;
+
+class Stream2 {
+public:
+ Stream2() : clientServerHdrContent(NULL) {}
+ ~Stream2() {delete [] clientServerHdrContent;}
+ int fromFile(ifstream *f);
+ void calcOffsets(ofstream *out);
+ void toFile(ofstream *out) const;
+ void toLua(ofstream *f, const string& binFileName, const string& streamTableName) const;
+ NetSocket getServerNetSocket() const;
+ NetSocket getClientNetSocket() const;
+ void setServerNetSocket(const NetSocket& netSocket);
+ void setClientNetSocket(const NetSocket& netSocket);
+ Stream::Header streamHdr;
+private:
+ int actionsFromFile(ifstream *f, size_t actionCount);
+ int setReferences(ifstream *f);
+
+ uint8_t *clientServerHdrContent;
+
+ uint32_t clientHdrBeg;
+ uint32_t serverHdrBeg;
+ uint32_t clientContentBeg;
+ uint32_t serverContentBeg;
+
+ vector<Stream::ActionEntry> m_actions;
+};
+
+#endif /* _STREAM2_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream3.cpp b/VNFs/DPPD-PROX/tools/flow_extract/stream3.cpp
new file mode 100644
index 00000000..30c166ae
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/stream3.cpp
@@ -0,0 +1,95 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+
+#include "stream3.hpp"
+
+Stream3::Stream3(uint32_t id, PcapPkt::L4Proto proto)
+ : m_id(id), m_proto(proto), m_pktCount(0), m_flushCount(0)
+{
+}
+
+void Stream3::writeHeader(ofstream *outputFile) const
+{
+ outputFile->write(reinterpret_cast<const char *>(&m_id), sizeof(m_id));
+ outputFile->write(reinterpret_cast<const char *>(&m_flushCount), sizeof(m_flushCount));
+}
+
+void Stream3::writePackets(ofstream *outputFile) const
+{
+ for (size_t i = 0; i < m_pkts.size(); ++i)
+ m_pkts[i]->toFile(outputFile);
+}
+
+void Stream3::clearPackets()
+{
+ for (size_t i = 0; i < m_pkts.size(); ++i)
+ delete m_pkts[i];
+ m_pkts.clear();
+ m_flushCount = 0;
+}
+
+void Stream3::flush(ofstream *outputFile)
+{
+ writeHeader(outputFile);
+ writePackets(outputFile);
+ clearPackets();
+}
+
+void Stream3::addPkt(const PcapPkt& pkt)
+{
+ m_pkts.push_back(new PcapPkt(pkt));
+ m_pktCount++;
+ m_flushCount++;
+}
+
+Timestamp Stream3::getTimeout() const
+{
+ uint32_t timeoutMinutes = m_proto == PcapPkt::PROTO_UDP? 10 : 5;
+
+ return Timestamp(timeoutMinutes * 60, 0);
+}
+
+uint32_t Stream3::getIDFromMem(uint8_t *mem)
+{
+ return *reinterpret_cast<uint32_t *>(mem);
+}
+
+void Stream3::addFromMemory(uint8_t *mem, size_t *len)
+{
+ uint32_t n_pkts;
+
+ mem += sizeof(m_id);
+ n_pkts = *reinterpret_cast<uint32_t *>(mem);
+ mem += sizeof(n_pkts);
+
+ *len = sizeof(m_id) + sizeof(n_pkts);
+ for (uint32_t i = 0; i < n_pkts; ++i) {
+ addPkt(PcapPkt(mem));
+ mem += m_pkts.back()->memSize();
+ *len += m_pkts.back()->memSize();
+ }
+}
+
+void Stream3::removeAllPackets()
+{
+ clearPackets();
+ m_pktCount = 0;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream3.hpp b/VNFs/DPPD-PROX/tools/flow_extract/stream3.hpp
new file mode 100644
index 00000000..7e94814e
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/stream3.hpp
@@ -0,0 +1,55 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _STREAM3_H_
+#define _STREAM3_H_
+
+#include <inttypes.h>
+#include <vector>
+
+#include "pcappkt.hpp"
+#include "timestamp.hpp"
+
+using namespace std;
+class Allocator;
+
+class Stream3 {
+public:
+ PcapPkt::L4Proto getProto(void) const {return m_proto;}
+ Stream3(uint32_t id, PcapPkt::L4Proto proto);
+ Stream3() : m_id(UINT32_MAX), m_proto(PcapPkt::PROTO_UDP), m_pktCount(0), m_flushCount(0) {}
+ void addPkt(const PcapPkt& pkt);
+ void flush(ofstream *outputFile);
+ void addFromMemory(uint8_t *mem, size_t *len);
+ static uint32_t getIDFromMem(uint8_t *mem);
+ bool hasFlushablePackets() const {return !!m_flushCount;}
+ Timestamp getTimeout() const;
+ uint32_t getID() const {return m_id;}
+ void removeAllPackets();
+ void setID(const uint32_t id) {m_id = id;}
+private:
+ void writeHeader(ofstream *outputFile) const;
+ void writePackets(ofstream *outputFile) const;
+ void clearPackets();
+
+ uint32_t m_id;
+ PcapPkt::L4Proto m_proto;
+ vector<PcapPkt *> m_pkts;
+ uint32_t m_pktCount;
+ uint32_t m_flushCount;
+};
+
+#endif /* _STREAM3_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamextract.cpp b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.cpp
new file mode 100644
index 00000000..e493ef3f
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.cpp
@@ -0,0 +1,406 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <inttypes.h>
+#include <string>
+#include <cstdio>
+#include <iostream>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sstream>
+#include <set>
+#include <arpa/inet.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <cerrno>
+#include <cstdlib>
+#include <map>
+
+#include "path.hpp"
+#include "bundle.hpp"
+#include "stream.hpp"
+#include "stream2.hpp"
+#include "allocator.hpp"
+#include "timestamp.hpp"
+#include "streamextract.hpp"
+#include "pcapreader.hpp"
+#include "pcapwriter.hpp"
+#include "flowtable.hpp"
+#include "stream3.hpp"
+#include "netsocket.hpp"
+#include "pcappktref.hpp"
+#include "progress.hpp"
+#include "mappedfile.hpp"
+#include "streamsorter.hpp"
+
+using namespace std;
+
+static bool is_dir(const string& path_dir_out)
+{
+ struct stat s = { 0 };
+
+ if (stat(path_dir_out.c_str(), &s)) {
+ return false;
+ }
+
+ return s.st_mode & S_IFDIR;
+}
+
+StreamExtract::StreamExtract(const ProgramConfig &cfg)
+ : ft2(cfg.flowTableSize),
+ streamSorter(cfg.flowTableSize, cfg.path_dir_out, 1024UL*1024*1024*8),
+ cfg(cfg)
+{
+}
+
+vector<Bundle> StreamExtract::createBundles(const string& streamPath)
+{
+ map<uint32_t, Bundle>::iterator iterBundle;
+ map<uint32_t, Bundle> bundles;
+ set<uint32_t> servers;
+
+ Stream2 s;
+ ifstream binIn;
+
+ binIn.open(streamPath.c_str());
+ binIn.seekg(0, binIn.end);
+ Progress progress(binIn.tellg());
+ binIn.seekg(0, binIn.beg);
+
+ while (!s.fromFile(&binIn)) {
+ if (progress.couldRefresh()) {
+ progress.setProgress(binIn.tellg());
+ progress.refresh();
+ }
+ if (!s.streamHdr.completedTCP)
+ continue;
+ if (!s.streamHdr.serverHdrLen)
+ continue;
+ /* The current implementation does not support clients
+ that are also servers. */
+ servers.insert(s.streamHdr.serverIP);
+ if (servers.find(s.streamHdr.clientIP) != servers.end())
+ continue;
+
+ /* Since each application is represented as a path
+ graph (there is only one reply for a given request
+ and only one request after a given reply), each
+ application must run on a unique server. For this
+ reason, check if the socket on the server already
+ is occupied and if so, keep incrementing the socket
+ until the collision is resolved. */
+ iterBundle = bundles.find(s.streamHdr.clientIP);
+
+ if (iterBundle == bundles.end()) {
+ bundles.insert(make_pair(s.streamHdr.clientIP, Bundle()));
+ iterBundle = bundles.find(s.streamHdr.clientIP);
+ }
+
+ (*iterBundle).second.addStream(s.streamHdr.streamId, s.getServerNetSocket().port);
+ }
+
+ progress.setProgress();
+ progress.refresh(true);
+
+ binIn.close();
+
+ vector<Bundle> ret;
+
+ ret.reserve(bundles.size());
+
+ for (map<uint32_t, Bundle>::const_iterator i = bundles.begin(); i != bundles.end(); ++i)
+ ret.push_back(i->second);
+
+ return ret;
+}
+
+set<uint32_t> StreamExtract::getBundleStreamIDs(const vector<Bundle*>& bundleSamples)
+{
+ set<uint32_t> streamIDs;
+
+ for (size_t i = 0; i < bundleSamples.size(); ++i) {
+ const vector<uint32_t> &bundleStreamIDs = bundleSamples[i]->getStream();
+
+ for (vector<uint32_t>::const_iterator j = bundleStreamIDs.begin(); j != bundleStreamIDs.end(); ++j) {
+ streamIDs.insert(*j);
+ }
+ }
+
+ return streamIDs;
+}
+
+static size_t getRandom(size_t limit)
+{
+ size_t r = rand();
+ size_t rand_limit = (RAND_MAX/limit)*limit;
+
+ while (r > rand_limit)
+ r = rand();
+
+ return r % limit;
+}
+
+static void removeFill(vector<Bundle*> *from, size_t idx)
+{
+ Bundle *last = from->back();
+ from->pop_back();
+
+ if (idx != from->size())
+ (*from)[idx] = last;
+}
+
+static vector<Bundle*> takeSamples(vector<Bundle>& bundles, size_t sampleCount)
+{
+ vector<Bundle*> bundleSamples;
+
+ bundleSamples.reserve(bundles.size());
+
+ cout << "Sampling " << sampleCount << " bundles out of " << bundles.size() << endl;
+ for (size_t i = 0; i < bundles.size(); ++i)
+ bundleSamples.push_back(&bundles[i]);
+
+ srand(1000);
+ while (bundleSamples.size() > sampleCount) {
+ size_t r = getRandom(bundleSamples.size());
+ removeFill(&bundleSamples, r);
+ }
+ return bundleSamples;
+}
+
+static size_t replaceWithRunningTotals(vector<size_t> *streamLength)
+{
+ size_t runningTotal = 0;
+ for (size_t i = 0; i < streamLength->size(); ++i) {
+ size_t len = (*streamLength)[i] + sizeof(uint32_t);
+ (*streamLength)[i] = runningTotal;
+ runningTotal += len;
+ }
+ return runningTotal;
+}
+
+static void printPorts(const vector<Bundle> &bundles)
+{
+ set<uint32_t> streamIDs;
+
+ for (size_t i = 0; i < bundles.size(); ++i) {
+ const vector<uint32_t> &ports = bundles[i].getPorts();
+
+ for (size_t j = 0; j < ports.size(); ++j) {
+ if (j + 1 == ports.size())
+ cout << ports[j] << ",END" << endl;
+ else
+ cout << ports[j] << "," << ports[j +1] << endl;
+ }
+ }
+}
+
+string StreamExtract::createStreamPcapFileName(int id)
+{
+ stringstream ss;
+
+ ss << cfg.path_dir_out << "/s" << id << ".pcap";
+
+ return ss.str();
+}
+
+int StreamExtract::writeToPcaps(const string &sourceFilePath, const set<uint32_t> &streamIDs)
+{
+ set<uint32_t>::const_iterator i = streamIDs.begin();
+
+ MappedFile mappedFile;
+ if (mappedFile.open(sourceFilePath)) {
+ cerr << "Failed to open file " << sourceFilePath << ":" << strerror(errno) << endl;
+ return -1;
+ }
+
+ PcapPkt::allocator = NULL;
+
+ Progress progress((uint64_t)mappedFile.getMapEnd() - (uint64_t)mappedFile.getMapBeg());
+ cout << "Writing " << streamIDs.size() << " streams to pcaps" << endl;
+ uint8_t *data2 = mappedFile.getMapBeg();
+ while (data2 < mappedFile.getMapEnd()) {
+ uint32_t id = *reinterpret_cast<uint32_t *>(data2);
+
+ data2 += sizeof(id);
+ uint32_t pktCount = *reinterpret_cast<uint32_t *>(data2);
+ data2 += sizeof(pktCount);
+ Stream s(id, pktCount);
+ while (pktCount--) {
+ PcapPkt p(data2);
+
+ data2 += p.memSize();
+ s.addPkt(p);
+ }
+
+ while (i != streamIDs.end() && (*i) < id)
+ i++;
+ if (i == streamIDs.end())
+ break;
+ if (*i > id)
+ continue;
+
+ const string pcapPath = createStreamPcapFileName(id);
+
+ s.toPcap(pcapPath);
+ if (progress.couldRefresh()) {
+ progress.setProgress((uint64_t)data2 - (uint64_t)mappedFile.getMapBeg());
+ progress.refresh();
+ mappedFile.sync();
+ }
+ }
+
+ progress.setProgress(data2 - mappedFile.getMapBeg());
+ progress.refresh(true);
+
+ mappedFile.close();
+ return 0;
+}
+
+int StreamExtract::writeToLua(const string& binFilePath, const Path &smallFinalBin, const string& luaFilePath, const string &orderedTemp)
+{
+ vector<Bundle> bundles = createBundles(binFilePath);
+ vector<Bundle*> bundleSamples = takeSamples(bundles, cfg.sampleCount);
+ set<uint32_t> streamIDs = getBundleStreamIDs(bundleSamples);
+
+ if (cfg.write_pcaps)
+ writeToPcaps(orderedTemp, streamIDs);
+
+ ofstream outLua;
+ ofstream outSmallBin;
+ outLua.open(luaFilePath.c_str());
+ outLua << "bf = \""<< smallFinalBin.getFileName() << "\"" << endl;
+ outLua << "s = {}\n";
+ set<uint32_t>::iterator i = streamIDs.begin();
+
+ set<NetSocket> serverSockets;
+ ifstream binIn;
+ Stream2 s;
+
+ outSmallBin.open(smallFinalBin.str().c_str());
+ binIn.open(binFilePath.c_str());
+ while (!s.fromFile(&binIn)) {
+ while (i != streamIDs.end() && (*i) < s.streamHdr.streamId)
+ i++;
+ if (i == streamIDs.end())
+ break;
+ if (*i > s.streamHdr.streamId)
+ continue;
+ s.calcOffsets(&outSmallBin);
+ s.toFile(&outSmallBin);
+ while (serverSockets.find(s.getServerNetSocket()) != serverSockets.end()) {
+ NetSocket ns = s.getServerNetSocket();
+
+ ns.port++;
+ s.setServerNetSocket(ns);
+ }
+ serverSockets.insert(s.getServerNetSocket());
+
+ s.toLua(&outLua, "bf", "s");
+ }
+ binIn.close();
+
+ uint32_t bundleCount = 0;
+
+ outLua << "bundles = {}" << endl;
+ for (size_t i = 0; i < bundleSamples.size(); ++i) {
+ bundleSamples[i]->toLua(&outLua, "s", ++bundleCount);
+ }
+ outLua << "return bundles" << endl;
+ outLua.close();
+ return 0;
+}
+
+int StreamExtract::writeFinalBin(const string& sourceFilePath, const string& destFilePath)
+{
+ MappedFile mappedFile;
+ if (mappedFile.open(sourceFilePath)) {
+ cerr << "Failed to open file " << sourceFilePath << ":" << strerror(errno) << endl;
+ return -1;
+ }
+ ofstream binOut;
+
+ binOut.open(destFilePath.c_str());
+ PcapPkt::allocator = NULL;
+
+ Progress progress((uint64_t)mappedFile.getMapEnd() - (uint64_t)mappedFile.getMapBeg());
+
+ int streamCount = 0;
+ uint8_t *data2 = mappedFile.getMapBeg();
+ while (data2 < mappedFile.getMapEnd()) {
+ uint32_t id = *reinterpret_cast<uint32_t *>(data2);
+
+ data2 += sizeof(id);
+ uint32_t pktCount = *reinterpret_cast<uint32_t *>(data2);
+ data2 += sizeof(pktCount);
+ Stream s(id, pktCount);
+ while (pktCount--) {
+ PcapPkt p(data2);
+
+ data2 += p.memSize();
+ s.addPkt(p);
+ }
+ s.toFile(&binOut);
+ streamCount++;
+ if (progress.couldRefresh()) {
+ progress.setProgress((uint64_t)data2 - (uint64_t)mappedFile.getMapBeg());
+ progress.refresh();
+ mappedFile.sync();
+ }
+ }
+
+ progress.setProgress(data2 - mappedFile.getMapBeg());
+ progress.refresh(true);
+
+ binOut.close();
+ mappedFile.close();
+ return 0;
+}
+
+int StreamExtract::run()
+{
+ Path p(cfg.path_dir_out);
+ p.mkdir();
+
+ string orderedTemp = p.add("/a").str();
+
+ string finalBin = p.add("/b").str();
+ Path smallfinalBin = p.add("/data.bin").str();
+ string luaFile = p.add("/cfg.lua").str();
+
+ cout << "Writing to directory '" << p.str() << "'" << endl;
+ cout << "Ordered streams '" << orderedTemp << "'" << endl;
+ cout << "Final binary output '" << finalBin << "'" << endl;
+ cout << "lua file '" << luaFile << "' will contain " << cfg.sampleCount << " bundles" << endl;
+
+ if (cfg.run_first_step) {
+ cout << "starting sorting" << endl;
+ streamSorter.sort(cfg.path_file_in_pcap, orderedTemp);
+ cout << "writing final binary file (converting format)" << endl;
+ if (writeFinalBin(orderedTemp, finalBin))
+ return -1;
+ } else {
+ cout << "Skipping first step" << endl;
+ if (!Path(finalBin).isFile()) {
+ cerr << "File is missing:" << finalBin << endl;
+ return -1;
+ }
+ }
+ cout << "writing Lua '" << luaFile << "'" << endl;
+ if (writeToLua(finalBin, smallfinalBin, luaFile, orderedTemp))
+ return -1;
+ return 0;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamextract.hpp b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.hpp
new file mode 100644
index 00000000..d5dbdb05
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.hpp
@@ -0,0 +1,55 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _STREAMEXTRACT_H_
+#define _STREAMEXTRACT_H_
+
+#include <string>
+#include <list>
+#include <map>
+#include <set>
+
+#include "programconfig.hpp"
+#include "bundle.hpp"
+#include "pcapreader.hpp"
+#include "flowtable.hpp"
+#include "pcappkt.hpp"
+#include "stream3.hpp"
+#include "streamsorter.hpp"
+#include "path.hpp"
+
+using namespace std;
+
+class StreamExtract {
+public:
+ /* The size of the flow table determines the number of flows
+ that can be active at a given time. When a flow expires, it
+ is written out to a file and the memory is freed. */
+ StreamExtract(const ProgramConfig &cfg);
+ int run();
+private:
+ int writeToPcaps(const string &sourceFilePath, const set<uint32_t> &streamIDs);
+ int writeToLua(const string& binFilePath, const Path &smallFinalBin, const string& luaFilePath, const string& orderedTemp);
+ int writeFinalBin(const string& sourceFilePath, const string& destFilePath);
+ string createStreamPcapFileName(int id);
+ vector<Bundle> createBundles(const string& streamPath);
+ set<uint32_t> getBundleStreamIDs(const vector<Bundle*>& bundleSamples);
+ FlowTable<pkt_tuple, Stream3> ft2;
+ StreamSorter streamSorter;
+ ProgramConfig cfg;
+};
+
+#endif /* _STREAMEXTRACT_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.cpp b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.cpp
new file mode 100644
index 00000000..65c645e1
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.cpp
@@ -0,0 +1,203 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+
+#include "mappedfile.hpp"
+#include "memreader.hpp"
+#include "streamsorter.hpp"
+#include "path.hpp"
+#include "allocator.hpp"
+#include "pcapreader.hpp"
+#include "progress.hpp"
+
+StreamSorter::StreamSorter(size_t flowTableSize, const string& workingDirectory, size_t memoryLimit)
+ : flowTableSize(flowTableSize),
+ workingDirectory(workingDirectory),
+ allocator(memoryLimit, 1024*10),
+ streamID(0)
+{
+}
+
+void StreamSorter::sort(const string &inputPcapFilePath, const string &outputBinFilePath)
+{
+ setTempFileName();
+ sortChunks(inputPcapFilePath);
+ mergeChunks(outputBinFilePath);
+}
+
+void StreamSorter::sortChunks(const string &inputPcapFilePath)
+{
+ ofstream outputTempFile;
+
+ outputTempFile.open(tempFilePath.c_str());
+
+ if (!outputTempFile.is_open())
+ return ;
+
+ PcapReader pr;
+ PcapPkt pkt;
+
+ if (pr.open(inputPcapFilePath)) {
+ pr.getError();
+ return;
+ }
+ PcapPkt::allocator = &allocator;
+
+ Progress progress(pr.end());
+ uint32_t packetDetail = progress.addDetail("packet count");
+
+ ft = new FlowTable<pkt_tuple, uint32_t>(flowTableSize);
+ resetStreams();
+
+ while (pr.read(&pkt)) {
+ processPkt(pkt);
+ if (progress.couldRefresh()) {
+ progress.setProgress(pr.pos());
+ progress.setDetail(packetDetail, pr.getPktReadCount());
+ progress.refresh();
+ }
+ if (allocator.lowThresholdReached()) {
+ flushStreams(&outputTempFile);
+ }
+ }
+ progress.setProgress();
+ progress.setDetail(packetDetail, pr.getPktReadCount());
+ progress.refresh(true);
+
+ pr.close();
+ flushStreams(&outputTempFile);
+ PcapPkt::allocator = NULL;
+ outputTempFile.close();
+ delete ft;
+}
+
+void StreamSorter::resetStreams()
+{
+ streams.clear();
+}
+
+void StreamSorter::flushStreams(ofstream *outputTempFile)
+{
+ size_t flushCount = 0;
+ size_t offset = outputTempFile->tellp();
+
+ Progress progress(streams.size());
+
+ cout << endl;
+ progress.setTitle("flush ");
+ for (size_t i = 0; i < streams.size(); ++i) {
+ if (streams[i].hasFlushablePackets()) {
+ streams[i].flush(outputTempFile);
+ flushCount++;
+ }
+
+ if (progress.couldRefresh()) {
+ progress.setProgress(i);
+ progress.refresh();
+ }
+ }
+ progress.setProgress();
+ progress.refresh(true);
+
+ if (flushCount)
+ flushOffsets.push_back(offset);
+ allocator.reset();
+}
+
+Stream3 *StreamSorter::addNewStream(PcapPkt::L4Proto proto)
+{
+ streams.push_back(Stream3(streamID++, proto));
+ return &streams.back();
+}
+
+FlowTable<pkt_tuple, uint32_t>::entry* StreamSorter::getFlowEntry(const PcapPkt &pkt)
+{
+ FlowTable<pkt_tuple, uint32_t>::entry *a;
+ struct pkt_tuple pt = pkt.parsePkt();
+ Stream3 *stream = NULL;
+
+ a = ft->lookup(pt.flip());
+ if (!a) {
+ a = ft->lookup(pt);
+ if (!a) {
+ stream = addNewStream(pkt.getProto());
+
+ a = ft->insert(pt, stream->getID(), pkt.ts());
+ }
+ }
+
+ if (a->expired(pkt.ts(), streams[a->value].getTimeout())) {
+ ft->remove(a);
+
+ stream = addNewStream(pkt.getProto());
+
+ a = ft->insert(pt, stream->getID(), pkt.ts());
+ }
+ return a;
+}
+
+void StreamSorter::processPkt(const PcapPkt &pkt)
+{
+ FlowTable<pkt_tuple, uint32_t>::entry *a;
+
+ a = getFlowEntry(pkt);
+ a->tv = pkt.ts();
+ streams[a->value].addPkt(pkt);
+}
+
+void StreamSorter::mergeChunks(const string &outputBinFile)
+{
+ cout << "merging chunks: " << tempFilePath << " to " << outputBinFile << endl;
+ cout << "have " << flushOffsets.size() << " parts to merge" << endl;
+ MappedFile tempFile;
+
+ if (tempFile.open(tempFilePath)) {
+ cerr << "failed to open temp file" << endl;
+ return;
+ }
+ ofstream file;
+
+ file.open(outputBinFile.c_str());
+
+ if (!file.is_open()) {
+ cerr << "failed top open file '" << outputBinFile << "'" << endl;
+ return;
+ }
+ MemReader memReader(&tempFile, flushOffsets);
+ Stream3 stream;
+
+ Progress progress(memReader.getTotalLength());
+
+ while (memReader.read(&stream)) {
+ stream.flush(&file);
+ if (progress.couldRefresh()) {
+ progress.setProgress(memReader.consumed());
+ progress.refresh();
+ }
+ }
+
+ progress.setProgress();
+ progress.refresh(true);
+ tempFile.close();
+}
+
+void StreamSorter::setTempFileName()
+{
+ tempFilePath = Path(workingDirectory).add("/tmp").str();
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.hpp b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.hpp
new file mode 100644
index 00000000..a6d3d6cd
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.hpp
@@ -0,0 +1,47 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _STREAMSORTER_H_
+#define _STREAMSORTER_H_
+
+#include "stream3.hpp"
+#include "flowtable.hpp"
+#include "allocator.hpp"
+
+class StreamSorter {
+public:
+ StreamSorter(size_t flowTableSize, const string& workingDirectory, size_t memoryLimit);
+ void sort(const string &inputPcapFile, const string &outputBinFile);
+private:
+ void sortChunks(const string &inputPcapFilePath);
+ void mergeChunks(const string &outputBinFilePath);
+ void setTempFileName();
+ void processPkt(const PcapPkt &pkt);
+ void resetStreams();
+ FlowTable<pkt_tuple, uint32_t>::entry* getFlowEntry(const PcapPkt &pkt);
+ void flushStreams(ofstream *outputTempFile);
+ Stream3 *addNewStream(PcapPkt::L4Proto proto);
+ size_t flowTableSize;
+ FlowTable<pkt_tuple, uint32_t> *ft;
+ vector<size_t> flushOffsets;
+ vector<Stream3> streams;
+ string tempFilePath;
+ const string workingDirectory;
+ Allocator allocator;
+ uint32_t streamID;
+};
+
+#endif /* _STREAMSORTER_H_ */
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/timestamp.cpp b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.cpp
new file mode 100644
index 00000000..9e91173d
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.cpp
@@ -0,0 +1,65 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include <cstdio>
+#include <iostream>
+#include <iomanip>
+
+#include "timestamp.hpp"
+
+Timestamp Timestamp::operator-(const Timestamp& other) const
+{
+ uint64_t sec;
+ uint64_t nsec;
+
+ if (other.m_nsec <= m_nsec) {
+ nsec = m_nsec - other.m_nsec;
+ sec = m_sec - other.m_sec;
+ } else {
+ nsec = (1000000000 + m_nsec) - other.m_nsec;
+ sec = m_sec - 1 - other.m_sec;
+ }
+
+ return Timestamp(sec, nsec);
+}
+
+bool Timestamp::operator>(const Timestamp& other)
+{
+ return m_sec > other.m_sec ||
+ (m_sec == other.m_sec && m_nsec > other.m_nsec);
+}
+
+bool Timestamp::operator<(const Timestamp& other)
+{
+ return m_sec < other.m_sec ||
+ (m_sec == other.m_sec && m_nsec < other.m_nsec);
+}
+
+ostream& operator<<(ostream& stream, const Timestamp& ts)
+{
+ stream << ts.m_sec << "." << setw(9) << setfill('0') << ts.m_nsec;
+ return stream;
+}
+
+double operator/(double d, const Timestamp &denominator)
+{
+ return d * 1000000000 / (denominator.m_sec * 1000000000 + denominator.m_nsec);
+}
+
+bool Timestamp::operator==(const Timestamp &other) const
+{
+ return m_sec == other.m_sec && m_nsec == other.m_nsec;
+}
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/timestamp.hpp b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.hpp
new file mode 100644
index 00000000..cf8ec5d4
--- /dev/null
+++ b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.hpp
@@ -0,0 +1,45 @@
+/*
+// Copyright (c) 2010-2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#ifndef _TIMESTAMP_H_
+#define _TIMESTAMP_H_
+
+#include <iostream>
+
+#include <sys/time.h>
+#include <inttypes.h>
+
+using namespace std;
+
+class Timestamp {
+public:
+ Timestamp(const uint64_t sec, const uint64_t nsec) : m_sec(sec), m_nsec(nsec) {}
+ Timestamp() {}
+ Timestamp(const struct timeval& tv) : m_sec(tv.tv_sec), m_nsec(tv.tv_usec) {}
+ Timestamp operator-(const Timestamp& other) const;
+ bool operator==(const Timestamp &other) const;
+ friend double operator/(double d, const Timestamp &denominator);
+ bool operator>(const Timestamp& other);
+ bool operator<(const Timestamp& other);
+ uint64_t sec() const {return m_sec;}
+ uint64_t nsec() const {return m_nsec;}
+ friend ostream& operator<<(ostream& stream, const Timestamp& ts);
+private:
+ uint64_t m_sec;
+ uint64_t m_nsec;
+};
+
+#endif /* _TIMESTAMP_H_ */