diff options
Diffstat (limited to 'VNFs/DPPD-PROX/tools')
45 files changed, 3696 insertions, 0 deletions
diff --git a/VNFs/DPPD-PROX/tools/flow_extract/Makefile b/VNFs/DPPD-PROX/tools/flow_extract/Makefile new file mode 100644 index 00000000..a772b8c5 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/Makefile @@ -0,0 +1,59 @@ +## +## Copyright (c) 2010-2017 Intel Corporation +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +SOURCES = main.cpp +SOURCES += streamextract.cpp +SOURCES += pcapreader.cpp +SOURCES += pcapwriter.cpp +SOURCES += timestamp.cpp +SOURCES += pcappkt.cpp +SOURCES += netsocket.cpp +SOURCES += stream3.cpp +SOURCES += stream2.cpp +SOURCES += stream.cpp +SOURCES += path.cpp +SOURCES += allocator.cpp +SOURCES += halfstream.cpp +SOURCES += bundle.cpp +SOURCES += progress.cpp +SOURCES += mappedfile.cpp +SOURCES += streamsorter.cpp +SOURCES += memreader.cpp +SOURCES += programconfig.cpp + +BUILD_DIR = build +OBJECTS = $(SOURCES:%.cpp=$(BUILD_DIR)/%.o) +PROG = flowextract + +CXXFLAGS += -D__STDC_LIMIT_MACROS -g -O2 -Wall -ansi -pedantic -Wno-unused -msse4.2 +LDFLAGS = -lpcap + +$(BUILD_DIR)/$(PROG): $(OBJECTS) + @echo -e "LD\t$<" + @$(CXX) $(CXXFLAGS) $(LDFLAGS) $(OBJECTS) -o $@ + +-include $(SOURCES:%.cpp=$(BUILD_DIR)/%.d) + +$(BUILD_DIR)/%.o: %.cpp + @mkdir -p $(BUILD_DIR) + @echo -e "CXX\t $<" + @$(CXX) -c $(CXXFLAGS) $*.cpp -o $@ + @$(CXX) -MM $(CXXFLAGS) $*.cpp -MT $(BUILD_DIR)/$*.o > $(BUILD_DIR)/$*.d + @cp -f $(BUILD_DIR)/$*.d $(BUILD_DIR)/$*.d.tmp + @sed -e 's/.*://' -e 's/\\$$//' < $(BUILD_DIR)/$*.d.tmp | fmt -1 | sed -e 's/^ *//' -e 's/$$/:/' >> $(BUILD_DIR)/$*.d + @rm -f $(BUILD_DIR)/$*.d.tmp +clean: + @rm -f $(BUILD_DIR)/$(PROG) $(BUILD_DIR)/*.o $(BUILD_DIR)/*.d diff --git a/VNFs/DPPD-PROX/tools/flow_extract/README b/VNFs/DPPD-PROX/tools/flow_extract/README new file mode 100644 index 00000000..fb8754b3 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/README @@ -0,0 +1,20 @@ +## +## Copyright (c) 2010-2017 Intel Corporation +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +The flow extract tool is meant a to be run as a first pass on a pcap +file. The output is a lua config file describing the relations between +flows together with a binary file that contains all the packet headers +and payload. diff --git a/VNFs/DPPD-PROX/tools/flow_extract/allocator.cpp b/VNFs/DPPD-PROX/tools/flow_extract/allocator.cpp new file mode 100644 index 00000000..c861ebfe --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/allocator.cpp @@ -0,0 +1,84 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <iostream> +#include <sys/mman.h> +#include <unistd.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <cstring> + +#define USEHP + +using namespace std; + +#include "allocator.hpp" + +Allocator::Allocator(size_t size, size_t threshold) + : m_size(size), m_threshold(threshold), m_alloc_offset(0) +{ +#ifdef USEHP + int fd = open("/mnt/huge/hp", O_CREAT | O_RDWR, 0755); + if (fd < 0) { + cerr << "Allocator failed to open huge page file descriptor: " << strerror(errno) << endl; + exit(EXIT_FAILURE); + } + m_mem = (uint8_t *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (m_mem == MAP_FAILED) { + perror("mmap"); + unlink("/mnt/huge"); + cerr << "Allocator mmap failed: " << strerror(errno) << endl; + exit (EXIT_FAILURE); + } +#else + m_mem = new uint8_t[size]; +#endif +} + +Allocator::~Allocator() +{ +#ifdef USEHP + munmap((void *)m_mem, m_size); +#else + delete[] m_mem; +#endif +} + +void *Allocator::alloc(size_t size) +{ + void *ret = &m_mem[m_alloc_offset]; + + m_alloc_offset += size; + return ret; +} + +void Allocator::reset() +{ + m_alloc_offset = 0; +} + +size_t Allocator::getFreeSize() const +{ + return m_size - m_alloc_offset; +} + +bool Allocator::lowThresholdReached() const +{ + return (m_size - m_alloc_offset) < m_threshold; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/allocator.hpp b/VNFs/DPPD-PROX/tools/flow_extract/allocator.hpp new file mode 100644 index 00000000..d3f1537e --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/allocator.hpp @@ -0,0 +1,38 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _ALLOCATOR_H_ +#define _ALLOCATOR_H_ + +#include <cstddef> +#include <inttypes.h> + +class Allocator { +public: + Allocator(size_t size, size_t lowThreshold); + ~Allocator(); + bool lowThresholdReached() const; + void *alloc(size_t size); + void reset(); + size_t getFreeSize() const; +private: + size_t m_size; + size_t m_threshold; + size_t m_alloc_offset; + uint8_t *m_mem; +}; + +#endif /* _ALLOCATOR_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/bundle.cpp b/VNFs/DPPD-PROX/tools/flow_extract/bundle.cpp new file mode 100644 index 00000000..abeaf14e --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/bundle.cpp @@ -0,0 +1,28 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include "bundle.hpp" + +void Bundle::toLua(ofstream *f, const string& streamTableName, uint32_t idx) const +{ + (*f) << "bundles[" << idx << "] = {"; + + for(vector<uint32_t>::const_iterator i = streams.begin(); i != streams.end(); ++i) { + (*f) << streamTableName << "[" << (*i) << "]," ; + } + + (*f) << "}" << endl; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/bundle.hpp b/VNFs/DPPD-PROX/tools/flow_extract/bundle.hpp new file mode 100644 index 00000000..cb5d81b6 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/bundle.hpp @@ -0,0 +1,38 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _BUNDLE_H_ +#define _BUNDLE_H_ + +#include <vector> +#include <inttypes.h> +#include <fstream> + +using namespace std; + +class Bundle +{ +public: + void addStream(uint32_t streamId, uint32_t port) {streams.push_back(streamId); ports.push_back(port);} + const vector<uint32_t>& getStream() const {return streams;} + const vector<uint32_t>& getPorts() const {return ports;} + void toLua(ofstream *f, const string& streamTableName, uint32_t idx) const; +private: + vector<uint32_t> streams; + vector<uint32_t> ports; +}; + +#endif /* _BUNDLE_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/crc.hpp b/VNFs/DPPD-PROX/tools/flow_extract/crc.hpp new file mode 100644 index 00000000..713b4abd --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/crc.hpp @@ -0,0 +1,51 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _CRC_H_ +#define _CRC_H_ + +static uint32_t crc32(const uint8_t *buf, size_t len, int init) +{ + uint32_t ret = init; + + while (len/8) { + ret = __builtin_ia32_crc32di(ret, *((uint64_t*)buf)); + len -= 8; + buf += 8; + } + + while (len/4) { + ret = __builtin_ia32_crc32si(ret, *((uint32_t*)buf)); + len -= 4; + buf += 4; + } + + while (len/2) { + ret = __builtin_ia32_crc32hi(ret, *((uint16_t*)buf)); + len -= 2; + buf += 2; + } + + while (len) { + ret = __builtin_ia32_crc32qi(ret, *((uint8_t*)buf)); + len -= 1; + buf += 1; + } + + return ret; +} + +#endif /* _CRC_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.cpp b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.cpp new file mode 100644 index 00000000..909fc94d --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.cpp @@ -0,0 +1,67 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <iostream> +#include <cstdlib> +#include <cstring> +#include <stdint.h> + +#include "csvfilereader.hpp" + +int CsvFileReader::open(const string& str) +{ + char *resolved_path = new char[1024]; + + memset(resolved_path, 0, 1024); + realpath(str.c_str(), resolved_path); + file.open(resolved_path); + + delete []resolved_path; + return file.is_open(); +} + +vector<string> CsvFileReader::read() +{ + vector<string> ret; + size_t prev = 0, cur = 0; + string line; + + if (file.eof()) + return vector<string>(); + + std::getline(file, line); + if (line.empty()) + return vector<string>(); + + while (true) { + cur = line.find_first_of(',', prev); + + if (cur != SIZE_MAX) { + ret.push_back(line.substr(prev, cur - prev)); + prev = cur + 1; + } + else { + ret.push_back(line.substr(prev, line.size() - prev)); + break; + } + } + return ret; +} + +void CsvFileReader::close() +{ + file.close(); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.hpp b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.hpp new file mode 100644 index 00000000..21f397a7 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/csvfilereader.hpp @@ -0,0 +1,35 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _CSVFILE_H_ +#define _CSVFILE_H_ + +#include <fstream> +#include <string> +#include <vector> + +using namespace std; + +class CsvFileReader { +public: + int open(const string& str); + vector<string> read(); + void close(); +private: + ifstream file; +}; + +#endif /* _CSVFILE_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/flowtable.hpp b/VNFs/DPPD-PROX/tools/flow_extract/flowtable.hpp new file mode 100644 index 00000000..ebb4d927 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/flowtable.hpp @@ -0,0 +1,174 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _FLOWTABLE_H_ +#define _FLOWTABLE_H_ + +#include <inttypes.h> +#include <sys/time.h> +#include <stdio.h> +#include <cstring> + +#include <vector> +#include <list> +#include <cstddef> +#include <utility> + +#include "crc.hpp" +#include "timestamp.hpp" + +using namespace std; + +template <typename K, typename T> +class FlowTable { +public: + struct entry { + entry(K key, T value, const struct timeval& tv, list<struct entry> *parent) : + key(key), value(value), tv(tv), parent(parent) {} + bool expired(const Timestamp &now, const Timestamp &maxDiff) const + { + return now - Timestamp(tv) > maxDiff; + } + K key; + T value; + struct timeval tv; /* List time entry has been hit */ + list<struct entry> *parent; + }; + class Iterator { + friend class FlowTable; + public: + bool operator!=(const Iterator& other) { + return m_v != other.m_v || + m_vec_pos != other.m_vec_pos || + m_a != other.m_a; + + } + Iterator& operator++() { + m_a++; + while (m_vec_pos != m_v->size() - 1 && m_a == (*m_v)[m_vec_pos].end()) { + m_vec_pos++; + m_a = (*m_v)[m_vec_pos].begin(); + } + + return *this; + } + struct entry &operator*() { + return *m_a; + } + private: + Iterator(uint32_t vec_pos, vector<list<struct entry> > *v) + : m_vec_pos(vec_pos), m_v(v) + { + m_a = (*m_v)[vec_pos].begin(); + while (m_vec_pos != m_v->size() - 1 && m_a == (*m_v)[m_vec_pos].end()) { + m_vec_pos++; + m_a = (*m_v)[m_vec_pos].begin(); + } + } + Iterator(uint32_t vec_pos, vector<list<struct entry> > *v, const typename list< struct entry>::iterator& a) + : m_vec_pos(vec_pos), m_v(v), m_a(a) + { } + uint32_t m_vec_pos; + vector<list<struct entry> > *m_v; + typename list<struct entry>::iterator m_a; + }; + uint32_t getEntryCount() const {return m_entryCount;} + FlowTable(uint32_t size); + void expire(const struct timeval& tv); + struct entry* lookup(const K& key); + void remove(struct FlowTable<K,T>::entry* entry); + struct entry* insert(const K& key, const T& value, const struct timeval& tv); + Iterator begin() {return Iterator(0, &m_elems);} + Iterator end() {return Iterator(m_elems.size() - 1, &m_elems, m_elems.back().end());} + void clear(); +private: + void clearBucket(list<struct entry> *l); + vector<list<struct entry> > m_elems; + uint32_t m_entryCount; +}; + +template <typename K, typename T> +FlowTable<K, T>::FlowTable(uint32_t size) + : m_elems(), m_entryCount(0) + +{ + m_elems.resize(size); +} + +template <typename K, typename T> +struct FlowTable<K, T>::entry* FlowTable<K, T>::lookup(const K& key) +{ + uint32_t ret = crc32((uint8_t*)&key, sizeof(K), 0); + + list<struct entry> &l = m_elems[ret % m_elems.size()]; + + if (l.empty()) + return NULL; + + for (typename list<struct entry>::iterator it = l.begin(); it != l.end(); ++it) { + if (memcmp(&((*it).key), &key, sizeof(key)) == 0) + return &(*it); + } + return NULL; +} + +template <typename K, typename T> +struct FlowTable<K, T>::entry *FlowTable<K, T>::insert(const K& key, const T& value, const struct timeval& tv) +{ + uint32_t ret = crc32((uint8_t*)&key, sizeof(K), 0); + list<struct entry> &l = m_elems[ret % m_elems.size()]; + + l.push_back(entry(key, value, tv, &l)); + + struct entry &n = l.back(); + m_entryCount++; + n.key = key; + n.value = value; + return &n; +} + +template <typename K, typename T> +void FlowTable<K, T>::remove(struct FlowTable<K,T>::entry* entry) +{ + list<struct entry> &l = *entry->parent; + + for (typename list<struct entry>::iterator it = l.begin(); it != l.end(); ++it) { + if (memcmp(&((*it).key), &entry->key, sizeof(entry->key)) == 0) { + l.erase(it); + m_entryCount--; + return ; + } + } +} + +template <typename K, typename T> +void FlowTable<K, T>::clearBucket(list<struct entry> *l) +{ + while (!l->empty()) { + m_entryCount--; + l->erase(l->begin()); + } +} + +template <typename K, typename T> +void FlowTable<K, T>::clear() +{ + for (size_t i = 0; i < m_elems.size(); ++i) { + clearBucket(&m_elems[i]); + } +} + +#endif /* _FLOWTABLE_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/halfstream.cpp b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.cpp new file mode 100644 index 00000000..7d8f1fe2 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.cpp @@ -0,0 +1,101 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <fstream> +#include <arpa/inet.h> + +#include "halfstream.hpp" + +HalfStream::Action::Part HalfStream::addPkt(const PcapPkt &pkt) +{ + const uint32_t pktId = pkts.size(); + const uint8_t *l5; + uint32_t l5Len; + uint16_t tmpHdrLen; + + const struct PcapPkt::tcp_hdr *tcp; + + struct pkt_tuple pt = pkt.parsePkt((const uint8_t **)&tcp, &tmpHdrLen, &l5, &l5Len); + + if (pt.proto_id == IPPROTO_TCP) { + if (tcp->tcp_flags & 0x02) + tcpOpen = true; + if (tcp->tcp_flags & 0x01) + tcpClose = true; + } + + if (pkts.empty()) { + first = pkt.ts(); + hdrLen = tmpHdrLen; + memcpy(hdr, pkt.payload(), hdrLen); + } + last = pkt.ts(); + totLen += pkt.len(); + contentLen += l5Len; + + pkts.push_back(pkt); + + return Action::Part(pktId, l5 - pkt.payload(), l5Len); +} + +double HalfStream::getRate() const +{ + if (pkts.empty()) + return 0; + if (first == last) + return 1250000000; + + return totLen / (last - first); +} + +HalfStream::Action::Action(HalfStream* stream, const Part &p, bool isClient) + : halfStream(stream), m_isClient(isClient) +{ + addPart(p); +} + +void HalfStream::Action::addPart(const Part &p) +{ + parts.push_back(p); +} + +uint32_t HalfStream::Action::totLen() const +{ + uint32_t ret = 0; + + for (list<Part>::const_iterator i = parts.begin(); i != parts.end(); ++i) { + ret += (*i).len; + } + + return ret; +} + +void HalfStream::Action::toFile(ofstream *f) const +{ + for (list<Part>::const_iterator i = parts.begin(); i != parts.end(); ++i) { + const PcapPkt &pkt = halfStream->pkts[i->pktId]; + const uint8_t *payload = &pkt.payload()[i->offset]; + const uint16_t len = i->len; + + f->write((const char *)payload, len); + } +} + +HalfStream::HalfStream() + : first(0, 0), last(0, 0), totLen(0), hdrLen(0), contentLen(0), tcpOpen(false), tcpClose(false) +{ + +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/halfstream.hpp b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.hpp new file mode 100644 index 00000000..6216979d --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/halfstream.hpp @@ -0,0 +1,63 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <inttypes.h> +#include <list> +#include <vector> + +#include "timestamp.hpp" +#include "pcappkt.hpp" + +struct HalfStream { + struct Action { + public: + struct Part { + Part(uint32_t pktId, uint32_t offset, uint32_t len) + : pktId(pktId), offset(offset), len(len) {} + uint32_t pktId; + uint32_t offset; + uint32_t len; + }; + + Action(HalfStream* stream, const Part &p, bool isClient); + void addPart(const Part& p); + bool isClient() const {return m_isClient;} + /* An action can consist of multiple + packets. The data is not stored in the + action. Instead, a packet id together with + an offset into the packet and a length is + kept to save space */ + void toFile(ofstream* f) const; + uint32_t totLen() const; + private: + HalfStream *halfStream; + bool m_isClient; + list<Part> parts; + }; + + HalfStream(); + Timestamp first; + Timestamp last; + uint64_t totLen; + uint64_t hdrLen; + uint8_t hdr[64]; + vector<PcapPkt> pkts; + uint64_t contentLen; + bool tcpOpen; + bool tcpClose; + Action::Part addPkt(const PcapPkt &pkt); + double getRate() const; +}; diff --git a/VNFs/DPPD-PROX/tools/flow_extract/main.cpp b/VNFs/DPPD-PROX/tools/flow_extract/main.cpp new file mode 100644 index 00000000..d1476c5f --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/main.cpp @@ -0,0 +1,37 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <inttypes.h> +#include <cstdlib> + +#include "streamextract.hpp" + +using namespace std; + +int main(int argc, char *argv[]) +{ + ProgramConfig programConfig; + + if (programConfig.parseOptions(argc, argv)) { + cerr << programConfig.getError() << endl; + cerr << programConfig.getUsage() << endl; + return EXIT_FAILURE; + } + + StreamExtract se(programConfig); + + return se.run(); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.cpp b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.cpp new file mode 100644 index 00000000..b2d1a9da --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.cpp @@ -0,0 +1,109 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <cstdlib> +#include <cstdio> +#include <unistd.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <iostream> +#include <cerrno> +#include <sys/mman.h> +#include <cstring> + +#include "mappedfile.hpp" + +static void zeroOutFile(int fd, size_t size) +{ + void *empty = calloc(1, 4096); + + while (size > 4096) { + write(fd, empty, 4096); + size -= 4096; + } + write(fd, empty, size); + free(empty); +} + +int MappedFile::open(const string& filePath, size_t size) +{ + mappedFileSize = size; + + fd = ::open(filePath.c_str(), O_RDWR | O_TRUNC | O_CREAT, S_IRUSR | S_IWUSR); + if (fd < 0) { + cerr << "Failed to open file " << filePath << ":" << strerror(errno) << endl; + return -1; + } + + zeroOutFile(fd, size); + data = mmap(NULL, mappedFileSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); + + if (data == MAP_FAILED) { + cerr << "Failed to map file: " << strerror(errno) << endl; + return -1; + } + return 0; +} + +static size_t getFileSize(const string& filePath) +{ + struct stat s; + if (stat(filePath.c_str(), &s)) + return -1; + + return s.st_size; +} + +int MappedFile::open(const string& filePath) +{ + mappedFileSize = getFileSize(filePath); + + fd = ::open(filePath.c_str(), O_RDONLY); + if (fd < 0) { + cerr << "Failed to open file " << filePath << ":" << strerror(errno) << endl; + return -1; + } + + data = mmap(NULL, mappedFileSize, PROT_READ, MAP_SHARED, fd, 0); + + if (data == MAP_FAILED) { + cerr << "Failed to map file: " << strerror(errno) << endl; + return -1; + } + return 0; +} + +int MappedFile::sync() +{ + if (msync(data, mappedFileSize, MS_SYNC) == -1) { + cerr << "Failed to sync: " << strerror(errno) << endl; + return -1; + } + return 0; +} + + +void MappedFile::close() +{ + sync(); + munmap(data, mappedFileSize); + ::close(fd); +} + +size_t MappedFile::size() const +{ + return mappedFileSize; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.hpp b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.hpp new file mode 100644 index 00000000..7bf79df5 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/mappedfile.hpp @@ -0,0 +1,40 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _MAPPEDFILE_H_ +#define _MAPPEDFILE_H_ + +#include <inttypes.h> +#include <string> + +using namespace std; + +class MappedFile { +public: + int open(const string& filePath, size_t size); + int open(const string& filePath); + void close(); + int sync(); + uint8_t* getMapBeg() {return (uint8_t *)data;} + uint8_t* getMapEnd() {return (uint8_t *)data + mappedFileSize;} + size_t size() const; +private: + int fd; + size_t mappedFileSize; + void *data; +}; + +#endif /* _MAPPEDFILE_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/memreader.cpp b/VNFs/DPPD-PROX/tools/flow_extract/memreader.cpp new file mode 100644 index 00000000..df77631c --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/memreader.cpp @@ -0,0 +1,106 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <cstdlib> + +#include "memreader.hpp" +#include "mappedfile.hpp" +#include "stream3.hpp" + +MemReader::MemReader(MappedFile *file, const vector<size_t> &offsets) +{ + initRanges(file->getMapBeg(), file->getMapEnd(), offsets); +} + +bool MemReader::read(Stream3 *stream) +{ + if (ranges.empty()) + return false; + + readStream(stream, getLowestID()); + removeEmptyRanges(); + return true; +} + +uint32_t MemReader::getLowestID() const +{ + uint32_t lowestID = UINT32_MAX; + uint32_t rangeID; + + for (size_t i = 0; i < ranges.size(); ++i) { + rangeID = Stream3::getIDFromMem(ranges[i].first); + if (rangeID < lowestID) + lowestID = rangeID; + } + return lowestID; +} + +void MemReader::readStream(Stream3 *stream, uint32_t id) +{ + stream->removeAllPackets(); + stream->setID(id); + + size_t len = 0; + for (size_t i = 0; i < ranges.size(); ++i) { + if (Stream3::getIDFromMem(ranges[i].first) == id) { + stream->addFromMemory(ranges[i].first, &len); + ranges[i].first += len; + } + } +} + +void MemReader::removeEmptyRanges() +{ + vector<pair <uint8_t *, uint8_t *> > original = ranges; + size_t destinationIdx = 0; + + for (size_t i = 0; i < original.size(); ++i) { + if (original[i].first < original[i].second) + ranges[destinationIdx++] = original[i]; + } + ranges.resize(destinationIdx); +} + +void MemReader::initRanges(uint8_t *begin, uint8_t *end, const vector<size_t> &offsets) +{ + ranges.resize(offsets.size()); + + totalLength = 0; + for (size_t i = 0; i < offsets.size(); ++i) { + ranges[i].first = begin + offsets[i]; + if (i != offsets.size() - 1) + ranges[i].second = begin + offsets[i + 1]; + else + ranges[i].second = end; + totalLength += ranges[i].second - ranges[i].first; + } + removeEmptyRanges(); +} + +size_t MemReader::getRangeLengths() const +{ + size_t total = 0; + + for (size_t i = 0; i < ranges.size(); ++i) { + total += ranges[i].second - ranges[i].first; + } + return total; +} + +size_t MemReader::consumed() const +{ + return totalLength - getRangeLengths(); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/memreader.hpp b/VNFs/DPPD-PROX/tools/flow_extract/memreader.hpp new file mode 100644 index 00000000..31be4c31 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/memreader.hpp @@ -0,0 +1,45 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _MEMREADER_H_ +#define _MEMREADER_H_ + +#include <vector> +#include <inttypes.h> + +using namespace std; + +class Stream3; +class MappedFile; + +class MemReader { +public: + MemReader(MappedFile *file, const vector<size_t> &offsets); + bool read(Stream3 *stream); + size_t getTotalLength() const {return totalLength;} + size_t consumed() const; +private: + size_t getRangeLengths() const; + uint32_t getLowestID() const; + void removeEmptyRanges(); + void readStream(Stream3 *stream, uint32_t id); + void initRanges(uint8_t *begin, uint8_t *end, const vector<size_t> &offsets); + + size_t totalLength; + vector<pair <uint8_t *, uint8_t *> > ranges; +}; + +#endif /* _MEMREADER_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/netsocket.cpp b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.cpp new file mode 100644 index 00000000..8c61ba7d --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.cpp @@ -0,0 +1,33 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include "netsocket.hpp" + +NetSocket::NetSocket(uint32_t host, uint16_t port) + : host(host), port(port) +{ + +} + +bool NetSocket::operator>(const NetSocket& other) const +{ + return host > other.host || (host == other.host && port > other.port); +} + +bool NetSocket::operator<(const NetSocket& other) const +{ + return host < other.host || (host == other.host && port < other.port); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/netsocket.hpp b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.hpp new file mode 100644 index 00000000..bfd6bec9 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/netsocket.hpp @@ -0,0 +1,31 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _NETSOCKET_H_ +#define _NETSOCKET_H_ + +#include <inttypes.h> + +struct NetSocket { + NetSocket() {} + NetSocket(uint32_t host, uint16_t port); + bool operator>(const NetSocket& other) const; + bool operator<(const NetSocket& other) const; + uint32_t host; + uint16_t port; +}; + +#endif /* _NETSOCKET_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/path.cpp b/VNFs/DPPD-PROX/tools/flow_extract/path.cpp new file mode 100644 index 00000000..7d94aae6 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/path.cpp @@ -0,0 +1,97 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <iostream> +#include <iomanip> +#include <sys/stat.h> +#include <sstream> +#include <fstream> + +#include "path.hpp" + +bool Path::isDir() const +{ + struct stat s = { 0 }; + + if (stat(path.c_str(), &s)) { + return false; + } + + return s.st_mode & S_IFDIR; +} + +bool Path::isFile() const +{ + struct stat s = { 0 }; + + if (stat(path.c_str(), &s)) { + return false; + } + + return s.st_mode & S_IFREG; +} + +Path Path::add(const string& str) const +{ + stringstream ss; + + ss << path << str; + + return Path(ss.str()); +} + +Path Path::add(int number) const +{ + stringstream ss; + + ss << path << number; + + return Path(ss.str()); +} + +Path &Path::concat(const string &add) +{ + stringstream ss; + + ss << path << add; + path = ss.str(); + + return *this; +} + +int Path::mkdir() const +{ + if (!isDir()) + return ::mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + return 0; +} + +std::ostream& operator<<(std::ofstream &stream, const Path &p) +{ + stream << p.path.c_str(); + + return stream; +} + +string Path::getFileName() const +{ + for (size_t i = path.size() - 1; i >= 0; --i) { + if (path[i] == '/') { + return path.substr(i + 1); + } + } + return path; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/path.hpp b/VNFs/DPPD-PROX/tools/flow_extract/path.hpp new file mode 100644 index 00000000..e56c9050 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/path.hpp @@ -0,0 +1,42 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _PATH_H_ +#define _PATH_H_ + +#include <string> + +using namespace std; + +class Path { +public: + Path(); + Path(const Path& other) : path(other.path) {} + Path(const string& str) : path(str) {} + Path add(const string& str) const; + Path add(int number) const; + Path &concat(const string &str); + const string& str() const {return path;} + bool isDir() const; + bool isFile() const; + string getFileName() const; + int mkdir() const; + friend std::ostream& operator<<(std::ofstream &stream, const Path &path); +private: + string path; +}; + +#endif /* _PATH_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.cpp new file mode 100644 index 00000000..91708bb1 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.cpp @@ -0,0 +1,266 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <pcap.h> +#include <inttypes.h> +#include <cstring> +#include <arpa/inet.h> +#include <iostream> +#include <fstream> +#include <cstdlib> +#include "allocator.hpp" +#include "pcappkt.hpp" + +Allocator *PcapPkt::allocator = NULL; + +void* PcapPkt::operator new(size_t size) +{ + if (allocator) + return allocator->alloc(size); + else + return ::operator new(size); +} + +void PcapPkt::operator delete(void *pointer) +{ + if (!allocator) + :: operator delete(pointer); +} + +PcapPkt::PcapPkt(uint8_t *mem) +{ + header = *(struct pcap_pkthdr *)mem; + mem += sizeof(header); + buf = new uint8_t[header.len]; + memcpy(buf, mem, header.len); +} + +PcapPkt::PcapPkt() +{ + buf = new uint8_t[1514]; + memset(&header, 0, sizeof(header)); +} + +PcapPkt::PcapPkt(const PcapPkt& other) +{ + if (!allocator) { + buf = new uint8_t[other.len()]; + } + else { + buf = (uint8_t *)allocator->alloc(other.len()); + } + + memcpy(buf, other.buf, other.len()); + header = other.header; +} + +PcapPkt::~PcapPkt() +{ + if (!allocator) + delete[] buf; +} + +#define ETYPE_IPv4 0x0008 /* IPv4 in little endian */ +#define ETYPE_IPv6 0xDD86 /* IPv6 in little endian */ +#define ETYPE_ARP 0x0608 /* ARP in little endian */ +#define ETYPE_VLAN 0x0081 /* 802-1aq - VLAN */ +#define ETYPE_MPLSU 0x4788 /* MPLS unicast */ +#define ETYPE_MPLSM 0x4888 /* MPLS multicast */ +#define ETYPE_8021ad 0xA888 /* Q-in-Q */ +#define ETYPE_LLDP 0xCC88 /* Link Layer Discovery Protocol (LLDP) */ +#define ETYPE_EoGRE 0x5865 /* EoGRE in little endian */ + +struct ipv4_hdr { + uint8_t version_ihl; /**< version and header length */ + uint8_t type_of_service; /**< type of service */ + uint16_t total_length; /**< length of packet */ + uint16_t packet_id; /**< packet ID */ + uint16_t fragment_offset; /**< fragmentation offset */ + uint8_t time_to_live; /**< time to live */ + uint8_t next_proto_id; /**< protocol ID */ + uint16_t hdr_checksum; /**< header checksum */ + uint32_t src_addr; /**< source address */ + uint32_t dst_addr; /**< destination address */ +} __attribute__((__packed__)); + +struct ether_addr { + uint8_t addr_bytes[6]; /**< Address bytes in transmission order */ +} __attribute__((__packed__)); + +struct ether_hdr { + struct ether_addr d_addr; /**< Destination address. */ + struct ether_addr s_addr; /**< Source address. */ + uint16_t ether_type; /**< Frame type. */ +} __attribute__((__packed__)); + +struct vlan_hdr { + uint16_t vlan_tci; /**< Priority (3) + CFI (1) + Identifier Code (12) */ + uint16_t eth_proto;/**< Ethernet type of encapsulated frame. */ +} __attribute__((__packed__)); + +struct udp_hdr { + uint16_t src_port; /**< UDP source port. */ + uint16_t dst_port; /**< UDP destination port. */ + uint16_t dgram_len; /**< UDP datagram length */ + uint16_t dgram_cksum; /**< UDP datagram checksum */ +} __attribute__((__packed__)); + +struct pkt_tuple PcapPkt::parsePkt(const uint8_t **l4_hdr, uint16_t *hdr_len, const uint8_t **l5, uint32_t *l5_len) const +{ + struct pkt_tuple pt = {0}; + + const struct ether_hdr *peth = (struct ether_hdr *)buf; + int l2_types_count = 0; + const struct ipv4_hdr* pip = 0; + + switch (peth->ether_type) { + case ETYPE_IPv4: + pip = (const struct ipv4_hdr *)(peth + 1); + break; + case ETYPE_VLAN: { + const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1); + if (vlan->eth_proto == ETYPE_IPv4) { + pip = (const struct ipv4_hdr *)(peth + 1); + } + else if (vlan->eth_proto == ETYPE_VLAN) { + const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1); + if (vlan->eth_proto == ETYPE_IPv4) { + pip = (const struct ipv4_hdr *)(peth + 1); + } + else if (vlan->eth_proto == ETYPE_IPv6) { + throw 0; + } + else { + /* TODO: handle BAD PACKET */ + throw 0; + } + } + } + break; + case ETYPE_8021ad: { + const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1); + if (vlan->eth_proto == ETYPE_VLAN) { + const struct vlan_hdr *vlan = (const struct vlan_hdr *)(peth + 1); + if (vlan->eth_proto == ETYPE_IPv4) { + pip = (const struct ipv4_hdr *)(peth + 1); + } + else { + throw 0; + } + } + else { + throw 0; + } + } + break; + case ETYPE_MPLSU: + break; + default: + break; + } + + /* L3 */ + if ((pip->version_ihl >> 4) == 4) { + + if ((pip->version_ihl & 0x0f) != 0x05) { + /* TODO: optional fields */ + throw 0; + } + + pt.proto_id = pip->next_proto_id; + pt.src_addr = pip->src_addr; + pt.dst_addr = pip->dst_addr; + } + else { + /* TODO: IPv6 and bad packets */ + throw 0; + } + + /* L4 parser */ + if (pt.proto_id == IPPROTO_UDP) { + const struct udp_hdr *udp = (const struct udp_hdr*)(pip + 1); + if (l4_hdr) + *l4_hdr = (const uint8_t*)udp; + if (hdr_len) + *hdr_len = (const uint8_t*)udp - buf; + pt.src_port = udp->src_port; + pt.dst_port = udp->dst_port; + if (l5) + *l5 = ((const uint8_t*)udp) + sizeof(struct udp_hdr); + if (l5_len) + *l5_len = ntohs(udp->dgram_len) - sizeof(struct udp_hdr); + } + else if (pt.proto_id == IPPROTO_TCP) { + const struct tcp_hdr *tcp = (const struct tcp_hdr *)(pip + 1); + if (l4_hdr) + *l4_hdr = (const uint8_t*)tcp; + if (hdr_len) + *hdr_len = (const uint8_t*)tcp - buf; + pt.src_port = tcp->src_port; + pt.dst_port = tcp->dst_port; + + if (l5) + *l5 = ((const uint8_t*)tcp) + ((tcp->data_off >> 4)*4); + if (l5_len) + *l5_len = ntohs(pip->total_length) - sizeof(struct ipv4_hdr) - ((tcp->data_off >> 4)*4); + } + else { + fprintf(stderr, "unsupported protocol %d\n", pt.proto_id); + throw 0; + } + + return pt; +} + +void PcapPkt::toMem(uint8_t *mem) const +{ + memcpy(mem, &header, sizeof(header)); + mem += sizeof(header); + memcpy(mem, buf, header.len); +} + +void PcapPkt::fromMem(uint8_t *mem) +{ + memcpy(&header, mem, sizeof(header)); + mem += sizeof(header); + memcpy(buf, mem, header.len); +} + +void PcapPkt::toFile(ofstream *file) const +{ + file->write(reinterpret_cast<const char *>(&header), sizeof(header)); + file->write(reinterpret_cast<const char *>(buf), header.len); +} +size_t PcapPkt::memSize() const +{ + return sizeof(header) + header.len; +} + +PcapPkt::L4Proto PcapPkt::getProto() const +{ + struct pkt_tuple pt = parsePkt(); + return pt.proto_id == IPPROTO_TCP? PROTO_TCP : PROTO_UDP; +} + +ostream& operator<<(ostream& stream, const pkt_tuple &other) +{ + stream << other.src_addr << "," + << other.dst_addr << "," + << (int)other.proto_id << "," + << other.src_port << "," + << other.dst_port; + return stream; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.hpp new file mode 100644 index 00000000..e437c790 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappkt.hpp @@ -0,0 +1,104 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _PCAPPKT_H_ +#define _PCAPPKT_H_ + +#include <inttypes.h> +#include <pcap.h> +#include <string> +#include <cstring> + +using namespace std; + +struct pkt_tuple { + uint32_t src_addr; + uint32_t dst_addr; + uint8_t proto_id; + uint16_t src_port; + uint16_t dst_port; + bool operator!=(const pkt_tuple& other) const + { + return src_addr != other.src_addr || + dst_addr != other.dst_addr || + proto_id != other.proto_id || + src_port != other.src_port || + dst_port != other.dst_port; + } + bool operator==(const pkt_tuple& other) const + { + return src_addr == other.src_addr && + dst_addr == other.dst_addr && + proto_id == other.proto_id && + src_port == other.src_port && + dst_port == other.dst_port; + } + friend ostream& operator<<(ostream& stream, const pkt_tuple &other); + struct pkt_tuple flip() const + { + struct pkt_tuple ret; + + ret = *this; + ret.src_addr = dst_addr; + ret.src_port = dst_port; + ret.dst_addr = src_addr; + ret.dst_port = src_port; + return ret; + } + +} __attribute__((packed)); + +class Allocator; + +class PcapPkt { + friend class PcapReader; +public: + struct tcp_hdr { + uint16_t src_port; /**< TCP source port. */ + uint16_t dst_port; /**< TCP destination port. */ + uint32_t sent_seq; /**< TX data sequence number. */ + uint32_t recv_ack; /**< RX data acknowledgement sequence number. */ + uint8_t data_off; /**< Data offset. */ + uint8_t tcp_flags; /**< TCP flags */ + uint16_t rx_win; /**< RX flow control window. */ + uint16_t cksum; /**< TCP checksum. */ + uint16_t tcp_urp; /**< TCP urgent pointer, if any. */ + } __attribute__((__packed__)); + + static Allocator *allocator; + enum L4Proto {PROTO_TCP, PROTO_UDP}; + PcapPkt(); + void* operator new(size_t size); + static void operator delete(void *pointer); + PcapPkt(const PcapPkt& other); + PcapPkt(uint8_t *mem); + void toMem(uint8_t *mem) const; + void fromMem(uint8_t *mem); + void toFile(ofstream *file) const; + size_t memSize() const; + const struct timeval &ts() const {return header.ts;} + const uint16_t len() const {return header.len;} + pkt_tuple parsePkt(const uint8_t **l4_hdr = NULL, uint16_t *hdr_len = NULL, const uint8_t **l5 = NULL, uint32_t *l5_len = NULL) const; + const struct pcap_pkthdr &hdr() const {return header;} + const uint8_t *payload() const {return buf;} + enum L4Proto getProto() const; + ~PcapPkt(); +private: + struct pcap_pkthdr header; + uint8_t *buf; +}; + +#endif /* _PCAPPKT_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.cpp new file mode 100644 index 00000000..2a0f2f05 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.cpp @@ -0,0 +1,32 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include "pcappktref.hpp" + +PcapPktRef::PcapPktRef(const PcapPktRef &other) + : pos(other.pos), pr(other.pr) +{ +} + +PcapPkt PcapPktRef::getPcapPkt() +{ + PcapPkt ret; + + if (!pr->readOnce(&ret, pos)) { + cerr << "failed to read pcap from pcap pkt ref" << endl; + } + return ret; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.hpp new file mode 100644 index 00000000..1afaf2a5 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcappktref.hpp @@ -0,0 +1,40 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _PCAPPKTREF_H_ +#define _PCAPPKTREF_H_ + +#include <iostream> + +#include "pcapreader.hpp" +#include "pcappkt.hpp" + +using namespace std; + +class PcapPktRef +{ +public: + PcapPktRef(uint64_t pos, PcapReader *pr) : pos(pos), pr(pr) {} + PcapPktRef(const PcapPktRef &other); + PcapPktRef() : pos(0), pr(0) {} + bool isValid() const {return pos != 0;} + PcapPkt getPcapPkt(); +private: + uint64_t pos; + PcapReader *pr; +}; + +#endif /* _PCAPPKTREF_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.cpp new file mode 100644 index 00000000..6b5a6734 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.cpp @@ -0,0 +1,76 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <pcap.h> +#include <cstring> +#include <linux/in.h> + +#include "pcapreader.hpp" + +int PcapReader::open(const string& file_path) +{ + char err_str[PCAP_ERRBUF_SIZE]; + + if (m_handle) { + m_error = "Pcap file already open"; + return -1; + } + + m_handle = pcap_open_offline_with_tstamp_precision(file_path.c_str(), + PCAP_TSTAMP_PRECISION_NANO, + err_str); + + if (!m_handle) { + m_error = "Failed to open pcap file"; + return -1; + } + + m_file_beg = ftell(pcap_file(m_handle)); + fseek(pcap_file(m_handle), 0, SEEK_END); + m_file_end = ftell(pcap_file(m_handle)); + fseek(pcap_file(m_handle), m_file_beg, SEEK_SET); + + return 0; +} + +int PcapReader::readOnce(PcapPkt *pkt, uint64_t pos) +{ + return -1; +} + +int PcapReader::read(PcapPkt *pkt) +{ + if (!m_handle) { + m_error = "No pcap file opened"; + } + + const uint8_t *buf = pcap_next(m_handle, &pkt->header); + + if (buf) { + memcpy(pkt->buf, buf, pkt->header.len); + pktReadCount++; + } + + return !!buf; +} + +void PcapReader::close() +{ + if (m_handle) + pcap_close(m_handle); + + m_handle = NULL; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.hpp new file mode 100644 index 00000000..3766c67b --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapreader.hpp @@ -0,0 +1,48 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _PCAPREADER_H_ +#define _PCAPREADER_H_ + +#include <inttypes.h> +#include <string> + +#include <pcap.h> + +#include "pcappkt.hpp" + +using namespace std; + +class PcapReader { +public: + PcapReader() : m_handle(NULL), pktReadCount(0) {} + int open(const string& file_path); + size_t pos() {return ftell(pcap_file(m_handle)) - m_file_beg;} + size_t end() {return m_file_end;} + int read(PcapPkt *pkt); + int readOnce(PcapPkt *pkt, uint64_t pos); + size_t getPktReadCount() const {return pktReadCount;} + void close(); + const string &getError() const {return m_error;} +private: + pcap_t *m_handle; + size_t m_file_beg; + size_t m_file_end; + size_t pktReadCount; + string m_error; +}; + +#endif /* _PCAPREADER_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.cpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.cpp new file mode 100644 index 00000000..4c7c4cea --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.cpp @@ -0,0 +1,46 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include "pcapwriter.hpp" + +int PcapWriter::open(const string& file_path) +{ + m_handle = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65536, PCAP_TSTAMP_PRECISION_NANO); + if (m_handle == NULL) + return -1; + + m_pcap_dumper = pcap_dump_open(m_handle, file_path.c_str()); + if (m_pcap_dumper == NULL) { + pcap_close(m_handle); + return -1; + } + + return 0; +} + +int PcapWriter::write(const PcapPkt& pkt) +{ + pcap_dump((unsigned char *)m_pcap_dumper, &pkt.hdr(), pkt.payload()); + return 0; +} + +void PcapWriter::close() +{ + if (m_pcap_dumper) + pcap_dump_close(m_pcap_dumper); + if (m_handle) + pcap_close(m_handle); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.hpp b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.hpp new file mode 100644 index 00000000..32f79369 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/pcapwriter.hpp @@ -0,0 +1,33 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _PCAPWRITER_H_ +#define _PCAPWRITER_H_ + +#include "pcappkt.hpp" + +class PcapWriter { +public: + PcapWriter() {} + int open(const string& file_path); + int write(const PcapPkt& pkt); + void close(); +private: + pcap_t *m_handle; + pcap_dumper_t *m_pcap_dumper; +}; + +#endif /* _PCAPWRITER_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/programconfig.cpp b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.cpp new file mode 100644 index 00000000..7b1e18e1 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.cpp @@ -0,0 +1,119 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <sstream> +#include <getopt.h> +#include <iostream> +#include <cstdlib> +#include "programconfig.hpp" + +ProgramConfig::ProgramConfig() + : path_file_in_pcap(""), path_dir_out("output"), + path_file_dest_lua("lua"), max_pkts(UINT32_MAX), + max_streams(UINT32_MAX), sampleCount(20000), flowTableSize(8*1024*1024), + run_first_step(true), write_pcaps(false) +{ +} + +string ProgramConfig::getUsage() const +{ + stringstream ret; + + ret << "Usage example: "<< m_programName << " -i in.pcap\n\n" + << "Flow Extract 2.0 analyzes and extracts a traffic profile\n" + << "configuration from a pcap file. The output is a lua\n" + << "configuration file and a binary file containing all the\n" + << "headers and payloads for each stream.\n\n" + + << "The program supports analyzing large pcap file (> 300 GB).\n" + << "For this, it uses a multi-pass approach. The output of \n" + << "intermediary steps is stored in the working directory. The\n" + << "algorithm can be described by the following steps:\n\n" + << " 1. The pcap file in read chunks of 16 GB. The packets in\n" + << " each chunk are associated with streams. The streams are\n" + << " ordered through a global ID. Each stream is stored as a" + << " sequence of packets that belong to that stream. The\n" + << " resulting file at 'DIR/tmp' where DIR is specified\n" + << " through -o options as shown below.\n" + << " Each chunk in tmp is merged and the result is written\n" + << " to file1. Reading the stream with a given ID from all chunks\n" + << " gets all the packets for the stream from the whole pcap in\n" + << " memory. This first step forms is implemented by an\n" + << " external sorting algorithm.\n" + << " 2. File2 is read and the source IP for each stream is used to\n" + << " associate each stream with a bundle. SAMPLE_COUNT samples\n" + << " are taken from the set of bundles. The set of streams that\n" + << " are still referenced by the sampled bundles extracted from\n" + << " file2 and written to the final binary file. This binary file\n" + << " is referenced from the lua configuration. The lua config file\n" + << " is written out as part of this step.\n" + << "Arguments:\n" + << "-i FILE Input pcap to process\n" + << "-o DIR output directory and working directory\n" + << "-s SAMPLE_COUNT Number of samples to take (default is 20K)\n" + << "-k Skip the first step as described above. Useful to\n" + << " adjust the number of samples without having to\n" + << " repeat the whole process\n"; + + + return ret.str(); +} + +int ProgramConfig::checkConfig() +{ + if (path_file_in_pcap.empty()) { + m_error = "Missing input pcap file\n"; + return -1; + } + return 0; +} + +int ProgramConfig::parseOptions(int argc, char *argv[]) +{ + char c; + + m_programName = argv[0]; + while ((c = getopt(argc, argv, "hki:o:s:p")) != -1) { + switch (c) { + case 'h': + return -1; + break; + case 'k': + run_first_step = false; + break; + case 'i': + path_file_in_pcap = optarg; + break; + case 'o': + path_dir_out = optarg; + break; + case 's': + sampleCount = atoi(optarg); + break; + case 'p': + write_pcaps = true; + break; + case '?': + cerr << getUsage() << endl; + return 0; + default: + m_error = "Invalid parameter\n"; + return -1; + } + } + + return checkConfig(); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/programconfig.hpp b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.hpp new file mode 100644 index 00000000..59b7104d --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/programconfig.hpp @@ -0,0 +1,47 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _PROGRAMCONFIG_H_ +#define _PROGRAMCONFIG_H_ + +#include <string> +#include <inttypes.h> + +using namespace std; + +class ProgramConfig { +public: + ProgramConfig(); + string getUsage() const; + int parseOptions(int argc, char *argv[]); + const string& getError() const {return m_error;} + + string path_file_in_pcap; + string path_dir_out; + string path_file_dest_lua; + uint32_t max_pkts; + uint32_t max_streams; + uint32_t sampleCount; + uint32_t flowTableSize; + bool run_first_step; + bool write_pcaps; +private: + int checkConfig(); + string m_error; + string m_programName; +}; + +#endif /* _PROGRAMCONFIG_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/progress.cpp b/VNFs/DPPD-PROX/tools/flow_extract/progress.cpp new file mode 100644 index 00000000..2c65960f --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/progress.cpp @@ -0,0 +1,96 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <sys/time.h> +#include <iostream> +#include <cstdio> +#include <sstream> + +#include "progress.hpp" + +static uint64_t getSec() +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return tv.tv_sec; +} + +Progress::Progress(size_t maxProgress, bool inPlace, bool showElapsedTime) + : maxProgress(maxProgress), curProgress(0), inPlace(inPlace), showElapsedTime(showElapsedTime), prevLength(0), title("Progress") +{ + lastRefresh = -1; + firstRefresh = getSec(); +} + +void Progress::setProgress(size_t curProgress) +{ + this->curProgress = curProgress; +} + +void Progress::setProgress() +{ + this->curProgress = maxProgress; +} + +uint32_t Progress::addDetail(const string& detail) +{ + details.push_back(make_pair(detail, 0)); + return details.size() - 1; +} + +void Progress::setDetail(uint32_t idx, uint32_t val) +{ + details[idx].second = val; +} + +bool Progress::couldRefresh() +{ + uint32_t cur = getSec(); + + return (lastRefresh != cur); +} + +void Progress::refresh(bool withNewLine) +{ + lastRefresh = getSec(); + uint64_t elapsed = lastRefresh - firstRefresh; + size_t progress = curProgress * 100 / maxProgress; + size_t remainingTime = curProgress? (elapsed * maxProgress - elapsed * curProgress) / curProgress : 0; + + stringstream ss; + + if (inPlace) + ss << "\r"; + ss << title << ": " << progress << "%"; + ss << ", remaining: " << remainingTime; + if (showElapsedTime) + ss << ", elapsed: " << elapsed; + for (size_t i = 0; i < details.size(); ++i) + ss << ", " << details[i].first << ": " << details[i].second; + + size_t prevLength2 = ss.str().size(); + + while (ss.str().size() < prevLength) + ss << " "; + prevLength = prevLength2; + + if (!inPlace || withNewLine) + ss << "\n"; + + cout << ss.str(); + cout.flush(); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/progress.hpp b/VNFs/DPPD-PROX/tools/flow_extract/progress.hpp new file mode 100644 index 00000000..7f55cf98 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/progress.hpp @@ -0,0 +1,50 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _PROGRESS_H_ +#define _PROGRESS_H_ + +#include <inttypes.h> +#include <vector> +#include <utility> +#include <string> + +using namespace std; + +class Progress { +public: + Progress(size_t maxProgress, bool inPlace = true, bool showElapsedTime = true); + void setTitle(const string &prefix) {this->title = title;} + void setProgress(size_t curProgress); + void setProgress(); + uint32_t addDetail(const string& detail); + void clearDetails() {details.clear();} + void setDetail(uint32_t idx, uint32_t val); + bool couldRefresh(); + void refresh(bool withNewLine = false); +private: + uint64_t firstRefresh; + uint64_t lastRefresh; + size_t maxProgress; + size_t curProgress; + bool inPlace; + bool showElapsedTime; + size_t prevLength; + string title; + vector<pair<string, uint32_t> > details; +}; + +#endif /* _PROGRESS_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream.cpp b/VNFs/DPPD-PROX/tools/flow_extract/stream.cpp new file mode 100644 index 00000000..b8056852 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/stream.cpp @@ -0,0 +1,171 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <iostream> +#include <iomanip> +#include <arpa/inet.h> + +#include "pcapwriter.hpp" +#include "stream.hpp" + +Stream::Stream(uint32_t id, uint32_t sizeHint) + : m_id(id), m_prevPktIsClient(false) +{ + m_client.pkts.reserve(sizeHint / 2); + m_server.pkts.reserve(sizeHint / 2); + m_pkts.reserve(sizeHint); +} + +bool Stream::isClient(const PcapPkt &pkt) const +{ + return m_pt == pkt.parsePkt(); +} + +size_t Stream::pktCount() const +{ + return m_client.pkts.size() + m_server.pkts.size(); +} + +void Stream::setTupleFromPkt(const PcapPkt &pkt) +{ + m_pt = pkt.parsePkt(); +} + +void Stream::addPkt(const PcapPkt &pkt) +{ + if (!pktCount()) + setTupleFromPkt(pkt); + + bool isClientPkt = isClient(pkt); + HalfStream *half; + + if (isClientPkt) + half = &m_client; + else + half = &m_server; + + HalfStream::Action::Part p = half->addPkt(pkt); + + if (p.len) { + addAction(half, p, isClientPkt); + } + + m_pkts.push_back(pkt); +} + +void Stream::addAction(HalfStream *half, HalfStream::Action::Part p, bool isClientPkt) +{ + if (m_actions.empty() || m_prevPktIsClient != isClientPkt || m_pt.proto_id == IPPROTO_UDP) + m_actions.push_back(HalfStream::Action(half, p, isClientPkt)); + else + m_actions.back().addPart(p); + m_prevPktIsClient = isClientPkt; +} + +Stream::Header Stream::getHeader() const +{ + Header h; + + h.streamId = m_id; + h.clientHdrLen = m_client.hdrLen; + h.clientContentLen = m_client.contentLen; + h.serverHdrLen = m_server.hdrLen; + h.serverContentLen = m_server.contentLen; + h.actionCount = m_actions.size(); + h.clientIP = m_pt.src_addr; + h.clientPort = m_pt.src_port; + h.serverIP = m_pt.dst_addr; + h.serverPort = m_pt.dst_port; + h.upRate = m_client.getRate(); + h.dnRate = m_server.getRate(); + h.protocol = m_pt.proto_id; + h.completedTCP = (m_client.tcpOpen && m_client.tcpClose && m_server.tcpOpen && m_server.tcpClose) || + (!m_client.tcpOpen && !m_client.tcpClose && !m_server.tcpOpen && !m_server.tcpClose); + + return h; +} + +void Stream::Header::toFile(ofstream *f) const +{ + f->write((const char *)this, sizeof(*this)); +} + +int Stream::Header::fromFile(ifstream *f) +{ + const size_t readSize = sizeof(*this); + + f->read((char *)this, readSize); + return f->gcount() == readSize? 0 : -1; +} + +size_t Stream::Header::getStreamLen() const +{ + return actionCount * sizeof(ActionEntry) + + clientHdrLen + clientContentLen + + serverHdrLen + serverContentLen; +} + +void Stream::actionsToFile(ofstream *f) const +{ + ActionEntry actionEntry; + uint32_t runningTotalLen[2] = {0}; + + for (size_t i = 0; i < m_actions.size(); ++i) { + actionEntry.peer = m_actions[i].isClient()? 0 : 1; + actionEntry.beg = runningTotalLen[actionEntry.peer]; + actionEntry.len = m_actions[i].totLen(); + + runningTotalLen[actionEntry.peer] += actionEntry.len; + f->write((const char *)&actionEntry, sizeof(actionEntry)); + } +} + +void Stream::clientHdrToFile(ofstream *f) const +{ + f->write((const char *)m_client.hdr, m_client.hdrLen); +} + +void Stream::serverHdrToFile(ofstream *f) const +{ + f->write((const char *)m_server.hdr, m_server.hdrLen); +} + +void Stream::contentsToFile(ofstream *f, bool isClient) const +{ + for (size_t i = 0; i < m_actions.size(); ++i) + if (m_actions[i].isClient() == isClient) + m_actions[i].toFile(f); +} + +void Stream::toFile(ofstream *f) +{ + getHeader().toFile(f); + actionsToFile(f); + clientHdrToFile(f); + serverHdrToFile(f); + contentsToFile(f, true); + contentsToFile(f, false); +} + +void Stream::toPcap(const string& outFile) +{ + PcapWriter pw; + + pw.open(outFile); + for (size_t i = 0; i < m_pkts.size(); ++i) + pw.write(m_pkts[i]); + pw.close(); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream.hpp b/VNFs/DPPD-PROX/tools/flow_extract/stream.hpp new file mode 100644 index 00000000..28547d18 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/stream.hpp @@ -0,0 +1,94 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _STREAM_H_ +#define _STREAM_H_ + +#include <list> +#include <string> +#include <fstream> +#include <cstring> +#include <vector> +#include <cstdlib> +#include <sys/time.h> + +#include "pcappktref.hpp" +#include "pcappkt.hpp" +#include "netsocket.hpp" +#include "timestamp.hpp" +#include "halfstream.hpp" + +using namespace std; + +class PcapReader; + +class Stream { +public: + struct Header { + uint32_t streamId; + uint16_t clientHdrLen; + uint32_t clientContentLen; + uint16_t serverHdrLen; + uint32_t serverContentLen; + uint32_t actionCount; + uint32_t clientIP; + uint16_t clientPort; + uint32_t serverIP; + uint16_t serverPort; + double upRate; + double dnRate; + uint8_t protocol; + uint8_t completedTCP; + void toFile(ofstream *f) const; + int fromFile(ifstream *f); + size_t getStreamLen() const; + }; + struct ActionEntry { + uint8_t peer; + uint32_t beg; + uint32_t len; + } __attribute__((packed)); + + Stream(uint32_t id = -1, uint32_t sizeHint = 0); + void addPkt(const PcapPkt &pkt); + void toFile(ofstream *f); + void toPcap(const string& outFile); + double getRate() const; + size_t actionCount() const {return m_actions.size();} + +private: + Header getHeader() const; + void actionsToFile(ofstream *f) const; + void clientHdrToFile(ofstream *f) const; + void serverHdrToFile(ofstream *f) const; + void contentsToFile(ofstream *f, bool isClient) const; + bool isClient(const PcapPkt &pkt) const; + size_t pktCount() const; + struct pkt_tuple m_pt; + void setTupleFromPkt(const PcapPkt &pkt); + void addToClient(const PcapPkt &pkt); + void addToServer(const PcapPkt &pkt); + void addAction(HalfStream *half, HalfStream::Action::Part p, bool isClientPkt); + + int m_id; + vector<PcapPkt> m_pkts; + vector<HalfStream::Action> m_actions; + HalfStream m_client; + HalfStream m_server; + bool m_prevPktIsClient; +}; + +#endif /* _STREAM_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream2.cpp b/VNFs/DPPD-PROX/tools/flow_extract/stream2.cpp new file mode 100644 index 00000000..51057e7d --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/stream2.cpp @@ -0,0 +1,151 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <iomanip> +#include <arpa/inet.h> +#include <sstream> + +#include "stream.hpp" +#include "stream2.hpp" + +int Stream2::fromFile(ifstream *f) +{ + m_actions.clear(); + if (streamHdr.fromFile(f)) + return -1; + if (actionsFromFile(f, streamHdr.actionCount)) + return -1; + if (setReferences(f)) + return -1; + + return 0; +} + +int Stream2::actionsFromFile(ifstream *f, size_t actionCount) +{ + m_actions.resize(actionCount); + for (size_t i = 0; i < actionCount; ++i) + f->read((char *)&m_actions[i], sizeof(Stream::ActionEntry)); + + return 0; +} + +int Stream2::setReferences(ifstream *f) +{ + size_t toRead = streamHdr.clientHdrLen + + streamHdr.serverHdrLen + + streamHdr.clientContentLen + + streamHdr.serverContentLen; + + delete [] clientServerHdrContent; + clientServerHdrContent = new uint8_t[toRead]; + f->read((char *)clientServerHdrContent, toRead); + return 0; +} + +void Stream2::calcOffsets(ofstream *out) +{ + size_t curPos = out->tellp(); + + clientHdrBeg = curPos; + serverHdrBeg = clientHdrBeg + streamHdr.clientHdrLen; + clientContentBeg = serverHdrBeg + streamHdr.serverHdrLen; + serverContentBeg = clientContentBeg + streamHdr.clientContentLen; +} + +void Stream2::toFile(ofstream *out) const +{ + size_t len = streamHdr.clientHdrLen + + streamHdr.serverHdrLen + + streamHdr.clientContentLen + + streamHdr.serverContentLen; + + out->write((const char *)clientServerHdrContent, len); +} + +static string ipToString(const uint32_t ip) +{ + uint32_t ip_ne = htonl(ip); + stringstream ss; + + ss << ((ip_ne >> 24) & 0xff) << "." + << ((ip_ne >> 16) & 0xff) << "." + << ((ip_ne >> 8) & 0xff) << "." + << (ip_ne & 0xff); + + return ss.str(); +} + +static string spaces(uint32_t count) +{ + stringstream ss; + + while (count--) + ss << " "; + return ss.str(); +} + +NetSocket Stream2::getServerNetSocket() const +{ + return NetSocket(streamHdr.serverIP, ntohs(streamHdr.serverPort)); +} + +NetSocket Stream2::getClientNetSocket() const +{ + return NetSocket(streamHdr.clientIP, ntohs(streamHdr.clientPort)); +} +void Stream2::setServerNetSocket(const NetSocket& netSocket) +{ + streamHdr.serverPort = htons(netSocket.port); + streamHdr.serverIP = netSocket.host; +} + +void Stream2::setClientNetSocket(const NetSocket& netSocket) +{ + streamHdr.clientPort = htons(netSocket.port); + streamHdr.clientIP = netSocket.host; +} +void Stream2::toLua(ofstream *f, const string& binFileName, const string& streamTableName) const + +{ + (*f) << std::fixed; + + (*f) << streamTableName << "[" << streamHdr.streamId << "] = {" << endl + << spaces(3) << "client_data = {" << endl + << spaces(6) << "header = bin_read(" << binFileName << "," << clientHdrBeg << "," << streamHdr.clientHdrLen << "), " << endl + << spaces(6) << "content = bin_read(" << binFileName << "," << clientContentBeg << "," << streamHdr.clientContentLen << "), " << endl + << spaces(3) << "}," << endl + << spaces(3) << "server_data = {" << endl + << spaces(6) << "header = bin_read(" << binFileName << "," << serverHdrBeg << "," << streamHdr.serverHdrLen << "), " << endl + << spaces(6) << "content = bin_read(" << binFileName << "," << serverContentBeg << "," << streamHdr.serverContentLen << "), " << endl + << spaces(3) << "}," << endl + << spaces(3) << "actions = {" << endl; + + for (size_t i = 0; i < m_actions.size(); ++i) { + const char *peer_str = m_actions[i].peer == 0? "client" : "server"; + + (*f) << spaces(6) << peer_str << "_content(" << m_actions[i].beg << "," << m_actions[i].len << ")," << endl; + } + + (*f) << spaces(3) << "}," << endl + << spaces(3) << "clients = {ip = ip(\"" << ipToString(streamHdr.clientIP) << "\"), port = " << ntohs(streamHdr.clientPort) << "}," << endl + << spaces(3) << "servers = {ip = ip(\"" << ipToString(streamHdr.serverIP) << "\"), port = " << ntohs(streamHdr.serverPort) << "}," << endl + << spaces(3) << "l4_proto = \"" << (streamHdr.protocol == 0x06? "tcp" : "udp") << "\"," << endl + << spaces(3) << "up_bps = " << setprecision(4) << streamHdr.upRate << "," << endl + << spaces(3) << "dn_bps = " << setprecision(4) << streamHdr.dnRate << "," << endl; + + (*f) << "}" << endl; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream2.hpp b/VNFs/DPPD-PROX/tools/flow_extract/stream2.hpp new file mode 100644 index 00000000..fd9d9c8c --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/stream2.hpp @@ -0,0 +1,54 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _STREAM2_H_ +#define _STREAM2_H_ + +#include <inttypes.h> +#include <fstream> + +#include "netsocket.hpp" + +using namespace std; + +class Stream2 { +public: + Stream2() : clientServerHdrContent(NULL) {} + ~Stream2() {delete [] clientServerHdrContent;} + int fromFile(ifstream *f); + void calcOffsets(ofstream *out); + void toFile(ofstream *out) const; + void toLua(ofstream *f, const string& binFileName, const string& streamTableName) const; + NetSocket getServerNetSocket() const; + NetSocket getClientNetSocket() const; + void setServerNetSocket(const NetSocket& netSocket); + void setClientNetSocket(const NetSocket& netSocket); + Stream::Header streamHdr; +private: + int actionsFromFile(ifstream *f, size_t actionCount); + int setReferences(ifstream *f); + + uint8_t *clientServerHdrContent; + + uint32_t clientHdrBeg; + uint32_t serverHdrBeg; + uint32_t clientContentBeg; + uint32_t serverContentBeg; + + vector<Stream::ActionEntry> m_actions; +}; + +#endif /* _STREAM2_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream3.cpp b/VNFs/DPPD-PROX/tools/flow_extract/stream3.cpp new file mode 100644 index 00000000..30c166ae --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/stream3.cpp @@ -0,0 +1,95 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <iostream> +#include <fstream> + +using namespace std; + +#include "stream3.hpp" + +Stream3::Stream3(uint32_t id, PcapPkt::L4Proto proto) + : m_id(id), m_proto(proto), m_pktCount(0), m_flushCount(0) +{ +} + +void Stream3::writeHeader(ofstream *outputFile) const +{ + outputFile->write(reinterpret_cast<const char *>(&m_id), sizeof(m_id)); + outputFile->write(reinterpret_cast<const char *>(&m_flushCount), sizeof(m_flushCount)); +} + +void Stream3::writePackets(ofstream *outputFile) const +{ + for (size_t i = 0; i < m_pkts.size(); ++i) + m_pkts[i]->toFile(outputFile); +} + +void Stream3::clearPackets() +{ + for (size_t i = 0; i < m_pkts.size(); ++i) + delete m_pkts[i]; + m_pkts.clear(); + m_flushCount = 0; +} + +void Stream3::flush(ofstream *outputFile) +{ + writeHeader(outputFile); + writePackets(outputFile); + clearPackets(); +} + +void Stream3::addPkt(const PcapPkt& pkt) +{ + m_pkts.push_back(new PcapPkt(pkt)); + m_pktCount++; + m_flushCount++; +} + +Timestamp Stream3::getTimeout() const +{ + uint32_t timeoutMinutes = m_proto == PcapPkt::PROTO_UDP? 10 : 5; + + return Timestamp(timeoutMinutes * 60, 0); +} + +uint32_t Stream3::getIDFromMem(uint8_t *mem) +{ + return *reinterpret_cast<uint32_t *>(mem); +} + +void Stream3::addFromMemory(uint8_t *mem, size_t *len) +{ + uint32_t n_pkts; + + mem += sizeof(m_id); + n_pkts = *reinterpret_cast<uint32_t *>(mem); + mem += sizeof(n_pkts); + + *len = sizeof(m_id) + sizeof(n_pkts); + for (uint32_t i = 0; i < n_pkts; ++i) { + addPkt(PcapPkt(mem)); + mem += m_pkts.back()->memSize(); + *len += m_pkts.back()->memSize(); + } +} + +void Stream3::removeAllPackets() +{ + clearPackets(); + m_pktCount = 0; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/stream3.hpp b/VNFs/DPPD-PROX/tools/flow_extract/stream3.hpp new file mode 100644 index 00000000..7e94814e --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/stream3.hpp @@ -0,0 +1,55 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _STREAM3_H_ +#define _STREAM3_H_ + +#include <inttypes.h> +#include <vector> + +#include "pcappkt.hpp" +#include "timestamp.hpp" + +using namespace std; +class Allocator; + +class Stream3 { +public: + PcapPkt::L4Proto getProto(void) const {return m_proto;} + Stream3(uint32_t id, PcapPkt::L4Proto proto); + Stream3() : m_id(UINT32_MAX), m_proto(PcapPkt::PROTO_UDP), m_pktCount(0), m_flushCount(0) {} + void addPkt(const PcapPkt& pkt); + void flush(ofstream *outputFile); + void addFromMemory(uint8_t *mem, size_t *len); + static uint32_t getIDFromMem(uint8_t *mem); + bool hasFlushablePackets() const {return !!m_flushCount;} + Timestamp getTimeout() const; + uint32_t getID() const {return m_id;} + void removeAllPackets(); + void setID(const uint32_t id) {m_id = id;} +private: + void writeHeader(ofstream *outputFile) const; + void writePackets(ofstream *outputFile) const; + void clearPackets(); + + uint32_t m_id; + PcapPkt::L4Proto m_proto; + vector<PcapPkt *> m_pkts; + uint32_t m_pktCount; + uint32_t m_flushCount; +}; + +#endif /* _STREAM3_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamextract.cpp b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.cpp new file mode 100644 index 00000000..e493ef3f --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.cpp @@ -0,0 +1,406 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <inttypes.h> +#include <string> +#include <cstdio> +#include <iostream> +#include <sys/stat.h> +#include <sys/types.h> +#include <sstream> +#include <set> +#include <arpa/inet.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/mman.h> +#include <cerrno> +#include <cstdlib> +#include <map> + +#include "path.hpp" +#include "bundle.hpp" +#include "stream.hpp" +#include "stream2.hpp" +#include "allocator.hpp" +#include "timestamp.hpp" +#include "streamextract.hpp" +#include "pcapreader.hpp" +#include "pcapwriter.hpp" +#include "flowtable.hpp" +#include "stream3.hpp" +#include "netsocket.hpp" +#include "pcappktref.hpp" +#include "progress.hpp" +#include "mappedfile.hpp" +#include "streamsorter.hpp" + +using namespace std; + +static bool is_dir(const string& path_dir_out) +{ + struct stat s = { 0 }; + + if (stat(path_dir_out.c_str(), &s)) { + return false; + } + + return s.st_mode & S_IFDIR; +} + +StreamExtract::StreamExtract(const ProgramConfig &cfg) + : ft2(cfg.flowTableSize), + streamSorter(cfg.flowTableSize, cfg.path_dir_out, 1024UL*1024*1024*8), + cfg(cfg) +{ +} + +vector<Bundle> StreamExtract::createBundles(const string& streamPath) +{ + map<uint32_t, Bundle>::iterator iterBundle; + map<uint32_t, Bundle> bundles; + set<uint32_t> servers; + + Stream2 s; + ifstream binIn; + + binIn.open(streamPath.c_str()); + binIn.seekg(0, binIn.end); + Progress progress(binIn.tellg()); + binIn.seekg(0, binIn.beg); + + while (!s.fromFile(&binIn)) { + if (progress.couldRefresh()) { + progress.setProgress(binIn.tellg()); + progress.refresh(); + } + if (!s.streamHdr.completedTCP) + continue; + if (!s.streamHdr.serverHdrLen) + continue; + /* The current implementation does not support clients + that are also servers. */ + servers.insert(s.streamHdr.serverIP); + if (servers.find(s.streamHdr.clientIP) != servers.end()) + continue; + + /* Since each application is represented as a path + graph (there is only one reply for a given request + and only one request after a given reply), each + application must run on a unique server. For this + reason, check if the socket on the server already + is occupied and if so, keep incrementing the socket + until the collision is resolved. */ + iterBundle = bundles.find(s.streamHdr.clientIP); + + if (iterBundle == bundles.end()) { + bundles.insert(make_pair(s.streamHdr.clientIP, Bundle())); + iterBundle = bundles.find(s.streamHdr.clientIP); + } + + (*iterBundle).second.addStream(s.streamHdr.streamId, s.getServerNetSocket().port); + } + + progress.setProgress(); + progress.refresh(true); + + binIn.close(); + + vector<Bundle> ret; + + ret.reserve(bundles.size()); + + for (map<uint32_t, Bundle>::const_iterator i = bundles.begin(); i != bundles.end(); ++i) + ret.push_back(i->second); + + return ret; +} + +set<uint32_t> StreamExtract::getBundleStreamIDs(const vector<Bundle*>& bundleSamples) +{ + set<uint32_t> streamIDs; + + for (size_t i = 0; i < bundleSamples.size(); ++i) { + const vector<uint32_t> &bundleStreamIDs = bundleSamples[i]->getStream(); + + for (vector<uint32_t>::const_iterator j = bundleStreamIDs.begin(); j != bundleStreamIDs.end(); ++j) { + streamIDs.insert(*j); + } + } + + return streamIDs; +} + +static size_t getRandom(size_t limit) +{ + size_t r = rand(); + size_t rand_limit = (RAND_MAX/limit)*limit; + + while (r > rand_limit) + r = rand(); + + return r % limit; +} + +static void removeFill(vector<Bundle*> *from, size_t idx) +{ + Bundle *last = from->back(); + from->pop_back(); + + if (idx != from->size()) + (*from)[idx] = last; +} + +static vector<Bundle*> takeSamples(vector<Bundle>& bundles, size_t sampleCount) +{ + vector<Bundle*> bundleSamples; + + bundleSamples.reserve(bundles.size()); + + cout << "Sampling " << sampleCount << " bundles out of " << bundles.size() << endl; + for (size_t i = 0; i < bundles.size(); ++i) + bundleSamples.push_back(&bundles[i]); + + srand(1000); + while (bundleSamples.size() > sampleCount) { + size_t r = getRandom(bundleSamples.size()); + removeFill(&bundleSamples, r); + } + return bundleSamples; +} + +static size_t replaceWithRunningTotals(vector<size_t> *streamLength) +{ + size_t runningTotal = 0; + for (size_t i = 0; i < streamLength->size(); ++i) { + size_t len = (*streamLength)[i] + sizeof(uint32_t); + (*streamLength)[i] = runningTotal; + runningTotal += len; + } + return runningTotal; +} + +static void printPorts(const vector<Bundle> &bundles) +{ + set<uint32_t> streamIDs; + + for (size_t i = 0; i < bundles.size(); ++i) { + const vector<uint32_t> &ports = bundles[i].getPorts(); + + for (size_t j = 0; j < ports.size(); ++j) { + if (j + 1 == ports.size()) + cout << ports[j] << ",END" << endl; + else + cout << ports[j] << "," << ports[j +1] << endl; + } + } +} + +string StreamExtract::createStreamPcapFileName(int id) +{ + stringstream ss; + + ss << cfg.path_dir_out << "/s" << id << ".pcap"; + + return ss.str(); +} + +int StreamExtract::writeToPcaps(const string &sourceFilePath, const set<uint32_t> &streamIDs) +{ + set<uint32_t>::const_iterator i = streamIDs.begin(); + + MappedFile mappedFile; + if (mappedFile.open(sourceFilePath)) { + cerr << "Failed to open file " << sourceFilePath << ":" << strerror(errno) << endl; + return -1; + } + + PcapPkt::allocator = NULL; + + Progress progress((uint64_t)mappedFile.getMapEnd() - (uint64_t)mappedFile.getMapBeg()); + cout << "Writing " << streamIDs.size() << " streams to pcaps" << endl; + uint8_t *data2 = mappedFile.getMapBeg(); + while (data2 < mappedFile.getMapEnd()) { + uint32_t id = *reinterpret_cast<uint32_t *>(data2); + + data2 += sizeof(id); + uint32_t pktCount = *reinterpret_cast<uint32_t *>(data2); + data2 += sizeof(pktCount); + Stream s(id, pktCount); + while (pktCount--) { + PcapPkt p(data2); + + data2 += p.memSize(); + s.addPkt(p); + } + + while (i != streamIDs.end() && (*i) < id) + i++; + if (i == streamIDs.end()) + break; + if (*i > id) + continue; + + const string pcapPath = createStreamPcapFileName(id); + + s.toPcap(pcapPath); + if (progress.couldRefresh()) { + progress.setProgress((uint64_t)data2 - (uint64_t)mappedFile.getMapBeg()); + progress.refresh(); + mappedFile.sync(); + } + } + + progress.setProgress(data2 - mappedFile.getMapBeg()); + progress.refresh(true); + + mappedFile.close(); + return 0; +} + +int StreamExtract::writeToLua(const string& binFilePath, const Path &smallFinalBin, const string& luaFilePath, const string &orderedTemp) +{ + vector<Bundle> bundles = createBundles(binFilePath); + vector<Bundle*> bundleSamples = takeSamples(bundles, cfg.sampleCount); + set<uint32_t> streamIDs = getBundleStreamIDs(bundleSamples); + + if (cfg.write_pcaps) + writeToPcaps(orderedTemp, streamIDs); + + ofstream outLua; + ofstream outSmallBin; + outLua.open(luaFilePath.c_str()); + outLua << "bf = \""<< smallFinalBin.getFileName() << "\"" << endl; + outLua << "s = {}\n"; + set<uint32_t>::iterator i = streamIDs.begin(); + + set<NetSocket> serverSockets; + ifstream binIn; + Stream2 s; + + outSmallBin.open(smallFinalBin.str().c_str()); + binIn.open(binFilePath.c_str()); + while (!s.fromFile(&binIn)) { + while (i != streamIDs.end() && (*i) < s.streamHdr.streamId) + i++; + if (i == streamIDs.end()) + break; + if (*i > s.streamHdr.streamId) + continue; + s.calcOffsets(&outSmallBin); + s.toFile(&outSmallBin); + while (serverSockets.find(s.getServerNetSocket()) != serverSockets.end()) { + NetSocket ns = s.getServerNetSocket(); + + ns.port++; + s.setServerNetSocket(ns); + } + serverSockets.insert(s.getServerNetSocket()); + + s.toLua(&outLua, "bf", "s"); + } + binIn.close(); + + uint32_t bundleCount = 0; + + outLua << "bundles = {}" << endl; + for (size_t i = 0; i < bundleSamples.size(); ++i) { + bundleSamples[i]->toLua(&outLua, "s", ++bundleCount); + } + outLua << "return bundles" << endl; + outLua.close(); + return 0; +} + +int StreamExtract::writeFinalBin(const string& sourceFilePath, const string& destFilePath) +{ + MappedFile mappedFile; + if (mappedFile.open(sourceFilePath)) { + cerr << "Failed to open file " << sourceFilePath << ":" << strerror(errno) << endl; + return -1; + } + ofstream binOut; + + binOut.open(destFilePath.c_str()); + PcapPkt::allocator = NULL; + + Progress progress((uint64_t)mappedFile.getMapEnd() - (uint64_t)mappedFile.getMapBeg()); + + int streamCount = 0; + uint8_t *data2 = mappedFile.getMapBeg(); + while (data2 < mappedFile.getMapEnd()) { + uint32_t id = *reinterpret_cast<uint32_t *>(data2); + + data2 += sizeof(id); + uint32_t pktCount = *reinterpret_cast<uint32_t *>(data2); + data2 += sizeof(pktCount); + Stream s(id, pktCount); + while (pktCount--) { + PcapPkt p(data2); + + data2 += p.memSize(); + s.addPkt(p); + } + s.toFile(&binOut); + streamCount++; + if (progress.couldRefresh()) { + progress.setProgress((uint64_t)data2 - (uint64_t)mappedFile.getMapBeg()); + progress.refresh(); + mappedFile.sync(); + } + } + + progress.setProgress(data2 - mappedFile.getMapBeg()); + progress.refresh(true); + + binOut.close(); + mappedFile.close(); + return 0; +} + +int StreamExtract::run() +{ + Path p(cfg.path_dir_out); + p.mkdir(); + + string orderedTemp = p.add("/a").str(); + + string finalBin = p.add("/b").str(); + Path smallfinalBin = p.add("/data.bin").str(); + string luaFile = p.add("/cfg.lua").str(); + + cout << "Writing to directory '" << p.str() << "'" << endl; + cout << "Ordered streams '" << orderedTemp << "'" << endl; + cout << "Final binary output '" << finalBin << "'" << endl; + cout << "lua file '" << luaFile << "' will contain " << cfg.sampleCount << " bundles" << endl; + + if (cfg.run_first_step) { + cout << "starting sorting" << endl; + streamSorter.sort(cfg.path_file_in_pcap, orderedTemp); + cout << "writing final binary file (converting format)" << endl; + if (writeFinalBin(orderedTemp, finalBin)) + return -1; + } else { + cout << "Skipping first step" << endl; + if (!Path(finalBin).isFile()) { + cerr << "File is missing:" << finalBin << endl; + return -1; + } + } + cout << "writing Lua '" << luaFile << "'" << endl; + if (writeToLua(finalBin, smallfinalBin, luaFile, orderedTemp)) + return -1; + return 0; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamextract.hpp b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.hpp new file mode 100644 index 00000000..d5dbdb05 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/streamextract.hpp @@ -0,0 +1,55 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _STREAMEXTRACT_H_ +#define _STREAMEXTRACT_H_ + +#include <string> +#include <list> +#include <map> +#include <set> + +#include "programconfig.hpp" +#include "bundle.hpp" +#include "pcapreader.hpp" +#include "flowtable.hpp" +#include "pcappkt.hpp" +#include "stream3.hpp" +#include "streamsorter.hpp" +#include "path.hpp" + +using namespace std; + +class StreamExtract { +public: + /* The size of the flow table determines the number of flows + that can be active at a given time. When a flow expires, it + is written out to a file and the memory is freed. */ + StreamExtract(const ProgramConfig &cfg); + int run(); +private: + int writeToPcaps(const string &sourceFilePath, const set<uint32_t> &streamIDs); + int writeToLua(const string& binFilePath, const Path &smallFinalBin, const string& luaFilePath, const string& orderedTemp); + int writeFinalBin(const string& sourceFilePath, const string& destFilePath); + string createStreamPcapFileName(int id); + vector<Bundle> createBundles(const string& streamPath); + set<uint32_t> getBundleStreamIDs(const vector<Bundle*>& bundleSamples); + FlowTable<pkt_tuple, Stream3> ft2; + StreamSorter streamSorter; + ProgramConfig cfg; +}; + +#endif /* _STREAMEXTRACT_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.cpp b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.cpp new file mode 100644 index 00000000..65c645e1 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.cpp @@ -0,0 +1,203 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <iostream> +#include <fstream> +#include <cstdlib> + +#include "mappedfile.hpp" +#include "memreader.hpp" +#include "streamsorter.hpp" +#include "path.hpp" +#include "allocator.hpp" +#include "pcapreader.hpp" +#include "progress.hpp" + +StreamSorter::StreamSorter(size_t flowTableSize, const string& workingDirectory, size_t memoryLimit) + : flowTableSize(flowTableSize), + workingDirectory(workingDirectory), + allocator(memoryLimit, 1024*10), + streamID(0) +{ +} + +void StreamSorter::sort(const string &inputPcapFilePath, const string &outputBinFilePath) +{ + setTempFileName(); + sortChunks(inputPcapFilePath); + mergeChunks(outputBinFilePath); +} + +void StreamSorter::sortChunks(const string &inputPcapFilePath) +{ + ofstream outputTempFile; + + outputTempFile.open(tempFilePath.c_str()); + + if (!outputTempFile.is_open()) + return ; + + PcapReader pr; + PcapPkt pkt; + + if (pr.open(inputPcapFilePath)) { + pr.getError(); + return; + } + PcapPkt::allocator = &allocator; + + Progress progress(pr.end()); + uint32_t packetDetail = progress.addDetail("packet count"); + + ft = new FlowTable<pkt_tuple, uint32_t>(flowTableSize); + resetStreams(); + + while (pr.read(&pkt)) { + processPkt(pkt); + if (progress.couldRefresh()) { + progress.setProgress(pr.pos()); + progress.setDetail(packetDetail, pr.getPktReadCount()); + progress.refresh(); + } + if (allocator.lowThresholdReached()) { + flushStreams(&outputTempFile); + } + } + progress.setProgress(); + progress.setDetail(packetDetail, pr.getPktReadCount()); + progress.refresh(true); + + pr.close(); + flushStreams(&outputTempFile); + PcapPkt::allocator = NULL; + outputTempFile.close(); + delete ft; +} + +void StreamSorter::resetStreams() +{ + streams.clear(); +} + +void StreamSorter::flushStreams(ofstream *outputTempFile) +{ + size_t flushCount = 0; + size_t offset = outputTempFile->tellp(); + + Progress progress(streams.size()); + + cout << endl; + progress.setTitle("flush "); + for (size_t i = 0; i < streams.size(); ++i) { + if (streams[i].hasFlushablePackets()) { + streams[i].flush(outputTempFile); + flushCount++; + } + + if (progress.couldRefresh()) { + progress.setProgress(i); + progress.refresh(); + } + } + progress.setProgress(); + progress.refresh(true); + + if (flushCount) + flushOffsets.push_back(offset); + allocator.reset(); +} + +Stream3 *StreamSorter::addNewStream(PcapPkt::L4Proto proto) +{ + streams.push_back(Stream3(streamID++, proto)); + return &streams.back(); +} + +FlowTable<pkt_tuple, uint32_t>::entry* StreamSorter::getFlowEntry(const PcapPkt &pkt) +{ + FlowTable<pkt_tuple, uint32_t>::entry *a; + struct pkt_tuple pt = pkt.parsePkt(); + Stream3 *stream = NULL; + + a = ft->lookup(pt.flip()); + if (!a) { + a = ft->lookup(pt); + if (!a) { + stream = addNewStream(pkt.getProto()); + + a = ft->insert(pt, stream->getID(), pkt.ts()); + } + } + + if (a->expired(pkt.ts(), streams[a->value].getTimeout())) { + ft->remove(a); + + stream = addNewStream(pkt.getProto()); + + a = ft->insert(pt, stream->getID(), pkt.ts()); + } + return a; +} + +void StreamSorter::processPkt(const PcapPkt &pkt) +{ + FlowTable<pkt_tuple, uint32_t>::entry *a; + + a = getFlowEntry(pkt); + a->tv = pkt.ts(); + streams[a->value].addPkt(pkt); +} + +void StreamSorter::mergeChunks(const string &outputBinFile) +{ + cout << "merging chunks: " << tempFilePath << " to " << outputBinFile << endl; + cout << "have " << flushOffsets.size() << " parts to merge" << endl; + MappedFile tempFile; + + if (tempFile.open(tempFilePath)) { + cerr << "failed to open temp file" << endl; + return; + } + ofstream file; + + file.open(outputBinFile.c_str()); + + if (!file.is_open()) { + cerr << "failed top open file '" << outputBinFile << "'" << endl; + return; + } + MemReader memReader(&tempFile, flushOffsets); + Stream3 stream; + + Progress progress(memReader.getTotalLength()); + + while (memReader.read(&stream)) { + stream.flush(&file); + if (progress.couldRefresh()) { + progress.setProgress(memReader.consumed()); + progress.refresh(); + } + } + + progress.setProgress(); + progress.refresh(true); + tempFile.close(); +} + +void StreamSorter::setTempFileName() +{ + tempFilePath = Path(workingDirectory).add("/tmp").str(); +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.hpp b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.hpp new file mode 100644 index 00000000..a6d3d6cd --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/streamsorter.hpp @@ -0,0 +1,47 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _STREAMSORTER_H_ +#define _STREAMSORTER_H_ + +#include "stream3.hpp" +#include "flowtable.hpp" +#include "allocator.hpp" + +class StreamSorter { +public: + StreamSorter(size_t flowTableSize, const string& workingDirectory, size_t memoryLimit); + void sort(const string &inputPcapFile, const string &outputBinFile); +private: + void sortChunks(const string &inputPcapFilePath); + void mergeChunks(const string &outputBinFilePath); + void setTempFileName(); + void processPkt(const PcapPkt &pkt); + void resetStreams(); + FlowTable<pkt_tuple, uint32_t>::entry* getFlowEntry(const PcapPkt &pkt); + void flushStreams(ofstream *outputTempFile); + Stream3 *addNewStream(PcapPkt::L4Proto proto); + size_t flowTableSize; + FlowTable<pkt_tuple, uint32_t> *ft; + vector<size_t> flushOffsets; + vector<Stream3> streams; + string tempFilePath; + const string workingDirectory; + Allocator allocator; + uint32_t streamID; +}; + +#endif /* _STREAMSORTER_H_ */ diff --git a/VNFs/DPPD-PROX/tools/flow_extract/timestamp.cpp b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.cpp new file mode 100644 index 00000000..9e91173d --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.cpp @@ -0,0 +1,65 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include <cstdio> +#include <iostream> +#include <iomanip> + +#include "timestamp.hpp" + +Timestamp Timestamp::operator-(const Timestamp& other) const +{ + uint64_t sec; + uint64_t nsec; + + if (other.m_nsec <= m_nsec) { + nsec = m_nsec - other.m_nsec; + sec = m_sec - other.m_sec; + } else { + nsec = (1000000000 + m_nsec) - other.m_nsec; + sec = m_sec - 1 - other.m_sec; + } + + return Timestamp(sec, nsec); +} + +bool Timestamp::operator>(const Timestamp& other) +{ + return m_sec > other.m_sec || + (m_sec == other.m_sec && m_nsec > other.m_nsec); +} + +bool Timestamp::operator<(const Timestamp& other) +{ + return m_sec < other.m_sec || + (m_sec == other.m_sec && m_nsec < other.m_nsec); +} + +ostream& operator<<(ostream& stream, const Timestamp& ts) +{ + stream << ts.m_sec << "." << setw(9) << setfill('0') << ts.m_nsec; + return stream; +} + +double operator/(double d, const Timestamp &denominator) +{ + return d * 1000000000 / (denominator.m_sec * 1000000000 + denominator.m_nsec); +} + +bool Timestamp::operator==(const Timestamp &other) const +{ + return m_sec == other.m_sec && m_nsec == other.m_nsec; +} diff --git a/VNFs/DPPD-PROX/tools/flow_extract/timestamp.hpp b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.hpp new file mode 100644 index 00000000..cf8ec5d4 --- /dev/null +++ b/VNFs/DPPD-PROX/tools/flow_extract/timestamp.hpp @@ -0,0 +1,45 @@ +/* +// Copyright (c) 2010-2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#ifndef _TIMESTAMP_H_ +#define _TIMESTAMP_H_ + +#include <iostream> + +#include <sys/time.h> +#include <inttypes.h> + +using namespace std; + +class Timestamp { +public: + Timestamp(const uint64_t sec, const uint64_t nsec) : m_sec(sec), m_nsec(nsec) {} + Timestamp() {} + Timestamp(const struct timeval& tv) : m_sec(tv.tv_sec), m_nsec(tv.tv_usec) {} + Timestamp operator-(const Timestamp& other) const; + bool operator==(const Timestamp &other) const; + friend double operator/(double d, const Timestamp &denominator); + bool operator>(const Timestamp& other); + bool operator<(const Timestamp& other); + uint64_t sec() const {return m_sec;} + uint64_t nsec() const {return m_nsec;} + friend ostream& operator<<(ostream& stream, const Timestamp& ts); +private: + uint64_t m_sec; + uint64_t m_nsec; +}; + +#endif /* _TIMESTAMP_H_ */ |