From bf9e27fe8c90143a78cb34644c0876458295e930 Mon Sep 17 00:00:00 2001 From: Jan Philipp Timme Date: Thu, 24 Nov 2016 10:49:47 +0100 Subject: [PATCH] Implement a more efficient way to create vertices --- .gitignore | 1 + src/.gitignore | 1 + .../de/hsh/inform/orientdb_project/Main.java | 13 +- ...anceKappaOrientDbNetdataImportService.java | 119 ++++++++++++++++++ .../orientdb/OrientDbHelperService.java | 24 +++- 5 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 src/.gitignore create mode 100644 src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java diff --git a/.gitignore b/.gitignore index 769b429..7d4b0bd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .classpath .settings .project +/target/ diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..ae3c172 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/src/main/java/de/hsh/inform/orientdb_project/Main.java b/src/main/java/de/hsh/inform/orientdb_project/Main.java index 42e9aca..b3defab 100644 --- a/src/main/java/de/hsh/inform/orientdb_project/Main.java +++ b/src/main/java/de/hsh/inform/orientdb_project/Main.java @@ -6,20 +6,27 @@ import java.util.concurrent.TimeoutException; import org.pcap4j.core.NotOpenException; import org.pcap4j.core.PcapNativeException; +import com.tinkerpop.blueprints.impls.orient.OrientConfigurableGraph.THREAD_MODE; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; + import de.hsh.inform.orientdb_project.netdata.AbstractNetdataImportService; -import de.hsh.inform.orientdb_project.orientdb.LowPerformanceOrientDbNetdataImportService; +import de.hsh.inform.orientdb_project.orientdb.HighPerformanceKappaOrientDbNetdataImportService; import de.hsh.inform.orientdb_project.orientdb.OrientDbHelperService; public class Main { public static void main(String[] args) { - OrientDbHelperService odhs = new OrientDbHelperService("192.168.0.110", "hshtest", "root", "root"); + OrientDbHelperService odhs = new OrientDbHelperService("127.0.0.1", "hshtest", "root", "root"); odhs.cleanUpServer(); odhs.setupSchema(); String filename = "/home/jpt/Temp/tcpdump_2"; + OrientGraphNoTx ogf = odhs.getOrientGraphFactory().getNoTx(); + ogf.setThreadMode(THREAD_MODE.MANUAL); + //AbstractNetdataImportService importService = new DummyImportService(filename); - AbstractNetdataImportService importService = new LowPerformanceOrientDbNetdataImportService(filename, odhs.getOrientGraphFactory().getNoTx()); + //AbstractNetdataImportService importService = new LowPerformanceOrientDbNetdataImportService(filename, ogf); + AbstractNetdataImportService importService = new HighPerformanceKappaOrientDbNetdataImportService(filename, ogf); try { System.out.println(System.currentTimeMillis() + ": Begin import of data ..."); importService.run(); diff --git a/src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java b/src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java new file mode 100644 index 0000000..54ab7b0 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java @@ -0,0 +1,119 @@ +package de.hsh.inform.orientdb_project.orientdb; + +import org.pcap4j.packet.ArpPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.packet.IcmpV4CommonPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; +import org.pcap4j.packet.UdpPacket; + +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertex; + +import de.hsh.inform.orientdb_project.netdata.AbstractNetdataImportService; + +public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNetdataImportService { + + private OrientGraphNoTx og; + + private Vertex ethernetFrame; + private Vertex arpPacket; + private Vertex ipPacket; + private Vertex udpPacket; + private Vertex tcpPacket; + private Vertex icmpPacket; + + public HighPerformanceKappaOrientDbNetdataImportService(String filename, OrientGraphNoTx orientGraph) { + super(filename); + this.og = orientGraph; + } + + public void handleEthernetPacket(EthernetPacket ether, long ts, int ms) { + // Clean up state vars before processing the next ethernet frame + this.ethernetFrame = null; + this.arpPacket = null; + this.ipPacket = null; + this.udpPacket = null; + this.tcpPacket = null; + this.icmpPacket = null; + // Okay, let's go! + Object[] arguments = { + "sourceMac", ether.getHeader().getSrcAddr().toString(), + "targetMac", ether.getHeader().getDstAddr().toString(), + "rawData", ether.getRawData(), + "size", ether.getRawData().length, + "payloadSize", ether.getRawData().length - ether.getHeader().length(), + "timestamp", ts, + "microseconds", ms, + }; + this.ethernetFrame = this.og.addVertex("class:EthernetFrame", arguments); + super.handleEthernetPacket(ether, ts, ms); + } + + public void handleArpPacket(ArpPacket arp, long ts, int ms) { + Object[] arguments = { + "size", arp.getRawData().length, + "payloadSize", arp.getRawData().length - arp.getHeader().length(), + }; + this.arpPacket = this.og.addVertex("class:ArpPacket", arguments); + // Wire up to its ethernet frame + Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrame, this.arpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.arpPacket, this.ethernetFrame, "isContainedIn"); + } + + public void handleIpV4Packet(IpV4Packet ipv4, long ts, int ms) { + Object[] arguments = { + "sourceIp", ipv4.getHeader().getSrcAddr().toString().split("/")[1], + "targetIp", ipv4.getHeader().getDstAddr().toString().split("/")[1], + "size", ipv4.getRawData().length, + "payloadSize", ipv4.getRawData().length - ipv4.getHeader().length(), + }; + this.ipPacket = this.og.addVertex("class:IpPacket", arguments); + // Wire up to its ethernet frame + Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrame, this.ipPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.ipPacket, this.ethernetFrame, "isContainedIn"); + super.handleIpV4Packet(ipv4, ts, ms); + } + + public void handleUdpPacket(UdpPacket udp, long ts, int ms) { + Object[] arguments = { + "sourcePort", udp.getHeader().getSrcPort().valueAsInt(), + "targetPort", udp.getHeader().getDstPort().valueAsInt(), + "size", udp.getRawData().length, + "payloadSize", udp.getRawData().length - udp.getHeader().length(), + }; + this.udpPacket = this.og.addVertex("class:UdpPacket"); + // Wire up to its ip packet + Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.udpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.udpPacket, this.ipPacket, "isContainedIn"); + } + + public void handleTcpPacket(TcpPacket tcp, long ts, int ms) { + Object[] arguments = { + "sourcePort", tcp.getHeader().getSrcPort().valueAsInt(), + "targetPort", tcp.getHeader().getDstPort().valueAsInt(), + "size", tcp.getRawData().length, + "payloadSize", tcp.getRawData().length - tcp.getHeader().length(), + }; + this.tcpPacket = this.og.addVertex("class:TcpPacket", arguments); + // Wire up to its ip packet + Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.tcpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.tcpPacket, this.ipPacket, "isContainedIn"); + } + + public void handleIcmpPacket(IcmpV4CommonPacket icmp, long ts, int ms) { + Object[] arguments = { + "size", icmp.getRawData().length, + "payloadSize", icmp.getRawData().length - icmp.getHeader().length(), + }; + this.icmpPacket = this.og.addVertex("class:IcmpPacket"); + // Wire up to its ip packet + Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.icmpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.icmpPacket, this.ipPacket, "isContainedIn"); + } + + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java index 7dc0525..6125ea5 100644 --- a/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java +++ b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java @@ -67,6 +67,11 @@ public class OrientDbHelperService { } public void setupSchema() { + this.createClasses(); + this.createClusters(); + } + + private void createClasses() { OrientGraphNoTx og = this.getOrientGraphFactory().getNoTx(); OrientVertexType ethernetFrameType = og.createVertexType("EthernetFrame", "V"); @@ -109,7 +114,6 @@ public class OrientDbHelperService { OrientVertexType hostType = og.createVertexType("Host", "V"); hostType.createProperty("ipAddress", OType.STRING); - hostType.createProperty("macAddress", OType.STRING); hostType.createProperty("internal", OType.BOOLEAN); OrientVertexType tcpConnectionType = og.createVertexType("TcpConnection", "V"); @@ -125,9 +129,23 @@ public class OrientDbHelperService { isContainedInType.setDescription("isContainedIn"); OrientEdgeType containsType = og.createEdgeType("contains", "E"); containsType.setDescription("contains"); - - // Uhm ... this should be okay ... ? + og.shutdown(); } + + private void createClusters() { + OServerAdmin admin = null; + try { + admin = new OServerAdmin(getDbUri(false)); + admin.connect(this.user, this.pass); + } catch (IOException e) { + try { + admin.createDatabase(this.db, "graph", "plocal"); + } catch (IOException e1) { + e1.printStackTrace(); + System.exit(1); + } + } + } }