From 2f996fc3c4e3468a93e0c64031e6148e61e32bad Mon Sep 17 00:00:00 2001 From: Jan Philipp Timme Date: Mon, 5 Dec 2016 16:29:54 +0100 Subject: [PATCH] Refactor tons of code into new model classes --- .../de/hsh/inform/orientdb_project/Main.java | 2 +- .../model/ArpPacketModel.java | 45 +++++ .../model/EthernetFrameModel.java | 63 +++++++ .../orientdb_project/model/HostModel.java | 35 ++++ .../model/IcmpPacketModel.java | 42 +++++ .../model/Ipv4PacketModel.java | 53 ++++++ ...onnection.java => TcpConnectionModel.java} | 26 ++- .../model/TcpPacketModel.java | 50 ++++++ .../model/UdpPacketModel.java | 50 ++++++ ...anceKappaOrientDbNetdataImportService.java | 169 +++++++++--------- .../orientdb/OrientDbHelperService.java | 73 +++----- .../util/SequenceProvider.java | 38 ++++ 12 files changed, 502 insertions(+), 144 deletions(-) create mode 100644 src/main/java/de/hsh/inform/orientdb_project/model/ArpPacketModel.java create mode 100644 src/main/java/de/hsh/inform/orientdb_project/model/EthernetFrameModel.java create mode 100644 src/main/java/de/hsh/inform/orientdb_project/model/HostModel.java create mode 100644 src/main/java/de/hsh/inform/orientdb_project/model/IcmpPacketModel.java create mode 100644 src/main/java/de/hsh/inform/orientdb_project/model/Ipv4PacketModel.java rename src/main/java/de/hsh/inform/orientdb_project/model/{TcpConnection.java => TcpConnectionModel.java} (63%) create mode 100644 src/main/java/de/hsh/inform/orientdb_project/model/TcpPacketModel.java create mode 100644 src/main/java/de/hsh/inform/orientdb_project/model/UdpPacketModel.java create mode 100644 src/main/java/de/hsh/inform/orientdb_project/util/SequenceProvider.java diff --git a/src/main/java/de/hsh/inform/orientdb_project/Main.java b/src/main/java/de/hsh/inform/orientdb_project/Main.java index d13d0e5..fb78031 100644 --- a/src/main/java/de/hsh/inform/orientdb_project/Main.java +++ b/src/main/java/de/hsh/inform/orientdb_project/Main.java @@ -17,7 +17,7 @@ public class Main { public static void main(String[] args) { // TODO: Make this configurable or easy to exchange. String filename = "/home/jpt/Temp/tcpdump_2"; - OrientDbHelperService odhs = new OrientDbHelperService("127.0.0.1", "hshtest", "root", "root"); + OrientDbHelperService odhs = new OrientDbHelperService("192.168.0.110", "hshtest", "root", "root"); // Clean up existing database and set up schema from scratch odhs.cleanUpServer(); diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/ArpPacketModel.java b/src/main/java/de/hsh/inform/orientdb_project/model/ArpPacketModel.java new file mode 100644 index 0000000..503cbcb --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/model/ArpPacketModel.java @@ -0,0 +1,45 @@ +package de.hsh.inform.orientdb_project.model; + +import org.pcap4j.packet.ArpPacket; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +// TODO: Not finished? +public class ArpPacketModel { + + public long ts; + public int ms; + + public int size; + public int payloadSize; + + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType arpPacketType = og.createVertexType("ArpPacket", "V"); + arpPacketType.createProperty("askedForIp", OType.STRING); + arpPacketType.createProperty("hasIp", OType.STRING); + arpPacketType.createProperty("size", OType.INTEGER); + arpPacketType.createProperty("payloadSize", OType.INTEGER); + } + + public ArpPacketModel(ArpPacket arp, long ts, int ms) { + this.ts = ts; + this.ms = ms; + this.size = arp.getRawData().length; + this.payloadSize = arp.getRawData().length - arp.getHeader().length(); + } + + public Object[] getArguments() { + Object[] arguments = { + "size", this.size, + "payloadSize", this.payloadSize, + }; + return arguments; + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/EthernetFrameModel.java b/src/main/java/de/hsh/inform/orientdb_project/model/EthernetFrameModel.java new file mode 100644 index 0000000..d190a85 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/model/EthernetFrameModel.java @@ -0,0 +1,63 @@ +package de.hsh.inform.orientdb_project.model; + +import org.pcap4j.packet.EthernetPacket; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +public class EthernetFrameModel { + + public long ts; + public int ms; + + public String sourceMac; + public String targetMac; + public byte[] rawData; + public int size; + public int payloadSize; + public long timestamp; + public int microseconds; + + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType ethernetFrameType = og.createVertexType("EthernetFrame", "V"); + ethernetFrameType.createProperty("sourceMac", OType.STRING); + ethernetFrameType.createProperty("targetMac", OType.STRING); + ethernetFrameType.createProperty("rawData", OType.BINARY); + ethernetFrameType.createProperty("size", OType.INTEGER); + ethernetFrameType.createProperty("payloadSize", OType.INTEGER); + ethernetFrameType.createProperty("timestamp", OType.LONG); + ethernetFrameType.createProperty("microseconds", OType.INTEGER); + } + + public EthernetFrameModel(EthernetPacket ether, long ts, int ms) { + this.ts = ts; + this.ms = ms; + this.sourceMac = ether.getHeader().getSrcAddr().toString(); + this.targetMac = ether.getHeader().getDstAddr().toString(); + this.rawData = ether.getRawData(); + this.size = ether.getRawData().length; + this.payloadSize = ether.getRawData().length - ether.getHeader().length(); + this.timestamp = ts; + this.microseconds = ms; + } + + public Object[] getArguments() { + Object[] arguments = { + "sourceMac", this.sourceMac, + "targetMac", this.targetMac, + "rawData", this.rawData, + "size", this.size, + "payloadSize", this.payloadSize, + "timestamp", this.timestamp, + "microseconds", this.microseconds, + }; + return arguments; + } + + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/HostModel.java b/src/main/java/de/hsh/inform/orientdb_project/model/HostModel.java new file mode 100644 index 0000000..2b500d0 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/model/HostModel.java @@ -0,0 +1,35 @@ +package de.hsh.inform.orientdb_project.model; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +public class HostModel { + + public String ipAddress; + public boolean internal; + + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType hostType = og.createVertexType("Host", "V"); + hostType.createProperty("ipAddress", OType.STRING); + hostType.createProperty("internal", OType.BOOLEAN); + } + + public HostModel(String ipAddress, boolean internal) { + this.ipAddress = ipAddress; + this.internal = internal; + } + + public Object[] getArguments() { + Object[] arguments = { + "ipAddress", this.ipAddress, + "internal", this.internal, + }; + return arguments; + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/IcmpPacketModel.java b/src/main/java/de/hsh/inform/orientdb_project/model/IcmpPacketModel.java new file mode 100644 index 0000000..4add045 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/model/IcmpPacketModel.java @@ -0,0 +1,42 @@ +package de.hsh.inform.orientdb_project.model; + +import org.pcap4j.packet.IcmpV4CommonPacket; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +public class IcmpPacketModel { + + public long ts; + public int ms; + + public int payloadSize; + public int size; + + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType icmpPacketType = og.createVertexType("IcmpPacket", "V"); + icmpPacketType.createProperty("size", OType.INTEGER); + icmpPacketType.createProperty("payloadSize", OType.INTEGER); + } + + public IcmpPacketModel(IcmpV4CommonPacket icmp, long ts, int ms) { + this.ts = ts; + this.ms = ms; + this.size = icmp.getRawData().length; + this.payloadSize = icmp.getRawData().length - icmp.getHeader().length(); + } + + public Object[] getArguments() { + Object[] arguments = { + "size", this.size, + "payloadSize", this.payloadSize, + }; + return arguments; + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/Ipv4PacketModel.java b/src/main/java/de/hsh/inform/orientdb_project/model/Ipv4PacketModel.java new file mode 100644 index 0000000..05364ae --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/model/Ipv4PacketModel.java @@ -0,0 +1,53 @@ +package de.hsh.inform.orientdb_project.model; + +import java.net.Inet4Address; + +import org.pcap4j.packet.IpV4Packet; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +public class Ipv4PacketModel { + + public long ts; + public int ms; + + public String sourceIp; + public String targetIp; + public int size; + public int payloadSize; + + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType ipPacketType = og.createVertexType("IpPacket", "V"); + ipPacketType.createProperty("sourceIp", OType.STRING); + ipPacketType.createProperty("targetIp", OType.STRING); + ipPacketType.createProperty("size", OType.INTEGER); + ipPacketType.createProperty("payloadSize", OType.INTEGER); + } + + public Ipv4PacketModel(IpV4Packet ipv4, Inet4Address sourceIp, Inet4Address targetIp, long ts, int ms) { + this.ts = ts; + this.ms = ms; + this.sourceIp = sourceIp.toString().split("/")[1]; + this.targetIp = targetIp.toString().split("/")[1]; + this.size = ipv4.getRawData().length; + this.payloadSize = ipv4.getRawData().length - ipv4.getHeader().length(); + + } + + public Object[] getArguments() { + Object[] arguments = { + "sourceIp", this.sourceIp, + "targetIp", this.targetIp, + "size", this.size, + "payloadSize", this.payloadSize, + }; + return arguments; + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/TcpConnection.java b/src/main/java/de/hsh/inform/orientdb_project/model/TcpConnectionModel.java similarity index 63% rename from src/main/java/de/hsh/inform/orientdb_project/model/TcpConnection.java rename to src/main/java/de/hsh/inform/orientdb_project/model/TcpConnectionModel.java index b12d07e..5651711 100644 --- a/src/main/java/de/hsh/inform/orientdb_project/model/TcpConnection.java +++ b/src/main/java/de/hsh/inform/orientdb_project/model/TcpConnectionModel.java @@ -4,10 +4,13 @@ import java.util.LinkedList; import org.pcap4j.packet.TcpPacket; +import com.orientechnologies.orient.core.metadata.schema.OType; import com.tinkerpop.blueprints.Vertex; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; -public class TcpConnection { +public class TcpConnectionModel { public long startTs; public int startMs; @@ -26,8 +29,27 @@ public class TcpConnection { public LinkedList knownTcpPacketVertices; + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType tcpConnectionType = og.createVertexType("TcpConnection", "V"); + tcpConnectionType.createProperty("startTs", OType.LONG); + tcpConnectionType.createProperty("startMs", OType.INTEGER); + tcpConnectionType.createProperty("endTs", OType.LONG); + tcpConnectionType.createProperty("endMs", OType.INTEGER); + tcpConnectionType.createProperty("sourceIp", OType.STRING); + tcpConnectionType.createProperty("sourcePort", OType.INTEGER); + tcpConnectionType.createProperty("targetIp", OType.STRING); + tcpConnectionType.createProperty("targetPort", OType.INTEGER); + tcpConnectionType.createProperty("volumeSourceToTarget", OType.LONG); + tcpConnectionType.createProperty("volumeTargetToSource", OType.LONG); + tcpConnectionType.createProperty("totalVolume", OType.LONG); + } - public TcpConnection(TcpPacket tcp, String sourceIp, String targetIp, long ts, int ms) { + + public TcpConnectionModel(TcpPacket tcp, String sourceIp, String targetIp, long ts, int ms) { this.setStart(ts, ms); this.setEnd(ts, ms); this.sourceIp = sourceIp; diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/TcpPacketModel.java b/src/main/java/de/hsh/inform/orientdb_project/model/TcpPacketModel.java new file mode 100644 index 0000000..68d249c --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/model/TcpPacketModel.java @@ -0,0 +1,50 @@ +package de.hsh.inform.orientdb_project.model; + +import org.pcap4j.packet.TcpPacket; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +public class TcpPacketModel { + + public long ts; + public int ms; + + public int sourcePort; + public int targetPort; + public int size; + public int payloadSize; + + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType tcpPacketType = og.createVertexType("TcpPacket", "V"); + tcpPacketType.createProperty("sourcePort", OType.INTEGER); + tcpPacketType.createProperty("targetPort", OType.INTEGER); + tcpPacketType.createProperty("size", OType.INTEGER); + tcpPacketType.createProperty("payloadSize", OType.INTEGER); + } + + public TcpPacketModel(TcpPacket tcp, long ts, int ms) { + this.ts = ts; + this.ms = ms; + this.sourcePort = tcp.getHeader().getSrcPort().valueAsInt(); + this.targetPort = tcp.getHeader().getDstPort().valueAsInt(); + this.size = tcp.getRawData().length; + this.payloadSize = tcp.getRawData().length - tcp.getHeader().length(); + } + + public Object[] getArguments() { + Object[] arguments = { + "sourcePort", this.sourcePort, + "targetPort", this.targetPort, + "size", this.size, + "payloadSize", this.payloadSize, + }; + return arguments; + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/model/UdpPacketModel.java b/src/main/java/de/hsh/inform/orientdb_project/model/UdpPacketModel.java new file mode 100644 index 0000000..64338d8 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/model/UdpPacketModel.java @@ -0,0 +1,50 @@ +package de.hsh.inform.orientdb_project.model; + +import org.pcap4j.packet.UdpPacket; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +public class UdpPacketModel { + + public long ts; + public int ms; + + public int sourcePort; + public int targetPort; + public int size; + public int payloadSize; + + /* + * Static helper method to create type for itself in OrientDb + * Check this.getArguments() to ensure completeness. + */ + public static void createType(OrientGraphNoTx og) { + OrientVertexType udpPacketType = og.createVertexType("UdpPacket", "V"); + udpPacketType.createProperty("sourcePort", OType.INTEGER); + udpPacketType.createProperty("targetPort", OType.INTEGER); + udpPacketType.createProperty("size", OType.INTEGER); + udpPacketType.createProperty("payloadSize", OType.INTEGER); + } + + public UdpPacketModel(UdpPacket udp, long ts, int ms) { + this.ts = ts; + this.ms = ms; + this.sourcePort = udp.getHeader().getSrcPort().valueAsInt(); + this.targetPort = udp.getHeader().getDstPort().valueAsInt(); + this.size = udp.getRawData().length; + this.payloadSize = udp.getRawData().length - udp.getHeader().length(); + } + + public Object[] getArguments() { + Object[] arguments = { + "sourcePort", this.sourcePort, + "targetPort", this.targetPort, + "size", this.size, + "payloadSize", this.payloadSize, + }; + return arguments; + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java b/src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java index 056ef69..a36b598 100644 --- a/src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java +++ b/src/main/java/de/hsh/inform/orientdb_project/orientdb/HighPerformanceKappaOrientDbNetdataImportService.java @@ -15,7 +15,14 @@ import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; -import de.hsh.inform.orientdb_project.model.TcpConnection; +import de.hsh.inform.orientdb_project.model.ArpPacketModel; +import de.hsh.inform.orientdb_project.model.EthernetFrameModel; +import de.hsh.inform.orientdb_project.model.HostModel; +import de.hsh.inform.orientdb_project.model.IcmpPacketModel; +import de.hsh.inform.orientdb_project.model.Ipv4PacketModel; +import de.hsh.inform.orientdb_project.model.TcpConnectionModel; +import de.hsh.inform.orientdb_project.model.TcpPacketModel; +import de.hsh.inform.orientdb_project.model.UdpPacketModel; import de.hsh.inform.orientdb_project.netdata.AbstractNetdataImportService; public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNetdataImportService { @@ -26,53 +33,58 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe private HashMap knownHosts; // To keep track of tcp connections - private HashMap> knownTcpConnections; + private HashMap> knownTcpConnections; - private Vertex ethernetFrame; - private Vertex arpPacket; - private Vertex ipPacket; - private Vertex udpPacket; - private Vertex tcpPacket; - private Vertex icmpPacket; + // References to already created model instances (these are reseted before processing a new ethernetFrame) + private EthernetFrameModel ethernetFrameModel; + private ArpPacketModel arpPacketModel; + private Ipv4PacketModel ipv4PacketModel; + private TcpPacketModel tcpPacketModel; + private UdpPacketModel udpPacketModel; + private IcmpPacketModel icmpPacketModel; + + // References to already created vertices (these are reseted before processing a new ethernetFrame) + private Vertex ethernetFrameVertex; + private Vertex arpPacketVertex; + private Vertex ipPacketVertex; + private Vertex udpPacketVertex; + private Vertex tcpPacketVertex; + private Vertex icmpPacketVertex; public HighPerformanceKappaOrientDbNetdataImportService(String filename, OrientGraphNoTx orientGraph) { super(filename); this.og = orientGraph; this.knownHosts = new HashMap(); - this.knownTcpConnections = new HashMap>(); + this.knownTcpConnections = new HashMap>(); } public void handleEthernetPacket(EthernetPacket ether, long ts, int ms) { // Clean up state vars before processing the next ethernet frame - this.ethernetFrame = null; - this.arpPacket = null; - this.ipPacket = null; - this.udpPacket = null; - this.tcpPacket = null; - this.icmpPacket = null; + this.ethernetFrameVertex = null; + this.arpPacketVertex = null; + this.ipPacketVertex = null; + this.udpPacketVertex = null; + this.tcpPacketVertex = null; + this.icmpPacketVertex = null; + // Also clean model instances + this.ethernetFrameModel = null; + this.arpPacketModel = null; + this.ipv4PacketModel = null; + this.tcpPacketModel = null; + this.udpPacketModel = null; + this.icmpPacketModel = null; // Okay, let's go! - Object[] arguments = { - "sourceMac", ether.getHeader().getSrcAddr().toString(), - "targetMac", ether.getHeader().getDstAddr().toString(), - "rawData", ether.getRawData(), - "size", ether.getRawData().length, - "payloadSize", ether.getRawData().length - ether.getHeader().length(), - "timestamp", ts, - "microseconds", ms, - }; - this.ethernetFrame = this.og.addVertex("class:EthernetFrame", arguments); + this.ethernetFrameModel = new EthernetFrameModel(ether, ts, ms); + this.ethernetFrameVertex = this.og.addVertex("class:EthernetFrame", this.ethernetFrameModel.getArguments()); super.handleEthernetPacket(ether, ts, ms); } public void handleArpPacket(ArpPacket arp, long ts, int ms) { - Object[] arguments = { - "size", arp.getRawData().length, - "payloadSize", arp.getRawData().length - arp.getHeader().length(), - }; - this.arpPacket = this.og.addVertex("class:ArpPacket", arguments); + this.arpPacketModel = new ArpPacketModel(arp, ts, ms); + this.arpPacketVertex = this.og.addVertex("class:ArpPacket", this.arpPacketModel.getArguments()); // Wire up to its ethernet frame - Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrame, this.arpPacket, "contains"); - Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.arpPacket, this.ethernetFrame, "isContainedIn"); + Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrameVertex, this.arpPacketVertex, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.arpPacketVertex, this.ethernetFrameVertex, "isContainedIn"); } public void handleIpV4Packet(IpV4Packet ipv4, long ts, int ms) { @@ -81,51 +93,36 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe // Add hosts to database if new this.addHostIfNew(sourceIp); this.addHostIfNew(targetIp); - Object[] arguments = { - "sourceIp", sourceIp.toString().split("/")[1], - "targetIp", targetIp.toString().split("/")[1], - "size", ipv4.getRawData().length, - "payloadSize", ipv4.getRawData().length - ipv4.getHeader().length(), - }; - this.ipPacket = this.og.addVertex("class:IpPacket", arguments); + this.ipv4PacketModel = new Ipv4PacketModel(ipv4, sourceIp, targetIp, ts, ms); + this.ipPacketVertex = this.og.addVertex("class:IpPacket", this.ipv4PacketModel.getArguments()); // Wire up to its ethernet frame - Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrame, this.ipPacket, "contains"); - Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.ipPacket, this.ethernetFrame, "isContainedIn"); + Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrameVertex, this.ipPacketVertex, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.ipPacketVertex, this.ethernetFrameVertex, "isContainedIn"); super.handleIpV4Packet(ipv4, ts, ms); } public void handleUdpPacket(UdpPacket udp, long ts, int ms) { - Object[] arguments = { - "sourcePort", udp.getHeader().getSrcPort().valueAsInt(), - "targetPort", udp.getHeader().getDstPort().valueAsInt(), - "size", udp.getRawData().length, - "payloadSize", udp.getRawData().length - udp.getHeader().length(), - }; - this.udpPacket = this.og.addVertex("class:UdpPacket"); + this.udpPacketModel = new UdpPacketModel(udp, ts, ms); + this.udpPacketVertex = this.og.addVertex("class:UdpPacket", this.udpPacketModel.getArguments()); // Wire up to its ip packet - Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.udpPacket, "contains"); - Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.udpPacket, this.ipPacket, "isContainedIn"); + Edge containsEdge = this.og.addEdge("class:contains", this.ipPacketVertex, this.udpPacketVertex, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.udpPacketVertex, this.ipPacketVertex, "isContainedIn"); } public void handleTcpPacket(TcpPacket tcp, long ts, int ms) { - Object[] arguments = { - "sourcePort", tcp.getHeader().getSrcPort().valueAsInt(), - "targetPort", tcp.getHeader().getDstPort().valueAsInt(), - "size", tcp.getRawData().length, - "payloadSize", tcp.getRawData().length - tcp.getHeader().length(), - }; - this.tcpPacket = this.og.addVertex("class:TcpPacket", arguments); + this.tcpPacketModel = new TcpPacketModel(tcp, ts, ms); + this.tcpPacketVertex = this.og.addVertex("class:TcpPacket", this.tcpPacketModel.getArguments()); // Wire up to its ip packet - Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.tcpPacket, "contains"); - Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.tcpPacket, this.ipPacket, "isContainedIn"); + Edge containsEdge = this.og.addEdge("class:contains", this.ipPacketVertex, this.tcpPacketVertex, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.tcpPacketVertex, this.ipPacketVertex, "isContainedIn"); // Track tcp connections - TcpConnection tcpConnection = this.getTcpConnectionFor(tcp); + TcpConnectionModel tcpConnection = this.getTcpConnectionFor(tcp); // If connection exists ... if(tcpConnection != null) { // ... and still "up to date" aka time difference < 2s if(ts - tcpConnection.endTs < 2) { // Update tcpConnection data - if(tcpConnection.sourceIp.equals(this.ipPacket.getProperty("sourceIp"))) { + if(tcpConnection.sourceIp.equals(this.ipPacketVertex.getProperty("sourceIp"))) { // SourceIp -> TargetIp tcpConnection.addVolumeSourceToTarget(tcp.getRawData().length - tcp.getHeader().length()); } else { @@ -136,24 +133,21 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe } } else { // Else create a new one and add it to the list. - String sourceIp = this.ipPacket.getProperty("sourceIp"); - String targetIp = this.ipPacket.getProperty("targetIp"); - tcpConnection = new TcpConnection(tcp, sourceIp, targetIp, ts, ms); + String sourceIp = this.ipPacketVertex.getProperty("sourceIp"); + String targetIp = this.ipPacketVertex.getProperty("targetIp"); + tcpConnection = new TcpConnectionModel(tcp, sourceIp, targetIp, ts, ms); this.addKnownTcpConnectionFor(tcpConnection, tcp); } // Remember tcpPacketVertex in tcpConnection for later edges - tcpConnection.addKnownTcpPacketVertex(this.tcpPacket); + tcpConnection.addKnownTcpPacketVertex(this.tcpPacketVertex); } public void handleIcmpPacket(IcmpV4CommonPacket icmp, long ts, int ms) { - Object[] arguments = { - "size", icmp.getRawData().length, - "payloadSize", icmp.getRawData().length - icmp.getHeader().length(), - }; - this.icmpPacket = this.og.addVertex("class:IcmpPacket"); + this.icmpPacketModel = new IcmpPacketModel(icmp, ts, ms); + this.icmpPacketVertex = this.og.addVertex("class:IcmpPacket", this.icmpPacketModel.getArguments()); // Wire up to its ip packet - Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.icmpPacket, "contains"); - Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.icmpPacket, this.ipPacket, "isContainedIn"); + Edge containsEdge = this.og.addEdge("class:contains", this.ipPacketVertex, this.icmpPacketVertex, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.icmpPacketVertex, this.ipPacketVertex, "isContainedIn"); } private void addHostIfNew(Inet4Address ipAddress) { @@ -165,11 +159,8 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe boolean isInternal = ipAddress.isSiteLocalAddress(); // TODO: VERIFY IF THIS IS CORRECT! // Create Vertex and add to HashMap String ipAddressStr = ipAddress.toString().split("/")[1]; - Object[] arguments = { - "ipAddress", ipAddressStr, - "internal", isInternal, - }; - Vertex host = this.og.addVertex("class:Host", arguments); + HostModel hostModel = new HostModel(ipAddressStr, isInternal); + Vertex host = this.og.addVertex("class:Host", hostModel.getArguments()); this.knownHosts.put(ipAddressStr, host); } } @@ -186,18 +177,18 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe } } - private TcpConnection getTcpConnectionFor(TcpPacket tcp) { + private TcpConnectionModel getTcpConnectionFor(TcpPacket tcp) { String source = ""; String target = ""; - String sourceIp = this.ipPacket.getProperty("sourceIp"); - String targetIp = this.ipPacket.getProperty("targetIp"); + String sourceIp = this.ipPacketVertex.getProperty("sourceIp"); + String targetIp = this.ipPacketVertex.getProperty("targetIp"); String sourcePort = tcp.getHeader().getSrcPort().valueAsString(); String targetPort = tcp.getHeader().getDstPort().valueAsString(); source = sourceIp + ":" + sourcePort; target = targetIp + ":" + targetPort; String connectionKey = this.buildConnectionKey(source, target); - TcpConnection tcpConnection = null; - LinkedList connectionList = null; + TcpConnectionModel tcpConnection = null; + LinkedList connectionList = null; // Get or create tcp connection list for connection key if(this.knownTcpConnections.containsKey(connectionKey)) { connectionList = this.knownTcpConnections.get(connectionKey); @@ -210,21 +201,21 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe return tcpConnection; } - private void addKnownTcpConnectionFor(TcpConnection tcpConnection, TcpPacket tcp) { + private void addKnownTcpConnectionFor(TcpConnectionModel tcpConnection, TcpPacket tcp) { String source = ""; String target = ""; - String sourceIp = this.ipPacket.getProperty("sourceIp"); - String targetIp = this.ipPacket.getProperty("targetIp"); + String sourceIp = this.ipPacketVertex.getProperty("sourceIp"); + String targetIp = this.ipPacketVertex.getProperty("targetIp"); String sourcePort = tcp.getHeader().getSrcPort().valueAsString(); String targetPort = tcp.getHeader().getDstPort().valueAsString(); source = sourceIp + ":" + sourcePort; target = targetIp + ":" + targetPort; String connectionKey = this.buildConnectionKey(source, target); - LinkedList connectionList = null; + LinkedList connectionList = null; if(this.knownTcpConnections.containsKey(connectionKey)) { connectionList = this.knownTcpConnections.get(connectionKey); } else { - connectionList = new LinkedList(); + connectionList = new LinkedList(); this.knownTcpConnections.put(connectionKey, connectionList); } // Put connection into list of known tcp connections @@ -234,8 +225,8 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe public void afterImport() { // TODO: Link TcpConnections up with their tcpPackets! System.out.println("All done. Processing collected TcpConnections ..."); - for(LinkedList connList : this.knownTcpConnections.values()) { - for(TcpConnection conn : connList) { + for(LinkedList connList : this.knownTcpConnections.values()) { + for(TcpConnectionModel conn : connList) { Vertex currentTcpConnection = this.og.addVertex("class:TcpConnection", conn.getArguments()); // Look up already created source and target host vertices Vertex sourceHost = this.knownHosts.get(conn.sourceIp); diff --git a/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java index b426226..10ed8d6 100644 --- a/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java +++ b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java @@ -3,7 +3,6 @@ package de.hsh.inform.orientdb_project.orientdb; import java.io.IOException; import com.orientechnologies.orient.client.remote.OServerAdmin; -import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; import com.orientechnologies.orient.core.intent.OIntentMassiveInsert; import com.orientechnologies.orient.core.metadata.schema.OType; import com.tinkerpop.blueprints.impls.orient.OrientConfigurableGraph.THREAD_MODE; @@ -12,6 +11,15 @@ import com.tinkerpop.blueprints.impls.orient.OrientGraphFactory; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; +import de.hsh.inform.orientdb_project.model.ArpPacketModel; +import de.hsh.inform.orientdb_project.model.EthernetFrameModel; +import de.hsh.inform.orientdb_project.model.HostModel; +import de.hsh.inform.orientdb_project.model.IcmpPacketModel; +import de.hsh.inform.orientdb_project.model.Ipv4PacketModel; +import de.hsh.inform.orientdb_project.model.TcpConnectionModel; +import de.hsh.inform.orientdb_project.model.TcpPacketModel; +import de.hsh.inform.orientdb_project.model.UdpPacketModel; + public class OrientDbHelperService { private String host; @@ -115,59 +123,19 @@ public class OrientDbHelperService { private void createClasses() { OrientGraphNoTx og = this.getOrientGraphFactory().getNoTx(); - OrientVertexType ethernetFrameType = og.createVertexType("EthernetFrame", "V"); - ethernetFrameType.createProperty("sourceMac", OType.STRING); - ethernetFrameType.createProperty("targetMac", OType.STRING); - ethernetFrameType.createProperty("rawData", OType.BINARY); - ethernetFrameType.createProperty("size", OType.INTEGER); - ethernetFrameType.createProperty("payloadSize", OType.INTEGER); - ethernetFrameType.createProperty("timestamp", OType.LONG); - ethernetFrameType.createProperty("microseconds", OType.INTEGER); + // Use methods integrated into the models to create their classes + // These are the vertex types used in our model + EthernetFrameModel.createType(og); + ArpPacketModel.createType(og); + Ipv4PacketModel.createType(og); + UdpPacketModel.createType(og); + TcpPacketModel.createType(og); + IcmpPacketModel.createType(og); - OrientVertexType arpPacketType = og.createVertexType("ArpPacket", "V"); - // TODO: Not finished! - arpPacketType.createProperty("askedForIp", OType.STRING); - arpPacketType.createProperty("hasIp", OType.STRING); - arpPacketType.createProperty("size", OType.INTEGER); - arpPacketType.createProperty("payloadSize", OType.INTEGER); - - OrientVertexType ipPacketType = og.createVertexType("IpPacket", "V"); - ipPacketType.createProperty("sourceIp", OType.STRING); - ipPacketType.createProperty("targetIp", OType.STRING); - ipPacketType.createProperty("size", OType.INTEGER); - ipPacketType.createProperty("payloadSize", OType.INTEGER); - - OrientVertexType udpPacketType = og.createVertexType("UdpPacket", "V"); - udpPacketType.createProperty("sourcePort", OType.INTEGER); - udpPacketType.createProperty("targetPort", OType.INTEGER); - udpPacketType.createProperty("size", OType.INTEGER); - udpPacketType.createProperty("payloadSize", OType.INTEGER); - - OrientVertexType tcpPacketType = og.createVertexType("TcpPacket", "V"); - tcpPacketType.createProperty("sourcePort", OType.INTEGER); - tcpPacketType.createProperty("targetPort", OType.INTEGER); - tcpPacketType.createProperty("size", OType.INTEGER); - tcpPacketType.createProperty("payloadSize", OType.INTEGER); - - OrientVertexType icmpPacketType = og.createVertexType("IcmpPacket", "V"); - icmpPacketType.createProperty("size", OType.INTEGER); - icmpPacketType.createProperty("payloadSize", OType.INTEGER); - - OrientVertexType hostType = og.createVertexType("Host", "V"); - hostType.createProperty("ipAddress", OType.STRING); - hostType.createProperty("internal", OType.BOOLEAN); - - OrientVertexType tcpConnectionType = og.createVertexType("TcpConnection", "V"); - tcpConnectionType.createProperty("startTs", OType.LONG); - tcpConnectionType.createProperty("startMs", OType.INTEGER); - tcpConnectionType.createProperty("endTs", OType.LONG); - tcpConnectionType.createProperty("endMs", OType.INTEGER); - tcpConnectionType.createProperty("sourcePort", OType.INTEGER); - tcpConnectionType.createProperty("targetPort", OType.INTEGER); - tcpConnectionType.createProperty("volumeSourceToTarget", OType.LONG); - tcpConnectionType.createProperty("volumeTargetToSource", OType.LONG); - tcpConnectionType.createProperty("totalVolume", OType.LONG); + HostModel.createType(og); + TcpConnectionModel.createType(og); + // Edges do not really need their own model (yet), they connect the vertex types OrientEdgeType isContainedInType = og.createEdgeType("isContainedIn", "E"); isContainedInType.setDescription("isContainedIn"); OrientEdgeType containsType = og.createEdgeType("contains", "E"); @@ -189,6 +157,7 @@ public class OrientDbHelperService { OrientEdgeType hasRelatedTcpPacketType = og.createEdgeType("hasRelatedTcpPacket", "E"); hasRelatedTcpPacketType.setDescription("hasRelatedTcpPacket"); + // We're done creating classes and types, shut down the database graph. og.shutdown(); } diff --git a/src/main/java/de/hsh/inform/orientdb_project/util/SequenceProvider.java b/src/main/java/de/hsh/inform/orientdb_project/util/SequenceProvider.java new file mode 100644 index 0000000..65e9688 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/util/SequenceProvider.java @@ -0,0 +1,38 @@ +package de.hsh.inform.orientdb_project.util; + +public class SequenceProvider { + + private static String NOT_LABELED = "Not labeled"; + + private long sequenceNumber; + private String label; + + public SequenceProvider() { + this(0, SequenceProvider.NOT_LABELED); + } + + public SequenceProvider(long startValue) { + this(startValue, SequenceProvider.NOT_LABELED); + } + + public SequenceProvider(long startValue, String label) { + this.sequenceNumber = startValue; + this.label = label; + } + + + public long getNextNumber() { + long result = this.sequenceNumber; + this.sequenceNumber++; + return result; + } + + public String toString() { + StringBuilder sb = new StringBuilder("[SequenceProvider '"); + sb.append(this.label); + sb.append("'] #="); + sb.append(this.sequenceNumber); + return sb.toString(); + } + +}