Refactor tons of code into new model classes

This commit is contained in:
Jan Philipp Timme 2016-12-05 16:29:54 +01:00
parent cec929a5f2
commit 2f996fc3c4
Signed by untrusted user: JPT
GPG Key ID: 5F2C85EC6F3754B7
12 changed files with 502 additions and 144 deletions

View File

@ -17,7 +17,7 @@ public class Main {
public static void main(String[] args) { public static void main(String[] args) {
// TODO: Make this configurable or easy to exchange. // TODO: Make this configurable or easy to exchange.
String filename = "/home/jpt/Temp/tcpdump_2"; String filename = "/home/jpt/Temp/tcpdump_2";
OrientDbHelperService odhs = new OrientDbHelperService("127.0.0.1", "hshtest", "root", "root"); OrientDbHelperService odhs = new OrientDbHelperService("192.168.0.110", "hshtest", "root", "root");
// Clean up existing database and set up schema from scratch // Clean up existing database and set up schema from scratch
odhs.cleanUpServer(); odhs.cleanUpServer();

View File

@ -0,0 +1,45 @@
package de.hsh.inform.orientdb_project.model;
import org.pcap4j.packet.ArpPacket;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
// TODO: Not finished?
public class ArpPacketModel {
public long ts;
public int ms;
public int size;
public int payloadSize;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType arpPacketType = og.createVertexType("ArpPacket", "V");
arpPacketType.createProperty("askedForIp", OType.STRING);
arpPacketType.createProperty("hasIp", OType.STRING);
arpPacketType.createProperty("size", OType.INTEGER);
arpPacketType.createProperty("payloadSize", OType.INTEGER);
}
public ArpPacketModel(ArpPacket arp, long ts, int ms) {
this.ts = ts;
this.ms = ms;
this.size = arp.getRawData().length;
this.payloadSize = arp.getRawData().length - arp.getHeader().length();
}
public Object[] getArguments() {
Object[] arguments = {
"size", this.size,
"payloadSize", this.payloadSize,
};
return arguments;
}
}

View File

@ -0,0 +1,63 @@
package de.hsh.inform.orientdb_project.model;
import org.pcap4j.packet.EthernetPacket;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
public class EthernetFrameModel {
public long ts;
public int ms;
public String sourceMac;
public String targetMac;
public byte[] rawData;
public int size;
public int payloadSize;
public long timestamp;
public int microseconds;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType ethernetFrameType = og.createVertexType("EthernetFrame", "V");
ethernetFrameType.createProperty("sourceMac", OType.STRING);
ethernetFrameType.createProperty("targetMac", OType.STRING);
ethernetFrameType.createProperty("rawData", OType.BINARY);
ethernetFrameType.createProperty("size", OType.INTEGER);
ethernetFrameType.createProperty("payloadSize", OType.INTEGER);
ethernetFrameType.createProperty("timestamp", OType.LONG);
ethernetFrameType.createProperty("microseconds", OType.INTEGER);
}
public EthernetFrameModel(EthernetPacket ether, long ts, int ms) {
this.ts = ts;
this.ms = ms;
this.sourceMac = ether.getHeader().getSrcAddr().toString();
this.targetMac = ether.getHeader().getDstAddr().toString();
this.rawData = ether.getRawData();
this.size = ether.getRawData().length;
this.payloadSize = ether.getRawData().length - ether.getHeader().length();
this.timestamp = ts;
this.microseconds = ms;
}
public Object[] getArguments() {
Object[] arguments = {
"sourceMac", this.sourceMac,
"targetMac", this.targetMac,
"rawData", this.rawData,
"size", this.size,
"payloadSize", this.payloadSize,
"timestamp", this.timestamp,
"microseconds", this.microseconds,
};
return arguments;
}
}

View File

@ -0,0 +1,35 @@
package de.hsh.inform.orientdb_project.model;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
public class HostModel {
public String ipAddress;
public boolean internal;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType hostType = og.createVertexType("Host", "V");
hostType.createProperty("ipAddress", OType.STRING);
hostType.createProperty("internal", OType.BOOLEAN);
}
public HostModel(String ipAddress, boolean internal) {
this.ipAddress = ipAddress;
this.internal = internal;
}
public Object[] getArguments() {
Object[] arguments = {
"ipAddress", this.ipAddress,
"internal", this.internal,
};
return arguments;
}
}

View File

@ -0,0 +1,42 @@
package de.hsh.inform.orientdb_project.model;
import org.pcap4j.packet.IcmpV4CommonPacket;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
public class IcmpPacketModel {
public long ts;
public int ms;
public int payloadSize;
public int size;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType icmpPacketType = og.createVertexType("IcmpPacket", "V");
icmpPacketType.createProperty("size", OType.INTEGER);
icmpPacketType.createProperty("payloadSize", OType.INTEGER);
}
public IcmpPacketModel(IcmpV4CommonPacket icmp, long ts, int ms) {
this.ts = ts;
this.ms = ms;
this.size = icmp.getRawData().length;
this.payloadSize = icmp.getRawData().length - icmp.getHeader().length();
}
public Object[] getArguments() {
Object[] arguments = {
"size", this.size,
"payloadSize", this.payloadSize,
};
return arguments;
}
}

View File

@ -0,0 +1,53 @@
package de.hsh.inform.orientdb_project.model;
import java.net.Inet4Address;
import org.pcap4j.packet.IpV4Packet;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
public class Ipv4PacketModel {
public long ts;
public int ms;
public String sourceIp;
public String targetIp;
public int size;
public int payloadSize;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType ipPacketType = og.createVertexType("IpPacket", "V");
ipPacketType.createProperty("sourceIp", OType.STRING);
ipPacketType.createProperty("targetIp", OType.STRING);
ipPacketType.createProperty("size", OType.INTEGER);
ipPacketType.createProperty("payloadSize", OType.INTEGER);
}
public Ipv4PacketModel(IpV4Packet ipv4, Inet4Address sourceIp, Inet4Address targetIp, long ts, int ms) {
this.ts = ts;
this.ms = ms;
this.sourceIp = sourceIp.toString().split("/")[1];
this.targetIp = targetIp.toString().split("/")[1];
this.size = ipv4.getRawData().length;
this.payloadSize = ipv4.getRawData().length - ipv4.getHeader().length();
}
public Object[] getArguments() {
Object[] arguments = {
"sourceIp", this.sourceIp,
"targetIp", this.targetIp,
"size", this.size,
"payloadSize", this.payloadSize,
};
return arguments;
}
}

View File

@ -4,10 +4,13 @@ import java.util.LinkedList;
import org.pcap4j.packet.TcpPacket; import org.pcap4j.packet.TcpPacket;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
public class TcpConnection { public class TcpConnectionModel {
public long startTs; public long startTs;
public int startMs; public int startMs;
@ -26,8 +29,27 @@ public class TcpConnection {
public LinkedList<Vertex> knownTcpPacketVertices; public LinkedList<Vertex> knownTcpPacketVertices;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType tcpConnectionType = og.createVertexType("TcpConnection", "V");
tcpConnectionType.createProperty("startTs", OType.LONG);
tcpConnectionType.createProperty("startMs", OType.INTEGER);
tcpConnectionType.createProperty("endTs", OType.LONG);
tcpConnectionType.createProperty("endMs", OType.INTEGER);
tcpConnectionType.createProperty("sourceIp", OType.STRING);
tcpConnectionType.createProperty("sourcePort", OType.INTEGER);
tcpConnectionType.createProperty("targetIp", OType.STRING);
tcpConnectionType.createProperty("targetPort", OType.INTEGER);
tcpConnectionType.createProperty("volumeSourceToTarget", OType.LONG);
tcpConnectionType.createProperty("volumeTargetToSource", OType.LONG);
tcpConnectionType.createProperty("totalVolume", OType.LONG);
}
public TcpConnection(TcpPacket tcp, String sourceIp, String targetIp, long ts, int ms) {
public TcpConnectionModel(TcpPacket tcp, String sourceIp, String targetIp, long ts, int ms) {
this.setStart(ts, ms); this.setStart(ts, ms);
this.setEnd(ts, ms); this.setEnd(ts, ms);
this.sourceIp = sourceIp; this.sourceIp = sourceIp;

View File

@ -0,0 +1,50 @@
package de.hsh.inform.orientdb_project.model;
import org.pcap4j.packet.TcpPacket;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
public class TcpPacketModel {
public long ts;
public int ms;
public int sourcePort;
public int targetPort;
public int size;
public int payloadSize;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType tcpPacketType = og.createVertexType("TcpPacket", "V");
tcpPacketType.createProperty("sourcePort", OType.INTEGER);
tcpPacketType.createProperty("targetPort", OType.INTEGER);
tcpPacketType.createProperty("size", OType.INTEGER);
tcpPacketType.createProperty("payloadSize", OType.INTEGER);
}
public TcpPacketModel(TcpPacket tcp, long ts, int ms) {
this.ts = ts;
this.ms = ms;
this.sourcePort = tcp.getHeader().getSrcPort().valueAsInt();
this.targetPort = tcp.getHeader().getDstPort().valueAsInt();
this.size = tcp.getRawData().length;
this.payloadSize = tcp.getRawData().length - tcp.getHeader().length();
}
public Object[] getArguments() {
Object[] arguments = {
"sourcePort", this.sourcePort,
"targetPort", this.targetPort,
"size", this.size,
"payloadSize", this.payloadSize,
};
return arguments;
}
}

View File

@ -0,0 +1,50 @@
package de.hsh.inform.orientdb_project.model;
import org.pcap4j.packet.UdpPacket;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
public class UdpPacketModel {
public long ts;
public int ms;
public int sourcePort;
public int targetPort;
public int size;
public int payloadSize;
/*
* Static helper method to create type for itself in OrientDb
* Check this.getArguments() to ensure completeness.
*/
public static void createType(OrientGraphNoTx og) {
OrientVertexType udpPacketType = og.createVertexType("UdpPacket", "V");
udpPacketType.createProperty("sourcePort", OType.INTEGER);
udpPacketType.createProperty("targetPort", OType.INTEGER);
udpPacketType.createProperty("size", OType.INTEGER);
udpPacketType.createProperty("payloadSize", OType.INTEGER);
}
public UdpPacketModel(UdpPacket udp, long ts, int ms) {
this.ts = ts;
this.ms = ms;
this.sourcePort = udp.getHeader().getSrcPort().valueAsInt();
this.targetPort = udp.getHeader().getDstPort().valueAsInt();
this.size = udp.getRawData().length;
this.payloadSize = udp.getRawData().length - udp.getHeader().length();
}
public Object[] getArguments() {
Object[] arguments = {
"sourcePort", this.sourcePort,
"targetPort", this.targetPort,
"size", this.size,
"payloadSize", this.payloadSize,
};
return arguments;
}
}

View File

@ -15,7 +15,14 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import de.hsh.inform.orientdb_project.model.TcpConnection; import de.hsh.inform.orientdb_project.model.ArpPacketModel;
import de.hsh.inform.orientdb_project.model.EthernetFrameModel;
import de.hsh.inform.orientdb_project.model.HostModel;
import de.hsh.inform.orientdb_project.model.IcmpPacketModel;
import de.hsh.inform.orientdb_project.model.Ipv4PacketModel;
import de.hsh.inform.orientdb_project.model.TcpConnectionModel;
import de.hsh.inform.orientdb_project.model.TcpPacketModel;
import de.hsh.inform.orientdb_project.model.UdpPacketModel;
import de.hsh.inform.orientdb_project.netdata.AbstractNetdataImportService; import de.hsh.inform.orientdb_project.netdata.AbstractNetdataImportService;
public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNetdataImportService { public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNetdataImportService {
@ -26,53 +33,58 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe
private HashMap<String, Vertex> knownHosts; private HashMap<String, Vertex> knownHosts;
// To keep track of tcp connections // To keep track of tcp connections
private HashMap<String, LinkedList<TcpConnection>> knownTcpConnections; private HashMap<String, LinkedList<TcpConnectionModel>> knownTcpConnections;
private Vertex ethernetFrame; // References to already created model instances (these are reseted before processing a new ethernetFrame)
private Vertex arpPacket; private EthernetFrameModel ethernetFrameModel;
private Vertex ipPacket; private ArpPacketModel arpPacketModel;
private Vertex udpPacket; private Ipv4PacketModel ipv4PacketModel;
private Vertex tcpPacket; private TcpPacketModel tcpPacketModel;
private Vertex icmpPacket; private UdpPacketModel udpPacketModel;
private IcmpPacketModel icmpPacketModel;
// References to already created vertices (these are reseted before processing a new ethernetFrame)
private Vertex ethernetFrameVertex;
private Vertex arpPacketVertex;
private Vertex ipPacketVertex;
private Vertex udpPacketVertex;
private Vertex tcpPacketVertex;
private Vertex icmpPacketVertex;
public HighPerformanceKappaOrientDbNetdataImportService(String filename, OrientGraphNoTx orientGraph) { public HighPerformanceKappaOrientDbNetdataImportService(String filename, OrientGraphNoTx orientGraph) {
super(filename); super(filename);
this.og = orientGraph; this.og = orientGraph;
this.knownHosts = new HashMap<String, Vertex>(); this.knownHosts = new HashMap<String, Vertex>();
this.knownTcpConnections = new HashMap<String, LinkedList<TcpConnection>>(); this.knownTcpConnections = new HashMap<String, LinkedList<TcpConnectionModel>>();
} }
public void handleEthernetPacket(EthernetPacket ether, long ts, int ms) { public void handleEthernetPacket(EthernetPacket ether, long ts, int ms) {
// Clean up state vars before processing the next ethernet frame // Clean up state vars before processing the next ethernet frame
this.ethernetFrame = null; this.ethernetFrameVertex = null;
this.arpPacket = null; this.arpPacketVertex = null;
this.ipPacket = null; this.ipPacketVertex = null;
this.udpPacket = null; this.udpPacketVertex = null;
this.tcpPacket = null; this.tcpPacketVertex = null;
this.icmpPacket = null; this.icmpPacketVertex = null;
// Also clean model instances
this.ethernetFrameModel = null;
this.arpPacketModel = null;
this.ipv4PacketModel = null;
this.tcpPacketModel = null;
this.udpPacketModel = null;
this.icmpPacketModel = null;
// Okay, let's go! // Okay, let's go!
Object[] arguments = { this.ethernetFrameModel = new EthernetFrameModel(ether, ts, ms);
"sourceMac", ether.getHeader().getSrcAddr().toString(), this.ethernetFrameVertex = this.og.addVertex("class:EthernetFrame", this.ethernetFrameModel.getArguments());
"targetMac", ether.getHeader().getDstAddr().toString(),
"rawData", ether.getRawData(),
"size", ether.getRawData().length,
"payloadSize", ether.getRawData().length - ether.getHeader().length(),
"timestamp", ts,
"microseconds", ms,
};
this.ethernetFrame = this.og.addVertex("class:EthernetFrame", arguments);
super.handleEthernetPacket(ether, ts, ms); super.handleEthernetPacket(ether, ts, ms);
} }
public void handleArpPacket(ArpPacket arp, long ts, int ms) { public void handleArpPacket(ArpPacket arp, long ts, int ms) {
Object[] arguments = { this.arpPacketModel = new ArpPacketModel(arp, ts, ms);
"size", arp.getRawData().length, this.arpPacketVertex = this.og.addVertex("class:ArpPacket", this.arpPacketModel.getArguments());
"payloadSize", arp.getRawData().length - arp.getHeader().length(),
};
this.arpPacket = this.og.addVertex("class:ArpPacket", arguments);
// Wire up to its ethernet frame // Wire up to its ethernet frame
Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrame, this.arpPacket, "contains"); Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrameVertex, this.arpPacketVertex, "contains");
Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.arpPacket, this.ethernetFrame, "isContainedIn"); Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.arpPacketVertex, this.ethernetFrameVertex, "isContainedIn");
} }
public void handleIpV4Packet(IpV4Packet ipv4, long ts, int ms) { public void handleIpV4Packet(IpV4Packet ipv4, long ts, int ms) {
@ -81,51 +93,36 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe
// Add hosts to database if new // Add hosts to database if new
this.addHostIfNew(sourceIp); this.addHostIfNew(sourceIp);
this.addHostIfNew(targetIp); this.addHostIfNew(targetIp);
Object[] arguments = { this.ipv4PacketModel = new Ipv4PacketModel(ipv4, sourceIp, targetIp, ts, ms);
"sourceIp", sourceIp.toString().split("/")[1], this.ipPacketVertex = this.og.addVertex("class:IpPacket", this.ipv4PacketModel.getArguments());
"targetIp", targetIp.toString().split("/")[1],
"size", ipv4.getRawData().length,
"payloadSize", ipv4.getRawData().length - ipv4.getHeader().length(),
};
this.ipPacket = this.og.addVertex("class:IpPacket", arguments);
// Wire up to its ethernet frame // Wire up to its ethernet frame
Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrame, this.ipPacket, "contains"); Edge containsEdge = this.og.addEdge("class:contains", this.ethernetFrameVertex, this.ipPacketVertex, "contains");
Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.ipPacket, this.ethernetFrame, "isContainedIn"); Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.ipPacketVertex, this.ethernetFrameVertex, "isContainedIn");
super.handleIpV4Packet(ipv4, ts, ms); super.handleIpV4Packet(ipv4, ts, ms);
} }
public void handleUdpPacket(UdpPacket udp, long ts, int ms) { public void handleUdpPacket(UdpPacket udp, long ts, int ms) {
Object[] arguments = { this.udpPacketModel = new UdpPacketModel(udp, ts, ms);
"sourcePort", udp.getHeader().getSrcPort().valueAsInt(), this.udpPacketVertex = this.og.addVertex("class:UdpPacket", this.udpPacketModel.getArguments());
"targetPort", udp.getHeader().getDstPort().valueAsInt(),
"size", udp.getRawData().length,
"payloadSize", udp.getRawData().length - udp.getHeader().length(),
};
this.udpPacket = this.og.addVertex("class:UdpPacket");
// Wire up to its ip packet // Wire up to its ip packet
Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.udpPacket, "contains"); Edge containsEdge = this.og.addEdge("class:contains", this.ipPacketVertex, this.udpPacketVertex, "contains");
Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.udpPacket, this.ipPacket, "isContainedIn"); Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.udpPacketVertex, this.ipPacketVertex, "isContainedIn");
} }
public void handleTcpPacket(TcpPacket tcp, long ts, int ms) { public void handleTcpPacket(TcpPacket tcp, long ts, int ms) {
Object[] arguments = { this.tcpPacketModel = new TcpPacketModel(tcp, ts, ms);
"sourcePort", tcp.getHeader().getSrcPort().valueAsInt(), this.tcpPacketVertex = this.og.addVertex("class:TcpPacket", this.tcpPacketModel.getArguments());
"targetPort", tcp.getHeader().getDstPort().valueAsInt(),
"size", tcp.getRawData().length,
"payloadSize", tcp.getRawData().length - tcp.getHeader().length(),
};
this.tcpPacket = this.og.addVertex("class:TcpPacket", arguments);
// Wire up to its ip packet // Wire up to its ip packet
Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.tcpPacket, "contains"); Edge containsEdge = this.og.addEdge("class:contains", this.ipPacketVertex, this.tcpPacketVertex, "contains");
Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.tcpPacket, this.ipPacket, "isContainedIn"); Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.tcpPacketVertex, this.ipPacketVertex, "isContainedIn");
// Track tcp connections // Track tcp connections
TcpConnection tcpConnection = this.getTcpConnectionFor(tcp); TcpConnectionModel tcpConnection = this.getTcpConnectionFor(tcp);
// If connection exists ... // If connection exists ...
if(tcpConnection != null) { if(tcpConnection != null) {
// ... and still "up to date" aka time difference < 2s // ... and still "up to date" aka time difference < 2s
if(ts - tcpConnection.endTs < 2) { if(ts - tcpConnection.endTs < 2) {
// Update tcpConnection data // Update tcpConnection data
if(tcpConnection.sourceIp.equals(this.ipPacket.getProperty("sourceIp"))) { if(tcpConnection.sourceIp.equals(this.ipPacketVertex.getProperty("sourceIp"))) {
// SourceIp -> TargetIp // SourceIp -> TargetIp
tcpConnection.addVolumeSourceToTarget(tcp.getRawData().length - tcp.getHeader().length()); tcpConnection.addVolumeSourceToTarget(tcp.getRawData().length - tcp.getHeader().length());
} else { } else {
@ -136,24 +133,21 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe
} }
} else { } else {
// Else create a new one and add it to the list. // Else create a new one and add it to the list.
String sourceIp = this.ipPacket.getProperty("sourceIp"); String sourceIp = this.ipPacketVertex.getProperty("sourceIp");
String targetIp = this.ipPacket.getProperty("targetIp"); String targetIp = this.ipPacketVertex.getProperty("targetIp");
tcpConnection = new TcpConnection(tcp, sourceIp, targetIp, ts, ms); tcpConnection = new TcpConnectionModel(tcp, sourceIp, targetIp, ts, ms);
this.addKnownTcpConnectionFor(tcpConnection, tcp); this.addKnownTcpConnectionFor(tcpConnection, tcp);
} }
// Remember tcpPacketVertex in tcpConnection for later edges // Remember tcpPacketVertex in tcpConnection for later edges
tcpConnection.addKnownTcpPacketVertex(this.tcpPacket); tcpConnection.addKnownTcpPacketVertex(this.tcpPacketVertex);
} }
public void handleIcmpPacket(IcmpV4CommonPacket icmp, long ts, int ms) { public void handleIcmpPacket(IcmpV4CommonPacket icmp, long ts, int ms) {
Object[] arguments = { this.icmpPacketModel = new IcmpPacketModel(icmp, ts, ms);
"size", icmp.getRawData().length, this.icmpPacketVertex = this.og.addVertex("class:IcmpPacket", this.icmpPacketModel.getArguments());
"payloadSize", icmp.getRawData().length - icmp.getHeader().length(),
};
this.icmpPacket = this.og.addVertex("class:IcmpPacket");
// Wire up to its ip packet // Wire up to its ip packet
Edge containsEdge = this.og.addEdge("class:contains", this.ipPacket, this.icmpPacket, "contains"); Edge containsEdge = this.og.addEdge("class:contains", this.ipPacketVertex, this.icmpPacketVertex, "contains");
Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.icmpPacket, this.ipPacket, "isContainedIn"); Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", this.icmpPacketVertex, this.ipPacketVertex, "isContainedIn");
} }
private void addHostIfNew(Inet4Address ipAddress) { private void addHostIfNew(Inet4Address ipAddress) {
@ -165,11 +159,8 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe
boolean isInternal = ipAddress.isSiteLocalAddress(); // TODO: VERIFY IF THIS IS CORRECT! boolean isInternal = ipAddress.isSiteLocalAddress(); // TODO: VERIFY IF THIS IS CORRECT!
// Create Vertex and add to HashMap // Create Vertex and add to HashMap
String ipAddressStr = ipAddress.toString().split("/")[1]; String ipAddressStr = ipAddress.toString().split("/")[1];
Object[] arguments = { HostModel hostModel = new HostModel(ipAddressStr, isInternal);
"ipAddress", ipAddressStr, Vertex host = this.og.addVertex("class:Host", hostModel.getArguments());
"internal", isInternal,
};
Vertex host = this.og.addVertex("class:Host", arguments);
this.knownHosts.put(ipAddressStr, host); this.knownHosts.put(ipAddressStr, host);
} }
} }
@ -186,18 +177,18 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe
} }
} }
private TcpConnection getTcpConnectionFor(TcpPacket tcp) { private TcpConnectionModel getTcpConnectionFor(TcpPacket tcp) {
String source = ""; String source = "";
String target = ""; String target = "";
String sourceIp = this.ipPacket.getProperty("sourceIp"); String sourceIp = this.ipPacketVertex.getProperty("sourceIp");
String targetIp = this.ipPacket.getProperty("targetIp"); String targetIp = this.ipPacketVertex.getProperty("targetIp");
String sourcePort = tcp.getHeader().getSrcPort().valueAsString(); String sourcePort = tcp.getHeader().getSrcPort().valueAsString();
String targetPort = tcp.getHeader().getDstPort().valueAsString(); String targetPort = tcp.getHeader().getDstPort().valueAsString();
source = sourceIp + ":" + sourcePort; source = sourceIp + ":" + sourcePort;
target = targetIp + ":" + targetPort; target = targetIp + ":" + targetPort;
String connectionKey = this.buildConnectionKey(source, target); String connectionKey = this.buildConnectionKey(source, target);
TcpConnection tcpConnection = null; TcpConnectionModel tcpConnection = null;
LinkedList<TcpConnection> connectionList = null; LinkedList<TcpConnectionModel> connectionList = null;
// Get or create tcp connection list for connection key // Get or create tcp connection list for connection key
if(this.knownTcpConnections.containsKey(connectionKey)) { if(this.knownTcpConnections.containsKey(connectionKey)) {
connectionList = this.knownTcpConnections.get(connectionKey); connectionList = this.knownTcpConnections.get(connectionKey);
@ -210,21 +201,21 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe
return tcpConnection; return tcpConnection;
} }
private void addKnownTcpConnectionFor(TcpConnection tcpConnection, TcpPacket tcp) { private void addKnownTcpConnectionFor(TcpConnectionModel tcpConnection, TcpPacket tcp) {
String source = ""; String source = "";
String target = ""; String target = "";
String sourceIp = this.ipPacket.getProperty("sourceIp"); String sourceIp = this.ipPacketVertex.getProperty("sourceIp");
String targetIp = this.ipPacket.getProperty("targetIp"); String targetIp = this.ipPacketVertex.getProperty("targetIp");
String sourcePort = tcp.getHeader().getSrcPort().valueAsString(); String sourcePort = tcp.getHeader().getSrcPort().valueAsString();
String targetPort = tcp.getHeader().getDstPort().valueAsString(); String targetPort = tcp.getHeader().getDstPort().valueAsString();
source = sourceIp + ":" + sourcePort; source = sourceIp + ":" + sourcePort;
target = targetIp + ":" + targetPort; target = targetIp + ":" + targetPort;
String connectionKey = this.buildConnectionKey(source, target); String connectionKey = this.buildConnectionKey(source, target);
LinkedList<TcpConnection> connectionList = null; LinkedList<TcpConnectionModel> connectionList = null;
if(this.knownTcpConnections.containsKey(connectionKey)) { if(this.knownTcpConnections.containsKey(connectionKey)) {
connectionList = this.knownTcpConnections.get(connectionKey); connectionList = this.knownTcpConnections.get(connectionKey);
} else { } else {
connectionList = new LinkedList<TcpConnection>(); connectionList = new LinkedList<TcpConnectionModel>();
this.knownTcpConnections.put(connectionKey, connectionList); this.knownTcpConnections.put(connectionKey, connectionList);
} }
// Put connection into list of known tcp connections // Put connection into list of known tcp connections
@ -234,8 +225,8 @@ public class HighPerformanceKappaOrientDbNetdataImportService extends AbstractNe
public void afterImport() { public void afterImport() {
// TODO: Link TcpConnections up with their tcpPackets! // TODO: Link TcpConnections up with their tcpPackets!
System.out.println("All done. Processing collected TcpConnections ..."); System.out.println("All done. Processing collected TcpConnections ...");
for(LinkedList<TcpConnection> connList : this.knownTcpConnections.values()) { for(LinkedList<TcpConnectionModel> connList : this.knownTcpConnections.values()) {
for(TcpConnection conn : connList) { for(TcpConnectionModel conn : connList) {
Vertex currentTcpConnection = this.og.addVertex("class:TcpConnection", conn.getArguments()); Vertex currentTcpConnection = this.og.addVertex("class:TcpConnection", conn.getArguments());
// Look up already created source and target host vertices // Look up already created source and target host vertices
Vertex sourceHost = this.knownHosts.get(conn.sourceIp); Vertex sourceHost = this.knownHosts.get(conn.sourceIp);

View File

@ -3,7 +3,6 @@ package de.hsh.inform.orientdb_project.orientdb;
import java.io.IOException; import java.io.IOException;
import com.orientechnologies.orient.client.remote.OServerAdmin; import com.orientechnologies.orient.client.remote.OServerAdmin;
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.intent.OIntentMassiveInsert; import com.orientechnologies.orient.core.intent.OIntentMassiveInsert;
import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.metadata.schema.OType;
import com.tinkerpop.blueprints.impls.orient.OrientConfigurableGraph.THREAD_MODE; import com.tinkerpop.blueprints.impls.orient.OrientConfigurableGraph.THREAD_MODE;
@ -12,6 +11,15 @@ import com.tinkerpop.blueprints.impls.orient.OrientGraphFactory;
import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
import de.hsh.inform.orientdb_project.model.ArpPacketModel;
import de.hsh.inform.orientdb_project.model.EthernetFrameModel;
import de.hsh.inform.orientdb_project.model.HostModel;
import de.hsh.inform.orientdb_project.model.IcmpPacketModel;
import de.hsh.inform.orientdb_project.model.Ipv4PacketModel;
import de.hsh.inform.orientdb_project.model.TcpConnectionModel;
import de.hsh.inform.orientdb_project.model.TcpPacketModel;
import de.hsh.inform.orientdb_project.model.UdpPacketModel;
public class OrientDbHelperService { public class OrientDbHelperService {
private String host; private String host;
@ -115,59 +123,19 @@ public class OrientDbHelperService {
private void createClasses() { private void createClasses() {
OrientGraphNoTx og = this.getOrientGraphFactory().getNoTx(); OrientGraphNoTx og = this.getOrientGraphFactory().getNoTx();
OrientVertexType ethernetFrameType = og.createVertexType("EthernetFrame", "V"); // Use methods integrated into the models to create their classes
ethernetFrameType.createProperty("sourceMac", OType.STRING); // These are the vertex types used in our model
ethernetFrameType.createProperty("targetMac", OType.STRING); EthernetFrameModel.createType(og);
ethernetFrameType.createProperty("rawData", OType.BINARY); ArpPacketModel.createType(og);
ethernetFrameType.createProperty("size", OType.INTEGER); Ipv4PacketModel.createType(og);
ethernetFrameType.createProperty("payloadSize", OType.INTEGER); UdpPacketModel.createType(og);
ethernetFrameType.createProperty("timestamp", OType.LONG); TcpPacketModel.createType(og);
ethernetFrameType.createProperty("microseconds", OType.INTEGER); IcmpPacketModel.createType(og);
OrientVertexType arpPacketType = og.createVertexType("ArpPacket", "V"); HostModel.createType(og);
// TODO: Not finished! TcpConnectionModel.createType(og);
arpPacketType.createProperty("askedForIp", OType.STRING);
arpPacketType.createProperty("hasIp", OType.STRING);
arpPacketType.createProperty("size", OType.INTEGER);
arpPacketType.createProperty("payloadSize", OType.INTEGER);
OrientVertexType ipPacketType = og.createVertexType("IpPacket", "V");
ipPacketType.createProperty("sourceIp", OType.STRING);
ipPacketType.createProperty("targetIp", OType.STRING);
ipPacketType.createProperty("size", OType.INTEGER);
ipPacketType.createProperty("payloadSize", OType.INTEGER);
OrientVertexType udpPacketType = og.createVertexType("UdpPacket", "V");
udpPacketType.createProperty("sourcePort", OType.INTEGER);
udpPacketType.createProperty("targetPort", OType.INTEGER);
udpPacketType.createProperty("size", OType.INTEGER);
udpPacketType.createProperty("payloadSize", OType.INTEGER);
OrientVertexType tcpPacketType = og.createVertexType("TcpPacket", "V");
tcpPacketType.createProperty("sourcePort", OType.INTEGER);
tcpPacketType.createProperty("targetPort", OType.INTEGER);
tcpPacketType.createProperty("size", OType.INTEGER);
tcpPacketType.createProperty("payloadSize", OType.INTEGER);
OrientVertexType icmpPacketType = og.createVertexType("IcmpPacket", "V");
icmpPacketType.createProperty("size", OType.INTEGER);
icmpPacketType.createProperty("payloadSize", OType.INTEGER);
OrientVertexType hostType = og.createVertexType("Host", "V");
hostType.createProperty("ipAddress", OType.STRING);
hostType.createProperty("internal", OType.BOOLEAN);
OrientVertexType tcpConnectionType = og.createVertexType("TcpConnection", "V");
tcpConnectionType.createProperty("startTs", OType.LONG);
tcpConnectionType.createProperty("startMs", OType.INTEGER);
tcpConnectionType.createProperty("endTs", OType.LONG);
tcpConnectionType.createProperty("endMs", OType.INTEGER);
tcpConnectionType.createProperty("sourcePort", OType.INTEGER);
tcpConnectionType.createProperty("targetPort", OType.INTEGER);
tcpConnectionType.createProperty("volumeSourceToTarget", OType.LONG);
tcpConnectionType.createProperty("volumeTargetToSource", OType.LONG);
tcpConnectionType.createProperty("totalVolume", OType.LONG);
// Edges do not really need their own model (yet), they connect the vertex types
OrientEdgeType isContainedInType = og.createEdgeType("isContainedIn", "E"); OrientEdgeType isContainedInType = og.createEdgeType("isContainedIn", "E");
isContainedInType.setDescription("isContainedIn"); isContainedInType.setDescription("isContainedIn");
OrientEdgeType containsType = og.createEdgeType("contains", "E"); OrientEdgeType containsType = og.createEdgeType("contains", "E");
@ -189,6 +157,7 @@ public class OrientDbHelperService {
OrientEdgeType hasRelatedTcpPacketType = og.createEdgeType("hasRelatedTcpPacket", "E"); OrientEdgeType hasRelatedTcpPacketType = og.createEdgeType("hasRelatedTcpPacket", "E");
hasRelatedTcpPacketType.setDescription("hasRelatedTcpPacket"); hasRelatedTcpPacketType.setDescription("hasRelatedTcpPacket");
// We're done creating classes and types, shut down the database graph.
og.shutdown(); og.shutdown();
} }

View File

@ -0,0 +1,38 @@
package de.hsh.inform.orientdb_project.util;
public class SequenceProvider {
private static String NOT_LABELED = "Not labeled";
private long sequenceNumber;
private String label;
public SequenceProvider() {
this(0, SequenceProvider.NOT_LABELED);
}
public SequenceProvider(long startValue) {
this(startValue, SequenceProvider.NOT_LABELED);
}
public SequenceProvider(long startValue, String label) {
this.sequenceNumber = startValue;
this.label = label;
}
public long getNextNumber() {
long result = this.sequenceNumber;
this.sequenceNumber++;
return result;
}
public String toString() {
StringBuilder sb = new StringBuilder("[SequenceProvider '");
sb.append(this.label);
sb.append("'] #=");
sb.append(this.sequenceNumber);
return sb.toString();
}
}