commit 3708c566bfec4e7df50760cfa25110712a3ab07a Author: Jan Philipp Timme Date: Sun Nov 6 20:20:14 2016 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..769b429 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.class + +.classpath +.settings +.project diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..4a03337 --- /dev/null +++ b/pom.xml @@ -0,0 +1,71 @@ + + 4.0.0 + + de.hsh.inform + dbp-project-readdata + 0.0.1-SNAPSHOT + jar + + dbp-project-readdata + http://maven.apache.org + + + UTF-8 + + + + + org.pcap4j + pcap4j-core + 1.4.0 + + + org.pcap4j + pcap4j-packetfactory-static + 1.4.0 + + + junit + junit + 3.8.1 + test + + + + com.orientechnologies + orientdb-core + 2.2.12 + + + + com.orientechnologies + orientdb-client + 2.2.12 + + + + com.orientechnologies + orientdb-graphdb + 2.2.12 + + + org.apache.tinkerpop + gremlin-core + 3.2.3 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + 1.8 + 1.8 + + + + + diff --git a/src/main/java/de/hsh/inform/orientdb_project/Main.java b/src/main/java/de/hsh/inform/orientdb_project/Main.java new file mode 100644 index 0000000..c2ee046 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/Main.java @@ -0,0 +1,30 @@ +package de.hsh.inform.orientdb_project; + +import java.io.EOFException; +import java.util.concurrent.TimeoutException; + +import org.pcap4j.core.NotOpenException; +import org.pcap4j.core.PcapNativeException; + +import de.hsh.inform.orientdb_project.orientdb.OrientDbHelperService; +import de.hsh.inform.orientdb_project.orientdb.OrientDbNetdataImportService; + +public class Main { + + public static void main(String[] args) { + OrientDbHelperService odhs = new OrientDbHelperService("192.168.0.110", "hshtest", "root", "root"); + odhs.cleanUpServer(); + odhs.setupSchema(); + + String filename = "/home/jpt/Temp/tcpdump_2"; + OrientDbNetdataImportService odbis = new OrientDbNetdataImportService(filename, odhs.getOrientGraphFactory().getNoTx()); + try { + System.out.println("Begin import of data ..."); + odbis.run(); + System.out.println("Import of data done!"); + } catch (EOFException | PcapNativeException | TimeoutException | NotOpenException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/netdata/AbstractNetdataImportService.java b/src/main/java/de/hsh/inform/orientdb_project/netdata/AbstractNetdataImportService.java new file mode 100644 index 0000000..30a3128 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/netdata/AbstractNetdataImportService.java @@ -0,0 +1,75 @@ +package de.hsh.inform.orientdb_project.netdata; + +import java.io.EOFException; +import java.util.concurrent.TimeoutException; + +import org.pcap4j.core.NotOpenException; +import org.pcap4j.core.PcapHandle; +import org.pcap4j.core.PcapNativeException; +import org.pcap4j.core.Pcaps; +import org.pcap4j.packet.ArpPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.packet.FragmentedPacket; +import org.pcap4j.packet.IcmpV4CommonPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.Packet; +import org.pcap4j.packet.TcpPacket; +import org.pcap4j.packet.UdpPacket; +import org.pcap4j.packet.namednumber.EtherType; +import org.pcap4j.packet.namednumber.IpNumber; + +/** + * Contains the logic to extract all the detailed stuff + */ +public abstract class AbstractNetdataImportService implements NetdataResultObserver { + + private String filename; + + public AbstractNetdataImportService(String filename) { + this.filename = filename; + } + + public final void run() throws PcapNativeException, EOFException, TimeoutException, NotOpenException { + PcapHandle handle = Pcaps.openOffline(this.filename); + for (;;) { + Packet packet = handle.getNextPacketEx(); + if(packet == null) break; + long ts = handle.getTimestampInts(); + int ms = handle.getTimestampMicros(); + EthernetPacket ether = packet.get(EthernetPacket.class); + this.handleEthernetPacket(ether, ts, ms); + } + } + + public void handleEthernetPacket(EthernetPacket ether, long ts, int ms) { + EtherType etherType = ether.getHeader().getType(); + if (etherType.equals(EtherType.IPV4)) { + IpV4Packet ipv4 = ether.getPayload().get(IpV4Packet.class); + this.handleIpV4Packet(ipv4, ts, ms); + } else if (ether.getHeader().getType().equals(EtherType.ARP)) { + ArpPacket arp = ether.getPayload().get(ArpPacket.class); + this.handleArpPacket(arp, ts, ms); + } else { + //System.out.println("Unknown ethernet frame type thing!"); + } + } + + public void handleIpV4Packet(IpV4Packet ipv4, long ts, int ms) { + IpNumber ipnum = ipv4.getHeader().getProtocol(); + if (ipv4.getPayload() instanceof FragmentedPacket) { + System.out.println("Fragmented IP Packet!"); + } else if (ipnum.equals(IpNumber.TCP)) { + TcpPacket tcp = ipv4.getPayload().get(TcpPacket.class); + this.handleTcpPacket(tcp, ts, ms); + } else if (ipnum.equals(IpNumber.UDP)) { + UdpPacket udp = ipv4.getPayload().get(UdpPacket.class); + this.handleUdpPacket(udp, ts, ms); + } else if (ipnum.equals(IpNumber.ICMPV4)) { + IcmpV4CommonPacket icmp = ipv4.getPayload().get(IcmpV4CommonPacket.class); + this.handleIcmpPacket(icmp, ts, ms); + } else { + //System.out.println("Unknown IP Packet!"); + } + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/netdata/NetdataResultObserver.java b/src/main/java/de/hsh/inform/orientdb_project/netdata/NetdataResultObserver.java new file mode 100644 index 0000000..a2537d2 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/netdata/NetdataResultObserver.java @@ -0,0 +1,24 @@ +package de.hsh.inform.orientdb_project.netdata; + +import org.pcap4j.packet.ArpPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.packet.IcmpV4CommonPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; +import org.pcap4j.packet.UdpPacket; + +public interface NetdataResultObserver { + + public abstract void handleEthernetPacket(EthernetPacket ether, long timestamp, int milliseconds); + + public abstract void handleArpPacket(ArpPacket arp, long timestamp, int milliseconds); + + public abstract void handleTcpPacket(TcpPacket tcp, long timestamp, int milliseconds); + + public abstract void handleUdpPacket(UdpPacket udp, long timestamp, int milliseconds); + + public abstract void handleIcmpPacket(IcmpV4CommonPacket icmp, long timestamp, int milliseconds); + + public abstract void handleIpV4Packet(IpV4Packet ipv4, long ts, int ms); + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java new file mode 100644 index 0000000..49198a4 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbHelperService.java @@ -0,0 +1,122 @@ +package de.hsh.inform.orientdb_project.orientdb; + +import java.io.IOException; + +import com.orientechnologies.orient.client.remote.OServerAdmin; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.tinkerpop.blueprints.impls.orient.OrientEdgeType; +import com.tinkerpop.blueprints.impls.orient.OrientGraphFactory; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +public class OrientDbHelperService { + + private String host; + private String db; + private String user; + private String pass; + + private OrientGraphFactory factory; + + + public OrientDbHelperService(String host, String db, String user, String pass) { + this.host = host; + this.db = db; + this.user = user; + this.pass = pass; + this.factory = null; + } + + public OrientGraphFactory getOrientGraphFactory() { + if(this.factory == null) { + this.factory = new OrientGraphFactory(getDbUri(true), this.user, this.pass); + } + return this.factory; + } + + public String getDbUri(boolean withDb) { + String uri = "remote:" + this.host; + if(withDb) { + uri += "/" + this.db; + } + return uri; + } + + public void cleanUpServer() { + //String storageType = "plocal"; + String storageType = "memory"; + // Drop old database and re-create it + try { + OServerAdmin admin = new OServerAdmin(getDbUri(false)); + admin.connect(this.user, this.pass); + admin.dropDatabase(this.db, storageType); + admin.createDatabase(this.db, "graph", storageType); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void setupSchema() { + OrientGraphNoTx og = this.getOrientGraphFactory().getNoTx(); + OrientVertexType ethernetFrameType = og.createVertexType("EthernetFrame", "V"); + ethernetFrameType.createProperty("sourceMac", OType.STRING); + ethernetFrameType.createProperty("targetMac", OType.STRING); + ethernetFrameType.createProperty("rawData", OType.BINARY); + ethernetFrameType.createProperty("size", OType.INTEGER); + ethernetFrameType.createProperty("payloadSize", OType.INTEGER); + ethernetFrameType.createProperty("timestamp", OType.LONG); + ethernetFrameType.createProperty("microseconds", OType.INTEGER); + + OrientVertexType arpPacketType = og.createVertexType("ArpPacket", "V"); + // TODO: Not finished! + arpPacketType.createProperty("askedForIp", OType.STRING); + arpPacketType.createProperty("hasIp", OType.STRING); + arpPacketType.createProperty("size", OType.INTEGER); + arpPacketType.createProperty("payloadSize", OType.INTEGER); + + OrientVertexType ipPacketType = og.createVertexType("IpPacket", "V"); + ipPacketType.createProperty("sourceIp", OType.STRING); + ipPacketType.createProperty("targetIp", OType.STRING); + ipPacketType.createProperty("size", OType.INTEGER); + ipPacketType.createProperty("payloadSize", OType.INTEGER); + + OrientVertexType udpPacketType = og.createVertexType("UdpPacket", "V"); + udpPacketType.createProperty("sourcePort", OType.INTEGER); + udpPacketType.createProperty("targetPort", OType.INTEGER); + udpPacketType.createProperty("size", OType.INTEGER); + udpPacketType.createProperty("payloadSize", OType.INTEGER); + + OrientVertexType tcpPacketType = og.createVertexType("TcpPacket", "V"); + tcpPacketType.createProperty("sourcePort", OType.INTEGER); + tcpPacketType.createProperty("targetPort", OType.INTEGER); + tcpPacketType.createProperty("size", OType.INTEGER); + tcpPacketType.createProperty("payloadSize", OType.INTEGER); + + OrientVertexType icmpPacketType = og.createVertexType("IcmpPacket", "V"); + icmpPacketType.createProperty("size", OType.INTEGER); + icmpPacketType.createProperty("payloadSize", OType.INTEGER); + + OrientVertexType hostType = og.createVertexType("Host", "V"); + hostType.createProperty("ipAddress", OType.STRING); + hostType.createProperty("macAddress", OType.STRING); + hostType.createProperty("internal", OType.BOOLEAN); + + OrientVertexType tcpConnectionType = og.createVertexType("TcpConnection", "V"); + tcpConnectionType.createProperty("start", OType.DATETIME); + tcpConnectionType.createProperty("end", OType.DATETIME); + tcpConnectionType.createProperty("sourcePort", OType.INTEGER); + tcpConnectionType.createProperty("targetPort", OType.INTEGER); + tcpConnectionType.createProperty("volumeSourceToTarget", OType.INTEGER); + tcpConnectionType.createProperty("volumeTargetToSource", OType.INTEGER); + tcpConnectionType.createProperty("totalVolume", OType.INTEGER); + + OrientEdgeType isContainedInType = og.createEdgeType("isContainedIn", "E"); + isContainedInType.setDescription("isContainedIn"); + OrientEdgeType containsType = og.createEdgeType("contains", "E"); + containsType.setDescription("contains"); + + // Uhm ... this should be okay ... ? + og.shutdown(); + } + +} diff --git a/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbNetdataImportService.java b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbNetdataImportService.java new file mode 100644 index 0000000..498f475 --- /dev/null +++ b/src/main/java/de/hsh/inform/orientdb_project/orientdb/OrientDbNetdataImportService.java @@ -0,0 +1,105 @@ +package de.hsh.inform.orientdb_project.orientdb; + +import org.pcap4j.packet.ArpPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.packet.IcmpV4CommonPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; +import org.pcap4j.packet.UdpPacket; + +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; + +import de.hsh.inform.orientdb_project.netdata.AbstractNetdataImportService; + +public class OrientDbNetdataImportService extends AbstractNetdataImportService { + + private OrientGraphNoTx og; + + public OrientDbNetdataImportService(String filename, OrientGraphNoTx orientGraph) { + super(filename); + this.og = orientGraph; + } + + public void handleEthernetPacket(EthernetPacket ether, long ts, int ms) { + Vertex ethernetFrame = this.og.addVertex("class:EthernetFrame"); + ethernetFrame.setProperty("sourceMac", ether.getHeader().getSrcAddr().toString()); + ethernetFrame.setProperty("targetMac", ether.getHeader().getDstAddr().toString()); + ethernetFrame.setProperty("rawData", ether.getRawData()); + ethernetFrame.setProperty("size", ether.getRawData().length); + ethernetFrame.setProperty("payloadSize", ether.getRawData().length - ether.getHeader().length()); + ethernetFrame.setProperty("timestamp", ts); + ethernetFrame.setProperty("microseconds", ms); + super.handleEthernetPacket(ether, ts, ms); + } + + public void handleArpPacket(ArpPacket arp, long ts, int ms) { + Vertex arpPacket = this.og.addVertex("class:ArpPacket"); + arpPacket.setProperty("size", arp.getRawData().length); + // TODO: Not finished yet! + arpPacket.setProperty("payloadSize", arp.getRawData().length - arp.getHeader().length()); + // Wire up to its ethernet frame + Iterable result = this.og.getVertices("EthernetFrame", new String[]{"microseconds", "timestamp"}, new Object[]{ms, ts}); + Vertex ethernetFrame = result.iterator().next(); + Edge containsEdge = this.og.addEdge("class:contains", ethernetFrame, arpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", arpPacket, ethernetFrame, "isContainedIn"); + } + + public void handleIpV4Packet(IpV4Packet ipv4, long ts, int ms) { + Vertex ipPacket = this.og.addVertex("class:IpPacket"); + ipPacket.setProperty("sourceIp", ipv4.getHeader().getSrcAddr().getAddress().toString()); + ipPacket.setProperty("targetIp", ipv4.getHeader().getDstAddr().getAddress().toString()); + ipPacket.setProperty("size", ipv4.getRawData().length); + ipPacket.setProperty("payloadSize", ipv4.getRawData().length - ipv4.getHeader().length()); + // Wire up to its ethernet frame + Iterable result = this.og.getVertices("EthernetFrame", new String[]{"microseconds", "timestamp"}, new Object[]{ms, ts}); + Vertex ethernetFrame = result.iterator().next(); + Edge containsEdge = this.og.addEdge("class:contains", ethernetFrame, ipPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", ipPacket, ethernetFrame, "isContainedIn"); + super.handleIpV4Packet(ipv4, ts, ms); + } + + public void handleUdpPacket(UdpPacket udp, long ts, int ms) { + Vertex udpPacket = this.og.addVertex("class:UdpPacket"); + udpPacket.setProperty("sourcePort", udp.getHeader().getSrcPort().valueAsInt()); + udpPacket.setProperty("targetPort", udp.getHeader().getDstPort().valueAsInt()); + udpPacket.setProperty("size", udp.getRawData().length); + udpPacket.setProperty("payloadSize", udp.getRawData().length - udp.getHeader().length()); + // Wire up to its ip packet + Iterable result = this.og.getVertices("EthernetFrame", new String[]{"microseconds", "timestamp"}, new Object[]{ms, ts}); + Vertex ethernetFrame = result.iterator().next(); + Vertex ipPacket = ethernetFrame.getEdges(Direction.OUT, "contains").iterator().next().getVertex(Direction.IN); + Edge containsEdge = this.og.addEdge("class:contains", ipPacket, udpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", udpPacket, ipPacket, "isContainedIn"); + } + + public void handleTcpPacket(TcpPacket tcp, long ts, int ms) { + Vertex tcpPacket = this.og.addVertex("class:TcpPacket"); + tcpPacket.setProperty("sourcePort", tcp.getHeader().getSrcPort().valueAsInt()); + tcpPacket.setProperty("targetPort", tcp.getHeader().getDstPort().valueAsInt()); + tcpPacket.setProperty("size", tcp.getRawData().length); + tcpPacket.setProperty("payloadSize", tcp.getRawData().length - tcp.getHeader().length()); + // Wire up to its ip packet + Iterable result = this.og.getVertices("EthernetFrame", new String[]{"microseconds", "timestamp"}, new Object[]{ms, ts}); + Vertex ethernetFrame = result.iterator().next(); + Vertex ipPacket = ethernetFrame.getEdges(Direction.OUT, "contains").iterator().next().getVertex(Direction.IN); + Edge containsEdge = this.og.addEdge("class:contains", ipPacket, tcpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", tcpPacket, ipPacket, "isContainedIn"); + } + + public void handleIcmpPacket(IcmpV4CommonPacket icmp, long ts, int ms) { + Vertex icmpPacket = this.og.addVertex("class:IcmpPacket"); + icmpPacket.setProperty("size", icmp.getRawData().length); + icmpPacket.setProperty("payloadSize", icmp.getRawData().length - icmp.getHeader().length()); + // Wire up to its ip packet + Iterable result = this.og.getVertices("EthernetFrame", new String[]{"microseconds", "timestamp"}, new Object[]{ms, ts}); + Vertex ethernetFrame = result.iterator().next(); + Vertex ipPacket = ethernetFrame.getEdges(Direction.OUT, "contains").iterator().next().getVertex(Direction.IN); + Edge containsEdge = this.og.addEdge("class:contains", ipPacket, icmpPacket, "contains"); + Edge isContainedInEdge = this.og.addEdge("class:isContainedIn", icmpPacket, ipPacket, "isContainedIn"); + } + + +}