mirror of https://github.com/40t/go-sniffer.git
Merge 0313eb2dff into a50ccb4caa
This commit is contained in:
hoffer
GitHub
commit
6d9b72393f
|
|
@ -2,13 +2,14 @@ package core
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/google/gopacket/pcap"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/google/gopacket/pcap"
|
||||
"github.com/google/gopacket/tcpassembly"
|
||||
"github.com/google/gopacket/tcpassembly/tcpreader"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Dispatch struct {
|
||||
|
|
@ -41,7 +42,7 @@ func (d *Dispatch) Capture() {
|
|||
|
||||
// Capture
|
||||
src := gopacket.NewPacketSource(handle, handle.LinkType())
|
||||
packets := src.Packets()
|
||||
packets := src.Packets() // get packet chan
|
||||
|
||||
// Set up assembly
|
||||
streamFactory := &ProtocolStreamFactory{
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -19,6 +19,27 @@ const (
|
|||
CreateTopicsReqKind = 19
|
||||
)
|
||||
|
||||
const ()
|
||||
|
||||
var RquestNameMap = map[int16]string{
|
||||
0: "ProduceRequest",
|
||||
1: "FetchRequest",
|
||||
2: "OffsetRequest",
|
||||
3: "MetadataRequest",
|
||||
//Non-user facing control APIs = 4-7
|
||||
8: "OffsetCommitRequest",
|
||||
9: "OffsetFetchRequest",
|
||||
10: "GroupCoordinatorRequest",
|
||||
11: "JoinGroupRequest",
|
||||
12: "HeartbeatRequest",
|
||||
13: "LeaveGroupRequest",
|
||||
14: "SyncGroupRequest",
|
||||
15: "DescribeGroupsRequest",
|
||||
16: "ListGroupsRequest",
|
||||
18: "APIVersionsReqKind",
|
||||
19: "CreateTopicsReqKind",
|
||||
}
|
||||
|
||||
const (
|
||||
ApiV0 = 0
|
||||
ApiV1 = 1
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@ package build
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/google/gopacket"
|
||||
"io"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -23,10 +25,10 @@ type Kafka struct {
|
|||
|
||||
type stream struct {
|
||||
packets chan *packet
|
||||
correlationMap map[int32]requestHeader
|
||||
}
|
||||
|
||||
type packet struct {
|
||||
|
||||
isClientFlow bool //客户端->服务器端流
|
||||
messageSize int32
|
||||
|
||||
|
|
@ -51,9 +53,10 @@ type messageSet struct {
|
|||
offset int64
|
||||
messageSize int32
|
||||
}
|
||||
|
||||
func newMessageSet(r io.Reader) messageSet {
|
||||
messageSet := messageSet{}
|
||||
messageSet.offset = ReadInt64(r)
|
||||
_, messageSet.offset = ReadInt64(r)
|
||||
messageSet.messageSize = ReadInt32(r)
|
||||
|
||||
return messageSet
|
||||
|
|
@ -69,6 +72,7 @@ type message struct {
|
|||
|
||||
var kafkaInstance *Kafka
|
||||
var once sync.Once
|
||||
|
||||
func NewInstance() *Kafka {
|
||||
once.Do(func() {
|
||||
kafkaInstance = &Kafka{
|
||||
|
|
@ -94,7 +98,7 @@ func (m *Kafka) SetFlag(flg []string) {
|
|||
|
||||
switch key {
|
||||
case CmdPort:
|
||||
p, err := strconv.Atoi(val);
|
||||
p, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
panic("端口数不正确")
|
||||
}
|
||||
|
|
@ -110,7 +114,7 @@ func (m *Kafka) SetFlag(flg []string) {
|
|||
}
|
||||
|
||||
func (m *Kafka) BPFFilter() string {
|
||||
return "tcp and port "+strconv.Itoa(m.port);
|
||||
return "tcp and port " + strconv.Itoa(m.port)
|
||||
}
|
||||
|
||||
func (m *Kafka) Version() string {
|
||||
|
|
@ -127,6 +131,7 @@ func (m *Kafka) ResolveStream(net, transport gopacket.Flow, buf io.Reader) {
|
|||
|
||||
var newStream = stream{
|
||||
packets: make(chan *packet, 100),
|
||||
correlationMap: make(map[int32]requestHeader),
|
||||
}
|
||||
|
||||
m.source[uuid] = &newStream
|
||||
|
|
@ -139,7 +144,7 @@ func (m *Kafka) ResolveStream(net, transport gopacket.Flow, buf io.Reader) {
|
|||
|
||||
newPacket := m.newPacket(net, transport, buf)
|
||||
if newPacket == nil {
|
||||
return
|
||||
continue
|
||||
}
|
||||
|
||||
m.source[uuid].packets <- newPacket
|
||||
|
|
@ -151,8 +156,22 @@ func (m *Kafka) newPacket(net, transport gopacket.Flow, r io.Reader) *packet {
|
|||
//read packet
|
||||
pk := packet{}
|
||||
|
||||
/*
|
||||
bs := make([]byte, 0)
|
||||
count, err := r.Read(bs)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Printf("read: %d, buffer: %b", count, bs)
|
||||
return nil
|
||||
*/
|
||||
|
||||
//read messageSize
|
||||
pk.messageSize = ReadInt32(r)
|
||||
if pk.messageSize == 0 {
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("pk.messageSize: %d\n", pk.messageSize)
|
||||
|
||||
//set flow direction
|
||||
if transport.Src().String() == strconv.Itoa(m.port) {
|
||||
|
|
@ -160,6 +179,7 @@ func (m *Kafka) newPacket(net, transport gopacket.Flow, r io.Reader) *packet {
|
|||
|
||||
respHeader := responseHeader{}
|
||||
respHeader.correlationId = ReadInt32(r)
|
||||
// TODO extract request
|
||||
pk.responseHeader = respHeader
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
|
@ -204,29 +224,68 @@ func (stm *stream) resolve() {
|
|||
select {
|
||||
case packet := <-stm.packets:
|
||||
if packet.isClientFlow {
|
||||
stm.correlationMap[packet.requestHeader.correlationId] = packet.requestHeader
|
||||
stm.resolveClientPacket(packet)
|
||||
} else {
|
||||
stm.resolveServerPacket(packet)
|
||||
if _, ok := stm.correlationMap[packet.responseHeader.correlationId]; ok {
|
||||
stm.resolveServerPacket(packet, stm.correlationMap[packet.responseHeader.correlationId])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (stm *stream) resolveServerPacket(pk *packet) {
|
||||
return
|
||||
func (stm *stream) resolveServerPacket(pk *packet, rh requestHeader) {
|
||||
var msg interface{}
|
||||
payload := pk.payload
|
||||
|
||||
action := Action{
|
||||
Request: GetRquestName(pk.apiKey),
|
||||
Direction: "isServer",
|
||||
ApiVersion: pk.apiVersion,
|
||||
}
|
||||
switch int(rh.apiKey) {
|
||||
case ProduceRequest:
|
||||
msg = ReadProduceResponse(payload, rh.apiVersion)
|
||||
case MetadataRequest:
|
||||
msg = ReadMetadataResponse(payload, rh.apiVersion)
|
||||
default:
|
||||
fmt.Printf("response ApiKey:%d TODO", rh.apiKey)
|
||||
}
|
||||
|
||||
if msg != nil {
|
||||
action.Message = msg
|
||||
}
|
||||
bs, err := json.Marshal(action)
|
||||
if err != nil {
|
||||
fmt.Printf("json marshal action failed, err: %+v\n", err)
|
||||
}
|
||||
fmt.Printf("%s\n", string(bs))
|
||||
}
|
||||
|
||||
func (stm *stream) resolveClientPacket(pk *packet) {
|
||||
var msg string
|
||||
var msg interface{}
|
||||
action := Action{
|
||||
Request: GetRquestName(pk.apiKey),
|
||||
Direction: "isClient",
|
||||
ApiVersion: pk.apiVersion,
|
||||
}
|
||||
payload := pk.payload
|
||||
|
||||
fmt.Println("apiKey:")
|
||||
fmt.Println(pk.apiKey)
|
||||
|
||||
switch int(pk.apiKey) {
|
||||
case ProduceRequest:
|
||||
msg = ReadProduceRequest(payload, pk.apiVersion)
|
||||
case MetadataRequest:
|
||||
msg = ReadMetadataRequest(payload, pk.apiVersion)
|
||||
default:
|
||||
fmt.Printf("ApiKey:%d TODO", pk.apiKey)
|
||||
}
|
||||
_=msg
|
||||
//fmt.Println(msg)
|
||||
|
||||
if msg != nil {
|
||||
action.Message = msg
|
||||
}
|
||||
bs, err := json.Marshal(action)
|
||||
if err != nil {
|
||||
fmt.Printf("json marshal action failed, err: %+v\n", err)
|
||||
}
|
||||
fmt.Printf("%s\n", string(bs))
|
||||
}
|
||||
|
|
@ -6,12 +6,13 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
|
||||
type Message struct {
|
||||
Key []byte
|
||||
Value []byte
|
||||
Offset int64
|
||||
Crc uint32
|
||||
Magic byte
|
||||
CompressCode byte
|
||||
Topic string
|
||||
Partition int32
|
||||
TipOffset int64
|
||||
|
|
@ -44,9 +45,8 @@ type ProduceReqPartition struct {
|
|||
Messages []*Message
|
||||
}
|
||||
|
||||
func ReadProduceRequest(r io.Reader, version int16) string {
|
||||
|
||||
var msg string
|
||||
func ReadProduceRequest(r io.Reader, version int16) *ProduceReq {
|
||||
// version == 1
|
||||
|
||||
produceReq := ProduceReq{}
|
||||
|
||||
|
|
@ -59,21 +59,157 @@ func ReadProduceRequest(r io.Reader, version int16) string {
|
|||
produceReq.Timeout = time.Duration(ReadInt32(r)) * time.Millisecond
|
||||
|
||||
l := ReadInt32(r)
|
||||
req := ProduceReq{}
|
||||
req.Topics = make([]ProduceReqTopic, l)
|
||||
produceReq.Topics = make([]ProduceReqTopic, l)
|
||||
|
||||
for ti := range req.Topics {
|
||||
var topic = &req.Topics[ti]
|
||||
for ti := range produceReq.Topics {
|
||||
var topic = &produceReq.Topics[ti]
|
||||
topic.Name, _ = ReadString(r)
|
||||
fmt.Println("msg")
|
||||
fmt.Println(topic.Name)
|
||||
|
||||
l := ReadInt32(r)
|
||||
topic.Partitions = make([]ProduceReqPartition, l)
|
||||
|
||||
for idx := 0; idx < int(l); idx++ {
|
||||
topic.Partitions[idx].ID = ReadInt32(r)
|
||||
_ = ReadInt32(r) // partitions size
|
||||
topic.Partitions[idx].Messages = ReadMessages(r, version)
|
||||
}
|
||||
}
|
||||
|
||||
return msg
|
||||
return &produceReq
|
||||
}
|
||||
|
||||
type ProduceRspPartitions struct {
|
||||
PartitionID int32
|
||||
Error int16
|
||||
Offset int64
|
||||
}
|
||||
|
||||
type ProduceRspTopic struct {
|
||||
TopicName string
|
||||
Partitions []ProduceRspPartitions
|
||||
ThrottleTime int32
|
||||
}
|
||||
|
||||
type ProduceRsp struct {
|
||||
Topics []ProduceRspTopic
|
||||
}
|
||||
|
||||
func ReadProduceResponse(r io.Reader, version int16) *ProduceRsp {
|
||||
// version == 1
|
||||
produceRsp := ProduceRsp{}
|
||||
l := ReadInt32(r)
|
||||
produceRsp.Topics = make([]ProduceRspTopic, 0)
|
||||
for i := 0; i < int(l); i++ {
|
||||
topic := ProduceRspTopic{}
|
||||
topic.TopicName, _ = ReadString(r)
|
||||
pl := ReadInt32(r)
|
||||
topic.Partitions = make([]ProduceRspPartitions, 0)
|
||||
for j := 0; j < int(pl); j++ {
|
||||
pt := ProduceRspPartitions{}
|
||||
pt.PartitionID = ReadInt32(r)
|
||||
pt.Error = ReadInt16(r)
|
||||
_, pt.Offset = ReadInt64(r)
|
||||
topic.Partitions = append(topic.Partitions, pt)
|
||||
}
|
||||
produceRsp.Topics = append(produceRsp.Topics, topic)
|
||||
}
|
||||
return &produceRsp
|
||||
}
|
||||
|
||||
type MetadataReq struct {
|
||||
TopicNames []string
|
||||
}
|
||||
|
||||
func ReadMetadataRequest(r io.Reader, version int16) *MetadataReq {
|
||||
// version == 0
|
||||
metadataReq := MetadataReq{}
|
||||
|
||||
l := ReadInt32(r)
|
||||
for i := 0; i < int(l); i++ {
|
||||
topicName, _ := ReadString(r)
|
||||
metadataReq.TopicNames = append(metadataReq.TopicNames, topicName)
|
||||
}
|
||||
|
||||
return &metadataReq
|
||||
}
|
||||
|
||||
type Broker struct {
|
||||
NodeID int32
|
||||
Host string
|
||||
Port int32
|
||||
}
|
||||
|
||||
type PartitionMetada struct {
|
||||
ErrorCode int16
|
||||
PartitionIndex int32
|
||||
LeaderID int32
|
||||
ReplicaNodes []int32
|
||||
IsrNodes []int32
|
||||
}
|
||||
|
||||
type TopicMetadata struct {
|
||||
ErrorCode int16
|
||||
Name string
|
||||
Partitions []PartitionMetada
|
||||
}
|
||||
|
||||
type MetadataRsp struct {
|
||||
Brokers []Broker
|
||||
Topics []TopicMetadata
|
||||
}
|
||||
|
||||
func ReadMetadataResponse(r io.Reader, version int16) *MetadataRsp {
|
||||
// version == 0
|
||||
metadataRsp := MetadataRsp{}
|
||||
|
||||
// read brokers
|
||||
metadataRsp.Brokers = make([]Broker, 0)
|
||||
l := ReadInt32(r)
|
||||
for i := 0; i < int(l); i++ {
|
||||
broker := Broker{}
|
||||
broker.NodeID = ReadInt32(r)
|
||||
broker.Host, _ = ReadString(r)
|
||||
broker.Port = ReadInt32(r)
|
||||
metadataRsp.Brokers = append(metadataRsp.Brokers, broker)
|
||||
}
|
||||
|
||||
// read topics
|
||||
metadataRsp.Topics = make([]TopicMetadata, 0)
|
||||
l = ReadInt32(r)
|
||||
for i := 0; i < int(l); i++ {
|
||||
topicMetadata := TopicMetadata{}
|
||||
topicMetadata.ErrorCode = ReadInt16(r)
|
||||
topicMetadata.Name, _ = ReadString(r)
|
||||
pl := ReadInt32(r)
|
||||
topicMetadata.Partitions = make([]PartitionMetada, 0)
|
||||
for j := 0; j < int(pl); j++ {
|
||||
pm := PartitionMetada{}
|
||||
pm.ErrorCode = ReadInt16(r)
|
||||
pm.PartitionIndex = ReadInt32(r)
|
||||
pm.LeaderID = ReadInt32(r)
|
||||
|
||||
pm.ReplicaNodes = make([]int32, 0)
|
||||
replicaLen := ReadInt32(r)
|
||||
for ri := 0; ri < int(replicaLen); ri++ {
|
||||
pm.ReplicaNodes = append(pm.ReplicaNodes, ReadInt32(r))
|
||||
}
|
||||
|
||||
pm.IsrNodes = make([]int32, 0)
|
||||
isrLen := ReadInt32(r)
|
||||
for ri := 0; ri < int(isrLen); ri++ {
|
||||
pm.IsrNodes = append(pm.IsrNodes, ReadInt32(r))
|
||||
}
|
||||
topicMetadata.Partitions = append(topicMetadata.Partitions, pm)
|
||||
}
|
||||
metadataRsp.Topics = append(metadataRsp.Topics, topicMetadata)
|
||||
}
|
||||
|
||||
return &metadataRsp
|
||||
}
|
||||
|
||||
type Action struct {
|
||||
Request string
|
||||
Direction string
|
||||
ApiVersion int16
|
||||
Message interface{}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ package build
|
|||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -31,6 +33,11 @@ func ReadOnce() {
|
|||
|
||||
}
|
||||
|
||||
func ReadByte(r io.Reader) (n byte) {
|
||||
binary.Read(r, binary.BigEndian, &n)
|
||||
return
|
||||
}
|
||||
|
||||
func ReadInt16(r io.Reader) (n int16) {
|
||||
binary.Read(r, binary.BigEndian, &n)
|
||||
return
|
||||
|
|
@ -41,11 +48,16 @@ func ReadInt32(r io.Reader) (n int32) {
|
|||
return
|
||||
}
|
||||
|
||||
func ReadInt64(r io.Reader) (n int64) {
|
||||
func ReadUint32(r io.Reader) (n uint32) {
|
||||
binary.Read(r, binary.BigEndian, &n)
|
||||
return
|
||||
}
|
||||
|
||||
func ReadInt64(r io.Reader) (err error, n int64) {
|
||||
err = binary.Read(r, binary.BigEndian, &n)
|
||||
return
|
||||
}
|
||||
|
||||
func ReadString(r io.Reader) (string, int) {
|
||||
|
||||
l := int(ReadInt16(r))
|
||||
|
|
@ -62,6 +74,7 @@ func ReadString(r io.Reader) (string, int) {
|
|||
|
||||
return string(str), l
|
||||
}
|
||||
|
||||
//
|
||||
//func TryReadInt16(r io.Reader) (n int16, err error) {
|
||||
//
|
||||
|
|
@ -76,19 +89,70 @@ func ReadString(r io.Reader) (string, int) {
|
|||
func ReadBytes(r io.Reader) []byte {
|
||||
|
||||
l := int(ReadInt32(r))
|
||||
result := make([]byte, 0)
|
||||
|
||||
if l <= 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
var result []byte
|
||||
var b = make([]byte, l)
|
||||
for i := 0; i < l; i++ {
|
||||
|
||||
_, err := r.Read(b)
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
|
||||
result = append(result, b[0])
|
||||
result = append(result, b...)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func ReadMessages(r io.Reader, version int16) []*Message {
|
||||
switch version {
|
||||
case 0:
|
||||
return ReadMessagesV1(r)
|
||||
case 1:
|
||||
return ReadMessagesV1(r)
|
||||
}
|
||||
|
||||
return make([]*Message, 0)
|
||||
}
|
||||
|
||||
func ReadMessagesV1(r io.Reader) []*Message {
|
||||
var err error
|
||||
messages := make([]*Message, 0)
|
||||
for {
|
||||
message := Message{}
|
||||
err, message.Offset = ReadInt64(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != io.ErrUnexpectedEOF {
|
||||
fmt.Printf("read message offset , err: %+v\n", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
_ = ReadInt32(r) // message size
|
||||
message.Crc = ReadUint32(r)
|
||||
message.Magic = ReadByte(r)
|
||||
message.CompressCode = ReadByte(r)
|
||||
message.Key = ReadBytes(r)
|
||||
message.Value = ReadBytes(r)
|
||||
messages = append(messages, &message)
|
||||
}
|
||||
return messages
|
||||
}
|
||||
|
||||
func GetRquestName(apiKey int16) string {
|
||||
if name, ok := RquestNameMap[apiKey]; ok {
|
||||
return name
|
||||
}
|
||||
return strconv.Itoa(int(apiKey))
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue