1
0
Fork 0

Add simple http server and dashboard prototype

This commit is contained in:
Gregory Eremin 2018-07-30 21:50:16 +02:00
parent 693c9765e8
commit bc1f5cf474
No known key found for this signature in database
GPG Key ID: 8CB79D42167BEB7F
4 changed files with 213 additions and 1 deletions

View File

@ -13,7 +13,7 @@ So far it is only cable of printing parsed messages. For usage eample take a
look at the main command. look at the main command.
``` ```
go run cmd/main.go -brokers 127.0.0.1:9092 go run cmd/printer/main.go -brokers 127.0.0.1:9092
``` ```
### Design ### Design

207
cmd/server/main.go Normal file
View File

@ -0,0 +1,207 @@
package main
import (
"bytes"
"flag"
"fmt"
"log"
"net/http"
"os"
"sort"
"strings"
"sync"
"time"
"html/template"
"github.com/localhots/koff"
)
type clusterState struct {
consumerOffsets map[string]koff.OffsetMessage
consumerGroups map[string]koff.GroupMessage
}
var state = &clusterState{
consumerOffsets: make(map[string]koff.OffsetMessage),
consumerGroups: make(map[string]koff.GroupMessage),
}
var lock sync.Mutex
func main() {
brokers := flag.String("brokers", "", "Comma separated list of brokers")
flag.Parse()
if *brokers == "" {
fmt.Println("Brokers list required")
flag.Usage()
os.Exit(1)
}
c, err := koff.NewConsumer(strings.Split(*brokers, ","), false)
if err != nil {
log.Fatalf("Failed to create consumer: %v", err)
}
defer c.Close()
go func() {
for msg := range c.Messages() {
state.add(msg)
}
}()
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Write(state.render())
})
http.ListenAndServe(":8080", nil)
}
func (s *clusterState) add(msg koff.Message) {
lock.Lock()
defer lock.Unlock()
if msg.OffsetMessage != nil {
cur, ok := s.consumerOffsets[msg.Consumer]
if !ok || cur.CommittedAt.Before(msg.OffsetMessage.CommittedAt) {
s.consumerOffsets[msg.Consumer] = *msg.OffsetMessage
}
} else {
cur, ok := s.consumerGroups[msg.Consumer]
if !ok || (msg.GroupMessage.GenerationID > cur.GenerationID && msg.GroupMessage.Complete()) {
s.consumerGroups[msg.Consumer] = *msg.GroupMessage
}
}
}
//
// Render
//
var htmlTpl = template.Must(template.New("main").Parse(`<!doctype html>
<html>
<head>
<title>Kafka Consumers</title>
<meta http-equiv="refresh" content="3">
<style type="text/css">
* { font: 14px Helvetica; color: #222; }
h3 { font-size: 24px; font-weight: 600; margin-bottom: 15px; }
table { margin:0; padding:0; border:none; border-collapse:collapse; border-spacing:0; width: 100%; }
th, td { text-align: left; padding: 10px; }
th { border-bottom: #444 1px solid; font-weight: 600; }
th.numeric, td.numeric { text-align: right; }
table.gr { border: #444 1px solid; }
th.title { border: none; font-size: 16px; }
</style>
</head>
<body>
<h3>Consumer Offsets</h3>
<table>
<tr>
<th width="40%">Consumer</th>
<th width="35%">Topic</th>
<th width="5%" class="numeric">Partition</th>
<th width="10%" class="numeric">Offset</th>
<th width="10%">Timestamp</th>
</tr>
{{range .ConsumerOffsets}}
<tr>
<td>{{.Consumer}}</td>
<td>{{.Topic}}</td>
<td class="numeric">{{.Partition}}</td>
<td class="numeric">{{.Offset}}</td>
<td>{{.Timestamp}}</td>
</tr>
{{end}}
</table>
<br/>
<h3>Consumer Groups</h3>
{{range .ConsumerGroups}}
<table class="gr">
<tr>
<th colspan="4" class="title">{{.Consumer}}</th>
</tr>
<tr>
<th width="40%">Consumer</th>
<th width="40%">Topic</th>
<th width="10%" class="numeric">Partition</th>
<th width="10%">Leader</th>
</tr>
{{range .Members}}
{{$id := .ID}}
{{range .Assignment}}
<tr>
<td>{{$id}}</td>
<td>{{.Topic}}</td>
<td class="numeric">{{.Partition}}</td>
<td>Yes</td>
</tr>
{{end}}
{{end}}
</table>
<br/>
{{end}}
</body>
</html>
`))
type offsetMessage struct {
Consumer string
Timestamp string
koff.OffsetMessage
}
type groupMessage struct {
Consumer string
koff.GroupMessage
}
func (s *clusterState) render() []byte {
var tpl struct {
ConsumerOffsets []offsetMessage
ConsumerGroups []groupMessage
}
lock.Lock()
defer lock.Unlock()
for k, m := range s.consumerOffsets {
if strings.HasPrefix(k, "console-consumer") {
continue
}
tpl.ConsumerOffsets = append(tpl.ConsumerOffsets, offsetMessage{
Consumer: k,
OffsetMessage: m,
Timestamp: m.CommittedAt.Format(time.Stamp),
})
}
sort.Slice(tpl.ConsumerOffsets, func(i, j int) bool {
return tpl.ConsumerOffsets[i].Consumer < tpl.ConsumerOffsets[j].Consumer
})
for k, m := range s.consumerGroups {
// if strings.HasPrefix(k, "console-consumer") {
// continue
// }
tpl.ConsumerGroups = append(tpl.ConsumerGroups, groupMessage{
Consumer: k,
GroupMessage: m,
})
}
sort.Slice(tpl.ConsumerGroups, func(i, j int) bool {
return tpl.ConsumerGroups[i].Consumer < tpl.ConsumerGroups[j].Consumer
})
var buf bytes.Buffer
err := htmlTpl.Execute(&buf, tpl)
if err != nil {
panic(err)
}
return buf.Bytes()
}

View File

@ -76,6 +76,11 @@ func Decode(ctx context.Context, key, val []byte) Message {
return m return m
} }
// Complete returns true if message is complete.
func (gm GroupMessage) Complete() bool {
return gm.LeaderID != ""
}
// Key structure: // Key structure:
// [2] Version, uint16 big endian // [2] Version, uint16 big endian
// [2] Consumer name length, uint16 big endian // [2] Consumer name length, uint16 big endian