Add prometheus instrumentation, count active ws conns (#8)

* Add prometheus instrumentation, count active ws conns

* Stop bum ticker

* Moving things around

* Handle unspecified host for ws proxy

* Add config reload counter

* metrics: rename conns -> players

example.yaml

@@ -11,6 +11,9 @@ spec:
     metadata:
       labels:
         run: quakejs
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '8080'
     spec:
       containers:
       - command:

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/gorilla/websocket v1.4.0
 	github.com/labstack/echo/v4 v4.1.16
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v0.9.3
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20200627165143-92b8a710ab6c // indirect
 	github.com/spf13/cobra v1.0.0

go.sum

@@ -8,6 +8,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -54,6 +55,7 @@ github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
@@ -104,6 +106,7 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -129,12 +132,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=

internal/quake/client/router.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	quakenet "github.com/criticalstack/quake-kube/internal/quake/net"
 )
@@ -52,6 +53,8 @@ func NewRouter(cfg *Config) (*echo.Echo, error) {
 		})
 	})
 
+	e.GET("/metrics", echo.WrapHandler(promhttp.Handler()))
+
 	e.GET("/info", func(c echo.Context) error {
 		m, err := quakenet.GetInfo(cfg.ServerAddr)
 		if err != nil {

internal/quake/client/server.go

@@ -32,7 +32,16 @@ func (s *Server) Serve(l net.Listener) error {
 		}
 	}()
 
-	wsproxy, err := NewProxy(s.ServerAddr)
+	host, port, err := net.SplitHostPort(s.ServerAddr)
+	if err != nil {
+		return err
+	}
+	proxyTarget := s.ServerAddr
+	if net.ParseIP(host).IsUnspecified() {
+		// handle case where host is 0.0.0.0
+		proxyTarget = net.JoinHostPort("127.0.0.1", port)
+	}
+	wsproxy, err := NewProxy(proxyTarget)
 	if err != nil {
 		return err
 	}

internal/quake/server/server.go

@@ -9,11 +9,33 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"sigs.k8s.io/yaml"
 
+	quakenet "github.com/criticalstack/quake-kube/internal/quake/net"
 	"github.com/criticalstack/quake-kube/internal/util/exec"
 )
 
+var (
+	actrvePlayers = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "quake_active_players",
+		Help: "The current number of active players",
+	})
+	scores = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "quake_player_scores",
+		Help: "Current scores by player, by map",
+	}, []string{"player", "map"})
+	pings = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "quake_player_pings",
+		Help: "Current ping by player",
+	}, []string{"player"})
+	configReloads = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "quake_config_reloads",
+		Help: "Config file reload count",
+	})
+)
+
 type Server struct {
 	Dir           string
 	WatchInterval time.Duration
@@ -68,6 +90,32 @@ func (s *Server) Start(ctx context.Context) error {
 		}
 	}()
 
+	go func() {
+		addr := s.Addr
+		if net.ParseIP(host).IsUnspecified() {
+			addr = net.JoinHostPort("127.0.0.1", port)
+		}
+		tick := time.NewTicker(5 * time.Second)
+		defer tick.Stop()
+		for {
+			select {
+			case <-tick.C:
+				status, err := quakenet.GetStatus(addr)
+				if err != nil {
+					log.Printf("metrics: get status failed %v", err)
+					continue
+				}
+				actrvePlayers.Set(float64(len(status.Players)))
+				for _, p := range status.Players {
+					scores.WithLabelValues(p.Name, status.Configuration["mapname"]).Set(float64(p.Score))
+					pings.WithLabelValues(p.Name).Set(float64(p.Ping))
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
 	ch, err := s.watch(ctx)
 	if err != nil {
 		return err
@@ -79,6 +127,7 @@ func (s *Server) Start(ctx context.Context) error {
 			if err := s.reload(); err != nil {
 				return err
 			}
+			configReloads.Inc()
 			if err := cmd.Restart(ctx); err != nil {
 				return err
 			}