From 1e1edac3a58e06605c46ac4d5aa838a348833428 Mon Sep 17 00:00:00 2001 From: Kyle Travis Date: Mon, 17 Aug 2020 10:59:30 -0400 Subject: [PATCH] Add prometheus instrumentation, count active ws conns (#8) * Add prometheus instrumentation, count active ws conns * Stop bum ticker * Moving things around * Handle unspecified host for ws proxy * Add config reload counter * metrics: rename conns -> players --- example.yaml | 3 ++ go.mod | 1 + go.sum | 7 +++++ internal/quake/client/router.go | 3 ++ internal/quake/client/server.go | 11 +++++++- internal/quake/server/server.go | 49 +++++++++++++++++++++++++++++++++ 6 files changed, 73 insertions(+), 1 deletion(-) diff --git a/example.yaml b/example.yaml index 198e6b9..7dace05 100644 --- a/example.yaml +++ b/example.yaml @@ -11,6 +11,9 @@ spec: metadata: labels: run: quakejs + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8080' spec: containers: - command: diff --git a/go.mod b/go.mod index e1c59df..ebc30c0 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gorilla/websocket v1.4.0 github.com/labstack/echo/v4 v4.1.16 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v0.9.3 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20200627165143-92b8a710ab6c // indirect github.com/spf13/cobra v1.0.0 diff --git a/go.sum b/go.sum index 2887df6..741ccbc 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -54,6 +55,7 @@ github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= @@ -104,6 +106,7 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -129,12 +132,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/internal/quake/client/router.go b/internal/quake/client/router.go index d01b1fc..329e00d 100644 --- a/internal/quake/client/router.go +++ b/internal/quake/client/router.go @@ -9,6 +9,7 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" + "github.com/prometheus/client_golang/prometheus/promhttp" quakenet "github.com/criticalstack/quake-kube/internal/quake/net" ) @@ -52,6 +53,8 @@ func NewRouter(cfg *Config) (*echo.Echo, error) { }) }) + e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) + e.GET("/info", func(c echo.Context) error { m, err := quakenet.GetInfo(cfg.ServerAddr) if err != nil { diff --git a/internal/quake/client/server.go b/internal/quake/client/server.go index db92c07..8e2e88a 100644 --- a/internal/quake/client/server.go +++ b/internal/quake/client/server.go @@ -32,7 +32,16 @@ func (s *Server) Serve(l net.Listener) error { } }() - wsproxy, err := NewProxy(s.ServerAddr) + host, port, err := net.SplitHostPort(s.ServerAddr) + if err != nil { + return err + } + proxyTarget := s.ServerAddr + if net.ParseIP(host).IsUnspecified() { + // handle case where host is 0.0.0.0 + proxyTarget = net.JoinHostPort("127.0.0.1", port) + } + wsproxy, err := NewProxy(proxyTarget) if err != nil { return err } diff --git a/internal/quake/server/server.go b/internal/quake/server/server.go index 69f7e19..cbaad58 100644 --- a/internal/quake/server/server.go +++ b/internal/quake/server/server.go @@ -9,11 +9,33 @@ import ( "path/filepath" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "sigs.k8s.io/yaml" + quakenet "github.com/criticalstack/quake-kube/internal/quake/net" "github.com/criticalstack/quake-kube/internal/util/exec" ) +var ( + actrvePlayers = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "quake_active_players", + Help: "The current number of active players", + }) + scores = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "quake_player_scores", + Help: "Current scores by player, by map", + }, []string{"player", "map"}) + pings = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "quake_player_pings", + Help: "Current ping by player", + }, []string{"player"}) + configReloads = promauto.NewCounter(prometheus.CounterOpts{ + Name: "quake_config_reloads", + Help: "Config file reload count", + }) +) + type Server struct { Dir string WatchInterval time.Duration @@ -68,6 +90,32 @@ func (s *Server) Start(ctx context.Context) error { } }() + go func() { + addr := s.Addr + if net.ParseIP(host).IsUnspecified() { + addr = net.JoinHostPort("127.0.0.1", port) + } + tick := time.NewTicker(5 * time.Second) + defer tick.Stop() + for { + select { + case <-tick.C: + status, err := quakenet.GetStatus(addr) + if err != nil { + log.Printf("metrics: get status failed %v", err) + continue + } + actrvePlayers.Set(float64(len(status.Players))) + for _, p := range status.Players { + scores.WithLabelValues(p.Name, status.Configuration["mapname"]).Set(float64(p.Score)) + pings.WithLabelValues(p.Name).Set(float64(p.Ping)) + } + case <-ctx.Done(): + return + } + } + }() + ch, err := s.watch(ctx) if err != nil { return err @@ -79,6 +127,7 @@ func (s *Server) Start(ctx context.Context) error { if err := s.reload(); err != nil { return err } + configReloads.Inc() if err := cmd.Restart(ctx); err != nil { return err }