diff --git a/.gitignore b/.gitignore index df45cc1..ee0d3d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea/ static/blog/ +geodb/ diff --git a/go.mod b/go.mod index e29a3c6..e76658e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emvi/api-go v0.2.2 github.com/emvi/logbuch v1.1.1 - github.com/emvi/pirsch v1.5.3-0.20200919134607-3930bc2da610 + github.com/emvi/pirsch v1.6.0 github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/golang-migrate/migrate v3.5.4+incompatible github.com/golang-migrate/migrate/v4 v4.12.2 @@ -16,7 +16,7 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/lib/pq v1.8.0 github.com/rs/cors v1.7.0 - github.com/stretchr/testify v1.5.1 // indirect + golang.org/x/sys v0.0.0-20200918174421-af09f7315aff // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index 48f1fa0..7413bce 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ github.com/emvi/pirsch v1.5.3-0.20200914194223-a990d20e91a2 h1:HoKxeUtA0E4QVuw2k github.com/emvi/pirsch v1.5.3-0.20200914194223-a990d20e91a2/go.mod h1:GDijqLHM331iWtmDmc7th19RxDrZadRkKoNvd9/kDX8= github.com/emvi/pirsch v1.5.3-0.20200919134607-3930bc2da610 h1:l6AgOUXkBYFOhD9I0YXp4J64udlkb2P6ZwAdMU7MG7o= github.com/emvi/pirsch v1.5.3-0.20200919134607-3930bc2da610/go.mod h1:GDijqLHM331iWtmDmc7th19RxDrZadRkKoNvd9/kDX8= +github.com/emvi/pirsch v1.6.0 h1:7uDT3h+rWEwAheMyNGJsxW65L3OMFRQpOpO9taOc5hg= +github.com/emvi/pirsch v1.6.0/go.mod h1:P+S4Zl4cL8ezh+6uqpLt59WBBnR6+v1VdsjzgMuFS38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -262,6 +264,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc= +github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -295,6 +299,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= @@ -416,6 +421,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -429,7 +435,10 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c= golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -577,6 +586,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index c3ed0a2..bcd4802 100644 --- a/main.go +++ b/main.go @@ -140,6 +140,7 @@ func serveTracking() http.HandlerFunc { Referrer []pirsch.ReferrerStats Browser []pirsch.BrowserStats OS []pirsch.OSStats + Countries []pirsch.CountryStats Platform *pirsch.VisitorStats HourlyVisitorsTodayLabels template.JS HourlyVisitorsTodayDps template.JS @@ -159,6 +160,7 @@ func serveTracking() http.HandlerFunc { tracking.GetReferrer(startDate, endDate), tracking.GetBrowser(startDate, endDate), tracking.GetOS(startDate, endDate), + tracking.GetCountry(startDate, endDate), tracking.GetPlatform(startDate, endDate), hourlyVisitorsTodayLabels, hourlyVisitorsTodayDps, diff --git a/schema/0003_pirsch_update.up.sql b/schema/0003_pirsch_update.up.sql new file mode 100644 index 0000000..cdac27c --- /dev/null +++ b/schema/0003_pirsch_update.up.sql @@ -0,0 +1,21 @@ +ALTER TABLE "hit" ADD COLUMN "country_code" character varying(2); + +CREATE TABLE "country_stats" ( + id bigint NOT NULL UNIQUE, + tenant_id bigint, + day date NOT NULL, + visitors integer NOT NULL, + country_code character varying(2) +); + +CREATE SEQUENCE country_stats_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE country_stats_id_seq OWNED BY "country_stats".id; +ALTER TABLE ONLY "country_stats" ALTER COLUMN id SET DEFAULT nextval('country_stats_id_seq'::regclass); +ALTER TABLE ONLY "country_stats" ADD CONSTRAINT country_stats_pkey PRIMARY KEY (id); +CREATE INDEX country_stats_day_index ON country_stats(day); diff --git a/secrets.env b/secrets.env index b8a1ca9..d8bd9e4 100644 --- a/secrets.env +++ b/secrets.env @@ -3,3 +3,4 @@ MB_DB_USER= MB_DB_PASSWORD= MB_DB_SCHEMA= MB_TRACKING_SALT= +MB_GEOLITE2_LICENSE_KEY= diff --git a/template/tracking.html b/template/tracking.html index d58fc7d..98e73e6 100644 --- a/template/tracking.html +++ b/template/tracking.html @@ -172,6 +172,27 @@ +
+

Countries

+ + + + + + + + + + {{range $data := .Countries}} + + + + + + {{end}} + +
CountryAbsoluteRelative
{{if $data.CountryCode.String}}{{$data.CountryCode.String}}{{else}}(unknown){{end}}{{$data.Visitors}}{{round (multiply $data.RelativeVisitors 100)}} %
+

Platform

diff --git a/tracking/statistics.go b/tracking/statistics.go index 6887a3c..7554891 100644 --- a/tracking/statistics.go +++ b/tracking/statistics.go @@ -138,6 +138,21 @@ func GetBrowser(startDate, endDate time.Time) []pirsch.BrowserStats { return browser } +func GetCountry(startDate, endDate time.Time) []pirsch.CountryStats { + countries, err := analyzer.Country(&pirsch.Filter{From: startDate, To: endDate}) + + if err != nil { + logbuch.Error("Error reading country statistics", logbuch.Fields{"err": err}) + return nil + } + + for i := range countries { + countries[i].CountryCode.String = strings.ToUpper(countries[i].CountryCode.String) + } + + return countries +} + func GetPlatform(startDate, endDate time.Time) *pirsch.VisitorStats { return analyzer.Platform(&pirsch.Filter{From: startDate, To: endDate}) } diff --git a/tracking/tracking.go b/tracking/tracking.go index 39e6c8e..1ebf39a 100644 --- a/tracking/tracking.go +++ b/tracking/tracking.go @@ -7,6 +7,11 @@ import ( "github.com/emvi/logbuch" "github.com/emvi/pirsch" "os" + "path/filepath" +) + +const ( + geodbPath = "geodb" ) var ( @@ -38,8 +43,10 @@ func NewTracker() (*pirsch.Tracker, context.CancelFunc) { processor := pirsch.NewProcessor(store) cancel := pirsch.RunAtMidnight(func() { processTrackingData(processor) + updateGeoDB(tracker) }) - processTrackingData(processor) // run on startup + processTrackingData(processor) + updateGeoDB(tracker) return tracker, cancel } @@ -58,3 +65,26 @@ func processTrackingData(processor *pirsch.Processor) { logbuch.Info("Done processing tracking data") } } + +func updateGeoDB(tracker *pirsch.Tracker) { + licenseKey := os.Getenv("MB_GEOLITE2_LICENSE_KEY") + + if licenseKey == "" { + return + } + + if err := pirsch.GetGeoLite2(geodbPath, licenseKey); err != nil { + logbuch.Error("Error loading GeoLite2", logbuch.Fields{"err": err}) + return + } + + geodb, err := pirsch.NewGeoDB(filepath.Join(geodbPath, pirsch.GeoLite2Filename)) + + if err != nil { + logbuch.Error("Error creating GeoDB", logbuch.Fields{"err": err}) + return + } + + tracker.SetGeoDB(geodb) + logbuch.Info("GeoDB updated") +}