diff --git a/Dockerfile b/Dockerfile index f374ebf..9874100 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,6 @@ WORKDIR /app ENV MB_LOGLEVEL=info ENV MB_ALLOWED_ORIGINS=* ENV MB_HOST=0.0.0.0:8888 -ENV MB_DB_PORT=5432 -ENV MB_DB_SSLMODE=disable EXPOSE 8888 CMD ["/app/main"] diff --git a/README.md b/README.md index da0a53d..cc8a0d6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # marvinblum.de + My website. diff --git a/db/db.go b/db/db.go deleted file mode 100644 index 98f156c..0000000 --- a/db/db.go +++ /dev/null @@ -1,29 +0,0 @@ -package db - -import ( - "fmt" - "github.com/emvi/logbuch" - "os" - "strconv" - "time" -) - -const ( - connectionString = `host=%s port=%s user=%s password=%s dbname=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s connectTimeout=%s timezone=%s` -) - -func GetConnectionString() string { - host := os.Getenv("MB_DB_HOST") - port := os.Getenv("MB_DB_PORT") - user := os.Getenv("MB_DB_USER") - password := os.Getenv("MB_DB_PASSWORD") - schema := os.Getenv("MB_DB_SCHEMA") - sslMode := os.Getenv("MB_DB_SSLMODE") - sslCert := os.Getenv("MB_DB_SSLCERT") - sslKey := os.Getenv("MB_DB_SSLKEY") - sslRootCert := os.Getenv("MB_DB_SSLROOTCERT") - zone, offset := time.Now().Zone() - timezone := zone + strconv.Itoa(-offset/3600) - logbuch.Info("Setting time zone", logbuch.Fields{"timezone": timezone}) - return fmt.Sprintf(connectionString, host, port, user, password, schema, sslMode, sslCert, sslKey, sslRootCert, "30", timezone) -} diff --git a/db/migrate.go b/db/migrate.go deleted file mode 100644 index 041d1fa..0000000 --- a/db/migrate.go +++ /dev/null @@ -1,48 +0,0 @@ -package db - -import ( - "fmt" - "github.com/emvi/logbuch" - "github.com/golang-migrate/migrate/v4" - _ "github.com/golang-migrate/migrate/v4/database/postgres" - _ "github.com/golang-migrate/migrate/v4/source/file" - "os" -) - -const ( - migrateConnectionString = `postgres://%s:%s/%s?user=%s&password=%s&sslmode=%s&sslcert=%s&sslkey=%s&sslrootcert=%s&connect_timeout=60` -) - -func Migrate() { - logbuch.Info("Migrating database schema (if required)") - m, err := migrate.New("file://schema", getMigrationConnectionString()) - - if err != nil { - logbuch.Fatal("Error migrating database schema", logbuch.Fields{"err": err}) - return - } - - if err := m.Up(); err != nil && err != migrate.ErrNoChange { - logbuch.Fatal("Error migrating database schema", logbuch.Fields{"err": err}) - return - } - - if sourceErr, dbErr := m.Close(); sourceErr != nil || dbErr != nil { - logbuch.Fatal("Error migrating database schema", logbuch.Fields{"source_err": sourceErr, "db_err": dbErr}) - } - - logbuch.Info("Done migrating database schema") -} - -func getMigrationConnectionString() string { - host := os.Getenv("MB_DB_HOST") - port := os.Getenv("MB_DB_PORT") - user := os.Getenv("MB_DB_USER") - password := os.Getenv("MB_DB_PASSWORD") - schema := os.Getenv("MB_DB_SCHEMA") - sslMode := os.Getenv("MB_DB_SSLMODE") - sslCert := os.Getenv("MB_DB_SSLCERT") - sslKey := os.Getenv("MB_DB_SSLKEY") - sslRootCert := os.Getenv("MB_DB_SSLROOTCERT") - return fmt.Sprintf(migrateConnectionString, host, port, schema, user, password, sslMode, sslCert, sslKey, sslRootCert) -} diff --git a/dev.sh b/dev.sh index 4bf3baa..c349c38 100755 --- a/dev.sh +++ b/dev.sh @@ -9,16 +9,10 @@ export MB_ALLOWED_ORIGINS=* export MB_HOST=localhost:8080 export MB_HOT_RELOAD=true export MB_EMVI_CLIENT_ID=3fBBn144yvSF9R3dPC8l -export MB_EMVI_CLIENT_SECRET=dw3FeshelTgdf1Gj13J7uF5FfdPDi40sQvvwqeFVKTTyIDuCdlAHhRY72csFL6yg +export MB_EMVI_CLIENT_SECRET= export MB_EMVI_ORGA=marvin -export MB_DB_HOST=localhost -export MB_DB_PORT=5432 -export MB_DB_USER=postgres -export MB_DB_PASSWORD=postgres -export MB_DB_SCHEMA=marvinblum -export MB_DB_SSLMODE=disable -export MB_DB_SSLCERT= -export MB_DB_SSLKEY= -export MB_DB_SSLROOTCERT= +export MB_PIRSCH_CLIENT_ID=gEb3pvgxZvZzFRlOTdMgPtyLvNYgeVKe +export MB_PIRSCH_CLIENT_SECRET=E7UqJehmxgnVuw81oq6ZhJAx9vCHqMimCUFfil7UFgbGhgQVVINqU7JqHBgaUvHg +export MB_PIRSCH_HOSTNAME=marvinblum.de go run main.go diff --git a/docker-compose.yml b/docker-compose.yml index 8d7a75b..61c6458 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,7 +52,8 @@ services: environment: MB_EMVI_CLIENT_ID: 3fBBn144yvSF9R3dPC8l MB_EMVI_ORGA: marvin - MB_DB_HOST: postgres + export MB_PIRSCH_CLIENT_ID: gEb3pvgxZvZzFRlOTdMgPtyLvNYgeVKe + export MB_PIRSCH_HOSTNAME: marvinblum.de labels: - "traefik.enable=true" - "traefik.port=8888" diff --git a/go.mod b/go.mod index ea7a781..289a3d1 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/lib/pq v1.8.0 github.com/pirsch-analytics/pirsch v1.8.1 + github.com/pirsch-analytics/pirsch-go-sdk v0.0.0-20201204224029-0ab7a9417d40 // indirect github.com/rs/cors v1.7.0 golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 // indirect ) diff --git a/go.sum b/go.sum index c943e2a..d002c8e 100644 --- a/go.sum +++ b/go.sum @@ -226,6 +226,8 @@ github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pirsch-analytics/pirsch v1.8.1 h1:07pFslS6gSTgGcu26camKeYhJ5G+KCwnqc6GGVRHcPU= github.com/pirsch-analytics/pirsch v1.8.1/go.mod h1:BPOASgFDyfVyXCXrrDvS5kVgc/eN2fM5cZAVj0fZGbI= +github.com/pirsch-analytics/pirsch-go-sdk v0.0.0-20201204224029-0ab7a9417d40 h1:wBNO4NcuRc8GL95QiewUkhIspx9wKg8qzwqBKHbMB44= +github.com/pirsch-analytics/pirsch-go-sdk v0.0.0-20201204224029-0ab7a9417d40/go.mod h1:PF2vnJw8FYcXQe6OTPQQcGn8l/agkpl7T4YO9d2aPSE= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/main.go b/main.go index e732b09..0694e58 100644 --- a/main.go +++ b/main.go @@ -3,21 +3,18 @@ package main import ( "context" "github.com/Kugelschieber/marvinblum/blog" - "github.com/Kugelschieber/marvinblum/db" "github.com/Kugelschieber/marvinblum/tpl" - "github.com/Kugelschieber/marvinblum/tracking" "github.com/NYTimes/gziphandler" emvi "github.com/emvi/api-go" "github.com/emvi/logbuch" "github.com/gorilla/mux" _ "github.com/lib/pq" - "github.com/pirsch-analytics/pirsch" + "github.com/pirsch-analytics/pirsch-go-sdk" "github.com/rs/cors" "html/template" "net/http" "os" "os/signal" - "strconv" "strings" "time" ) @@ -31,14 +28,14 @@ const ( ) var ( - tracker *pirsch.Tracker + client *pirsch.Client tplCache *tpl.Cache blogInstance *blog.Blog ) func serveAbout() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - go tracker.Hit(r, nil) + go hit(r) tplCache.Render(w, "about.html", struct { Articles []emvi.Article }{ @@ -49,14 +46,14 @@ func serveAbout() http.HandlerFunc { func serveLegal() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - go tracker.Hit(r, nil) + go hit(r) tplCache.Render(w, "legal.html", nil) } } func serveBlogPage() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - go tracker.Hit(r, nil) + go hit(r) tplCache.Render(w, "blog.html", struct { Articles map[int][]emvi.Article }{ @@ -83,7 +80,7 @@ func serveBlogArticle() http.HandlerFunc { } // track the hit if the article was found, otherwise we don't care - go tracker.Hit(r, nil) + go hit(r) tplCache.RenderWithoutCache(w, "article.html", struct { Title string @@ -99,79 +96,7 @@ func serveBlogArticle() http.HandlerFunc { func serveTracking() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - go tracker.Hit(r, nil) - start, _ := strconv.Atoi(r.URL.Query().Get("start")) - - if start > 365 { - start = 365 - } else if start < 7 { - start = 7 - } - - var startDate, endDate time.Time - - if err := r.ParseForm(); err != nil { - logbuch.Warn("Error parsing tracking form", logbuch.Fields{"err": err}) - } else { - startDate, _ = time.Parse("2006-01-02", r.FormValue("start-date")) - endDate, _ = time.Parse("2006-01-02", r.FormValue("end-date")) - } - - if startDate.IsZero() || endDate.IsZero() { - startDate = time.Now().UTC().Add(-time.Hour * 24 * time.Duration(start)) - endDate = time.Now().UTC() - } - - activeVisitorPages, activeVisitors := tracking.GetActiveVisitors() - totalVisitorsLabels, totalVisitorsDps, sessionsDps, bouncesDps := tracking.GetTotalVisitors(startDate, endDate) - hourlyVisitorsTodayLabels, hourlyVisitorsTodayDps := tracking.GetHourlyVisitorsToday() - pageVisitors, pageRank := tracking.GetPageVisits(startDate, endDate) - timeOfDay, timeOfDayMax := tracking.GetVisitorTimeOfDay(startDate, endDate) - tplCache.RenderWithoutCache(w, "tracking.html", struct { - Start int - StartDate time.Time - EndDate time.Time - TotalVisitorsLabels template.JS - TotalVisitorsDps template.JS - SessionsDps template.JS - BouncesDps template.JS - PageVisitors []tracking.PageVisitors - PageRank []tracking.PageVisitors - Languages []pirsch.LanguageStats - Referrer []pirsch.ReferrerStats - Browser []pirsch.BrowserStats - OS []pirsch.OSStats - Countries []pirsch.CountryStats - Platform *pirsch.VisitorStats - TimeOfDay []pirsch.TimeOfDayVisitors - TimeOfDayMax float64 - HourlyVisitorsTodayLabels template.JS - HourlyVisitorsTodayDps template.JS - ActiveVisitors int - ActiveVisitorPages []pirsch.Stats - }{ - start, - startDate, - endDate, - totalVisitorsLabels, - totalVisitorsDps, - sessionsDps, - bouncesDps, - pageVisitors, - pageRank, - tracking.GetLanguages(startDate, endDate), - tracking.GetReferrer(startDate, endDate), - tracking.GetBrowser(startDate, endDate), - tracking.GetOS(startDate, endDate), - tracking.GetCountry(startDate, endDate), - tracking.GetPlatform(startDate, endDate), - timeOfDay, - float64(timeOfDayMax), - hourlyVisitorsTodayLabels, - hourlyVisitorsTodayDps, - activeVisitors, - activeVisitorPages, - }) + http.Redirect(w, r, "https://marvinblum.pirsch.io/", http.StatusFound) } } @@ -230,7 +155,7 @@ func configureCors(router *mux.Router) http.Handler { return c.Handler(router) } -func start(handler http.Handler, trackingCancel context.CancelFunc) { +func start(handler http.Handler) { logbuch.Info("Starting server...") var server http.Server server.Handler = handler @@ -241,8 +166,6 @@ func start(handler http.Handler, trackingCancel context.CancelFunc) { signal.Notify(sigint, os.Interrupt) <-sigint logbuch.Info("Shutting down server...") - trackingCancel() - tracker.Stop() ctx, _ := context.WithTimeout(context.Background(), shutdownTimeout) if err := server.Shutdown(ctx); err != nil { @@ -257,15 +180,22 @@ func start(handler http.Handler, trackingCancel context.CancelFunc) { logbuch.Info("Server shut down") } +func hit(r *http.Request) { + if err := client.Hit(r); err != nil { + logbuch.Warn("Error sending page hit to pirsch", logbuch.Fields{"err": err}) + } +} + func main() { configureLog() logEnvConfig() - db.Migrate() - var trackingCancel context.CancelFunc - tracker, trackingCancel = tracking.NewTracker() + client = pirsch.NewClient(os.Getenv("MB_PIRSCH_CLIENT_ID"), + os.Getenv("MB_PIRSCH_CLIENT_SECRET"), + os.Getenv("MB_PIRSCH_HOSTNAME"), + nil) tplCache = tpl.NewCache() blogInstance = blog.NewBlog(tplCache) router := setupRouter() corsConfig := configureCors(router) - start(corsConfig, trackingCancel) + start(corsConfig) } diff --git a/postgres/clear_logs.sh b/postgres/clear_logs.sh deleted file mode 100644 index 08ea2b0..0000000 --- a/postgres/clear_logs.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo "Clearing docker logs..." -echo "" > $(docker inspect --format='{{.LogPath}}' postgres) diff --git a/postgres/docker-compose.yml b/postgres/docker-compose.yml deleted file mode 100644 index 2f3f5fc..0000000 --- a/postgres/docker-compose.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: "3" - -services: - postgres: - image: postgres:12-alpine - container_name: postgres - restart: always - command: -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key - networks: - - db-internal - environment: - POSTGRES_PASSWORD: - ports: - - "5432:5432" - volumes: - - /root/postgres/data:/var/lib/postgresql/data - - /root/postgres/cert/server.crt:/var/lib/postgresql/server.crt - - /root/postgres/cert/server.key:/var/lib/postgresql/server.key - -networks: - db-internal: - driver: bridge diff --git a/postgres/gen_cert.sh b/postgres/gen_cert.sh deleted file mode 100755 index 7a75bb4..0000000 --- a/postgres/gen_cert.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -mkdir cert -cd cert -openssl req -new -text -passout pass:$1 -subj /CN=localhost -out server.req -keyout privkey.pem -openssl rsa -in privkey.pem -passin pass:$1 -out server.key -openssl req -x509 -in server.req -text -key server.key -out server.crt -chown 0:70 server.key -chmod 640 server.key -echo "done" diff --git a/schema/0001_baseline.up.sql b/schema/0001_baseline.up.sql deleted file mode 100644 index 445caa1..0000000 --- a/schema/0001_baseline.up.sql +++ /dev/null @@ -1,171 +0,0 @@ -CREATE TABLE "hit" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - fingerprint varchar(32) NOT NULL, - "session" timestamp without time zone, - path varchar(2000), - url varchar(2000), - language varchar(10), - user_agent varchar(200), - referrer varchar(200), - "os" character varying(20), - "os_version" character varying(20), - "browser" character varying(20), - "browser_version" character varying(20), - "desktop" boolean DEFAULT FALSE, - "mobile" boolean DEFAULT FALSE, - time timestamp without time zone NOT NULL -); - -CREATE SEQUENCE hit_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE hit_id_seq OWNED BY "hit".id; -ALTER TABLE ONLY "hit" ALTER COLUMN id SET DEFAULT nextval('hit_id_seq'::regclass); -ALTER TABLE ONLY "hit" ADD CONSTRAINT hit_pkey PRIMARY KEY (id); -CREATE INDEX hit_fingerprint_index ON hit(fingerprint); -CREATE INDEX hit_path_index ON hit(path); -CREATE INDEX hit_time_index ON hit(time); - -CREATE TABLE "visitor_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - path varchar(2000) NOT NULL, - visitors integer NOT NULL, - sessions integer NOT NULL DEFAULT 0, - bounces integer NOT NULL DEFAULT 0, - platform_desktop integer NOT NULL, - platform_mobile integer NOT NULL, - platform_unknown integer NOT NULL -); - -CREATE SEQUENCE visitor_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE visitor_stats_id_seq OWNED BY "visitor_stats".id; -ALTER TABLE ONLY "visitor_stats" ALTER COLUMN id SET DEFAULT nextval('visitor_stats_id_seq'::regclass); -ALTER TABLE ONLY "visitor_stats" ADD CONSTRAINT visitor_stats_pkey PRIMARY KEY (id); -CREATE INDEX visitor_stats_day_index ON visitor_stats(day); -CREATE INDEX visitor_stats_path_index ON visitor_stats(path); - -CREATE TABLE "visitor_time_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - path varchar(2000) NOT NULL, - hour smallint NOT NULL, - visitors integer NOT NULL, - sessions integer NOT NULL DEFAULT 0 -); - -CREATE SEQUENCE visitor_time_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE visitor_time_stats_id_seq OWNED BY "visitor_time_stats".id; -ALTER TABLE ONLY "visitor_time_stats" ALTER COLUMN id SET DEFAULT nextval('visitor_time_stats_id_seq'::regclass); -ALTER TABLE ONLY "visitor_time_stats" ADD CONSTRAINT visitor_time_stats_pkey PRIMARY KEY (id); -CREATE INDEX visitor_time_stats_day_index ON visitor_time_stats(day); -CREATE INDEX visitor_time_stats_path_index ON visitor_time_stats(path); - -CREATE TABLE "language_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - path varchar(2000) NOT NULL, - language varchar(10), - visitors integer NOT NULL -); - -CREATE SEQUENCE language_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE language_stats_id_seq OWNED BY "language_stats".id; -ALTER TABLE ONLY "language_stats" ALTER COLUMN id SET DEFAULT nextval('language_stats_id_seq'::regclass); -ALTER TABLE ONLY "language_stats" ADD CONSTRAINT language_stats_pkey PRIMARY KEY (id); -CREATE INDEX language_stats_day_index ON language_stats(day); -CREATE INDEX language_stats_path_index ON language_stats(path); - -CREATE TABLE "referrer_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - path varchar(2000) NOT NULL, - referrer varchar(2000), - visitors integer NOT NULL -); - -CREATE SEQUENCE referrer_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE referrer_stats_id_seq OWNED BY "referrer_stats".id; -ALTER TABLE ONLY "referrer_stats" ALTER COLUMN id SET DEFAULT nextval('referrer_stats_id_seq'::regclass); -ALTER TABLE ONLY "referrer_stats" ADD CONSTRAINT referrer_stats_pkey PRIMARY KEY (id); -CREATE INDEX referrer_stats_day_index ON referrer_stats(day); -CREATE INDEX referrer_stats_path_index ON referrer_stats(path); - -CREATE TABLE "os_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - path varchar(2000) NOT NULL, - os character varying(20), - os_version character varying(20), - visitors integer NOT NULL -); - -CREATE SEQUENCE os_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE os_stats_id_seq OWNED BY "os_stats".id; -ALTER TABLE ONLY "os_stats" ALTER COLUMN id SET DEFAULT nextval('os_stats_id_seq'::regclass); -ALTER TABLE ONLY "os_stats" ADD CONSTRAINT os_stats_pkey PRIMARY KEY (id); -CREATE INDEX os_stats_day_index ON os_stats(day); -CREATE INDEX os_stats_path_index ON os_stats(path); - -CREATE TABLE "browser_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - path varchar(2000) NOT NULL, - browser character varying(20), - browser_version character varying(20), - visitors integer NOT NULL -); - -CREATE SEQUENCE browser_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE browser_stats_id_seq OWNED BY "browser_stats".id; -ALTER TABLE ONLY "browser_stats" ALTER COLUMN id SET DEFAULT nextval('browser_stats_id_seq'::regclass); -ALTER TABLE ONLY "browser_stats" ADD CONSTRAINT browser_stats_pkey PRIMARY KEY (id); -CREATE INDEX browser_stats_day_index ON browser_stats(day); -CREATE INDEX browser_stats_path_index ON browser_stats(path); diff --git a/schema/0002_pirsch_update.up.sql b/schema/0002_pirsch_update.up.sql deleted file mode 100644 index e5786a2..0000000 --- a/schema/0002_pirsch_update.up.sql +++ /dev/null @@ -1,23 +0,0 @@ -ALTER TABLE "hit" ADD COLUMN "screen_width" integer DEFAULT 0; -ALTER TABLE "hit" ADD COLUMN "screen_height" integer DEFAULT 0; - -CREATE TABLE "screen_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - visitors integer NOT NULL, - width integer NOT NULL, - height integer NOT NULL -); - -CREATE SEQUENCE screen_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE screen_stats_id_seq OWNED BY "screen_stats".id; -ALTER TABLE ONLY "screen_stats" ALTER COLUMN id SET DEFAULT nextval('screen_stats_id_seq'::regclass); -ALTER TABLE ONLY "screen_stats" ADD CONSTRAINT screen_stats_pkey PRIMARY KEY (id); -CREATE INDEX screen_stats_day_index ON screen_stats(day); diff --git a/schema/0003_pirsch_update.up.sql b/schema/0003_pirsch_update.up.sql deleted file mode 100644 index cdac27c..0000000 --- a/schema/0003_pirsch_update.up.sql +++ /dev/null @@ -1,21 +0,0 @@ -ALTER TABLE "hit" ADD COLUMN "country_code" character varying(2); - -CREATE TABLE "country_stats" ( - id bigint NOT NULL UNIQUE, - tenant_id bigint, - day date NOT NULL, - visitors integer NOT NULL, - country_code character varying(2) -); - -CREATE SEQUENCE country_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE country_stats_id_seq OWNED BY "country_stats".id; -ALTER TABLE ONLY "country_stats" ALTER COLUMN id SET DEFAULT nextval('country_stats_id_seq'::regclass); -ALTER TABLE ONLY "country_stats" ADD CONSTRAINT country_stats_pkey PRIMARY KEY (id); -CREATE INDEX country_stats_day_index ON country_stats(day); diff --git a/secrets.env b/secrets.env index d8bd9e4..3183321 100644 --- a/secrets.env +++ b/secrets.env @@ -1,6 +1,2 @@ MB_EMVI_CLIENT_SECRET= -MB_DB_USER= -MB_DB_PASSWORD= -MB_DB_SCHEMA= -MB_TRACKING_SALT= -MB_GEOLITE2_LICENSE_KEY= +MB_PIRSCH_CLIENT_SECRET= diff --git a/template/tracking.html b/template/tracking.html deleted file mode 100644 index 30a387a..0000000 --- a/template/tracking.html +++ /dev/null @@ -1,353 +0,0 @@ -{{template "head.html"}} -{{template "menu.html"}} - -
-

Tracking

-

- This page shows tracking statistics for my website using Pirsch and Chart.Js. The data shows unique visitors. All times and dates are UTC. -

-

- Week - Month - Quarter - Half Year - Year -

-
- - - -
-
-
-

Active Visitors

-

- Active visitors within the last ten minutes: {{.ActiveVisitors}} -

-

- The next diagram shows active visitors for each hour of today. -

- -

- The next table shows where the active visitors are for the past 10 minutes. Visitors switching between pages fast do create duplicate entries in the table. -

- - - - - - - - - {{range $data := .ActiveVisitorPages}} - - - - - {{end}} - -
PathVisitors
- {{$data.Path}} - {{$data.Visitors}}
-
-
-

Total Visitors

- -
-
-

Pages

-

- Here are the top 10 visited pages. -

- - - - - - - - - {{range $data := .PageRank}} - - - - - {{end}} - -
PathVisitors
- {{$data.Path}} - {{$data.Visitors}}
-
-
-

Languages

-

- Here are the top 10 languages used by my visitors. -

- - - - - - - - - - {{range $data := .Languages}} - - - - - - {{end}} - -
LanguageAbsoluteRelative
{{if $data.Language.String}}{{$data.Language.String}}{{else}}(not set){{end}}{{$data.Visitors}}{{round (multiply $data.RelativeVisitors 100)}} %
-
-
-

Referrer

-

- Here are the top 10 referrer. -

- - - - - - - - - {{range $data := .Referrer}} - - - - - {{end}} - -
ReferrerVisitors
- {{if $data.Referrer.String}} - {{$data.Referrer.String}} - {{else}} - (unknown) - {{end}} - {{$data.Visitors}}
-
-
-

Browser

- - - - - - - - - - {{range $data := .Browser}} - - - - - - {{end}} - -
BrowserAbsoluteRelative
{{if $data.Browser.String}}{{$data.Browser.String}}{{else}}(unknown){{end}}{{$data.Visitors}}{{round (multiply $data.RelativeVisitors 100)}} %
-
-
-

Operating System

- - - - - - - - - - {{range $data := .OS}} - - - - - - {{end}} - -
OSAbsoluteRelative
{{if $data.OS.String}}{{$data.OS.String}}{{else}}(unknown){{end}}{{$data.Visitors}}{{round (multiply $data.RelativeVisitors 100)}} %
-
-
-

Countries

- - - - - - - - - - {{range $data := .Countries}} - - - - - - {{end}} - -
CountryAbsoluteRelative
{{if $data.CountryCode.String}}{{$data.CountryCode.String}}{{else}}(unknown){{end}}{{$data.Visitors}}{{round (multiply $data.RelativeVisitors 100)}} %
-
-
-

Platform

- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
PlatformAbsoluteRelative
Desktop{{.Platform.PlatformDesktop}}{{round (multiply .Platform.RelativePlatformDesktop 100)}} %
Mobile{{.Platform.PlatformMobile}}{{round (multiply .Platform.RelativePlatformMobile 100)}} %
(unknown){{.Platform.PlatformUnknown}}{{round (multiply .Platform.RelativePlatformUnknown 100)}} %
-
-
-

Time of Day

- - - - - {{range $day := .TimeOfDay}} - - {{end}} - - - - {{$global := .}} - {{range $i := (intRange 0 24)}} - - - {{range $j, $day := $global.TimeOfDay}} - {{$visitors := float64 (index (index $global.TimeOfDay $j).Stats $i).Visitors}} - - {{end}} - - {{end}} - -
Time{{format $day.Day "Mon 01/02"}}
{{$i}} - {{$visitors}} -
-
-
-

Page Visits

-
- -{{range $i, $data := .PageVisitors}} -
-

{{$data.Path}}

- -
-{{end}} - - - - -{{template "end.html"}} diff --git a/tracking/statistics.go b/tracking/statistics.go deleted file mode 100644 index f83ba4b..0000000 --- a/tracking/statistics.go +++ /dev/null @@ -1,241 +0,0 @@ -package tracking - -import ( - "fmt" - "github.com/emvi/logbuch" - "github.com/pirsch-analytics/pirsch" - "html/template" - "sort" - "strings" - "time" -) - -const ( - statisticsDateFormat = "2006-01-02" -) - -type PageVisitors struct { - Path string - Visitors int - Labels template.JS - Data template.JS - Sessions template.JS - Bounces template.JS -} - -func GetActiveVisitors() ([]pirsch.Stats, int) { - visitors, total, err := analyzer.ActiveVisitors(nil, time.Minute*10) - - if err != nil { - logbuch.Error("Error reading active visitors", logbuch.Fields{"err": err}) - return nil, 0 - } - - return visitors, total -} - -func GetHourlyVisitorsToday() (template.JS, template.JS) { - visitors, err := analyzer.VisitorHours(&pirsch.Filter{From: today(), To: today()}) - - if err != nil { - logbuch.Error("Error reading hourly visitors for today", logbuch.Fields{"err": err}) - return "", "" - } - - return getLabelsAndDataHourly(visitors) -} - -func GetTotalVisitors(startDate, endDate time.Time) (template.JS, template.JS, template.JS, template.JS) { - visitors, err := analyzer.Visitors(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading visitor statistics", logbuch.Fields{"err": err}) - return "", "", "", "" - } - - return getLabelsAndData(visitors) -} - -func GetPageVisits(startDate, endDate time.Time) ([]PageVisitors, []PageVisitors) { - visits, err := analyzer.PageVisitors(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading page statistics", logbuch.Fields{"err": err}) - return nil, nil - } - - pageVisitors := make([]PageVisitors, len(visits)) - - for i, visit := range visits { - labels, data, sessions, bounces := getLabelsAndData(visit.Stats) - pageVisitors[i] = PageVisitors{ - Path: visit.Path, - Visitors: sumVisitors(visit.Stats), - Labels: labels, - Data: data, - Sessions: sessions, - Bounces: bounces, - } - } - - pageRank := make([]PageVisitors, len(pageVisitors)) - copy(pageRank, pageVisitors) - sort.Slice(pageRank, func(i, j int) bool { - return pageRank[i].Visitors > pageRank[j].Visitors - }) - return pageVisitors, pageRank -} - -func GetLanguages(startDate, endDate time.Time) []pirsch.LanguageStats { - languages, err := analyzer.Languages(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading language statistics", logbuch.Fields{"err": err}) - return nil - } - - if len(languages) > 10 { - return languages[:10] - } - - return languages -} - -func GetReferrer(startDate, endDate time.Time) []pirsch.ReferrerStats { - referrer, err := analyzer.Referrer(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading referrer statistics", logbuch.Fields{"err": err}) - return nil - } - - if len(referrer) > 10 { - return referrer[:10] - } - - return referrer -} - -func GetOS(startDate, endDate time.Time) []pirsch.OSStats { - os, err := analyzer.OS(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading OS statistics", logbuch.Fields{"err": err}) - return nil - } - - return os -} - -func GetBrowser(startDate, endDate time.Time) []pirsch.BrowserStats { - browser, err := analyzer.Browser(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading browser statistics", logbuch.Fields{"err": err}) - return nil - } - - return browser -} - -func GetCountry(startDate, endDate time.Time) []pirsch.CountryStats { - countries, err := analyzer.Country(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading country statistics", logbuch.Fields{"err": err}) - return nil - } - - for i := range countries { - countries[i].CountryCode.String = strings.ToUpper(countries[i].CountryCode.String) - } - - if len(countries) > 10 { - return countries[:10] - } - - return countries -} - -func GetPlatform(startDate, endDate time.Time) *pirsch.VisitorStats { - return analyzer.Platform(&pirsch.Filter{From: startDate, To: endDate}) -} - -func GetVisitorTimeOfDay(startDate, endDate time.Time) ([]pirsch.TimeOfDayVisitors, int) { - min := endDate.Add(-time.Hour * 24 * 7) - - if startDate.Before(min) { - startDate = min - } - - visitors, err := analyzer.TimeOfDay(&pirsch.Filter{From: startDate, To: endDate}) - - if err != nil { - logbuch.Error("Error reading visitor time of day statistics", logbuch.Fields{"err": err}) - return nil, 0 - } - - maxVisitors := 0 - - for _, v := range visitors { - for _, s := range v.Stats { - if maxVisitors < s.Visitors { - maxVisitors = s.Visitors - } - } - } - - return visitors, maxVisitors -} - -func sumVisitors(stats []pirsch.Stats) int { - sum := 0 - - for _, s := range stats { - sum += s.Visitors - } - - return sum -} - -func getLabelsAndData(visitors []pirsch.Stats) (template.JS, template.JS, template.JS, template.JS) { - var labels strings.Builder - var dp strings.Builder - var sessions strings.Builder - var bounces strings.Builder - - for _, point := range visitors { - labels.WriteString(fmt.Sprintf("'%s',", point.Day.Format(statisticsDateFormat))) - dp.WriteString(fmt.Sprintf("%d,", point.Visitors)) - sessions.WriteString(fmt.Sprintf("%d,", point.Sessions)) - bounces.WriteString(fmt.Sprintf("%d,", point.Bounces)) - } - - labelsStr := labels.String() - dataStr := dp.String() - sessionsStr := sessions.String() - bouncesStr := bounces.String() - return template.JS(labelsStr[:len(labelsStr)-1]), - template.JS(dataStr[:len(dataStr)-1]), - template.JS(sessionsStr[:len(sessionsStr)-1]), - template.JS(bouncesStr[:len(bouncesStr)-1]) -} - -func getLabelsAndDataHourly(visitors []pirsch.VisitorTimeStats) (template.JS, template.JS) { - var labels strings.Builder - var dp strings.Builder - - for _, point := range visitors { - labels.WriteString(fmt.Sprintf("'%d',", point.Hour)) - dp.WriteString(fmt.Sprintf("%d,", point.Visitors)) - } - - labelsStr := labels.String() - dataStr := dp.String() - return template.JS(labelsStr[:len(labelsStr)-1]), template.JS(dataStr[:len(dataStr)-1]) -} - -func today() time.Time { - now := time.Now() - return time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) -} diff --git a/tracking/tracking.go b/tracking/tracking.go deleted file mode 100644 index 0ec5a99..0000000 --- a/tracking/tracking.go +++ /dev/null @@ -1,90 +0,0 @@ -package tracking - -import ( - "context" - "database/sql" - "github.com/Kugelschieber/marvinblum/db" - "github.com/emvi/logbuch" - "github.com/pirsch-analytics/pirsch" - "os" - "path/filepath" -) - -const ( - geodbPath = "geodb" -) - -var ( - store pirsch.Store - analyzer *pirsch.Analyzer -) - -func NewTracker() (*pirsch.Tracker, context.CancelFunc) { - logbuch.Info("Connecting to database...") - conn, err := sql.Open("postgres", db.GetConnectionString()) - - if err != nil { - logbuch.Fatal("Error connecting to database", logbuch.Fields{"err": err}) - return nil, nil - } - - if err := conn.Ping(); err != nil { - logbuch.Fatal("Error pinging database", logbuch.Fields{"err": err}) - return nil, nil - } - - store = pirsch.NewPostgresStore(conn, nil) - tracker := pirsch.NewTracker(store, os.Getenv("MB_TRACKING_SALT"), &pirsch.TrackerConfig{ - ReferrerDomainBlacklist: []string{"marvinblum.de"}, // I don't care about traffic from my own website - ReferrerDomainBlacklistIncludesSubdomains: true, - Sessions: true, - }) - analyzer = pirsch.NewAnalyzer(store, nil) - processor := pirsch.NewProcessor(store) - cancel := pirsch.RunAtMidnight(func() { - processTrackingData(processor) - updateGeoDB(tracker) - }) - processTrackingData(processor) - updateGeoDB(tracker) - return tracker, cancel -} - -func processTrackingData(processor *pirsch.Processor) { - logbuch.Info("Processing tracking data...") - - defer func() { - if err := recover(); err != nil { - logbuch.Error("Error processing tracking data", logbuch.Fields{"err": err}) - } - }() - - if err := processor.Process(); err != nil { - logbuch.Error("Error processing tracking data", logbuch.Fields{"err": err}) - } else { - logbuch.Info("Done processing tracking data") - } -} - -func updateGeoDB(tracker *pirsch.Tracker) { - licenseKey := os.Getenv("MB_GEOLITE2_LICENSE_KEY") - - if licenseKey == "" { - return - } - - if err := pirsch.GetGeoLite2(geodbPath, licenseKey); err != nil { - logbuch.Error("Error loading GeoLite2", logbuch.Fields{"err": err}) - return - } - - geodb, err := pirsch.NewGeoDB(filepath.Join(geodbPath, pirsch.GeoLite2Filename)) - - if err != nil { - logbuch.Error("Error creating GeoDB", logbuch.Fields{"err": err}) - return - } - - tracker.SetGeoDB(geodb) - logbuch.Info("GeoDB updated") -}