marvin/marvinblum
1
0
Fork 0
mirror of https://github.com/Kugelschieber/marvinblum.git synced 2026-01-18 06:40:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removed pirsch library and use pirsch.io instead.

Browse Source
This commit is contained in:
Marvin Blum
2020-12-04 23:53:31 +01:00
committed by Marvin Blum
parent 9389b9a605
commit 159c199f2f
19 changed files with 30 additions and 1119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@@ -20,8 +20,6 @@ WORKDIR /app
ENV MB_LOGLEVEL=info
ENV MB_ALLOWED_ORIGINS=*
ENV MB_HOST=0.0.0.0:8888
ENV MB_DB_PORT=5432
ENV MB_DB_SSLMODE=disable
EXPOSE 8888
CMD ["/app/main"]

1
README.md
View File

@@ -1,2 +1,3 @@
# marvinblum.de
My website.

29
db/db.go
View File

@@ -1,29 +0,0 @@
package db
import (
"fmt"
"github.com/emvi/logbuch"
"os"
"strconv"
"time"
)
const (
connectionString = `host=%s port=%s user=%s password=%s dbname=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s connectTimeout=%s timezone=%s`
)
func GetConnectionString() string {
host := os.Getenv("MB_DB_HOST")
port := os.Getenv("MB_DB_PORT")
user := os.Getenv("MB_DB_USER")
password := os.Getenv("MB_DB_PASSWORD")
schema := os.Getenv("MB_DB_SCHEMA")
sslMode := os.Getenv("MB_DB_SSLMODE")
sslCert := os.Getenv("MB_DB_SSLCERT")
sslKey := os.Getenv("MB_DB_SSLKEY")
sslRootCert := os.Getenv("MB_DB_SSLROOTCERT")
zone, offset := time.Now().Zone()
timezone := zone + strconv.Itoa(-offset/3600)
logbuch.Info("Setting time zone", logbuch.Fields{"timezone": timezone})
return fmt.Sprintf(connectionString, host, port, user, password, schema, sslMode, sslCert, sslKey, sslRootCert, "30", timezone)
}

48
db/migrate.go
View File

@@ -1,48 +0,0 @@
package db
import (
"fmt"
"github.com/emvi/logbuch"
"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
"os"
)
const (
migrateConnectionString = `postgres://%s:%s/%s?user=%s&password=%s&sslmode=%s&sslcert=%s&sslkey=%s&sslrootcert=%s&connect_timeout=60`
)
func Migrate() {
logbuch.Info("Migrating database schema (if required)")
m, err := migrate.New("file://schema", getMigrationConnectionString())
if err != nil {
logbuch.Fatal("Error migrating database schema", logbuch.Fields{"err": err})
return
}
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
logbuch.Fatal("Error migrating database schema", logbuch.Fields{"err": err})
return
}
if sourceErr, dbErr := m.Close(); sourceErr != nil || dbErr != nil {
logbuch.Fatal("Error migrating database schema", logbuch.Fields{"source_err": sourceErr, "db_err": dbErr})
}
logbuch.Info("Done migrating database schema")
}
func getMigrationConnectionString() string {
host := os.Getenv("MB_DB_HOST")
port := os.Getenv("MB_DB_PORT")
user := os.Getenv("MB_DB_USER")
password := os.Getenv("MB_DB_PASSWORD")
schema := os.Getenv("MB_DB_SCHEMA")
sslMode := os.Getenv("MB_DB_SSLMODE")
sslCert := os.Getenv("MB_DB_SSLCERT")
sslKey := os.Getenv("MB_DB_SSLKEY")
sslRootCert := os.Getenv("MB_DB_SSLROOTCERT")
return fmt.Sprintf(migrateConnectionString, host, port, schema, user, password, sslMode, sslCert, sslKey, sslRootCert)
}

14
dev.sh
View File

@@ -9,16 +9,10 @@ export MB_ALLOWED_ORIGINS=*
export MB_HOST=localhost:8080
export MB_HOT_RELOAD=true
export MB_EMVI_CLIENT_ID=3fBBn144yvSF9R3dPC8l
export MB_EMVI_CLIENT_SECRET=dw3FeshelTgdf1Gj13J7uF5FfdPDi40sQvvwqeFVKTTyIDuCdlAHhRY72csFL6yg
export MB_EMVI_CLIENT_SECRET=
export MB_EMVI_ORGA=marvin
export MB_DB_HOST=localhost
export MB_DB_PORT=5432
export MB_DB_USER=postgres
export MB_DB_PASSWORD=postgres
export MB_DB_SCHEMA=marvinblum
export MB_DB_SSLMODE=disable
export MB_DB_SSLCERT=
export MB_DB_SSLKEY=
export MB_DB_SSLROOTCERT=
export MB_PIRSCH_CLIENT_ID=gEb3pvgxZvZzFRlOTdMgPtyLvNYgeVKe
export MB_PIRSCH_CLIENT_SECRET=E7UqJehmxgnVuw81oq6ZhJAx9vCHqMimCUFfil7UFgbGhgQVVINqU7JqHBgaUvHg
export MB_PIRSCH_HOSTNAME=marvinblum.de
go run main.go

3
docker-compose.yml
View File

@@ -52,7 +52,8 @@ services:
environment:
MB_EMVI_CLIENT_ID: 3fBBn144yvSF9R3dPC8l
MB_EMVI_ORGA: marvin
MB_DB_HOST: postgres
export MB_PIRSCH_CLIENT_ID: gEb3pvgxZvZzFRlOTdMgPtyLvNYgeVKe
export MB_PIRSCH_HOSTNAME: marvinblum.de
labels:
- "traefik.enable=true"
- "traefik.port=8888"

1
go.mod
View File

@@ -12,6 +12,7 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/lib/pq v1.8.0
github.com/pirsch-analytics/pirsch v1.8.1
github.com/pirsch-analytics/pirsch-go-sdk v0.0.0-20201204224029-0ab7a9417d40 // indirect
github.com/rs/cors v1.7.0
golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 // indirect
)

2
go.sum
View File

@@ -226,6 +226,8 @@ github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pirsch-analytics/pirsch v1.8.1 h1:07pFslS6gSTgGcu26camKeYhJ5G+KCwnqc6GGVRHcPU=
github.com/pirsch-analytics/pirsch v1.8.1/go.mod h1:BPOASgFDyfVyXCXrrDvS5kVgc/eN2fM5cZAVj0fZGbI=
github.com/pirsch-analytics/pirsch-go-sdk v0.0.0-20201204224029-0ab7a9417d40 h1:wBNO4NcuRc8GL95QiewUkhIspx9wKg8qzwqBKHbMB44=
github.com/pirsch-analytics/pirsch-go-sdk v0.0.0-20201204224029-0ab7a9417d40/go.mod h1:PF2vnJw8FYcXQe6OTPQQcGn8l/agkpl7T4YO9d2aPSE=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

108
main.go
View File

@@ -3,21 +3,18 @@ package main
import (
"context"
"github.com/Kugelschieber/marvinblum/blog"
"github.com/Kugelschieber/marvinblum/db"
"github.com/Kugelschieber/marvinblum/tpl"
"github.com/Kugelschieber/marvinblum/tracking"
"github.com/NYTimes/gziphandler"
emvi "github.com/emvi/api-go"
"github.com/emvi/logbuch"
"github.com/gorilla/mux"
_ "github.com/lib/pq"
"github.com/pirsch-analytics/pirsch"
"github.com/pirsch-analytics/pirsch-go-sdk"
"github.com/rs/cors"
"html/template"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"time"
)
@@ -31,14 +28,14 @@ const (
)
var (
tracker *pirsch.Tracker
client *pirsch.Client
tplCache *tpl.Cache
blogInstance *blog.Blog
)
func serveAbout() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
go tracker.Hit(r, nil)
go hit(r)
tplCache.Render(w, "about.html", struct {
Articles []emvi.Article
}{
@@ -49,14 +46,14 @@ func serveAbout() http.HandlerFunc {
func serveLegal() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
go tracker.Hit(r, nil)
go hit(r)
tplCache.Render(w, "legal.html", nil)
}
}
func serveBlogPage() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
go tracker.Hit(r, nil)
go hit(r)
tplCache.Render(w, "blog.html", struct {
Articles map[int][]emvi.Article
}{
@@ -83,7 +80,7 @@ func serveBlogArticle() http.HandlerFunc {
}
// track the hit if the article was found, otherwise we don't care
go tracker.Hit(r, nil)
go hit(r)
tplCache.RenderWithoutCache(w, "article.html", struct {
Title string
@@ -99,79 +96,7 @@ func serveBlogArticle() http.HandlerFunc {
func serveTracking() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
go tracker.Hit(r, nil)
start, _ := strconv.Atoi(r.URL.Query().Get("start"))
if start > 365 {
start = 365
} else if start < 7 {
start = 7
}
var startDate, endDate time.Time
if err := r.ParseForm(); err != nil {
logbuch.Warn("Error parsing tracking form", logbuch.Fields{"err": err})
} else {
startDate, _ = time.Parse("2006-01-02", r.FormValue("start-date"))
endDate, _ = time.Parse("2006-01-02", r.FormValue("end-date"))
}
if startDate.IsZero() || endDate.IsZero() {
startDate = time.Now().UTC().Add(-time.Hour * 24 * time.Duration(start))
endDate = time.Now().UTC()
}
activeVisitorPages, activeVisitors := tracking.GetActiveVisitors()
totalVisitorsLabels, totalVisitorsDps, sessionsDps, bouncesDps := tracking.GetTotalVisitors(startDate, endDate)
hourlyVisitorsTodayLabels, hourlyVisitorsTodayDps := tracking.GetHourlyVisitorsToday()
pageVisitors, pageRank := tracking.GetPageVisits(startDate, endDate)
timeOfDay, timeOfDayMax := tracking.GetVisitorTimeOfDay(startDate, endDate)
tplCache.RenderWithoutCache(w, "tracking.html", struct {
Start int
StartDate time.Time
EndDate time.Time
TotalVisitorsLabels template.JS
TotalVisitorsDps template.JS
SessionsDps template.JS
BouncesDps template.JS
PageVisitors []tracking.PageVisitors
PageRank []tracking.PageVisitors
Languages []pirsch.LanguageStats
Referrer []pirsch.ReferrerStats
Browser []pirsch.BrowserStats
OS []pirsch.OSStats
Countries []pirsch.CountryStats
Platform *pirsch.VisitorStats
TimeOfDay []pirsch.TimeOfDayVisitors
TimeOfDayMax float64
HourlyVisitorsTodayLabels template.JS
HourlyVisitorsTodayDps template.JS
ActiveVisitors int
ActiveVisitorPages []pirsch.Stats
}{
start,
startDate,
endDate,
totalVisitorsLabels,
totalVisitorsDps,
sessionsDps,
bouncesDps,
pageVisitors,
pageRank,
tracking.GetLanguages(startDate, endDate),
tracking.GetReferrer(startDate, endDate),
tracking.GetBrowser(startDate, endDate),
tracking.GetOS(startDate, endDate),
tracking.GetCountry(startDate, endDate),
tracking.GetPlatform(startDate, endDate),
timeOfDay,
float64(timeOfDayMax),
hourlyVisitorsTodayLabels,
hourlyVisitorsTodayDps,
activeVisitors,
activeVisitorPages,
})
http.Redirect(w, r, "https://marvinblum.pirsch.io/", http.StatusFound)
}
}
@@ -230,7 +155,7 @@ func configureCors(router *mux.Router) http.Handler {
return c.Handler(router)
}
func start(handler http.Handler, trackingCancel context.CancelFunc) {
func start(handler http.Handler) {
logbuch.Info("Starting server...")
var server http.Server
server.Handler = handler
@@ -241,8 +166,6 @@ func start(handler http.Handler, trackingCancel context.CancelFunc) {
signal.Notify(sigint, os.Interrupt)
<-sigint
logbuch.Info("Shutting down server...")
trackingCancel()
tracker.Stop()
ctx, _ := context.WithTimeout(context.Background(), shutdownTimeout)
if err := server.Shutdown(ctx); err != nil {
@@ -257,15 +180,22 @@ func start(handler http.Handler, trackingCancel context.CancelFunc) {
logbuch.Info("Server shut down")
}
func hit(r *http.Request) {
if err := client.Hit(r); err != nil {
logbuch.Warn("Error sending page hit to pirsch", logbuch.Fields{"err": err})
}
}
func main() {
configureLog()
logEnvConfig()
db.Migrate()
var trackingCancel context.CancelFunc
tracker, trackingCancel = tracking.NewTracker()
client = pirsch.NewClient(os.Getenv("MB_PIRSCH_CLIENT_ID"),
os.Getenv("MB_PIRSCH_CLIENT_SECRET"),
os.Getenv("MB_PIRSCH_HOSTNAME"),
nil)
tplCache = tpl.NewCache()
blogInstance = blog.NewBlog(tplCache)
router := setupRouter()
corsConfig := configureCors(router)
start(corsConfig, trackingCancel)
start(corsConfig)
}

4
postgres/clear_logs.sh
View File

@@ -1,4 +0,0 @@
#!/bin/bash
echo "Clearing docker logs..."
echo "" > $(docker inspect --format='{{.LogPath}}' postgres)

22
postgres/docker-compose.yml
View File

@@ -1,22 +0,0 @@
version: "3"
services:
postgres:
image: postgres:12-alpine
container_name: postgres
restart: always
command: -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key
networks:
- db-internal
environment:
POSTGRES_PASSWORD:
ports:
- "5432:5432"
volumes:
- /root/postgres/data:/var/lib/postgresql/data
- /root/postgres/cert/server.crt:/var/lib/postgresql/server.crt
- /root/postgres/cert/server.key:/var/lib/postgresql/server.key
networks:
db-internal:
driver: bridge

10
postgres/gen_cert.sh
View File

@@ -1,10 +0,0 @@
#!/bin/bash
mkdir cert
cd cert
openssl req -new -text -passout pass:$1 -subj /CN=localhost -out server.req -keyout privkey.pem
openssl rsa -in privkey.pem -passin pass:$1 -out server.key
openssl req -x509 -in server.req -text -key server.key -out server.crt
chown 0:70 server.key
chmod 640 server.key
echo "done"

171
schema/0001_baseline.up.sql
View File

@@ -1,171 +0,0 @@
CREATE TABLE "hit" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
fingerprint varchar(32) NOT NULL,
"session" timestamp without time zone,
path varchar(2000),
url varchar(2000),
language varchar(10),
user_agent varchar(200),
referrer varchar(200),
"os" character varying(20),
"os_version" character varying(20),
"browser" character varying(20),
"browser_version" character varying(20),
"desktop" boolean DEFAULT FALSE,
"mobile" boolean DEFAULT FALSE,
time timestamp without time zone NOT NULL
);
CREATE SEQUENCE hit_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE hit_id_seq OWNED BY "hit".id;
ALTER TABLE ONLY "hit" ALTER COLUMN id SET DEFAULT nextval('hit_id_seq'::regclass);
ALTER TABLE ONLY "hit" ADD CONSTRAINT hit_pkey PRIMARY KEY (id);
CREATE INDEX hit_fingerprint_index ON hit(fingerprint);
CREATE INDEX hit_path_index ON hit(path);
CREATE INDEX hit_time_index ON hit(time);
CREATE TABLE "visitor_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
path varchar(2000) NOT NULL,
visitors integer NOT NULL,
sessions integer NOT NULL DEFAULT 0,
bounces integer NOT NULL DEFAULT 0,
platform_desktop integer NOT NULL,
platform_mobile integer NOT NULL,
platform_unknown integer NOT NULL
);
CREATE SEQUENCE visitor_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE visitor_stats_id_seq OWNED BY "visitor_stats".id;
ALTER TABLE ONLY "visitor_stats" ALTER COLUMN id SET DEFAULT nextval('visitor_stats_id_seq'::regclass);
ALTER TABLE ONLY "visitor_stats" ADD CONSTRAINT visitor_stats_pkey PRIMARY KEY (id);
CREATE INDEX visitor_stats_day_index ON visitor_stats(day);
CREATE INDEX visitor_stats_path_index ON visitor_stats(path);
CREATE TABLE "visitor_time_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
path varchar(2000) NOT NULL,
hour smallint NOT NULL,
visitors integer NOT NULL,
sessions integer NOT NULL DEFAULT 0
);
CREATE SEQUENCE visitor_time_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE visitor_time_stats_id_seq OWNED BY "visitor_time_stats".id;
ALTER TABLE ONLY "visitor_time_stats" ALTER COLUMN id SET DEFAULT nextval('visitor_time_stats_id_seq'::regclass);
ALTER TABLE ONLY "visitor_time_stats" ADD CONSTRAINT visitor_time_stats_pkey PRIMARY KEY (id);
CREATE INDEX visitor_time_stats_day_index ON visitor_time_stats(day);
CREATE INDEX visitor_time_stats_path_index ON visitor_time_stats(path);
CREATE TABLE "language_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
path varchar(2000) NOT NULL,
language varchar(10),
visitors integer NOT NULL
);
CREATE SEQUENCE language_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE language_stats_id_seq OWNED BY "language_stats".id;
ALTER TABLE ONLY "language_stats" ALTER COLUMN id SET DEFAULT nextval('language_stats_id_seq'::regclass);
ALTER TABLE ONLY "language_stats" ADD CONSTRAINT language_stats_pkey PRIMARY KEY (id);
CREATE INDEX language_stats_day_index ON language_stats(day);
CREATE INDEX language_stats_path_index ON language_stats(path);
CREATE TABLE "referrer_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
path varchar(2000) NOT NULL,
referrer varchar(2000),
visitors integer NOT NULL
);
CREATE SEQUENCE referrer_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE referrer_stats_id_seq OWNED BY "referrer_stats".id;
ALTER TABLE ONLY "referrer_stats" ALTER COLUMN id SET DEFAULT nextval('referrer_stats_id_seq'::regclass);
ALTER TABLE ONLY "referrer_stats" ADD CONSTRAINT referrer_stats_pkey PRIMARY KEY (id);
CREATE INDEX referrer_stats_day_index ON referrer_stats(day);
CREATE INDEX referrer_stats_path_index ON referrer_stats(path);
CREATE TABLE "os_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
path varchar(2000) NOT NULL,
os character varying(20),
os_version character varying(20),
visitors integer NOT NULL
);
CREATE SEQUENCE os_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE os_stats_id_seq OWNED BY "os_stats".id;
ALTER TABLE ONLY "os_stats" ALTER COLUMN id SET DEFAULT nextval('os_stats_id_seq'::regclass);
ALTER TABLE ONLY "os_stats" ADD CONSTRAINT os_stats_pkey PRIMARY KEY (id);
CREATE INDEX os_stats_day_index ON os_stats(day);
CREATE INDEX os_stats_path_index ON os_stats(path);
CREATE TABLE "browser_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
path varchar(2000) NOT NULL,
browser character varying(20),
browser_version character varying(20),
visitors integer NOT NULL
);
CREATE SEQUENCE browser_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE browser_stats_id_seq OWNED BY "browser_stats".id;
ALTER TABLE ONLY "browser_stats" ALTER COLUMN id SET DEFAULT nextval('browser_stats_id_seq'::regclass);
ALTER TABLE ONLY "browser_stats" ADD CONSTRAINT browser_stats_pkey PRIMARY KEY (id);
CREATE INDEX browser_stats_day_index ON browser_stats(day);
CREATE INDEX browser_stats_path_index ON browser_stats(path);

23
schema/0002_pirsch_update.up.sql
View File

@@ -1,23 +0,0 @@
ALTER TABLE "hit" ADD COLUMN "screen_width" integer DEFAULT 0;
ALTER TABLE "hit" ADD COLUMN "screen_height" integer DEFAULT 0;
CREATE TABLE "screen_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
visitors integer NOT NULL,
width integer NOT NULL,
height integer NOT NULL
);
CREATE SEQUENCE screen_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE screen_stats_id_seq OWNED BY "screen_stats".id;
ALTER TABLE ONLY "screen_stats" ALTER COLUMN id SET DEFAULT nextval('screen_stats_id_seq'::regclass);
ALTER TABLE ONLY "screen_stats" ADD CONSTRAINT screen_stats_pkey PRIMARY KEY (id);
CREATE INDEX screen_stats_day_index ON screen_stats(day);

21
schema/0003_pirsch_update.up.sql
View File

@@ -1,21 +0,0 @@
ALTER TABLE "hit" ADD COLUMN "country_code" character varying(2);
CREATE TABLE "country_stats" (
id bigint NOT NULL UNIQUE,
tenant_id bigint,
day date NOT NULL,
visitors integer NOT NULL,
country_code character varying(2)
);
CREATE SEQUENCE country_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE country_stats_id_seq OWNED BY "country_stats".id;
ALTER TABLE ONLY "country_stats" ALTER COLUMN id SET DEFAULT nextval('country_stats_id_seq'::regclass);
ALTER TABLE ONLY "country_stats" ADD CONSTRAINT country_stats_pkey PRIMARY KEY (id);
CREATE INDEX country_stats_day_index ON country_stats(day);

6
secrets.env
View File

@@ -1,6 +1,2 @@
MB_EMVI_CLIENT_SECRET=
MB_DB_USER=
MB_DB_PASSWORD=
MB_DB_SCHEMA=
MB_TRACKING_SALT=
MB_GEOLITE2_LICENSE_KEY=
MB_PIRSCH_CLIENT_SECRET=

353
template/tracking.html
View File

@@ -1,353 +0,0 @@
{{template "head.html"}}
{{template "menu.html"}}
<section>
<h1>Tracking</h1>
<p>
This page shows tracking statistics for my website using <a href="https://github.com/pirsch-analytics/pirsch" target="_blank">Pirsch</a> and <a href="https://www.chartjs.org/" target="_blank">Chart.Js</a>. The data shows unique visitors. All times and dates are UTC.
</p>
<p>
<a href="/tracking?start=7" class="button {{if eq .Start 7}}filled{{end}}">Week</a>
<a href="/tracking?start=30" class="button {{if eq .Start 30}}filled{{end}}">Month</a>
<a href="/tracking?start=90" class="button {{if eq .Start 90}}filled{{end}}">Quarter</a>
<a href="/tracking?start=182" class="button {{if eq .Start 182}}filled{{end}}">Half Year</a>
<a href="/tracking?start=365" class="button {{if eq .Start 365}}filled{{end}}">Year</a>
</p>
<form class="tracking-form">
<input type="date" name="start-date" value="{{format .StartDate "2006-01-02"}}" />
<input type="date" name="end-date" value="{{format .EndDate "2006-01-02"}}" />
<input type="submit" value="Update" />
</form>
</section>
<section>
<h2>Active Visitors</h2>
<p>
Active visitors within the last ten minutes: {{.ActiveVisitors}}
</p>
<p>
The next diagram shows active visitors for each hour of today.
</p>
<canvas id="hourlyVisitorsToday" class="tracking"></canvas>
<p>
The next table shows where the active visitors are for the past 10 minutes. Visitors switching between pages fast do create duplicate entries in the table.
</p>
<table>
<thead>
<tr>
<th>Path</th>
<th>Visitors</th>
</tr>
</thead>
<tbody>
{{range $data := .ActiveVisitorPages}}
<tr>
<td class="break-line-anywhere">
<a href="{{$data.Path}}" target="_blank">{{$data.Path}}</a>
</td>
<td>{{$data.Visitors}}</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Total Visitors</h2>
<canvas id="totalVisitors" class="tracking"></canvas>
</section>
<section>
<h2>Pages</h2>
<p>
Here are the top 10 visited pages.
</p>
<table>
<thead>
<tr>
<th>Path</th>
<th>Visitors</th>
</tr>
</thead>
<tbody>
{{range $data := .PageRank}}
<tr>
<td class="break-line-anywhere">
<a href="{{$data.Path}}" target="_blank">{{$data.Path}}</a>
</td>
<td>{{$data.Visitors}}</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Languages</h2>
<p>
Here are the top 10 languages used by my visitors.
</p>
<table>
<thead>
<tr>
<th>Language</th>
<th>Absolute</th>
<th>Relative</th>
</tr>
</thead>
<tbody>
{{range $data := .Languages}}
<tr>
<td>{{if $data.Language.String}}{{$data.Language.String}}{{else}}(not set){{end}}</td>
<td>{{$data.Visitors}}</td>
<td>{{round (multiply $data.RelativeVisitors 100)}} %</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Referrer</h2>
<p>
Here are the top 10 referrer.
</p>
<table>
<thead>
<tr>
<th>Referrer</th>
<th>Visitors</th>
</tr>
</thead>
<tbody>
{{range $data := .Referrer}}
<tr>
<td class="break-line-anywhere">
{{if $data.Referrer.String}}
<a href="{{$data.Referrer.String}}" target="_blank">{{$data.Referrer.String}}</a>
{{else}}
(unknown)
{{end}}
</td>
<td>{{$data.Visitors}}</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Browser</h2>
<table>
<thead>
<tr>
<th>Browser</th>
<th>Absolute</th>
<th>Relative</th>
</tr>
</thead>
<tbody>
{{range $data := .Browser}}
<tr>
<td>{{if $data.Browser.String}}{{$data.Browser.String}}{{else}}(unknown){{end}}</td>
<td>{{$data.Visitors}}</td>
<td>{{round (multiply $data.RelativeVisitors 100)}} %</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Operating System</h2>
<table>
<thead>
<tr>
<th>OS</th>
<th>Absolute</th>
<th>Relative</th>
</tr>
</thead>
<tbody>
{{range $data := .OS}}
<tr>
<td>{{if $data.OS.String}}{{$data.OS.String}}{{else}}(unknown){{end}}</td>
<td>{{$data.Visitors}}</td>
<td>{{round (multiply $data.RelativeVisitors 100)}} %</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Countries</h2>
<table>
<thead>
<tr>
<th>Country</th>
<th>Absolute</th>
<th>Relative</th>
</tr>
</thead>
<tbody>
{{range $data := .Countries}}
<tr>
<td>{{if $data.CountryCode.String}}{{$data.CountryCode.String}}{{else}}(unknown){{end}}</td>
<td>{{$data.Visitors}}</td>
<td>{{round (multiply $data.RelativeVisitors 100)}} %</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Platform</h2>
<canvas id="platform" class="tracking"></canvas>
</section>
<section>
<table>
<thead>
<tr>
<th>Platform</th>
<th>Absolute</th>
<th>Relative</th>
</tr>
</thead>
<tbody>
<tr>
<td>Desktop</td>
<td>{{.Platform.PlatformDesktop}}</td>
<td>{{round (multiply .Platform.RelativePlatformDesktop 100)}} %</td>
</tr>
<tr>
<td>Mobile</td>
<td>{{.Platform.PlatformMobile}}</td>
<td>{{round (multiply .Platform.RelativePlatformMobile 100)}} %</td>
</tr>
<tr>
<td>(unknown)</td>
<td>{{.Platform.PlatformUnknown}}</td>
<td>{{round (multiply .Platform.RelativePlatformUnknown 100)}} %</td>
</tr>
</tbody>
</table>
</section>
<section>
<h2>Time of Day</h2>
<table>
<thead>
<tr>
<th>Time</th>
{{range $day := .TimeOfDay}}
<th>{{format $day.Day "Mon 01/02"}}</th>
{{end}}
</tr>
</thead>
<tbody>
{{$global := .}}
{{range $i := (intRange 0 24)}}
<tr>
<td>{{$i}}</td>
{{range $j, $day := $global.TimeOfDay}}
{{$visitors := float64 (index (index $global.TimeOfDay $j).Stats $i).Visitors}}
<td style="background: rgba(81, 81, 81, {{divide $visitors $global.TimeOfDayMax}});color: #fff;text-align: center;">
{{$visitors}}
</td>
{{end}}
</tr>
{{end}}
</tbody>
</table>
</section>
<section>
<h2>Page Visits</h2>
</section>
{{range $i, $data := .PageVisitors}}
<section>
<h3>{{$data.Path}}</h3>
<canvas id="pageVisits{{$i}}" class="tracking"></canvas>
</section>
{{end}}
<script type="text/javascript" src="/static/js/Chart-v2.9.3.bundle.min.js"></script>
<script type="text/javascript">
new Chart(document.getElementById('hourlyVisitorsToday').getContext('2d'), {
type: "bar",
data: {
labels: [{{.HourlyVisitorsTodayLabels}}],
datasets: [{
backgroundColor: "#7f7f7f",
borderColor: "#7f7f7f",
label: "Hourly Visitors for Today",
data: [{{.HourlyVisitorsTodayDps}}]
}]
}
});
new Chart(document.getElementById('totalVisitors').getContext('2d'), {
type: "line",
data: {
labels: [{{.TotalVisitorsLabels}}],
datasets: [
{
backgroundColor: "rgb(43, 180, 0, 0.02)",
borderColor: "rgb(40, 152, 0, 0.5)",
label: "Total Visitors",
data: [{{.TotalVisitorsDps}}]
},
{
backgroundColor: "rgb(0, 63, 197, 0.02)",
borderColor: "rgb(0, 53, 159, 0.5)",
label: "Sessions",
data: [{{.SessionsDps}}]
},
{
backgroundColor: "rgba(194, 0, 0, 0.02)",
borderColor: "rgb(152, 0, 0, 0.5)",
label: "Bounces",
data: [{{.BouncesDps}}]
}
]
}
});
new Chart(document.getElementById('platform').getContext('2d'), {
type: "doughnut",
data: {
labels: ["Desktop", "Mobile", "(unknown)"],
datasets: [{
backgroundColor: ["#515151", "#7f7f7f", "#dbdbdb"],
borderColor: ["#515151", "#7f7f7f", "#dbdbdb"],
data: [
{{round (multiply .Platform.RelativePlatformDesktop 100)}},
{{round (multiply .Platform.RelativePlatformMobile 100)}},
{{round (multiply .Platform.RelativePlatformUnknown 100)}}]
}]
}
});
{{range $i, $data := .PageVisitors}}
new Chart(document.getElementById('pageVisits{{$i}}').getContext('2d'), {
type: "line",
data: {
labels: [{{$data.Labels}}],
datasets: [
{
backgroundColor: "rgb(43, 180, 0, 0.02)",
borderColor: "rgb(40, 152, 0, 0.5)",
label: "Page Visits",
data: [{{$data.Data}}]
},
{
backgroundColor: "rgb(0, 63, 197, 0.02)",
borderColor: "rgb(0, 53, 159, 0.5)",
label: "Sessions",
data: [{{$data.Sessions}}]
},
{
backgroundColor: "rgba(194, 0, 0, 0.02)",
borderColor: "rgb(152, 0, 0, 0.5)",
label: "Bounces",
data: [{{$data.Bounces}}]
}
]
}
});
{{end}}
</script>
{{template "end.html"}}

241
tracking/statistics.go
View File

@@ -1,241 +0,0 @@
package tracking
import (
"fmt"
"github.com/emvi/logbuch"
"github.com/pirsch-analytics/pirsch"
"html/template"
"sort"
"strings"
"time"
)
const (
statisticsDateFormat = "2006-01-02"
)
type PageVisitors struct {
Path string
Visitors int
Labels template.JS
Data template.JS
Sessions template.JS
Bounces template.JS
}
func GetActiveVisitors() ([]pirsch.Stats, int) {
visitors, total, err := analyzer.ActiveVisitors(nil, time.Minute*10)
if err != nil {
logbuch.Error("Error reading active visitors", logbuch.Fields{"err": err})
return nil, 0
}
return visitors, total
}
func GetHourlyVisitorsToday() (template.JS, template.JS) {
visitors, err := analyzer.VisitorHours(&pirsch.Filter{From: today(), To: today()})
if err != nil {
logbuch.Error("Error reading hourly visitors for today", logbuch.Fields{"err": err})
return "", ""
}
return getLabelsAndDataHourly(visitors)
}
func GetTotalVisitors(startDate, endDate time.Time) (template.JS, template.JS, template.JS, template.JS) {
visitors, err := analyzer.Visitors(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading visitor statistics", logbuch.Fields{"err": err})
return "", "", "", ""
}
return getLabelsAndData(visitors)
}
func GetPageVisits(startDate, endDate time.Time) ([]PageVisitors, []PageVisitors) {
visits, err := analyzer.PageVisitors(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading page statistics", logbuch.Fields{"err": err})
return nil, nil
}
pageVisitors := make([]PageVisitors, len(visits))
for i, visit := range visits {
labels, data, sessions, bounces := getLabelsAndData(visit.Stats)
pageVisitors[i] = PageVisitors{
Path: visit.Path,
Visitors: sumVisitors(visit.Stats),
Labels: labels,
Data: data,
Sessions: sessions,
Bounces: bounces,
}
}
pageRank := make([]PageVisitors, len(pageVisitors))
copy(pageRank, pageVisitors)
sort.Slice(pageRank, func(i, j int) bool {
return pageRank[i].Visitors > pageRank[j].Visitors
})
return pageVisitors, pageRank
}
func GetLanguages(startDate, endDate time.Time) []pirsch.LanguageStats {
languages, err := analyzer.Languages(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading language statistics", logbuch.Fields{"err": err})
return nil
}
if len(languages) > 10 {
return languages[:10]
}
return languages
}
func GetReferrer(startDate, endDate time.Time) []pirsch.ReferrerStats {
referrer, err := analyzer.Referrer(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading referrer statistics", logbuch.Fields{"err": err})
return nil
}
if len(referrer) > 10 {
return referrer[:10]
}
return referrer
}
func GetOS(startDate, endDate time.Time) []pirsch.OSStats {
os, err := analyzer.OS(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading OS statistics", logbuch.Fields{"err": err})
return nil
}
return os
}
func GetBrowser(startDate, endDate time.Time) []pirsch.BrowserStats {
browser, err := analyzer.Browser(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading browser statistics", logbuch.Fields{"err": err})
return nil
}
return browser
}
func GetCountry(startDate, endDate time.Time) []pirsch.CountryStats {
countries, err := analyzer.Country(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading country statistics", logbuch.Fields{"err": err})
return nil
}
for i := range countries {
countries[i].CountryCode.String = strings.ToUpper(countries[i].CountryCode.String)
}
if len(countries) > 10 {
return countries[:10]
}
return countries
}
func GetPlatform(startDate, endDate time.Time) *pirsch.VisitorStats {
return analyzer.Platform(&pirsch.Filter{From: startDate, To: endDate})
}
func GetVisitorTimeOfDay(startDate, endDate time.Time) ([]pirsch.TimeOfDayVisitors, int) {
min := endDate.Add(-time.Hour * 24 * 7)
if startDate.Before(min) {
startDate = min
}
visitors, err := analyzer.TimeOfDay(&pirsch.Filter{From: startDate, To: endDate})
if err != nil {
logbuch.Error("Error reading visitor time of day statistics", logbuch.Fields{"err": err})
return nil, 0
}
maxVisitors := 0
for _, v := range visitors {
for _, s := range v.Stats {
if maxVisitors < s.Visitors {
maxVisitors = s.Visitors
}
}
}
return visitors, maxVisitors
}
func sumVisitors(stats []pirsch.Stats) int {
sum := 0
for _, s := range stats {
sum += s.Visitors
}
return sum
}
func getLabelsAndData(visitors []pirsch.Stats) (template.JS, template.JS, template.JS, template.JS) {
var labels strings.Builder
var dp strings.Builder
var sessions strings.Builder
var bounces strings.Builder
for _, point := range visitors {
labels.WriteString(fmt.Sprintf("'%s',", point.Day.Format(statisticsDateFormat)))
dp.WriteString(fmt.Sprintf("%d,", point.Visitors))
sessions.WriteString(fmt.Sprintf("%d,", point.Sessions))
bounces.WriteString(fmt.Sprintf("%d,", point.Bounces))
}
labelsStr := labels.String()
dataStr := dp.String()
sessionsStr := sessions.String()
bouncesStr := bounces.String()
return template.JS(labelsStr[:len(labelsStr)-1]),
template.JS(dataStr[:len(dataStr)-1]),
template.JS(sessionsStr[:len(sessionsStr)-1]),
template.JS(bouncesStr[:len(bouncesStr)-1])
}
func getLabelsAndDataHourly(visitors []pirsch.VisitorTimeStats) (template.JS, template.JS) {
var labels strings.Builder
var dp strings.Builder
for _, point := range visitors {
labels.WriteString(fmt.Sprintf("'%d',", point.Hour))
dp.WriteString(fmt.Sprintf("%d,", point.Visitors))
}
labelsStr := labels.String()
dataStr := dp.String()
return template.JS(labelsStr[:len(labelsStr)-1]), template.JS(dataStr[:len(dataStr)-1])
}
func today() time.Time {
now := time.Now()
return time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
}

90
tracking/tracking.go
View File

@@ -1,90 +0,0 @@
package tracking
import (
"context"
"database/sql"
"github.com/Kugelschieber/marvinblum/db"
"github.com/emvi/logbuch"
"github.com/pirsch-analytics/pirsch"
"os"
"path/filepath"
)
const (
geodbPath = "geodb"
)
var (
store pirsch.Store
analyzer *pirsch.Analyzer
)
func NewTracker() (*pirsch.Tracker, context.CancelFunc) {
logbuch.Info("Connecting to database...")
conn, err := sql.Open("postgres", db.GetConnectionString())
if err != nil {
logbuch.Fatal("Error connecting to database", logbuch.Fields{"err": err})
return nil, nil
}
if err := conn.Ping(); err != nil {
logbuch.Fatal("Error pinging database", logbuch.Fields{"err": err})
return nil, nil
}
store = pirsch.NewPostgresStore(conn, nil)
tracker := pirsch.NewTracker(store, os.Getenv("MB_TRACKING_SALT"), &pirsch.TrackerConfig{
ReferrerDomainBlacklist: []string{"marvinblum.de"}, // I don't care about traffic from my own website
ReferrerDomainBlacklistIncludesSubdomains: true,
Sessions: true,
})
analyzer = pirsch.NewAnalyzer(store, nil)
processor := pirsch.NewProcessor(store)
cancel := pirsch.RunAtMidnight(func() {
processTrackingData(processor)
updateGeoDB(tracker)
})
processTrackingData(processor)
updateGeoDB(tracker)
return tracker, cancel
}
func processTrackingData(processor *pirsch.Processor) {
logbuch.Info("Processing tracking data...")
defer func() {
if err := recover(); err != nil {
logbuch.Error("Error processing tracking data", logbuch.Fields{"err": err})
}
}()
if err := processor.Process(); err != nil {
logbuch.Error("Error processing tracking data", logbuch.Fields{"err": err})
} else {
logbuch.Info("Done processing tracking data")
}
}
func updateGeoDB(tracker *pirsch.Tracker) {
licenseKey := os.Getenv("MB_GEOLITE2_LICENSE_KEY")
if licenseKey == "" {
return
}
if err := pirsch.GetGeoLite2(geodbPath, licenseKey); err != nil {
logbuch.Error("Error loading GeoLite2", logbuch.Fields{"err": err})
return
}
geodb, err := pirsch.NewGeoDB(filepath.Join(geodbPath, pirsch.GeoLite2Filename))
if err != nil {
logbuch.Error("Error creating GeoDB", logbuch.Fields{"err": err})
return
}
tracker.SetGeoDB(geodb)
logbuch.Info("GeoDB updated")
}