From 949a78da3350bf061c29248b38e0e23f234dab06 Mon Sep 17 00:00:00 2001 From: "A. Svensson" Date: Wed, 14 Sep 2016 17:07:00 +0200 Subject: [PATCH] Dumped gorm and refactored database... --- src/db.go | 195 +++++++++++++++++++++++++++++++------------------ src/models.go | 37 ++++------ src/updater.go | 42 +++++++++-- src/web.go | 32 ++++++-- 4 files changed, 199 insertions(+), 107 deletions(-) diff --git a/src/db.go b/src/db.go index 5f2998b..ec06546 100644 --- a/src/db.go +++ b/src/db.go @@ -1,99 +1,150 @@ package ss13 import ( - "fmt" - "log" "time" - "github.com/jinzhu/gorm" + "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" ) -type DB struct { - *gorm.DB +const SCHEMA string = ` +CREATE TABLE IF NOT EXISTS servers ( + id INTEGER PRIMARY KEY, + last_updated DATETIME, + title TEXT UNIQUE, + game_url TEXT, + site_url TEXT, + players_current INTEGER, + players_avg INTEGER, + players_min INTEGER, + players_max INTEGER, + players_mon INTEGER, + players_tue INTEGER, + players_wed INTEGER, + players_thu INTEGER, + players_fri INTEGER, + players_sat INTEGER, + players_sun INTEGER +); + +CREATE TABLE IF NOT EXISTS server_populations ( + id INTEGER PRIMARY KEY, + timestamp DATETIME, + players INTEGER, + server_id INTEGER REFERENCES servers(id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS server_pop_index ON server_populations(server_id); +` + +type Database struct { + *sqlx.DB } -func OpenSqliteDB(args ...interface{}) (*DB, error) { - db, e := gorm.Open("sqlite3", args...) +type TX struct { + *sqlx.Tx +} + +func OpenDatabase(path string) (*Database, error) { + db, e := sqlx.Connect("sqlite3", path) if e != nil { return nil, e } - db.AutoMigrate(&Server{}) - db.AutoMigrate(&ServerPopulation{}) - return &DB{db}, nil + _, e = db.Exec(SCHEMA) + if e != nil { + return nil, e + } + return &Database{db}, nil } -func (db *DB) NewTransaction() *DB { - return &DB{db.Begin()} -} - -func (db *DB) AllServers() []*Server { +func (db *Database) AllServers() ([]*Server, error) { var tmp []*Server - db.Order("last_updated desc, players_current desc, players_avg desc, title").Find(&tmp) - return tmp + e := db.Select(&tmp, `SELECT * FROM servers ORDER BY + last_updated DESC, + players_current DESC, + players_avg DESC, + title;`) + if e != nil { + return nil, e + } + return tmp, nil } -func (db *DB) GetServer(id int) (*Server, error) { +func (db *Database) GetServer(id int) (*Server, error) { var tmp Server - if db.First(&tmp, id).RecordNotFound() { - return nil, fmt.Errorf("Server not found") + e := db.Get(&tmp, "SELECT * FROM servers WHERE id = ? LIMIT 1;", id) + if e != nil { + return nil, e } return &tmp, nil } -func (db *DB) GetOldServers(ts time.Time) []*Server { +func (db *Database) GetOldServers(ts time.Time) ([]*Server, error) { var tmp []*Server - db.Where("last_updated < ?", ts).Find(&tmp) - return tmp + e := db.Select(&tmp, "SELECT * FROM servers WHERE last_updated < ?;", ts) + if e != nil { + return nil, e + } + return tmp, nil } -func (db *DB) RemoveOldServers(ts time.Time) { - db.Where("last_updated < datetime(?, '-7 days')", ts).Delete(Server{}) +func (db *Database) RemoveOldServers(ts time.Time) error { + _, e := db.Exec("DELETE FROM servers WHERE last_updated < datetime(?, '-7 days');", ts) + return e } -func (db *DB) GetServerPopulation(id int, d time.Duration) []*ServerPopulation { +func (db *Database) GetServerPopulation(id int, d time.Duration) ([]*ServerPopulation, error) { var tmp []*ServerPopulation t := time.Now().Add(-d) - db.Order("timestamp desc, server_id").Where("server_id = ? and timestamp > ?", id, t).Find(&tmp) - return tmp -} - -func (db *DB) InsertOrSelect(s *RawServerData) int { - var tmp Server - newserver := Server{ - LastUpdated: s.Timestamp, - Title: s.Title, - GameUrl: s.Game_url, - SiteUrl: s.Site_url, - PlayersCurrent: s.Players, - PlayersAvg: s.Players, - PlayersMin: s.Players, - PlayersMax: s.Players, + e := db.Select(&tmp, `SELECT * FROM server_populations WHERE + server_id = ? AND timestamp > ? + ORDER BY timestamp DESC, server_id;`, id, t) + if e != nil { + return nil, e } - db.Where("title = ?", s.Title).Attrs(newserver).FirstOrCreate(&tmp) - return tmp.ID + return tmp, nil } -func (db *DB) AddServerPopulation(id int, s *RawServerData, now time.Time) { +func (db *TX) InsertOrSelect(s *RawServerData) (int, error) { var tmp Server - db.Where("id = ?", id).First(&tmp) - pop := ServerPopulation{ - Timestamp: now, - Players: s.Players, - Server: tmp, + e := db.Get(&tmp, "SELECT * FROM servers WHERE title = ? LIMIT 1;", s.Title) + if e == nil { + return tmp.ID, nil } - db.Create(&pop) + + r, e := db.Exec(`INSERT INTO servers ( + last_updated, title, game_url, site_url, players_current, + players_avg, players_min, players_max + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?);`, + s.Timestamp, + s.Title, + s.Game_url, + s.Site_url, + s.Players, s.Players, s.Players, s.Players) + if e != nil { + return -1, e + } + id, e := r.LastInsertId() + if e != nil { + return -1, e + } + return int(id), nil } -func (db *DB) UpdateServerStats(id int, s *RawServerData, now time.Time) { - var tmp Server +func (db *TX) AddServerPopulation(id int, s *RawServerData, now time.Time) error { + _, e := db.Exec(`INSERT INTO server_populations ( + timestamp, players, server_id + ) VALUES (?, ?, ?);`, now, s.Players, id) + return e +} +func (db *TX) UpdateServerStats(id int, s *RawServerData, now time.Time) error { period := now.Add(-time.Duration(30*24) * time.Hour) - db.Where("id = ?", id).First(&tmp) - rows, err := db.Table("server_populations").Where("server_id = ? AND timestamp > ?", tmp.ID, period).Select("timestamp, players").Order("timestamp desc").Rows() - if err != nil { - log.Panic(err) - return + rows, e := db.Queryx(`SELECT timestamp, players FROM server_populations + WHERE server_id = ? AND timestamp > ? + ORDER BY timestamp DESC`, id, period) + if e != nil { + return e } defer rows.Close() @@ -119,20 +170,18 @@ func (db *DB) UpdateServerStats(id int, s *RawServerData, now time.Time) { day_counts[day]++ } - tmp.LastUpdated = s.Timestamp - tmp.Title = s.Title - tmp.GameUrl = s.Game_url - tmp.SiteUrl = s.Site_url - tmp.PlayersCurrent = s.Players - tmp.PlayersAvg = sum / count - tmp.PlayersMin = min - tmp.PlayersMax = max - tmp.PlayersMon = day_sums[1] / day_counts[1] - tmp.PlayersTue = day_sums[2] / day_counts[2] - tmp.PlayersWed = day_sums[3] / day_counts[3] - tmp.PlayersThu = day_sums[4] / day_counts[4] - tmp.PlayersFri = day_sums[5] / day_counts[5] - tmp.PlayersSat = day_sums[6] / day_counts[6] - tmp.PlayersSun = day_sums[0] / day_counts[0] - db.Save(&tmp) + _, e = db.Exec(`UPDATE servers SET last_updated = ?, players_current = ?, + players_avg = ?, players_min = ?, players_max = ?, players_mon = ?, + players_tue = ?, players_wed = ?, players_thu = ?, players_fri = ?, + players_sat = ?, players_sun = ? + WHERE id = ?;`, s.Timestamp, s.Players, sum/count, min, max, + day_sums[1]/day_counts[1], + day_sums[2]/day_counts[2], + day_sums[3]/day_counts[3], + day_sums[4]/day_counts[4], + day_sums[5]/day_counts[5], + day_sums[6]/day_counts[6], + day_sums[0]/day_counts[0], + id) + return e } diff --git a/src/models.go b/src/models.go index d64f21b..3ec3483 100644 --- a/src/models.go +++ b/src/models.go @@ -9,23 +9,23 @@ import ( type Server struct { ID int - LastUpdated time.Time - Title string `sql:"type:varchar(64);unique"` - GameUrl string - SiteUrl string + LastUpdated time.Time `db:"last_updated"` + Title string + GameUrl string `db:"game_url"` + SiteUrl string `db:"site_url"` - PlayersCurrent int - PlayersAvg int - PlayersMin int - PlayersMax int + PlayersCurrent int `db:"players_current"` + PlayersAvg int `db:"players_avg"` + PlayersMin int `db:"players_min"` + PlayersMax int `db:"players_max"` - PlayersMon int - PlayersTue int - PlayersWed int - PlayersThu int - PlayersFri int - PlayersSat int - PlayersSun int + PlayersMon int `db:"players_mon"` + PlayersTue int `db:"players_tue"` + PlayersWed int `db:"players_wed"` + PlayersThu int `db:"players_thu"` + PlayersFri int `db:"players_fri"` + PlayersSat int `db:"players_sat"` + PlayersSun int `db:"players_sun"` } // Check if a server's last_updated time since now is greater or equal to X hours. @@ -63,10 +63,5 @@ type ServerPopulation struct { ID int Timestamp time.Time Players int - ServerID int `sql:"index;type:integer REFERENCES servers(id) ON DELETE CASCADE ON UPDATE CASCADE"` - Server Server + ServerID int `db:"server_id"` } - -// See https://github.com/jinzhu/gorm/issues/635 for why we have to manually add -// in a raw REFERENCES statement here. -// Hint: Foreign key creation is bugged when using gorm with sqlite. diff --git a/src/updater.go b/src/updater.go index 51b6e3a..ed26e15 100644 --- a/src/updater.go +++ b/src/updater.go @@ -51,24 +51,50 @@ func (i *Instance) UpdateServers() { addServer(s) } - tx := i.db.NewTransaction() + // TODO: move this block into db.go + //tx, e := i.db.NewTransaction() + t, e := i.db.Beginx() + if e != nil { + panic(e) // TODO + } + tx := &TX{t} + for _, s := range servers { fmt.Println("Updating:", s.Title) // get server's db id (or create) - id := tx.InsertOrSelect(s) + id, e := tx.InsertOrSelect(s) + if e != nil { + continue + } // create new player history point - tx.AddServerPopulation(id, s, now) + e = tx.AddServerPopulation(id, s, now) + if e != nil { + continue + } // update server (urls and player stats) - tx.UpdateServerStats(id, s, now) + e = tx.UpdateServerStats(id, s, now) + if e != nil { + continue + } + } + e = tx.Commit() + if e != nil { + panic(e) // TODO + } + + e = i.db.RemoveOldServers(Now()) + if e != nil { + panic(e) // TODO } - tx.RemoveOldServers(Now()) - tx.Commit() } func (i *Instance) getOldServers() []*RawServerData { - // TODO: there's some bug that makes this func return all servers? var tmp []*RawServerData - for _, old := range i.db.GetOldServers(Now()) { + servers, e := i.db.GetOldServers(Now()) + if e != nil { + panic(e) // TODO + } + for _, old := range servers { s := RawServerData{ Title: old.Title, Game_url: old.GameUrl, diff --git a/src/web.go b/src/web.go index 4ab3da0..1736f12 100644 --- a/src/web.go +++ b/src/web.go @@ -18,7 +18,7 @@ type Instance struct { Config *Config Debug bool - db *DB + db *Database router *mux.Router tmpls *template.Template } @@ -43,7 +43,7 @@ func New(debug bool, path string) (*Instance, error) { return nil, e } - db, e := OpenSqliteDB(c.DatabasePath) + db, e := OpenDatabase(c.DatabasePath) if e != nil { return nil, e } @@ -89,6 +89,11 @@ func (i *Instance) page_404(w http.ResponseWriter, r *http.Request) { i.tmpls.ExecuteTemplate(w, "page_404.html", nil) } +func (i *Instance) page_500(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + //i.tmpls.ExecuteTemplate(w, "page_500.html", nil) +} + func (i *Instance) page_static(w http.ResponseWriter, r *http.Request) { p := path.Join("static", mux.Vars(r)["file"]) b, e := Asset(p) @@ -106,7 +111,12 @@ func (i *Instance) page_static(w http.ResponseWriter, r *http.Request) { } func (i *Instance) page_index(w http.ResponseWriter, r *http.Request) { - servers := i.db.AllServers() + servers, e := i.db.AllServers() + if e != nil { + Log("Error for AllServers(): %s", e) + i.page_500(w, r) + return + } i.tmpls.ExecuteTemplate(w, "page_index.html", D{ "pagetitle": "Index", "servers": servers, @@ -142,11 +152,23 @@ func (i *Instance) page_server(w http.ResponseWriter, r *http.Request) { weekday{"Saturday", s.PlayersSat}, weekday{"Sunday", s.PlayersSun}, } + weekly, e := i.db.GetServerPopulation(int(id), time.Duration(7*24+12)*time.Hour) + if e != nil { + Log("Error for GetServerPopulation(): %s", e) + i.page_500(w, r) + return + } + monthly, e := i.db.GetServerPopulation(int(id), time.Duration(31*24)*time.Hour) + if e != nil { + Log("Error for GetServerPopulation(): %s", e) + i.page_500(w, r) + return + } i.tmpls.ExecuteTemplate(w, "page_server.html", D{ "pagetitle": s.Title, "server": s, - "weekhistory": i.db.GetServerPopulation(int(id), time.Duration(7*24+12)*time.Hour), - "monthhistory": i.db.GetServerPopulation(int(id), time.Duration(31*24)*time.Hour), + "weekhistory": weekly, + "monthhistory": monthly, "weekdayavg": weekdayavg, }) }