Adds the new and freshly refactored source

This commit is contained in:
A. Svensson 2017-07-15 17:53:29 +02:00
parent 3e68474e3c
commit c6f73275b4
9 changed files with 1027 additions and 0 deletions

164
charts.go Normal file
View File

@ -0,0 +1,164 @@
package ss13hub
import (
"bytes"
"fmt"
"io"
"net/http"
"time"
chart "github.com/wcharczuk/go-chart"
)
type renderableChart interface {
Render(chart.RendererProvider, io.Writer) error
}
func (a *App) renderChart(w http.ResponseWriter, c renderableChart) error {
buf := &bytes.Buffer{}
err := c.Render(chart.PNG, buf)
if err != nil {
//a.Log("Error while rendering chart: %s", err)
return HttpError{
Status: http.StatusInternalServerError,
Err: fmt.Errorf("error while rendering chart"),
}
}
w.Header().Add("Content-Type", "image/png")
_, err = io.Copy(w, buf)
if err != nil {
a.Log("Error while sending chart: %s", err)
return HttpError{
Status: http.StatusInternalServerError,
Err: fmt.Errorf("error while sending chart"),
}
}
return nil
}
func makeHistoryChart(title string, points []ServerPoint) chart.Chart {
var xVals []time.Time
var yVals []float64
for _, p := range points {
xVals = append(xVals, p.Time)
yVals = append(yVals, float64(p.Players))
}
series := chart.TimeSeries{
Name: "Players",
XValues: xVals,
YValues: yVals,
}
li := &chart.LinearRegressionSeries{
Name: "Linear regression",
InnerSeries: series,
}
sma := &chart.SMASeries{
Name: "Simple moving avg.",
InnerSeries: series,
}
c := chart.Chart{
Title: title,
TitleStyle: chart.Style{
Show: true,
},
Background: chart.Style{
Padding: chart.Box{
Left: 120,
},
},
XAxis: chart.XAxis{
Style: chart.Style{
Show: true,
},
},
YAxis: chart.YAxis{
Style: chart.Style{
Show: true,
},
},
Series: []chart.Series{
series,
li,
sma,
},
}
// Add a legend
c.Elements = []chart.Renderable{
chart.LegendLeft(&c),
}
return c
}
// NOTE: The chart won't be renderable unless we've got at least two days of history
func makeDayAverageChart(title string, points []ServerPoint) chart.BarChart {
days := make(map[time.Weekday][]int)
for _, p := range points {
day := p.Time.Weekday()
days[day] = append(days[day], p.Players)
}
avgDays := make(map[time.Weekday]float64)
for day, vals := range days {
sum := 0
for _, v := range vals {
sum += v
}
avg := sum / len(vals)
avgDays[day] = float64(avg)
}
prettyName := func(d time.Weekday) string {
return fmt.Sprintf("%s (%.0f)", d, avgDays[d])
}
return chart.BarChart{
Title: title,
TitleStyle: chart.Style{
Show: true,
},
BarWidth: 60,
XAxis: chart.Style{
Show: true,
},
YAxis: chart.YAxis{
Style: chart.Style{
Show: true,
},
},
Bars: []chart.Value{
chart.Value{
Label: prettyName(time.Monday),
Value: avgDays[time.Monday],
},
chart.Value{
Label: prettyName(time.Tuesday),
Value: avgDays[time.Tuesday],
},
chart.Value{
Label: prettyName(time.Wednesday),
Value: avgDays[time.Wednesday],
},
chart.Value{
Label: prettyName(time.Thursday),
Value: avgDays[time.Thursday],
},
chart.Value{
Label: prettyName(time.Friday),
Value: avgDays[time.Friday],
},
chart.Value{
Label: prettyName(time.Saturday),
Value: avgDays[time.Saturday],
},
chart.Value{
Label: prettyName(time.Sunday),
Value: avgDays[time.Sunday],
},
},
}
}

29
cmd/server/server.go Normal file
View File

@ -0,0 +1,29 @@
package main
import (
"time"
"github.com/lmas/ss13hub"
)
func main() {
// TODO: load config from a toml file
conf := ss13hub.Conf{
WebAddr: ":8000",
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
ScrapeTimeout: 10 * time.Minute,
Storage: &ss13hub.StorageSqlite{
Path: "./tmp/servers.db",
},
}
app, err := ss13hub.New(conf)
if err != nil {
panic(err)
}
err = app.Run()
if err != nil {
panic(err)
}
}

49
handler_helpers.go Normal file
View File

@ -0,0 +1,49 @@
package ss13hub
import (
"fmt"
"log"
"net/http"
"time"
"github.com/gorilla/mux"
)
type HttpError struct {
Status int
Err error
}
func (s HttpError) Error() string {
return fmt.Sprintf("%d %s", s.Status, s.Err.Error())
}
type handlerVars map[string]string
type handler func(http.ResponseWriter, *http.Request, handlerVars) error
func (h handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
start := time.Now()
err := h(rw, req, mux.Vars(req))
dur := time.Since(start)
if err != nil {
switch e := err.(type) {
case HttpError:
http.Error(rw, e.Error(), e.Status)
default:
http.Error(rw, http.StatusText(http.StatusInternalServerError),
http.StatusInternalServerError)
}
}
log.Printf("%s %s \t%s \terr: %v\n",
//req.RemoteAddr,
req.Method,
req.URL.String(),
//req.UserAgent(),
dur,
//resp.Status,
err,
)
}

104
handlers.go Normal file
View File

@ -0,0 +1,104 @@
package ss13hub
import (
"fmt"
"net/http"
)
func (a *App) pageIndex(w http.ResponseWriter, r *http.Request, vars handlerVars) error {
servers, err := a.store.GetServers()
if err != nil {
return err
}
// Remove the internal entry used to count total players
index := -1
for i, s := range servers {
if s.Title == internalServerTitle {
index = i
break
}
}
var hub ServerEntry
if index > -1 {
hub = servers[index]
servers = append(servers[:index], servers[index+1:]...)
}
return a.templates["index"].Execute(w, map[string]interface{}{
"Servers": servers,
"Hub": hub,
"TotalServers": len(servers),
})
}
func (a *App) pageServer(w http.ResponseWriter, r *http.Request, vars handlerVars) error {
id := vars["id"]
server, err := a.store.GetServer(id)
if err != nil {
// TODO: handle and log the error properly
return HttpError{
Status: 404,
Err: fmt.Errorf("server not found"),
}
}
if server.Title == internalServerTitle {
server.Title = "Global stats"
}
return a.templates["server"].Execute(w, map[string]interface{}{
"Server": server,
})
}
func (a *App) pageDailyChart(w http.ResponseWriter, r *http.Request, vars handlerVars) error {
id := vars["id"]
points, err := a.store.GetSingleServerHistory(id, 1)
if err != nil {
return err
}
if len(points) < 1 {
return HttpError{
Status: 404,
Err: fmt.Errorf("server not found"),
}
}
c := makeHistoryChart("Daily history", points)
return a.renderChart(w, c)
}
func (a *App) pageWeeklyChart(w http.ResponseWriter, r *http.Request, vars handlerVars) error {
id := vars["id"]
points, err := a.store.GetSingleServerHistory(id, 7)
if err != nil {
return err
}
if len(points) < 1 {
return HttpError{
Status: 404,
Err: fmt.Errorf("server not found"),
}
}
c := makeHistoryChart("Weekly history", points)
return a.renderChart(w, c)
}
func (a *App) pageAverageChart(w http.ResponseWriter, r *http.Request, vars handlerVars) error {
id := vars["id"]
points, err := a.store.GetSingleServerHistory(id, 7)
if err != nil {
return err
}
if len(points) < 1 {
return HttpError{
Status: 404,
Err: fmt.Errorf("server not found"),
}
}
c := makeDayAverageChart("Average per day", points)
return a.renderChart(w, c)
}

184
main.go Normal file
View File

@ -0,0 +1,184 @@
package ss13hub
import (
"html/template"
"log"
"net/http"
"strings"
"time"
"github.com/gorilla/mux"
)
// Used internally for logging a global # of players
const internalServerTitle string = "_ss13.se"
type Conf struct {
// Web stuff
WebAddr string
ReadTimeout time.Duration
WriteTimeout time.Duration
// Scraper stuff
ScrapeTimeout time.Duration
// Misc.
Storage Storage
}
type App struct {
conf Conf
web *http.Server
store Storage
templates map[string]*template.Template
}
func New(c Conf) (*App, error) {
templates, err := loadTemplates()
if err != nil {
return nil, err
}
w := &http.Server{
Addr: c.WebAddr,
ReadTimeout: c.ReadTimeout,
WriteTimeout: c.WriteTimeout,
}
a := &App{
conf: c,
web: w,
store: c.Storage,
templates: templates,
}
r := mux.NewRouter()
r.Handle("/", handler(a.pageIndex))
r.Handle("/server/{id}", handler(a.pageServer))
r.Handle("/server/{id}/daily", handler(a.pageDailyChart))
r.Handle("/server/{id}/weekly", handler(a.pageWeeklyChart))
r.Handle("/server/{id}/average", handler(a.pageAverageChart))
a.web.Handler = r
return a, nil
}
func (a *App) Log(msg string, args ...interface{}) {
log.Printf(msg+"\n", args...)
}
func (a *App) Run() error {
a.Log("Opening storage...")
err := a.store.Open()
if err != nil {
return err
}
a.Log("Running updater")
go a.runUpdater()
a.Log("Running server on %s", a.conf.WebAddr)
return a.web.ListenAndServe()
}
func (a *App) runUpdater() {
webClient := &http.Client{
Timeout: 60 * time.Second,
}
for {
now := time.Now()
servers, err := scrapeByond(webClient, now)
dur := time.Since(now)
a.Log("Scrape done in %s, errors: %v", dur, err)
if err == nil {
servers = append(servers, a.makeHubEntry(now, servers))
if err := a.store.SaveServers(servers); err != nil {
a.Log("Error saving servers: %s", err)
}
if err := a.updateHistory(now, servers); err != nil {
a.Log("Error saving server history: %s", err)
}
if err := a.updateOldServers(now, servers); err != nil {
a.Log("Error updating old servers: %s", err)
}
}
time.Sleep(a.conf.ScrapeTimeout)
}
}
func (a *App) updateHistory(t time.Time, servers []ServerEntry) error {
var history []ServerPoint
for _, s := range servers {
history = append(history, ServerPoint{
Time: t,
ServerID: s.ID,
Players: s.Players,
})
}
return a.store.SaveServerHistory(history)
}
func (a *App) updateOldServers(t time.Time, servers []ServerEntry) error {
var old []ServerEntry
for _, s := range servers {
if !s.Time.Equal(t) {
s.Players = 0
old = append(old, s)
}
}
var remove []ServerEntry
for index, s := range old {
delta := t.Sub(s.Time)
if delta.Hours() > 24*1 { // TODO: CHANGE TO A WEEK AFTER TESTING
remove = append(remove, s)
old = append(old[:index], old[index+1:]...)
}
}
if len(remove) > 0 {
a.Log("Removing servers: %s", serverNameList(remove)) // TODO: remove after testing?
if err := a.store.RemoveServers(remove); err != nil {
return err
}
}
if len(old) > 0 {
a.Log("Old servers: %s", serverNameList(old)) // TODO: remove after testing
if err := a.updateHistory(t, old); err != nil {
return err
}
}
return nil
}
// TODO: can probably remove this func after we're done testing
func serverNameList(servers []ServerEntry) string {
var names []string
for _, s := range servers {
names = append(names, s.Title)
}
return strings.Join(names, ", ")
}
func (a *App) makeHubEntry(t time.Time, servers []ServerEntry) ServerEntry {
var totalPlayers int
for _, s := range servers {
totalPlayers += s.Players
}
return ServerEntry{
ID: makeID(internalServerTitle),
Title: internalServerTitle,
SiteURL: "",
GameURL: "",
Time: t,
Players: totalPlayers,
}
}

155
scraper.go Normal file
View File

@ -0,0 +1,155 @@
package ss13hub
import (
"crypto/sha256"
"fmt"
"io"
"log"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"golang.org/x/text/encoding/charmap"
)
const (
byondURL string = "http://www.byond.com/games/Exadv1/SpaceStation13"
//byondURL string = "./tmp/dump.html" // For testing
userAgent string = "ss13hub/2.0pre"
)
var (
rePlayers = regexp.MustCompile(`Logged in: (\d+) player`)
//rePlayers = regexp.MustCompile(`<br/>\s*<br/>\s*Logged in: (\d+) player.*<a href`)
)
func scrapeByond(webClient *http.Client, now time.Time) ([]ServerEntry, error) {
var body io.ReadCloser
if byondURL == "./tmp/dump.html" {
r, err := os.Open(byondURL)
if err != nil {
return nil, err
}
body = r
} else {
r, err := openPage(webClient, byondURL)
if err != nil {
return nil, err
}
body = r
}
defer body.Close()
servers, err := parseByondPage(now, body)
if err != nil {
return nil, err
}
return servers, nil
}
func openPage(webClient *http.Client, url string) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Add("User-Agent", userAgent)
resp, err := webClient.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("bad http.Response.Status: %s", resp.Status)
}
return resp.Body, nil
}
func parseByondPage(now time.Time, body io.Reader) ([]ServerEntry, error) {
// Yep, Byond serves it's pages with Windows-1252 encoding...
r := charmap.Windows1252.NewDecoder().Reader(body)
doc, err := goquery.NewDocumentFromReader(r)
if err != nil {
return nil, err
}
var servers []ServerEntry
doc.Find(".live_game_entry").Each(func(i int, s *goquery.Selection) {
entry, err := parseEntry(s.Find(".live_game_status"))
if err != nil {
log.Println("Error parsing entry:", err)
return
}
if entry.IsZero() {
return
}
// Make sure we only try to add only one instance of a server.
// And since byond orders the most popular servers up on top,
// we get a small protection from bad guys who's trying to
// influent the history of a server.
for _, s := range servers {
if s.ID == entry.ID {
return
}
}
entry.Time = now
servers = append(servers, entry)
})
return servers, nil
}
func parseEntry(s *goquery.Selection) (ServerEntry, error) {
// Try find a player count (really tricky since it's not in a valid
// html tag by itself)
tmp := strings.TrimSpace(strings.Replace(s.Text(), "\n", "", -1))
r := rePlayers.FindStringSubmatch(tmp)
// 2 == because the regexp returns wholestring + matched part
// If it's less than 2 we couldn't find a match and if it's greater
// than 2 there's multiple matches, which is fishy...
players := 0
if len(r) == 2 {
p, err := strconv.Atoi(r[1])
if err != nil {
return ServerEntry{}, err
}
players = p
}
// Grab and sanitize the server's name
title := s.Find("b").First().Text()
title = strings.Replace(strings.TrimSpace(title), "\n", "", -1)
if len(title) < 1 {
// the byond page sometimes has server entries that's basiclly
// blank, no server name or player count (just some byond url)
return ServerEntry{}, nil
}
id := makeID(title)
gameURL := s.Find("span.smaller").Find("nobr").Text()
siteURL := s.Find("a").First().AttrOr("href", "")
if siteURL == "http://" {
siteURL = ""
}
return ServerEntry{
ID: id,
Title: title,
SiteURL: siteURL,
GameURL: gameURL,
Players: players,
}, nil
}
func makeID(title string) string {
return fmt.Sprintf("%x", sha256.Sum256([]byte(title)))
}

59
storage.go Normal file
View File

@ -0,0 +1,59 @@
package ss13hub
import (
"html/template"
"net/url"
"time"
)
type ServerEntry struct {
ID string `db:"id"`
Title string `db:"title"`
SiteURL string `db:"site_url"`
GameURL string `db:"game_url"`
Time time.Time `db:"time"`
Players int `db:"players"`
}
func (e ServerEntry) IsZero() bool {
return e.ID == ""
}
func (e ServerEntry) LastUpdated() string {
return e.Time.Format("2006-01-02 15:04 MST")
}
func (e ServerEntry) ByondURL() template.URL {
u, err := url.Parse(e.GameURL)
if err != nil {
return ""
}
if u.Scheme != "byond" {
return ""
}
return template.URL(u.String())
}
type ServerPoint struct {
Time time.Time `db:"time"`
ServerID string `db:"server_id"`
Players int `db:"players"`
}
func (p ServerPoint) IsZero() bool {
return p.ServerID == "" && p.Time.IsZero()
}
type Storage interface {
Open() error
SaveServers([]ServerEntry) error
GetServer(string) (ServerEntry, error)
GetServers() ([]ServerEntry, error)
RemoveServers([]ServerEntry) error
SaveServerHistory([]ServerPoint) error
GetServerHistory(int) ([]ServerPoint, error)
GetSingleServerHistory(string, int) ([]ServerPoint, error)
}

153
storage_sqlite.go Normal file
View File

@ -0,0 +1,153 @@
package ss13hub
import (
"time"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
)
const sqliteScheme string = `
CREATE TABLE IF NOT EXISTS server_entry(
id TEXT UNIQUE,
title STRING,
site_url STRING,
game_url STRING,
time DATETIME,
players INTEGER
);
CREATE INDEX IF NOT EXISTS idx_server_entry ON server_entry(time, players, title);
CREATE TABLE IF NOT EXISTS server_history (
id INTEGER PRIMARY KEY,
time DATETIME,
server_id TEXT,
players INTEGER
);
CREATE INDEX IF NOT EXISTS idx_server_history ON server_history(time, server_id);
`
type StorageSqlite struct {
*sqlx.DB
Path string
}
func (store *StorageSqlite) Open() error {
db, err := sqlx.Connect("sqlite3", store.Path)
if err != nil {
return err
}
_, err = db.Exec(sqliteScheme)
if err != nil {
return err
}
store.DB = db
return nil
}
func (store *StorageSqlite) SaveServers(servers []ServerEntry) error {
tx, err := store.Begin()
if err != nil {
return err
}
q := `INSERT OR REPLACE INTO server_entry (id, title, site_url, game_url, time, players) VALUES(?, ?, ?, ?, ?, ?);`
for _, s := range servers {
_, err := tx.Exec(q, s.ID, s.Title, s.SiteURL, s.GameURL, s.Time, s.Players)
if err != nil {
tx.Rollback() // TODO: handle error?
return err
}
}
return tx.Commit()
}
func (store *StorageSqlite) GetServer(id string) (ServerEntry, error) {
var server ServerEntry
q := `SELECT * FROM server_entry WHERE id = ? LIMIT 1;`
err := store.Get(&server, q, id)
if err != nil {
return ServerEntry{}, err
}
return server, nil
}
func (store *StorageSqlite) GetServers() ([]ServerEntry, error) {
var servers []ServerEntry
q := `SELECT * FROM server_entry ORDER BY players DESC, id ASC;`
err := store.Select(&servers, q)
if err != nil {
return nil, err
}
return servers, nil
}
func (store *StorageSqlite) RemoveServers(servers []ServerEntry) error {
tx, err := store.Begin()
if err != nil {
return err
}
qHistory := `DELETE FROM server_history WHERE server_id = ?;`
qEntry := `DELETE FROM server_entry WHERE id = ?;`
for _, s := range servers {
_, err := tx.Exec(qHistory, s.ID)
if err != nil {
tx.Rollback() // TODO: handle error?
return err
}
_, err = tx.Exec(qEntry, s.ID)
if err != nil {
tx.Rollback() // TODO: handle error?
return err
}
}
return tx.Commit()
}
func (store *StorageSqlite) SaveServerHistory(points []ServerPoint) error {
tx, err := store.Begin()
if err != nil {
return err
}
q := `INSERT INTO server_history (time, server_id, players) VALUES(?, ?, ?);`
for _, p := range points {
_, err := tx.Exec(q, p.Time, p.ServerID, p.Players)
if err != nil {
tx.Rollback() // TODO: handle error?
return err
}
}
return tx.Commit()
}
func (store *StorageSqlite) GetServerHistory(days int) ([]ServerPoint, error) {
var points []ServerPoint
delta := time.Now().AddDate(0, 0, -days)
q := `SELECT time,server_id,players FROM server_history WHERE time > ? ORDER BY time DESC, server_id ASC;`
err := store.Select(&points, q, delta)
if err != nil {
return nil, err
}
return points, nil
}
func (store *StorageSqlite) GetSingleServerHistory(id string, days int) ([]ServerPoint, error) {
var points []ServerPoint
delta := time.Now().AddDate(0, 0, -days)
q := `SELECT time,server_id,players FROM server_history WHERE server_id = ? AND time > ? ORDER BY time DESC;`
err := store.Select(&points, q, id, delta)
if err != nil {
return nil, err
}
return points, nil
}

130
templates.go Normal file
View File

@ -0,0 +1,130 @@
package ss13hub
import (
"html/template"
)
func loadTemplates() (map[string]*template.Template, error) {
tmpls := make(map[string]*template.Template)
for name, src := range tmplList {
t, err := parseTemplate(tmplBase, src)
if err != nil {
return nil, err
}
tmpls[name] = t
}
return tmpls, nil
}
func parseTemplate(src ...string) (*template.Template, error) {
var err error
t := template.New("*")
for _, s := range src {
t, err = t.Parse(s)
if err != nil {
return nil, err
}
}
return t, nil
}
const tmplBase string = `<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>
{{block "title" .}}NO TITLE{{end}} | ss13.se
</title>
<style type="text/css">
img {
display: block;
margin: auto;
}
footer {
text-align: center;
}
.left {
float: left;
}
.right {
float: right;
}
</style>
</head>
<body>
<header>
<h2><a href="/">SS13.se</a></h2>
</header>
<section id="body">
{{block "body" .}}NO BODY{{end}}
</section>
<footer>
<p>
<span class="left">
Source code at
<a href="https://github.com/lmas/ss13_se">Github</a>
</span>
{{/* TODO: not sure about the copyright stuff when fetching ext. data */}}
Copyright © 2017 A. Svensson
<span class="right">
Using raw data from
<a href="http://www.byond.com/games/exadv1/spacestation13">Byond</a>
</span>
</p>
</footer>
</body>
</html>`
var tmplList = map[string]string{
"index": `{{define "title"}}Index{{end}}
{{define "body"}}
<p>Last updated: {{.Hub.LastUpdated}}</p>
<p>Current # of servers: {{.TotalServers}}</p>
<p>Current # of players: {{.Hub.Players}}</p>
<a href="/server/{{.Hub.ID}}">Global stats</a><br />
<br />
<table>
<thead><tr>
<td>Players</td>
<td>Server</td>
</tr></thead>
<tbody>
{{range .Servers}}
<tr>
<td>{{.Players}}</td>
<td><a href="/server/{{.ID}}">{{.Title}}</a></td>
</tr>
{{else}}
<tr><td>0</td><td>Sorry, no servers yet!</td></tr>
{{end}}
</tbody>
</table>
{{end}}
`,
"server": `{{define "title"}}{{.Server.Title}}{{end}}
{{define "body"}}
<h1>{{.Server.Title}}</h1>
<p>Last updated: {{.Server.LastUpdated}}</p>
<p>Current players: {{.Server.Players}}</p>
{{if .Server.SiteURL}}
<a href="{{.Server.SiteURL}}">Web site</a><br />
{{end}}
{{if .Server.ByondURL}}
<a href="{{.Server.ByondURL}}">Join game</a><br />
{{end}}
<br />
<img src="/server/{{.Server.ID}}/daily" alt="Daily history"><br />
<img src="/server/{{.Server.ID}}/weekly" alt="Weekly history"><br />
<img src="/server/{{.Server.ID}}/average" alt="Average per day"><br />
{{end}}
`,
}