diff --git a/README.md b/README.md index 1e2075c..074d46d 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,6 @@ Todo - Update static files to newer versions. -- Use unicode for server names in the templates. - - Fix and clean up the tooltips in the server details template. - Use the same format for the verbose timestamps. diff --git a/src/scraper.go b/src/scraper.go index 6227e6b..8f7ba5d 100644 --- a/src/scraper.go +++ b/src/scraper.go @@ -2,10 +2,15 @@ package ss13 import ( "fmt" + "io" + "net/http" "os" "regexp" "strconv" "strings" + "time" + + "golang.org/x/text/encoding/charmap" "github.com/PuerkitoBio/goquery" ) @@ -20,21 +25,26 @@ func ScrapePage() []*RawServerData { } func download_data() *goquery.Document { - var ( - doc *goquery.Document - err error - ) + var r io.Reader if IsDebugging() { fmt.Println("Scraper data source: ./dump.html") f, err := os.Open("./tmp/dump.html") check_error(err) defer f.Close() - doc, err = goquery.NewDocumentFromReader(f) - check_error(err) + r = charmap.Windows1252.NewDecoder().Reader(f) } else { - doc, err = goquery.NewDocument("http://www.byond.com/games/exadv1/spacestation13") - check_error(err) + client := &http.Client{ + Timeout: time.Duration(1) * time.Minute, + } + resp, e := client.Get("http://www.byond.com/games/exadv1/spacestation13") + check_error(e) + defer resp.Body.Close() + // Yep, Byond serve's it's pages with Windows-1252 encoding... + r = charmap.Windows1252.NewDecoder().Reader(resp.Body) + } + doc, e := goquery.NewDocumentFromReader(r) + check_error(e) return doc } @@ -57,6 +67,7 @@ func parse_server_data(raw *goquery.Selection) *RawServerData { t = t.Find("b").First() } title := strings.TrimSpace(t.Text()) + //title = toUtf8([]byte(title)) title = strings.Replace(title, "\n", "", -1) if len(title) < 1 { // Yes, someone has made a public server without a server name at least once diff --git a/src/web.go b/src/web.go index 2118fd7..0cd1705 100644 --- a/src/web.go +++ b/src/web.go @@ -11,6 +11,7 @@ import ( func (i *Instance) Init() { InitSchema(i.DB) + SetDebug(i.Debug) // TODO: get rid of this stupid debug thing } func (i *Instance) Serve(addr string) error {