From afa8e74b6f49b1e7d5d044b0f44613e947062aef Mon Sep 17 00:00:00 2001 From: "A. Svensson" Date: Sun, 8 Mar 2015 12:56:59 +0100 Subject: [PATCH 1/5] Removed the redis stuff and replaced with new django model. --HG-- branch : history_model --- requirements.txt | 1 - .../migrations/0011_serverhistory.py | 28 +++++++++++++++ src/gameservers/models.py | 36 +++++-------------- 3 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 src/gameservers/migrations/0011_serverhistory.py diff --git a/requirements.txt b/requirements.txt index 49520f6..60efde2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,5 @@ django django-debug-toolbar requests beautifulsoup4 -redis gunicorn fabric diff --git a/src/gameservers/migrations/0011_serverhistory.py b/src/gameservers/migrations/0011_serverhistory.py new file mode 100644 index 0000000..8b432a7 --- /dev/null +++ b/src/gameservers/migrations/0011_serverhistory.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('gameservers', '0010_auto_20150223_1927'), + ] + + operations = [ + migrations.CreateModel( + name='ServerHistory', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('created', models.DateTimeField(default=django.utils.timezone.now)), + ('players', models.PositiveIntegerField(default=0)), + ('server', models.ForeignKey(to='gameservers.Server')), + ], + options={ + 'ordering': ['-created', 'server'], + }, + bases=(models.Model,), + ), + ] diff --git a/src/gameservers/models.py b/src/gameservers/models.py index 1f270ca..eecc1dc 100644 --- a/src/gameservers/models.py +++ b/src/gameservers/models.py @@ -2,8 +2,6 @@ from django.db import models from django.utils import timezone -import redis - class Server(models.Model): title = models.CharField(max_length=255) @@ -28,32 +26,14 @@ class Server(models.Model): server.delete() -class PlayerHistory(object): - def __init__(self, redis_settings=dict(host='localhost', port=6379, db=0)): - self.redis = redis.StrictRedis(**redis_settings) +class ServerHistory(models.Model): + server = models.ForeignKey(Server) + created = models.DateTimeField(default=timezone.now) + players = models.PositiveIntegerField(default=0) - # 32256 = 4 times per hour * 24 hours * 7 days * 4 weeks * 12 months - self.max_items = 32256 + class Meta: + ordering = ['-created', 'server'] - def add_point(self, server, time, players): - '''Add a new point in the player history.''' - self.redis.lpush(server, '{},{}'.format(time, players)) - - def trim_points(self, server): - '''Trim away too old points and servers in the player history.''' - self.redis.ltrim(server, 0, self.max_items) - # let the list expire after a week without updates - self.redis.expire(server, 604800) - - def get_history(self, server, days=7): - '''Get a range of days in a server's player history.''' - # 96 = 4 times per hour * 24 hours - max_items = days * 96 - - items = [] - for tmp in self.redis.lrange(server, 0, max_items): - time, players = tmp.split(',') - time, players = float(time), int(players) - items.append((time, players)) - return items + def __str__(self): + return 'History for {} at {}.'.format(self.server, self.created) From 65d1b043bf54e8a0cee95091b4667fab5631c166 Mon Sep 17 00:00:00 2001 From: "A. Svensson" Date: Sun, 8 Mar 2015 13:00:39 +0100 Subject: [PATCH 2/5] Use the new model in the views. --HG-- branch : history_model --- src/gameservers/views.py | 26 +++++++++++--------- src/templates/gameservers/server_detail.html | 10 ++------ 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/gameservers/views.py b/src/gameservers/views.py index d42bc66..25c88aa 100644 --- a/src/gameservers/views.py +++ b/src/gameservers/views.py @@ -1,8 +1,11 @@ +from datetime import timedelta + from django.shortcuts import render from django.views import generic +from django.utils import timezone -from .models import Server, PlayerHistory +from .models import Server, ServerHistory class ServerListView(generic.ListView): model = Server @@ -13,25 +16,24 @@ class ServerDetailView(generic.DetailView): def get_context_data(self, **kwargs): context = super(ServerDetailView, self).get_context_data(**kwargs) server = context['server'] - history = PlayerHistory() - items = history.get_history(server) - context['player_history'] = items + + weekly_history = ServerHistory.objects.filter( + server=server, + created__gte=timezone.now() - timedelta(days=7), + ) + context['weekly_history'] = weekly_history # Moving average for the last day - # TODO: remove the hardcoded value - tmp = [players for time, players in items[-96:]] + tmp = [tmp.players for tmp in ServerHistory.objects.filter( + server=server, + created__gte=timezone.now() - timedelta(days=1))] context['daily_average'] = sum(tmp) / float(len(tmp)) context['daily_min'] = min(tmp) context['daily_max'] = max(tmp) - tmp = [players for time, players in items[-96*7:]] + tmp = [tmp.players for tmp in weekly_history] context['weekly_average'] = sum(tmp) / float(len(tmp)) context['weekly_min'] = min(tmp) context['weekly_max'] = max(tmp) - - tmp = [players for time, players in items] - context['total_average'] = sum(tmp) / float(len(tmp)) - context['total_min'] = min(tmp) - context['total_max'] = max(tmp) return context diff --git a/src/templates/gameservers/server_detail.html b/src/templates/gameservers/server_detail.html index 500d732..67d1954 100644 --- a/src/templates/gameservers/server_detail.html +++ b/src/templates/gameservers/server_detail.html @@ -45,12 +45,6 @@ {{weekly_min}} {{weekly_max}} - - Total - {{total_average|floatformat}} - {{total_min}} - {{total_max}} -
@@ -62,8 +56,8 @@ $(function() { var series = [ {# convert timestamp to ms, because javascript... #} - {% for timestamp, players in player_history %} - [{{timestamp}} * 1000, {{players}}], + {% for item in weekly_history %} + [{{item.created|date:'U'}} * 1000, {{item.players}}], {% endfor %} ]; From a532543121864a0913a46fa4ff3c2cad54c8cf1d Mon Sep 17 00:00:00 2001 From: "A. Svensson" Date: Sun, 8 Mar 2015 13:11:46 +0100 Subject: [PATCH 3/5] update_population uses the new model too. --HG-- branch : history_model --- .../management/commands/update_population.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/gameservers/management/commands/update_population.py b/src/gameservers/management/commands/update_population.py index 784a861..7d1fd04 100755 --- a/src/gameservers/management/commands/update_population.py +++ b/src/gameservers/management/commands/update_population.py @@ -1,12 +1,10 @@ #!/usr/bin/env python import re -import time from django.core.management.base import BaseCommand -from django.utils import timezone -from gameservers.models import Server, PlayerHistory +from gameservers.models import Server, ServerHistory import requests from bs4 import BeautifulSoup @@ -94,9 +92,8 @@ class Command(BaseCommand): parser = ServerParser() #parser.url = './dump.html' # Use a local file instead when testing servers = parser.run() - history = PlayerHistory() - now = time.mktime(timezone.now().timetuple()) servers_handled = [] + new_items = [] for data in servers: # Prevent empty servers with identical names to other, active servers @@ -106,10 +103,6 @@ class Command(BaseCommand): else: servers_handled.append(data['title']) - # Keep the amount of data down in redis - history.trim_points(data['title']) - - # TODO: do bulk insert instead! server, created = Server.objects.update_or_create( title=data['title'], defaults= dict( @@ -119,8 +112,9 @@ class Command(BaseCommand): ) ) - # Update the player history - history.add_point(server, now, data['player_count']) + history = ServerHistory(server=server, players=data['player_count']) + new_items.append(history) + ServerHistory.objects.bulk_create(new_items) Server.remove_old_servers() From 936a2fca086086af176b32f149c21f84e1c31b7e Mon Sep 17 00:00:00 2001 From: "A. Svensson" Date: Sun, 8 Mar 2015 13:14:01 +0100 Subject: [PATCH 4/5] Renamed the update script to a better name. --HG-- branch : history_model --- .../commands/{update_population.py => update_server_stats.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/gameservers/management/commands/{update_population.py => update_server_stats.py} (98%) diff --git a/src/gameservers/management/commands/update_population.py b/src/gameservers/management/commands/update_server_stats.py similarity index 98% rename from src/gameservers/management/commands/update_population.py rename to src/gameservers/management/commands/update_server_stats.py index 7d1fd04..0011774 100755 --- a/src/gameservers/management/commands/update_population.py +++ b/src/gameservers/management/commands/update_server_stats.py @@ -86,7 +86,7 @@ class ServerParser(object): class Command(BaseCommand): - help = 'Update population stats for all ss13 servers.' + help = 'Update history stats for all ss13 servers.' def handle(self, *args, **kwargs): parser = ServerParser() From e452d68675baedadf9a5b339ee970dd9adf76511 Mon Sep 17 00:00:00 2001 From: "A. Svensson" Date: Sun, 8 Mar 2015 13:27:13 +0100 Subject: [PATCH 5/5] Closed branch history_model. --HG-- branch : history_model