From 026858da0e74c933256bc61d678f13bf225dbd02 Mon Sep 17 00:00:00 2001 From: "A. Svensson" Date: Sun, 5 Apr 2015 13:49:47 +0200 Subject: [PATCH] Moved the poller and scaper to new file. --- src/gameservers/data_sources.py | 163 +++++++++++++++++ .../commands/update_server_stats.py | 165 +----------------- 2 files changed, 164 insertions(+), 164 deletions(-) create mode 100644 src/gameservers/data_sources.py diff --git a/src/gameservers/data_sources.py b/src/gameservers/data_sources.py new file mode 100644 index 0000000..9a07887 --- /dev/null +++ b/src/gameservers/data_sources.py @@ -0,0 +1,163 @@ + +from multiprocessing import Pool +import socket +import struct + +import re +import requests +from bs4 import BeautifulSoup + + +def poll_ss13_server(host, port, timeout=10): + # Thanks to /u/headswe for showing how to poll servers. + # Source: http://www.reddit.com/r/SS13/comments/31b5im/a_bunch_of_graphs_for_all_servers/cq11nld + print 'polling:', host, port + cmd = '?players' + query = '\x00\x83{0}\x00\x00\x00\x00\x00{1}\x00'.format( + struct.pack('>H', len(cmd) + 6), cmd + ) + + try: + sock = socket.create_connection((host, port), timeout=timeout) + except socket.timeout: + return + + try: + sock.sendall(query) + response = sock.recv(1024) + except socket.timeout: + response = '' + + sock.close() + print 'done:', host, port + if len(response) < 1: + return + else: + if not response[:5] == '\x00\x83\x00\x05\x2a': + return + tmp = struct.unpack('f', response[5:9]) + return (int(tmp[0]), host, port) + + +class ServerPoller(object): + def __init__(self, timeout=30): + self.timeout = timeout + self.workers = 5 + + def run(self): + targets = self._get_servers() + servers = self._poll_servers(targets) + return servers + + def _get_servers(self): + return [ + ('baystation12.net', 8000), + ('8.8.4.4', 3333), + ('ss13.lljk.net', 26100), + ('204.152.219.158', 3333), + ] + + def _poll_servers(self, targets): + pool = Pool(processes=self.workers) + results = [] + for (host, port) in targets: + future = pool.apply_async(poll_ss13_server, (host, port)) + results.append(future) + + pool.close() + pool.join() + + servers = [] + for future in results: + server = self._handle_future(future) + if server: + servers.append(server) + return servers + + def _handle_future(self, future): + tmp = future.get() + if not tmp: + return + (players, host, port) = tmp + server = dict( + title = host, + game_url = 'byond://{}:{}'.format(host, port), + site_url = '', + player_count = players, + ) + return server + + +class ServerScraper(object): + def __init__(self): + self.url = 'http://www.byond.com/games/exadv1/spacestation13' + # TODO: Better regexp that can't be spoofed by server names + self.PLAYERS = re.compile('Logged in: (\d+) player') + + + + def run(self): + '''Run the parser and return a neat list of dicts containing server data.''' + raw_data = self._download_data() + servers = self._parse_data(raw_data) + return servers + + def _download_data(self): + '''Download raw data, either from local file or a web page.''' + if self.url.startswith('http://') or self.url.startswith('https://'): + raw_data = requests.get(self.url).text.strip() + else: + # HACK: In case of local testing or debugging, since requests can't + # handle local files. + with open(self.url, 'r') as f: + raw_data = f.read().strip() + return raw_data + + def _parse_data(self, raw_data): + '''Parse the raw data and run through all servers.''' + servers = [] + soup_data = BeautifulSoup(raw_data) + for server_data in soup_data.find_all('div', 'live_game_status'): + server = self._parse_server_data(server_data) + if server: + servers.append(server) + + return servers + + def _parse_server_data(self, data): + '''Parse the individual parts of each server.''' + try: + title = data.find('b').get_text().splitlines()[0].strip().encode('utf-8') + except AttributeError: + # HACK: I think this happends because the raw data was incomplete. + # No complete data, no server update. + return None + game_url = data.find('span', 'smaller').text.encode('utf-8') + + tmp = data.find('a') + site_url = None + # Default means the server hasn't set a custom site url + if tmp and not tmp.text == 'Default': + try: + site_url = tmp['href'].encode('utf-8') + # Handle some funky servers... + if site_url == 'http://': + site_url = '' + except KeyError: + # Sometimes there's a tag without a href attribute + pass + + tmp = data.text + player_count = 0 + if tmp.find('No players.') == -1: + data = self.PLAYERS.search(tmp) + player_count = int(data.group(1)) + + server = dict( + title = title, + game_url = game_url, + site_url = site_url, + player_count = player_count, + ) + return server + diff --git a/src/gameservers/management/commands/update_server_stats.py b/src/gameservers/management/commands/update_server_stats.py index 6455167..154c1ed 100755 --- a/src/gameservers/management/commands/update_server_stats.py +++ b/src/gameservers/management/commands/update_server_stats.py @@ -1,174 +1,11 @@ #!/usr/bin/env python -import re - from django.core.management.base import BaseCommand from django.utils import timezone from gameservers.models import Server, ServerHistory -import requests -from bs4 import BeautifulSoup - - -URL = 'http://www.byond.com/games/exadv1/spacestation13' -# TODO: Better regexp that can't be spoofed by server names -PLAYER_COUNT = re.compile('Logged in: (\d+) player') - - -class ServerScraper(object): - def __init__(self): - self.url = URL - - def run(self): - '''Run the parser and return a neat list of dicts containing server data.''' - raw_data = self._download_data() - servers = self._parse_data(raw_data) - return servers - - def _download_data(self): - '''Download raw data, either from local file or a web page.''' - if self.url.startswith('http://') or self.url.startswith('https://'): - raw_data = requests.get(self.url).text.strip() - else: - # HACK: In case of local testing or debugging, since requests can't - # handle local files. - with open(self.url, 'r') as f: - raw_data = f.read().strip() - return raw_data - - def _parse_data(self, raw_data): - '''Parse the raw data and run through all servers.''' - servers = [] - soup_data = BeautifulSoup(raw_data) - for server_data in soup_data.find_all('div', 'live_game_status'): - server = self._parse_server_data(server_data) - if server: - servers.append(server) - - return servers - - def _parse_server_data(self, data): - '''Parse the individual parts of each server.''' - try: - title = data.find('b').get_text().splitlines()[0].strip().encode('utf-8') - except AttributeError: - # HACK: I think this happends because the raw data was incomplete. - # No complete data, no server update. - return None - game_url = data.find('span', 'smaller').text.encode('utf-8') - - tmp = data.find('a') - site_url = None - # Default means the server hasn't set a custom site url - if tmp and not tmp.text == 'Default': - try: - site_url = tmp['href'].encode('utf-8') - # Handle some funky servers... - if site_url == 'http://': - site_url = '' - except KeyError: - # Sometimes there's a tag without a href attribute - pass - - tmp = data.text - player_count = 0 - if tmp.find('No players.') == -1: - data = PLAYER_COUNT.search(tmp) - player_count = int(data.group(1)) - - server = dict( - title = title, - game_url = game_url, - site_url = site_url, - player_count = player_count, - ) - return server - - -from multiprocessing import Pool -import socket -import struct - - -def poll_ss13_server(host, port, timeout=10): - # Thanks to /u/headswe for showing how to poll servers. - # Source: http://www.reddit.com/r/SS13/comments/31b5im/a_bunch_of_graphs_for_all_servers/cq11nld - print 'polling:', host, port - cmd = '?players' - query = '\x00\x83{0}\x00\x00\x00\x00\x00{1}\x00'.format( - struct.pack('>H', len(cmd) + 6), cmd - ) - - try: - sock = socket.create_connection((host, port), timeout=timeout) - except socket.timeout: - return - - try: - sock.sendall(query) - response = sock.recv(1024) - except socket.timeout: - response = '' - - sock.close() - print 'done:', host, port - if len(response) < 1: - return - else: - if not response[:5] == '\x00\x83\x00\x05\x2a': - return - tmp = struct.unpack('f', response[5:9]) - return (int(tmp[0]), host, port) - -class ServerPoller(object): - def __init__(self, timeout=30): - self.timeout = timeout - self.workers = 5 - - def run(self): - targets = self._get_servers() - servers = self._poll_servers(targets) - return servers - - def _get_servers(self): - return [ - ('baystation12.net', 8000), - ('8.8.4.4', 3333), - ('ss13.lljk.net', 26100), - ('204.152.219.158', 3333), - ] - - def _poll_servers(self, targets): - pool = Pool(processes=self.workers) - results = [] - for (host, port) in targets: - future = pool.apply_async(poll_ss13_server, (host, port)) - results.append(future) - - pool.close() - pool.join() - - servers = [] - for future in results: - server = self._handle_future(future) - if server: - servers.append(server) - return servers - - def _handle_future(self, future): - tmp = future.get() - if not tmp: - return - (players, host, port) = tmp - server = dict( - title = host, - game_url = 'byond://{}:{}'.format(host, port), - site_url = '', - player_count = players, - ) - return server - +from gameservers.data_sources import ServerPoller, ServerScraper class Command(BaseCommand): help = 'Update history stats for all ss13 servers.'