From ab79098e9a0de03647f006e34decc214c50c2dce Mon Sep 17 00:00:00 2001 From: lelo Date: Tue, 18 Mar 2025 22:38:11 +0000 Subject: [PATCH] improve dashboard --- app.py | 79 +++++++++++++++++++++++++-------- check_ssh_tunnel.sh | 2 + docker-compose.yml | 8 +++- templates/dashboard.html | 96 ++++++++++++++++++++++++++-------------- 4 files changed, 133 insertions(+), 52 deletions(-) diff --git a/app.py b/app.py index d0302c7..3859229 100755 --- a/app.py +++ b/app.py @@ -6,18 +6,16 @@ from functools import wraps import mimetypes import sqlite3 from datetime import datetime, date, timedelta -from urllib.parse import unquote import diskcache import json import geoip2.database +from urllib.parse import urlparse, unquote from werkzeug.middleware.proxy_fix import ProxyFix -cache = diskcache.Cache('./filecache', size_limit= 32 * 1024**3) # 32 GB limit +cache = diskcache.Cache('./filecache', size_limit= 48 * 1024**3) # 32 GB limit app = Flask(__name__) app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) -# Use a raw string for the default FILE_ROOT path. -app.config['FILE_ROOT'] = r'/mp3_root' app.config['SECRET_KEY'] = '85c1117eb3a5f2c79f0ff395bada8ff8d9a257b99ef5e143' app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=90) app.config['SESSION_COOKIE_SAMESITE'] = 'None' @@ -131,8 +129,9 @@ def list_directory_contents(directory, subpath): full_path = os.path.join(directory, item) # Process directories. if os.path.isdir(full_path): - # Also skip the "Transkription" folder. - if item.lower() == "transkription": + # skip folder + skip_folder = ["Transkription", "@eaDir"] + if item in skip_folder: continue rel_path = os.path.join(subpath, item) if subpath else item rel_path = rel_path.replace(os.sep, '/') @@ -211,6 +210,28 @@ def lookup_location(ip, reader): except Exception: return "Unknown", "Unknown" +# Helper function to classify device type based on user agent string +def get_device_type(user_agent): + if 'Android' in user_agent: + return 'Android' + elif 'iPhone' in user_agent or 'iPad' in user_agent: + return 'iOS' + elif 'Windows' in user_agent: + return 'Windows' + elif 'Macintosh' in user_agent or 'Mac OS' in user_agent: + return 'MacOS' + elif 'Linux' in user_agent: + return 'Linux' + else: + return 'Other' + +def shorten_referrer(url): + segments = [seg for seg in url.split('/') if seg] + segment = segments[-1] + # Decode all percent-encoded characters (like %20, %2F, etc.) + segment_decoded = unquote(segment) + return segment_decoded + @app.route("/dashboard") @require_secret def dashboard(): @@ -238,6 +259,7 @@ def dashboard(): WHERE timestamp >= ? GROUP BY full_path ORDER BY access_count DESC + LIMIT 20 ''', (start.isoformat(),)) rows = cursor.fetchall() @@ -262,18 +284,23 @@ def dashboard(): ''', (start.isoformat(),)) top_files_data = [dict(full_path=row[0], access_count=row[1]) for row in cursor.fetchall()] - # User agent distribution (limit to 10) + # User agent distribution (aggregate by device type) cursor.execute(''' SELECT user_agent, COUNT(*) as count FROM file_access_log WHERE timestamp >= ? GROUP BY user_agent ORDER BY count DESC - LIMIT 10 ''', (start.isoformat(),)) - user_agent_data = [dict(user_agent=row[0], count=row[1]) for row in cursor.fetchall()] + raw_user_agents = [dict(user_agent=row[0], count=row[1]) for row in cursor.fetchall()] + device_counts = {} + for entry in raw_user_agents: + device = get_device_type(entry['user_agent']) + device_counts[device] = device_counts.get(device, 0) + entry['count'] + # Rename to user_agent_data for compatibility with the frontend + user_agent_data = [dict(device=device, count=count) for device, count in device_counts.items()] - # Referrer distribution (limit to 10) + # Referrer distribution (shorten links) cursor.execute(''' SELECT referrer, COUNT(*) as count FROM file_access_log @@ -282,7 +309,11 @@ def dashboard(): ORDER BY count DESC LIMIT 10 ''', (start.isoformat(),)) - referrer_data = [dict(referrer=row[0] if row[0] else "Direct/None", count=row[1]) for row in cursor.fetchall()] + referrer_data = [] + for row in cursor.fetchall(): + raw_ref = row[0] + shortened = shorten_referrer(raw_ref) if raw_ref else "Direct/None" + referrer_data.append(dict(referrer=shortened, count=row[1])) # Aggregate IP addresses with counts cursor.execute(''' @@ -291,6 +322,7 @@ def dashboard(): WHERE timestamp >= ? GROUP BY ip_address ORDER BY count DESC + LIMIT 20 ''', (start.isoformat(),)) ip_rows = cursor.fetchall() @@ -302,6 +334,13 @@ def dashboard(): ip_data.append(dict(ip=ip, count=count, country=country, city=city)) reader.close() + # Aggregate by city (ignoring entries without a city) + city_counts = {} + for entry in ip_data: + if entry['city']: + city_counts[entry['city']] = city_counts.get(entry['city'], 0) + entry['count'] + city_data = [dict(city=city, count=count) for city, count in city_counts.items()] + # Summary stats total_accesses = sum([row[1] for row in rows]) unique_files = len(rows) @@ -317,6 +356,7 @@ def dashboard(): user_agent_data=user_agent_data, referrer_data=referrer_data, ip_data=ip_data, + city_data=city_data, total_accesses=total_accesses, unique_files=unique_files, unique_ips=unique_ips) @@ -364,18 +404,21 @@ def serve_file(filename): app.logger.error(f"File not found: {full_path}") return "File not found", 404 - # HEAD request are coming in to initiate server caching. - # only log initial hits and not the reload of further file parts - range_header = request.headers.get('Range') - if request.method != 'HEAD' and (not range_header or range_header.startswith("bytes=0-")): - log_file_access(full_path) - mime, _ = mimetypes.guess_type(full_path) mime = mime or 'application/octet-stream' - response = None + if mime and mime.startswith('image/'): + pass # do not log access to images + + else: + # HEAD request are coming in to initiate server caching. + # only log initial hits and not the reload of further file parts + range_header = request.headers.get('Range') + if request.method != 'HEAD' and (not range_header or range_header.startswith("bytes=0-")): + log_file_access(full_path) # Check cache first (using diskcache) + response = None cached = cache.get(filename) if cached: cached_file_bytes, mime = cached diff --git a/check_ssh_tunnel.sh b/check_ssh_tunnel.sh index bef5df7..bbcc0e7 100755 --- a/check_ssh_tunnel.sh +++ b/check_ssh_tunnel.sh @@ -16,11 +16,13 @@ SERVER1_MOUNT_POINTS=( "/mnt/app.bethaus/Gottesdienste Speyer" "/mnt/app.bethaus/Besondere Gottesdienste" "/mnt/app.bethaus/Liedersammlung" + "/mnt/app.share" ) SERVER1_NFS_SHARES=( "/volume1/Aufnahme-stereo/010 Gottesdienste ARCHIV" "/volume1/Aufnahme-stereo/013 Besondere Gottesdienste" "/volume1/Aufnahme-stereo/014 Liedersammlung" + "/volume1/app.share" ) # Server 2 Configuration diff --git a/docker-compose.yml b/docker-compose.yml index ac08c07..848e88d 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,18 @@ services: - ./GeoLite2-City.mmdb:/app/GeoLite2-City.mmdb:ro - type: bind source: /mnt/app.bethaus - target: /mp3_root + target: /main + bind: + propagation: rshared + - type: bind + source: /mnt/app.share + target: /share bind: propagation: rshared environment: - FLASK_APP=app.py - FLASK_RUN_HOST=0.0.0.0 - FLASK_ENV=production - - MP3_ROOT=/mp3_root networks: - traefik labels: diff --git a/templates/dashboard.html b/templates/dashboard.html index d37d7f3..edc3ea4 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -91,35 +91,6 @@ - -
-
- Detailed File Access Data -
-
- - - - - - - - - {% for row in rows %} - - - - - {% else %} - - - - {% endfor %} - -
File PathAccess Count
{{ row[0] }}{{ row[1] }}
No data available for the selected timeframe.
-
-
-
@@ -152,13 +123,74 @@
+ + +
+
+ City Access Distribution +
+
+ + + + + + + + + {% for city in city_data %} + + + + + {% else %} + + + + {% endfor %} + +
CityAccess Count
{{ city.city }}{{ city.count }}
No city access data available for the selected timeframe.
+
+
+ + +
+
+ Detailed File Access Data +
+
+ + + + + + + + + {% for row in rows %} + + + + + {% else %} + + + + {% endfor %} + +
File PathAccess Count
{{ row[0] }}{{ row[1] }}
No data available for the selected timeframe.
+
+
+ +