improve dashboard

app.py

@@ -6,18 +6,16 @@ from functools import wraps
 import mimetypes
 import sqlite3
 from datetime import datetime, date, timedelta
-from urllib.parse import unquote
 import diskcache
 import json
 import geoip2.database
+from urllib.parse import urlparse, unquote
 from werkzeug.middleware.proxy_fix import ProxyFix
-cache = diskcache.Cache('./filecache', size_limit= 32 * 1024**3)  # 32 GB limit
+cache = diskcache.Cache('./filecache', size_limit= 48 * 1024**3)  # 32 GB limit
 
 app = Flask(__name__)
 app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
 
-# Use a raw string for the default FILE_ROOT path.
-app.config['FILE_ROOT'] = r'/mp3_root'
 app.config['SECRET_KEY'] = '85c1117eb3a5f2c79f0ff395bada8ff8d9a257b99ef5e143'
 app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=90)
 app.config['SESSION_COOKIE_SAMESITE'] = 'None'
@@ -131,8 +129,9 @@ def list_directory_contents(directory, subpath):
             full_path = os.path.join(directory, item)
             # Process directories.
             if os.path.isdir(full_path):
-                # Also skip the "Transkription" folder.
+                # skip folder
-                if item.lower() == "transkription":
+                skip_folder = ["Transkription", "@eaDir"]
+                if item in skip_folder:
                     continue
                 rel_path = os.path.join(subpath, item) if subpath else item
                 rel_path = rel_path.replace(os.sep, '/')
@@ -211,6 +210,28 @@ def lookup_location(ip, reader):
     except Exception:
         return "Unknown", "Unknown"
 
+# Helper function to classify device type based on user agent string
+def get_device_type(user_agent):
+    if 'Android' in user_agent:
+        return 'Android'
+    elif 'iPhone' in user_agent or 'iPad' in user_agent:
+        return 'iOS'
+    elif 'Windows' in user_agent:
+        return 'Windows'
+    elif 'Macintosh' in user_agent or 'Mac OS' in user_agent:
+        return 'MacOS'
+    elif 'Linux' in user_agent:
+        return 'Linux'
+    else:
+        return 'Other'
+
+def shorten_referrer(url):
+    segments = [seg for seg in url.split('/') if seg]
+    segment = segments[-1]
+    # Decode all percent-encoded characters (like %20, %2F, etc.)
+    segment_decoded = unquote(segment)
+    return segment_decoded
+
 @app.route("/dashboard")
 @require_secret
 def dashboard():
@@ -238,6 +259,7 @@ def dashboard():
         WHERE timestamp >= ?
         GROUP BY full_path
         ORDER BY access_count DESC
+        LIMIT 20
     ''', (start.isoformat(),))
     rows = cursor.fetchall()
 
@@ -262,18 +284,23 @@ def dashboard():
     ''', (start.isoformat(),))
     top_files_data = [dict(full_path=row[0], access_count=row[1]) for row in cursor.fetchall()]
 
-    # User agent distribution (limit to 10)
+    # User agent distribution (aggregate by device type)
     cursor.execute('''
         SELECT user_agent, COUNT(*) as count
         FROM file_access_log
         WHERE timestamp >= ?
         GROUP BY user_agent
         ORDER BY count DESC
-        LIMIT 10
     ''', (start.isoformat(),))
-    user_agent_data = [dict(user_agent=row[0], count=row[1]) for row in cursor.fetchall()]
+    raw_user_agents = [dict(user_agent=row[0], count=row[1]) for row in cursor.fetchall()]
+    device_counts = {}
+    for entry in raw_user_agents:
+        device = get_device_type(entry['user_agent'])
+        device_counts[device] = device_counts.get(device, 0) + entry['count']
+    # Rename to user_agent_data for compatibility with the frontend
+    user_agent_data = [dict(device=device, count=count) for device, count in device_counts.items()]
 
-    # Referrer distribution (limit to 10)
+    # Referrer distribution (shorten links)
     cursor.execute('''
         SELECT referrer, COUNT(*) as count
         FROM file_access_log
@@ -282,7 +309,11 @@ def dashboard():
         ORDER BY count DESC
         LIMIT 10
     ''', (start.isoformat(),))
-    referrer_data = [dict(referrer=row[0] if row[0] else "Direct/None", count=row[1]) for row in cursor.fetchall()]
+    referrer_data = []
+    for row in cursor.fetchall():
+        raw_ref = row[0]
+        shortened = shorten_referrer(raw_ref) if raw_ref else "Direct/None"
+        referrer_data.append(dict(referrer=shortened, count=row[1]))
 
     # Aggregate IP addresses with counts
     cursor.execute('''
@@ -291,6 +322,7 @@ def dashboard():
         WHERE timestamp >= ?
         GROUP BY ip_address
         ORDER BY count DESC
+        LIMIT 20
     ''', (start.isoformat(),))
     ip_rows = cursor.fetchall()
 
@@ -302,6 +334,13 @@ def dashboard():
         ip_data.append(dict(ip=ip, count=count, country=country, city=city))
     reader.close()
 
+    # Aggregate by city (ignoring entries without a city)
+    city_counts = {}
+    for entry in ip_data:
+        if entry['city']:
+            city_counts[entry['city']] = city_counts.get(entry['city'], 0) + entry['count']
+    city_data = [dict(city=city, count=count) for city, count in city_counts.items()]
+
     # Summary stats
     total_accesses = sum([row[1] for row in rows])
     unique_files = len(rows)
@@ -317,6 +356,7 @@ def dashboard():
                            user_agent_data=user_agent_data,
                            referrer_data=referrer_data,
                            ip_data=ip_data,
+                           city_data=city_data,
                            total_accesses=total_accesses,
                            unique_files=unique_files,
                            unique_ips=unique_ips)
@@ -364,18 +404,21 @@ def serve_file(filename):
         app.logger.error(f"File not found: {full_path}")
         return "File not found", 404
 
-    # HEAD request are coming in to initiate server caching.
-    # only log initial hits and not the reload of further file parts
-    range_header = request.headers.get('Range')
-    if request.method != 'HEAD' and (not range_header or range_header.startswith("bytes=0-")):
-        log_file_access(full_path)
-
     mime, _ = mimetypes.guess_type(full_path)
     mime = mime or 'application/octet-stream'
 
-    response = None
+    if mime and mime.startswith('image/'):
+        pass # do not log access to images
+    
+    else:
+        # HEAD request are coming in to initiate server caching.
+        # only log initial hits and not the reload of further file parts
+        range_header = request.headers.get('Range')
+        if request.method != 'HEAD' and (not range_header or range_header.startswith("bytes=0-")):
+            log_file_access(full_path)
 
     # Check cache first (using diskcache)
+    response = None
     cached = cache.get(filename)
     if cached:
         cached_file_bytes, mime = cached

check_ssh_tunnel.sh

@@ -16,11 +16,13 @@ SERVER1_MOUNT_POINTS=(
     "/mnt/app.bethaus/Gottesdienste Speyer"
     "/mnt/app.bethaus/Besondere Gottesdienste"
     "/mnt/app.bethaus/Liedersammlung"
+    "/mnt/app.share"
 )
 SERVER1_NFS_SHARES=(
     "/volume1/Aufnahme-stereo/010 Gottesdienste ARCHIV"
     "/volume1/Aufnahme-stereo/013 Besondere Gottesdienste"
     "/volume1/Aufnahme-stereo/014 Liedersammlung"
+    "/volume1/app.share"
 )
 
 # Server 2 Configuration

docker-compose.yml

@@ -11,14 +11,18 @@ services:
       - ./GeoLite2-City.mmdb:/app/GeoLite2-City.mmdb:ro
       - type: bind
         source: /mnt/app.bethaus
-        target: /mp3_root
+        target: /main
+        bind:
+          propagation: rshared
+      - type: bind
+        source: /mnt/app.share
+        target: /share
         bind:
           propagation: rshared
     environment:
       - FLASK_APP=app.py
       - FLASK_RUN_HOST=0.0.0.0
       - FLASK_ENV=production
-      - MP3_ROOT=/mp3_root
     networks:
       - traefik
     labels:

templates/dashboard.html

@@ -91,35 +91,6 @@
       </div>
     </div>
 
-    <!-- Detailed Table of Top File Accesses -->
-    <div class="card mb-4">
-      <div class="card-header">
-        Detailed File Access Data
-      </div>
-      <div class="card-body">
-        <table class="table table-striped">
-          <thead>
-            <tr>
-              <th>File Path</th>
-              <th>Access Count</th>
-            </tr>
-          </thead>
-          <tbody>
-            {% for row in rows %}
-            <tr>
-              <td>{{ row[0] }}</td>
-              <td>{{ row[1] }}</td>
-            </tr>
-            {% else %}
-            <tr>
-              <td colspan="2">No data available for the selected timeframe.</td>
-            </tr>
-            {% endfor %}
-          </tbody>
-        </table>
-      </div>
-    </div>
-
     <!-- New Section: IP Address Access Data -->
     <div class="card">
       <div class="card-header">
@@ -152,13 +123,74 @@
         </table>
       </div>
     </div>
+
+    <!-- New Section: Aggregated City Access Data -->
+    <div class="card">
+      <div class="card-header">
+        City Access Distribution
+      </div>
+      <div class="card-body">
+        <table class="table table-bordered">
+          <thead>
+            <tr>
+              <th>City</th>
+              <th>Access Count</th>
+            </tr>
+          </thead>
+          <tbody>
+            {% for city in city_data %}
+            <tr>
+              <td>{{ city.city }}</td>
+              <td>{{ city.count }}</td>
+            </tr>
+            {% else %}
+            <tr>
+              <td colspan="2">No city access data available for the selected timeframe.</td>
+            </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+      </div>
+    </div>
+
+    <!-- Detailed Table of Top File Accesses -->
+    <div class="card mb-4">
+      <div class="card-header">
+        Detailed File Access Data
+      </div>
+      <div class="card-body">
+        <table class="table table-striped">
+          <thead>
+            <tr>
+              <th>File Path</th>
+              <th>Access Count</th>
+            </tr>
+          </thead>
+          <tbody>
+            {% for row in rows %}
+            <tr>
+              <td>{{ row[0] }}</td>
+              <td>{{ row[1] }}</td>
+            </tr>
+            {% else %}
+            <tr>
+              <td colspan="2">No data available for the selected timeframe.</td>
+            </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+      </div>
+    </div>
+
   </div>
+
   <!-- Load Chart.js from CDN -->
   <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
   <script>
     // Data passed from the backend as JSON
     const dailyAccessData = {{ daily_access_data|tojson }};
     const topFilesData = {{ top_files_data|tojson }};
+    // Note: user_agent_data now contains 'device' and 'count'
     const userAgentData = {{ user_agent_data|tojson }};
     const referrerData = {{ referrer_data|tojson }};
 
@@ -208,12 +240,12 @@
       }
     });
 
-    // User Agent Distribution - Pie Chart
+    // User Agent Distribution - Pie Chart (using aggregated device data)
     const ctxUserAgent = document.getElementById('userAgentChart').getContext('2d');
     new Chart(ctxUserAgent, {
       type: 'pie',
       data: {
-        labels: userAgentData.map(item => item.user_agent),
+        labels: userAgentData.map(item => item.device),
         datasets: [{
           data: userAgentData.map(item => item.count)
         }]
@@ -221,7 +253,7 @@
       options: { responsive: true }
     });
 
-    // Referrer Distribution - Pie Chart
+    // Referrer Distribution - Pie Chart (with shortened referrers)
     const ctxReferrer = document.getElementById('referrerChart').getContext('2d');
     new Chart(ctxReferrer, {
       type: 'pie',