improve song analytics

analytics.py

@@ -3,6 +3,8 @@ from flask import render_template, request, session
 from datetime import datetime, timedelta, timezone
 import geoip2.database
 from auth import require_secret
+from collections import defaultdict
+import json
 import os
 
 file_access_temp = []
@@ -12,6 +14,7 @@ DB_NAME = 'access_log.db'
 
 # Create a single global connection to SQLite
 log_db = sqlite3.connect(DB_NAME, check_same_thread=False)
+search_db = sqlite3.connect("search.db", check_same_thread=False)
 
 # geo location
 geoReader = geoip2.database.Reader('GeoLite2-City.mmdb')
@@ -137,6 +140,57 @@ def return_file_access():
     else:
         return []
 
+def songs_dashboard():
+    if 'songs_dashboard_timeframe' not in session:
+        session['songs_dashboard_timeframe'] = "30"
+    timeframe_param = request.args.get("timeframe", session['songs_dashboard_timeframe'])
+    session['songs_dashboard_timeframe'] = timeframe_param
+    
+    if 'songs_dashboard_category' not in session:
+        session['songs_dashboard_category'] = "Gemeinsamer Gesang"
+    category = request.args.get("category", session['songs_dashboard_category'])
+    session['songs_dashboard_category'] = category
+    
+    if 'songs_dashboard_site' not in session:
+        session['songs_dashboard_site'] = "Speyer"
+    site = request.args.get("site", session['songs_dashboard_site'])
+    session['songs_dashboard_site'] = site
+    
+    # Determine cutoff_date based on the days parameter
+    if timeframe_param == "all":
+        cutoff_date = None  # No date filtering when analyzing all time
+        timeframe = "all"   # Pass the string to the template if needed
+    else:
+        timeframe = int(timeframe_param)
+        now = datetime.now()
+        cutoff_date = now - timedelta(days=timeframe)
+    
+    cursor = search_db.cursor()
+    # Query rows with category "Gemeinsamer Gesang"
+    query = "SELECT titel, performance_date FROM files WHERE category = ? and site = ?"
+    cursor.execute(query, (category, site))
+    rows = cursor.fetchall()
+    
+    # Group and count performances per titel (only if performance_date is within the timeframe,
+    # or count all if cutoff_date is None)
+    performance_counts = defaultdict(int)
+    for titel, performance_date in rows:
+        if performance_date:
+            try:
+                # Convert date from "dd.mm.yyyy" format
+                date_obj = datetime.strptime(performance_date, "%d.%m.%Y")
+            except ValueError:
+                continue
+            # If cutoff_date is None, count all dates; otherwise, filter by cutoff_date.
+            if cutoff_date is None or date_obj >= cutoff_date:
+                performance_counts[titel] += 1
+    
+    # Create a list of tuples: (count, titel), sorted in descending order by count.
+    performance_data = [(count, titel) for titel, count in performance_counts.items()]
+    performance_data.sort(reverse=True, key=lambda x: x[0])
+    
+    return render_template('songs_dashboard.html', timeframe=timeframe_param, performance_data=performance_data, site=site, category=category)
+
 @require_secret
 def connections():
     return render_template('connections.html')
@@ -424,3 +478,31 @@ def dashboard():
         cached_percentage=cached_percentage,
         timeframe_data=timeframe_data
     )
+
+def export_to_excel():
+    """Export search_db to an Excel file and store it locally."""
+    import pandas as pd
+
+    # Query all data from the search_db
+    query = "SELECT * FROM files"
+    cursor = search_db.cursor()
+    cursor.execute(query)
+    rows = cursor.fetchall()
+
+    # Get column names from the cursor description
+    column_names = [description[0] for description in cursor.description]
+
+    # Create a DataFrame and save it to an Excel file
+    df = pd.DataFrame(rows, columns=column_names)
+    df = df.drop(columns=['transcript'], errors='ignore')  # Drop the 'id' column if it exists   
+    df.to_excel("search_db.xlsx", index=False)
+
+    # Close the cursor and database connection
+    cursor.close()
+
+if __name__ == "__main__":
+    print("Running as a standalone script.")
+    export_to_excel()
+    print("Exported search_db to search_db.xlsx")
+    
+    

app.py

@@ -41,6 +41,8 @@ app.add_url_rule('/remove_secret', view_func=auth.remove_secret, methods=['POST'
 app.add_url_rule('/search', view_func=search.search, methods=['GET'])
 app.add_url_rule('/searchcommand', view_func=search.searchcommand, methods=['POST'])
 
+app.add_url_rule('/songs_dashboard', view_func=a.songs_dashboard)
+
 
 # Grab the HOST_RULE environment variable
 host_rule = os.getenv("HOST_RULE", "")

index_for_search.py

@@ -1,6 +1,8 @@
 import os
 import json
 import sqlite3
+from datetime import datetime
+import re
 
 SEARCH_DB_NAME = 'search.db'
 ACCESS_LOG_DB_NAME = 'access_log.db'
@@ -25,6 +27,11 @@ def init_db():
             basefolder TEXT,
             filename TEXT,
             filetype TEXT,
+            category TEXT,
+            titel TEXT,
+            name TEXT,
+            performance_date TEXT,
+            site TEXT,
             transcript TEXT,
             hitcount INTEGER DEFAULT 0,
             UNIQUE(relative_path, filename)
@@ -42,6 +49,31 @@ def init_db():
     except sqlite3.OperationalError:
         # Likely the column already exists, so we ignore this error.
         pass
+    try:
+        cursor.execute("ALTER TABLE files ADD COLUMN category TEXT")
+    except sqlite3.OperationalError:
+        # Likely the column already exists, so we ignore this error.
+        pass
+    try:
+        cursor.execute("ALTER TABLE files ADD COLUMN titel TEXT")
+    except sqlite3.OperationalError:
+        # Likely the column already exists, so we ignore this error.
+        pass
+    try:
+        cursor.execute("ALTER TABLE files ADD COLUMN name TEXT")
+    except sqlite3.OperationalError:
+        # Likely the column already exists, so we ignore this error.
+        pass
+    try:
+        cursor.execute("ALTER TABLE files ADD COLUMN performance_date TEXT")
+    except sqlite3.OperationalError:
+        # Likely the column already exists, so we ignore this error.
+        pass
+    try:
+        cursor.execute("ALTER TABLE files ADD COLUMN site TEXT")
+    except sqlite3.OperationalError:
+        # Likely the column already exists, so we ignore this error.
+        pass
     search_db.commit()
 
 def scan_dir(directory):
@@ -86,7 +118,6 @@ def updatefileindex():
             # Accumulate scanned file data and keys for this base folder.
             scanned_files = []  # Each entry: (relative_path, basefolder, filename, filetype, transcript, hitcount)
             current_keys = set()
-
             for entry in scan_dir(norm_folderpath):
                 entry_path = os.path.normpath(entry.path)
                 # Get relative part by slicing if possible.
@@ -97,24 +128,92 @@ def updatefileindex():
                 # Prepend the foldername so it becomes part of the stored relative path.
                 relative_path = os.path.join(foldername, rel_part).replace(os.sep, '/')
                 filetype = os.path.splitext(entry.name)[1].lower()
-                transcript = None
-
-                # Check for a corresponding transcript file in a sibling "Transkription" folder.
-                parent_dir = os.path.dirname(entry_path)
-                transcript_dir = os.path.join(parent_dir, "Transkription")
-                transcript_filename = os.path.splitext(entry.name)[0] + ".md"
-                transcript_path = os.path.join(transcript_dir, transcript_filename)
-                if os.path.exists(transcript_path):
-                    try:
-                        with open(transcript_path, 'r', encoding='utf-8') as tf:
-                            transcript = tf.read()
-                    except Exception:
-                        transcript = None
-
+                
                 # Retrieve the hit count for this file.
                 hit_count = get_hit_count(relative_path)
+                
+                category, titel, name, performance_date, site = None, None, None, None, None
+                
+                # Determine the site
+                if foldername == 'Gottesdienste Speyer':
+                    site = 'Speyer'
+                elif foldername == 'Gottesdienste Schwegenheim':
+                    site = 'Schwegenheim'
+                
+                if filetype == '.mp3':
+                    transcript = None
 
-                scanned_files.append((relative_path, foldername, entry.name, filetype, transcript, hit_count))
+                    # Check for a corresponding transcript file in a sibling "Transkription" folder.
+                    parent_dir = os.path.dirname(entry_path)
+                    transcript_dir = os.path.join(parent_dir, "Transkription")
+                    transcript_filename = os.path.splitext(entry.name)[0] + ".md"
+                    transcript_path = os.path.join(transcript_dir, transcript_filename)
+                    if os.path.exists(transcript_path):
+                        try:
+                            with open(transcript_path, 'r', encoding='utf-8') as tf:
+                                transcript = tf.read()
+                        except Exception:
+                            transcript = None
+
+                    # extract category and titel from filename
+                    filename_ext = os.path.splitext(entry.name)[0]
+                    left_side, right_side = filename_ext.split('-', 1) if '-' in filename_ext else (filename_ext, None)
+                    try:
+                        int(left_side.strip())
+                        # first part is only a number
+                        previous_right_side = right_side
+                        left_side, right_side = previous_right_side.split('-', 1) if '-' in previous_right_side else (previous_right_side, None)
+                    except:
+                        # first part not a number
+                        continue
+                    
+                    if 'predig' in left_side.lower():
+                        category = 'Predigt'
+                    elif 'wort' in left_side.lower() or 'einladung' in left_side.lower():
+                        category = 'Vorwort'
+                    elif 'chor' in left_side.lower():
+                        category = 'Chor'
+                    elif 'orchester' in left_side.lower():
+                        category = 'Orchester'
+                    elif 'gruppenlied' in left_side.lower() or 'jugendlied' in left_side.lower():
+                        category = 'Gruppenlied'
+                    elif 'gemeinsam' in left_side.lower() or 'gesang' in left_side.lower() or 'lied' in left_side.lower():
+                        category = 'Gemeinsamer Gesang'
+                    elif 'gedicht' in left_side.lower():
+                        category = 'Gedicht'
+                    elif 'instrumental' in left_side.lower() or 'musikstück' in left_side.lower():
+                        category = 'Instrumental'
+                    else:
+                        category = None
+
+                    if right_side:
+                        titel, name = right_side.split('-', 1) if '-' in right_side else (right_side, None)
+                        if category == 'Predigt' or category == 'Vorwort' or category == 'Gedicht':
+                            if not name: # kein Titel, nur Name
+                                name = titel
+                                titel = None
+                    else:
+                        titel = None
+                        name = None
+                    
+                # extract the date from path using regex (dd.mm.yyyy or dd.mm.yy)
+                date_match = re.search(r'(\d{1,2}\.\d{1,2}\.\d{2,4})', relative_path)
+                if date_match:
+                    date_str = date_match.group(1)
+                    # Convert to YYYY-MM-DD format
+                    try:
+                        date_obj = datetime.strptime(date_str, '%d.%m.%Y')
+                        performance_date = date_obj.strftime('%d.%m.%Y')
+                    except ValueError:
+                        try:
+                            date_obj = datetime.strptime(date_str, '%d.%m.%y')
+                            performance_date = date_obj.strftime('%d.%m.%Y')
+                        except ValueError:
+                            performance_date = None
+                else:
+                    performance_date = None
+
+                scanned_files.append((relative_path, foldername, entry.name, filetype, category, titel, name, performance_date, site, transcript, hit_count))
                 current_keys.add((relative_path, entry.name))
 
             # Remove database entries for files under this base folder that are no longer on disk.
@@ -128,7 +227,7 @@ def updatefileindex():
 
             # Bulk write the scanned files using INSERT OR REPLACE.
             cursor.executemany(
-                "INSERT OR REPLACE INTO files (relative_path, basefolder, filename, filetype, transcript, hitcount) VALUES (?, ?, ?, ?, ?, ?)",
+                "INSERT OR REPLACE INTO files (relative_path, basefolder, filename, filetype, category, titel, name, performance_date, site, transcript, hitcount) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                 scanned_files
             )
 

requirements.txt

@@ -4,5 +4,7 @@ pillow
 qrcode
 diskcache
 geoip2
+pandas
+openpyxl
 gunicorn
 eventlet

templates/connections.html

@@ -67,18 +67,11 @@
         <a class="navbar-brand" href="#">Downloads der letzten 10 Minuten</a>
         <div class="navbar-collapse" id="navbarNav">
           <ul class="navbar-nav ms-auto">
-            <li class="nav-item">
-              <a href="{{ url_for('index') }}" class="nav-link">App</a>
-            </li>
-            <li class="nav-item">
-              <a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a>
-            </li>
-            <li class="nav-item">
-              <a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a>
-            </li>
-            <li class="nav-item">
-              <a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung</a>
-            </li>
+            <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
+            <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
+            <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
+            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
+            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
           </ul>
         </div>
       </div>

templates/dashboard.html

@@ -31,7 +31,8 @@
             <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
             <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
             <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
-            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung</a></li>
+            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
+            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
           </ul>
         </div>
       </div>
@@ -272,8 +273,8 @@
     </div>
 
     <!-- Scripts -->
-    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
     <script>
       // Data passed from the backend as JSON
       let distinctDeviceData = {{ distinct_device_data|tojson }};

templates/mylinks.html

@@ -33,7 +33,8 @@
             <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
             <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
             <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
-            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung</a></li>
+            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
+            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
           </ul>
         </div>
       </div>

templates/songs_dashboard.html

@@ -0,0 +1,182 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Dashboard - Verbindungsanalyse</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet">
+    <style>
+      html, body {
+        height: 100%;
+        margin: 0;
+        padding: 0;
+      }
+      .container-fluid {
+        height: 100%;
+        display: flex;
+        flex-direction: column;
+      }
+      .card {
+        margin-bottom: 1.5rem;
+      }
+    </style>
+  </head>
+  <body>
+    <!-- Navigation Bar -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-secondary mb-3">
+      <div class="container-fluid">
+        <a class="navbar-brand" href="#">Dashboard - Analyse Wiederholungen</a>
+        <div class="navbar-collapse" id="navbarNav">
+          <ul class="navbar-nav ms-auto">
+            <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
+            <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
+            <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
+            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
+            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
+          </ul>
+        </div>
+      </div>
+    </nav>
+
+    <!-- Main Container -->
+    <div class="container-fluid px-4">
+      <!-- Dropdown Controls -->
+      <div class="mb-4 d-flex flex-wrap gap-2">
+
+        <!-- Site Dropdown -->
+        <div class="dropdown">
+          <button class="btn btn-secondary dropdown-toggle" 
+                  type="button" id="siteDropdown" data-bs-toggle="dropdown" aria-expanded="false">
+            {% if session['songs_dashboard_site'] == 'Speyer' %}
+              Speyer
+            {% elif session['songs_dashboard_site'] == 'Schwegenheim' %}
+              Schwegenheim
+            {% else %}
+              Select Site
+            {% endif %}
+          </button>
+          <ul class="dropdown-menu" aria-labelledby="siteDropdown">
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_site'] == 'Speyer' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', site='Speyer') }}">
+                Speyer
+              </a>
+            </li>
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_site'] == 'Schwegenheim' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', site='Schwegenheim') }}">
+                Schwegenheim
+              </a>
+            </li>
+          </ul>
+        </div>
+        
+        <!-- Timeframe Dropdown -->
+        <div class="dropdown">
+          <button class="btn btn-secondary dropdown-toggle" 
+                  type="button" id="timeframeDropdown" data-bs-toggle="dropdown" aria-expanded="false">
+            {% if session['songs_dashboard_timeframe'] == '7' %}
+              Last 7 Days
+            {% elif session['songs_dashboard_timeframe'] == '30' %}
+              Last 30 Days
+            {% elif session['songs_dashboard_timeframe'] == '90' %}
+              Last 90 Days
+            {% elif session['songs_dashboard_timeframe'] == '365' %}
+              Last 365 Days
+            {% elif session['songs_dashboard_timeframe'] == 'all' %}
+              All Data
+            {% else %}
+              Select Timeframe
+            {% endif %}
+          </button>
+          <ul class="dropdown-menu" aria-labelledby="timeframeDropdown">
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '7' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', timeframe='7') }}">
+                Last 7 Days
+              </a>
+            </li>
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '30' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', timeframe='30') }}">
+                Last 30 Days
+              </a>
+            </li>
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '90' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', timeframe='90') }}">
+                Last 90 Days
+              </a>
+            </li>
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '365' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', timeframe='365') }}">
+                Last 365 Days
+              </a>
+            </li>
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == 'all' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', timeframe='all') }}">
+                All Data
+              </a>
+            </li>
+          </ul>
+        </div>
+
+        <!-- category Dropdown -->
+        <div class="dropdown">
+          <button class="btn btn-secondary dropdown-toggle" type="button" id="categoryDropdown" data-bs-toggle="dropdown" aria-expanded="false">
+            {% if session['songs_dashboard_category'] == 'Gemeinsamer Gesang' %}
+            Gemeinsamer Gesang
+            {% elif session['songs_dashboard_category'] == 'Chor' %}
+            Chor
+            {% elif session['songs_dashboard_category'] == 'Gruppenlied' %}
+            Gruppenlied
+            {% else %}
+            Select Category
+            {% endif %}
+          </button>
+          <ul class="dropdown-menu" aria-labelledby="categoryDropdown">
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_category'] == 'Gemeinsamer Gesang' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', category='Gemeinsamer Gesang') }}">
+                Gemeinsamer Gesang
+              </a>
+            </li>
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_category'] == 'Chor' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', category='Chor') }}">
+                Chor
+              </a>
+            </li>
+            <li>
+              <a class="dropdown-item {% if session['songs_dashboard_category'] == 'Gruppenlied' %}active{% endif %}" 
+                href="{{ url_for('songs_dashboard', category='Gruppenlied') }}">
+                Gruppenlied
+              </a>
+            </li>
+          </ul>
+        </div>
+      </div> 
+  
+    <!-- Table Output -->
+    <table class="table table-bordered">
+      <thead>
+        <tr>
+          <th>Anzahl</th>
+          <th>Titel</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for count, titel in performance_data %}
+          <tr>
+            <td>{{ count }}</td>
+            <td>{{ titel }}</td>
+          </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+</body>
+</html>