analytics.py

@@ -3,8 +3,6 @@ from flask import render_template, request, session
 from datetime import datetime, timedelta, timezone
 import geoip2.database
 from auth import require_secret
-from collections import defaultdict
-import json
 import os
 
 file_access_temp = []
@@ -14,7 +12,6 @@ DB_NAME = 'access_log.db'
 
 # Create a single global connection to SQLite
 log_db = sqlite3.connect(DB_NAME, check_same_thread=False)
-search_db = sqlite3.connect("search.db", check_same_thread=False)
 
 # geo location
 geoReader = geoip2.database.Reader('GeoLite2-City.mmdb')
@@ -140,57 +137,6 @@ def return_file_access():
     else:
         return []
 
-def songs_dashboard():
-    if 'songs_dashboard_timeframe' not in session:
-        session['songs_dashboard_timeframe'] = "30"
-    timeframe_param = request.args.get("timeframe", session['songs_dashboard_timeframe'])
-    session['songs_dashboard_timeframe'] = timeframe_param
-    
-    if 'songs_dashboard_category' not in session:
-        session['songs_dashboard_category'] = "Gemeinsamer Gesang"
-    category = request.args.get("category", session['songs_dashboard_category'])
-    session['songs_dashboard_category'] = category
-    
-    if 'songs_dashboard_site' not in session:
-        session['songs_dashboard_site'] = "Speyer"
-    site = request.args.get("site", session['songs_dashboard_site'])
-    session['songs_dashboard_site'] = site
-    
-    # Determine cutoff_date based on the days parameter
-    if timeframe_param == "all":
-        cutoff_date = None  # No date filtering when analyzing all time
-        timeframe = "all"   # Pass the string to the template if needed
-    else:
-        timeframe = int(timeframe_param)
-        now = datetime.now()
-        cutoff_date = now - timedelta(days=timeframe)
-    
-    cursor = search_db.cursor()
-    # Query rows with category "Gemeinsamer Gesang"
-    query = "SELECT titel, performance_date FROM files WHERE category = ? and site = ?"
-    cursor.execute(query, (category, site))
-    rows = cursor.fetchall()
-    
-    # Group and count performances per titel (only if performance_date is within the timeframe,
-    # or count all if cutoff_date is None)
-    performance_counts = defaultdict(int)
-    for titel, performance_date in rows:
-        if performance_date:
-            try:
-                # Convert date from "dd.mm.yyyy" format
-                date_obj = datetime.strptime(performance_date, "%d.%m.%Y")
-            except ValueError:
-                continue
-            # If cutoff_date is None, count all dates; otherwise, filter by cutoff_date.
-            if cutoff_date is None or date_obj >= cutoff_date:
-                performance_counts[titel] += 1
-    
-    # Create a list of tuples: (count, titel), sorted in descending order by count.
-    performance_data = [(count, titel) for titel, count in performance_counts.items()]
-    performance_data.sort(reverse=True, key=lambda x: x[0])
-    
-    return render_template('songs_dashboard.html', timeframe=timeframe_param, performance_data=performance_data, site=site, category=category)
-
 @require_secret
 def connections():
     return render_template('connections.html')
@@ -478,31 +424,3 @@ def dashboard():
         cached_percentage=cached_percentage,
         timeframe_data=timeframe_data
     )
-
-def export_to_excel():
-    """Export search_db to an Excel file and store it locally."""
-    import pandas as pd
-
-    # Query all data from the search_db
-    query = "SELECT * FROM files"
-    cursor = search_db.cursor()
-    cursor.execute(query)
-    rows = cursor.fetchall()
-
-    # Get column names from the cursor description
-    column_names = [description[0] for description in cursor.description]
-
-    # Create a DataFrame and save it to an Excel file
-    df = pd.DataFrame(rows, columns=column_names)
-    df = df.drop(columns=['transcript'], errors='ignore')  # Drop the 'id' column if it exists   
-    df.to_excel("search_db.xlsx", index=False)
-
-    # Close the cursor and database connection
-    cursor.close()
-
-if __name__ == "__main__":
-    print("Running as a standalone script.")
-    export_to_excel()
-    print("Exported search_db to search_db.xlsx")
-    
-    

app.py

@@ -41,8 +41,6 @@ app.add_url_rule('/remove_secret', view_func=auth.remove_secret, methods=['POST'
 app.add_url_rule('/search', view_func=search.search, methods=['GET'])
 app.add_url_rule('/searchcommand', view_func=search.searchcommand, methods=['POST'])
 
-app.add_url_rule('/songs_dashboard', view_func=a.songs_dashboard)
-
 
 # Grab the HOST_RULE environment variable
 host_rule = os.getenv("HOST_RULE", "")

index_for_search.py

@@ -1,8 +1,6 @@
 import os
 import json
 import sqlite3
-from datetime import datetime
-import re
 
 SEARCH_DB_NAME = 'search.db'
 ACCESS_LOG_DB_NAME = 'access_log.db'
@@ -27,11 +25,6 @@ def init_db():
             basefolder TEXT,
             filename TEXT,
             filetype TEXT,
-            category TEXT,
-            titel TEXT,
-            name TEXT,
-            performance_date TEXT,
-            site TEXT,
             transcript TEXT,
             hitcount INTEGER DEFAULT 0,
             UNIQUE(relative_path, filename)
@@ -49,31 +42,6 @@ def init_db():
     except sqlite3.OperationalError:
         # Likely the column already exists, so we ignore this error.
         pass
-    try:
-        cursor.execute("ALTER TABLE files ADD COLUMN category TEXT")
-    except sqlite3.OperationalError:
-        # Likely the column already exists, so we ignore this error.
-        pass
-    try:
-        cursor.execute("ALTER TABLE files ADD COLUMN titel TEXT")
-    except sqlite3.OperationalError:
-        # Likely the column already exists, so we ignore this error.
-        pass
-    try:
-        cursor.execute("ALTER TABLE files ADD COLUMN name TEXT")
-    except sqlite3.OperationalError:
-        # Likely the column already exists, so we ignore this error.
-        pass
-    try:
-        cursor.execute("ALTER TABLE files ADD COLUMN performance_date TEXT")
-    except sqlite3.OperationalError:
-        # Likely the column already exists, so we ignore this error.
-        pass
-    try:
-        cursor.execute("ALTER TABLE files ADD COLUMN site TEXT")
-    except sqlite3.OperationalError:
-        # Likely the column already exists, so we ignore this error.
-        pass
     search_db.commit()
 
 def scan_dir(directory):
@@ -118,6 +86,7 @@ def updatefileindex():
             # Accumulate scanned file data and keys for this base folder.
             scanned_files = []  # Each entry: (relative_path, basefolder, filename, filetype, transcript, hitcount)
             current_keys = set()
+
             for entry in scan_dir(norm_folderpath):
                 entry_path = os.path.normpath(entry.path)
                 # Get relative part by slicing if possible.
@@ -128,92 +97,24 @@ def updatefileindex():
                 # Prepend the foldername so it becomes part of the stored relative path.
                 relative_path = os.path.join(foldername, rel_part).replace(os.sep, '/')
                 filetype = os.path.splitext(entry.name)[1].lower()
-                
+                transcript = None
+
+                # Check for a corresponding transcript file in a sibling "Transkription" folder.
+                parent_dir = os.path.dirname(entry_path)
+                transcript_dir = os.path.join(parent_dir, "Transkription")
+                transcript_filename = os.path.splitext(entry.name)[0] + ".md"
+                transcript_path = os.path.join(transcript_dir, transcript_filename)
+                if os.path.exists(transcript_path):
+                    try:
+                        with open(transcript_path, 'r', encoding='utf-8') as tf:
+                            transcript = tf.read()
+                    except Exception:
+                        transcript = None
+
                 # Retrieve the hit count for this file.
                 hit_count = get_hit_count(relative_path)
-                
-                category, titel, name, performance_date, site = None, None, None, None, None
-                
-                # Determine the site
-                if foldername == 'Gottesdienste Speyer':
-                    site = 'Speyer'
-                elif foldername == 'Gottesdienste Schwegenheim':
-                    site = 'Schwegenheim'
-                
-                if filetype == '.mp3':
-                    transcript = None
 
-                    # Check for a corresponding transcript file in a sibling "Transkription" folder.
-                    parent_dir = os.path.dirname(entry_path)
-                    transcript_dir = os.path.join(parent_dir, "Transkription")
-                    transcript_filename = os.path.splitext(entry.name)[0] + ".md"
-                    transcript_path = os.path.join(transcript_dir, transcript_filename)
-                    if os.path.exists(transcript_path):
-                        try:
-                            with open(transcript_path, 'r', encoding='utf-8') as tf:
-                                transcript = tf.read()
-                        except Exception:
-                            transcript = None
-
-                    # extract category and titel from filename
-                    filename_ext = os.path.splitext(entry.name)[0]
-                    left_side, right_side = filename_ext.split('-', 1) if '-' in filename_ext else (filename_ext, None)
-                    try:
-                        int(left_side.strip())
-                        # first part is only a number
-                        previous_right_side = right_side
-                        left_side, right_side = previous_right_side.split('-', 1) if '-' in previous_right_side else (previous_right_side, None)
-                    except:
-                        # first part not a number
-                        continue
-                    
-                    if 'predig' in left_side.lower():
-                        category = 'Predigt'
-                    elif 'wort' in left_side.lower() or 'einladung' in left_side.lower():
-                        category = 'Vorwort'
-                    elif 'chor' in left_side.lower():
-                        category = 'Chor'
-                    elif 'orchester' in left_side.lower():
-                        category = 'Orchester'
-                    elif 'gruppenlied' in left_side.lower() or 'jugendlied' in left_side.lower():
-                        category = 'Gruppenlied'
-                    elif 'gemeinsam' in left_side.lower() or 'gesang' in left_side.lower() or 'lied' in left_side.lower():
-                        category = 'Gemeinsamer Gesang'
-                    elif 'gedicht' in left_side.lower():
-                        category = 'Gedicht'
-                    elif 'instrumental' in left_side.lower() or 'musikstück' in left_side.lower():
-                        category = 'Instrumental'
-                    else:
-                        category = None
-
-                    if right_side:
-                        titel, name = right_side.split('-', 1) if '-' in right_side else (right_side, None)
-                        if category == 'Predigt' or category == 'Vorwort' or category == 'Gedicht':
-                            if not name: # kein Titel, nur Name
-                                name = titel
-                                titel = None
-                    else:
-                        titel = None
-                        name = None
-                    
-                # extract the date from path using regex (dd.mm.yyyy or dd.mm.yy)
-                date_match = re.search(r'(\d{1,2}\.\d{1,2}\.\d{2,4})', relative_path)
-                if date_match:
-                    date_str = date_match.group(1)
-                    # Convert to YYYY-MM-DD format
-                    try:
-                        date_obj = datetime.strptime(date_str, '%d.%m.%Y')
-                        performance_date = date_obj.strftime('%d.%m.%Y')
-                    except ValueError:
-                        try:
-                            date_obj = datetime.strptime(date_str, '%d.%m.%y')
-                            performance_date = date_obj.strftime('%d.%m.%Y')
-                        except ValueError:
-                            performance_date = None
-                else:
-                    performance_date = None
-
-                scanned_files.append((relative_path, foldername, entry.name, filetype, category, titel, name, performance_date, site, transcript, hit_count))
+                scanned_files.append((relative_path, foldername, entry.name, filetype, transcript, hit_count))
                 current_keys.add((relative_path, entry.name))
 
             # Remove database entries for files under this base folder that are no longer on disk.
@@ -227,7 +128,7 @@ def updatefileindex():
 
             # Bulk write the scanned files using INSERT OR REPLACE.
             cursor.executemany(
-                "INSERT OR REPLACE INTO files (relative_path, basefolder, filename, filetype, category, titel, name, performance_date, site, transcript, hitcount) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                "INSERT OR REPLACE INTO files (relative_path, basefolder, filename, filetype, transcript, hitcount) VALUES (?, ?, ?, ?, ?, ?)",
                 scanned_files
             )
 

requirements.txt

@@ -4,7 +4,5 @@ pillow
 qrcode
 diskcache
 geoip2
-pandas
-openpyxl
 gunicorn
 eventlet

templates/connections.html

@@ -67,11 +67,18 @@
         <a class="navbar-brand" href="#">Downloads der letzten 10 Minuten</a>
         <div class="navbar-collapse" id="navbarNav">
           <ul class="navbar-nav ms-auto">
-            <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
-            <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
-            <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
-            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
-            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
+            <li class="nav-item">
+              <a href="{{ url_for('index') }}" class="nav-link">App</a>
+            </li>
+            <li class="nav-item">
+              <a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a>
+            </li>
+            <li class="nav-item">
+              <a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a>
+            </li>
+            <li class="nav-item">
+              <a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung</a>
+            </li>
           </ul>
         </div>
       </div>

templates/dashboard.html

@@ -31,8 +31,7 @@
             <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
             <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
             <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
-            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
-            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
+            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung</a></li>
           </ul>
         </div>
       </div>
@@ -273,8 +272,8 @@
     </div>
 
     <!-- Scripts -->
-    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
     <script>
       // Data passed from the backend as JSON
       let distinctDeviceData = {{ distinct_device_data|tojson }};

templates/mylinks.html

@@ -33,8 +33,7 @@
             <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
             <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
             <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
-            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
-            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
+            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung</a></li>
           </ul>
         </div>
       </div>

templates/songs_dashboard.html

@@ -1,182 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>Dashboard - Verbindungsanalyse</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet">
-    <style>
-      html, body {
-        height: 100%;
-        margin: 0;
-        padding: 0;
-      }
-      .container-fluid {
-        height: 100%;
-        display: flex;
-        flex-direction: column;
-      }
-      .card {
-        margin-bottom: 1.5rem;
-      }
-    </style>
-  </head>
-  <body>
-    <!-- Navigation Bar -->
-    <nav class="navbar navbar-expand-lg navbar-dark bg-secondary mb-3">
-      <div class="container-fluid">
-        <a class="navbar-brand" href="#">Dashboard - Analyse Wiederholungen</a>
-        <div class="navbar-collapse" id="navbarNav">
-          <ul class="navbar-nav ms-auto">
-            <li class="nav-item"><a href="{{ url_for('index') }}" class="nav-link">App</a></li>
-            <li class="nav-item"><a href="{{ url_for('mylinks') }}" class="nav-link">Meine Links</a></li>
-            <li class="nav-item"><a href="{{ url_for('connections') }}" class="nav-link">Verbindungen</a></li>
-            <li class="nav-item"><a href="{{ url_for('dashboard') }}" class="nav-link">Auswertung-Downloads</a></li>
-            <li class="nav-item"><a href="{{ url_for('songs_dashboard') }}" class="nav-link">Auswertung-Wiederholungen</a></li>
-          </ul>
-        </div>
-      </div>
-    </nav>
-
-    <!-- Main Container -->
-    <div class="container-fluid px-4">
-      <!-- Dropdown Controls -->
-      <div class="mb-4 d-flex flex-wrap gap-2">
-
-        <!-- Site Dropdown -->
-        <div class="dropdown">
-          <button class="btn btn-secondary dropdown-toggle" 
-                  type="button" id="siteDropdown" data-bs-toggle="dropdown" aria-expanded="false">
-            {% if session['songs_dashboard_site'] == 'Speyer' %}
-              Speyer
-            {% elif session['songs_dashboard_site'] == 'Schwegenheim' %}
-              Schwegenheim
-            {% else %}
-              Select Site
-            {% endif %}
-          </button>
-          <ul class="dropdown-menu" aria-labelledby="siteDropdown">
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_site'] == 'Speyer' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', site='Speyer') }}">
-                Speyer
-              </a>
-            </li>
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_site'] == 'Schwegenheim' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', site='Schwegenheim') }}">
-                Schwegenheim
-              </a>
-            </li>
-          </ul>
-        </div>
-        
-        <!-- Timeframe Dropdown -->
-        <div class="dropdown">
-          <button class="btn btn-secondary dropdown-toggle" 
-                  type="button" id="timeframeDropdown" data-bs-toggle="dropdown" aria-expanded="false">
-            {% if session['songs_dashboard_timeframe'] == '7' %}
-              Last 7 Days
-            {% elif session['songs_dashboard_timeframe'] == '30' %}
-              Last 30 Days
-            {% elif session['songs_dashboard_timeframe'] == '90' %}
-              Last 90 Days
-            {% elif session['songs_dashboard_timeframe'] == '365' %}
-              Last 365 Days
-            {% elif session['songs_dashboard_timeframe'] == 'all' %}
-              All Data
-            {% else %}
-              Select Timeframe
-            {% endif %}
-          </button>
-          <ul class="dropdown-menu" aria-labelledby="timeframeDropdown">
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '7' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', timeframe='7') }}">
-                Last 7 Days
-              </a>
-            </li>
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '30' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', timeframe='30') }}">
-                Last 30 Days
-              </a>
-            </li>
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '90' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', timeframe='90') }}">
-                Last 90 Days
-              </a>
-            </li>
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == '365' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', timeframe='365') }}">
-                Last 365 Days
-              </a>
-            </li>
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_timeframe'] == 'all' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', timeframe='all') }}">
-                All Data
-              </a>
-            </li>
-          </ul>
-        </div>
-
-        <!-- category Dropdown -->
-        <div class="dropdown">
-          <button class="btn btn-secondary dropdown-toggle" type="button" id="categoryDropdown" data-bs-toggle="dropdown" aria-expanded="false">
-            {% if session['songs_dashboard_category'] == 'Gemeinsamer Gesang' %}
-            Gemeinsamer Gesang
-            {% elif session['songs_dashboard_category'] == 'Chor' %}
-            Chor
-            {% elif session['songs_dashboard_category'] == 'Gruppenlied' %}
-            Gruppenlied
-            {% else %}
-            Select Category
-            {% endif %}
-          </button>
-          <ul class="dropdown-menu" aria-labelledby="categoryDropdown">
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_category'] == 'Gemeinsamer Gesang' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', category='Gemeinsamer Gesang') }}">
-                Gemeinsamer Gesang
-              </a>
-            </li>
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_category'] == 'Chor' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', category='Chor') }}">
-                Chor
-              </a>
-            </li>
-            <li>
-              <a class="dropdown-item {% if session['songs_dashboard_category'] == 'Gruppenlied' %}active{% endif %}" 
-                href="{{ url_for('songs_dashboard', category='Gruppenlied') }}">
-                Gruppenlied
-              </a>
-            </li>
-          </ul>
-        </div>
-      </div> 
-  
-    <!-- Table Output -->
-    <table class="table table-bordered">
-      <thead>
-        <tr>
-          <th>Anzahl</th>
-          <th>Titel</th>
-        </tr>
-      </thead>
-      <tbody>
-        {% for count, titel in performance_data %}
-          <tr>
-            <td>{{ count }}</td>
-            <td>{{ titel }}</td>
-          </tr>
-        {% endfor %}
-      </tbody>
-    </table>
-  </div>
-<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
-</body>
-</html>