Merge branch 'development' of gitea.centx.de:lelo/cdh-merger into development

app/app.py

@@ -6,23 +6,26 @@ from flask_session import Session
 
 app = Flask(__name__)
 app.secret_key = "gfbierpf934hftrntr45otgß45890tfh34gft45rw"  # replace with a secure random key
+app.secret_key = "gfbierpf934hftrntr45otgß45890tfh34gft45rw"  # replace with a secure random key
 app.config['SESSION_TYPE'] = 'filesystem'
 app.config['SESSION_FILE_DIR'] = './.flask_session/'
 Session(app)
 
 STRIPE_COLS = ['Type', 'ID', 'Created', 'Description', 'Amount', 'Currency', 'Converted Amount', 'Fees', 'Net', 'Converted Currency', 'Details']
 RAISENOW_COLS = ['Identifikationsnummer', 'Erstellt', 'UTC-Offset', 'Status', 'Betrag', 'Währung', 'Übernommene Gebühren - Betrag', 'Übernommene Gebühren - Währung', 'Zahlungsmethode', 'Zahlungsanbieter', 'Nettobetrag', 'Auszahlungswährung']
+STRIPE_COLS = ['Type', 'ID', 'Created', 'Description', 'Amount', 'Currency', 'Converted Amount', 'Fees', 'Net', 'Converted Currency', 'Details']
+RAISENOW_COLS = ['Identifikationsnummer', 'Erstellt', 'UTC-Offset', 'Status', 'Betrag', 'Währung', 'Übernommene Gebühren - Betrag', 'Übernommene Gebühren - Währung', 'Zahlungsmethode', 'Zahlungsanbieter', 'Nettobetrag', 'Auszahlungswährung']
 
 
-def get_dataframe(key, cols):
+def get_dataframe(key):
     """
-    Load a DataFrame from session or create an empty one with the given columns.
+    Load a DataFrame from session.
     """
     records = session.get(key, [])
     if records:
         df = pd.DataFrame(records)
     else:
-        df = pd.DataFrame(columns=cols)
+        df = pd.DataFrame()
     return df
 
 
@@ -37,11 +40,16 @@ def get_merged_df(table_name):
     """
 
     # --- load & normalize Stripe ---
+    stripe = get_dataframe('stripe_import')
+    if not stripe.empty:
         stripe = (
-        get_dataframe('stripe_import', STRIPE_COLS)
+            stripe
             .query("Type == 'Charge'")
             .copy()
         )
+    else:
+        return stripe
+    
     stripe['idx_stripe']  = stripe.index
     stripe['norm_date']   = pd.to_datetime(stripe['Created'],    format='%Y-%m-%d %H:%M')
     stripe['norm_amount'] = stripe['Amount'].astype(str).str.replace(',', '.').astype(float)
@@ -51,12 +59,16 @@ def get_merged_df(table_name):
     )
     
     # --- load & normalize Raisenow ---
+    raisenow = get_dataframe('raiseNow_import')
+    if not raisenow.empty:
         raisenow = (
-        get_dataframe('raiseNow_import', RAISENOW_COLS)
+            raisenow
             .query("Zahlungsmethode != 'paypal'")
             .query("Status == 'succeeded'")
             .copy()
         )
+    else:
+        return raisenow
     
     raisenow['idx_raisenow'] = raisenow.index
     raisenow['norm_date']    = pd.to_datetime(raisenow['Erstellt'], format='%Y-%m-%d %H:%M')
@@ -77,6 +89,18 @@ def get_merged_df(table_name):
             .str.extract(r'https?://[^/]+/([^/?#]+)')[0]
     )
 
+    # --- return raw tables if requested ---
+    if table_name == 'stripe_import':
+        return stripe.dropna(axis=1, how='all')
+    if table_name == 'raiseNow_import':
+        return raisenow.dropna(axis=1, how='all')
+    # additional assignment: build a mask of rows where norm_zweck is still empty/NaN
+    mask = raisenow['norm_zweck'].isna() | (raisenow['norm_zweck'] == '')
+    raisenow.loc[mask, 'norm_zweck'] = (
+        raisenow.loc[mask, 'raisenow_parameters.product.source_url']
+            .str.extract(r'https?://[^/]+/([^/?#]+)')[0]
+    )
+
     # --- return raw tables if requested ---
     if table_name == 'stripe_import':
         return stripe.dropna(axis=1, how='all')
@@ -87,6 +111,26 @@ def get_merged_df(table_name):
     pairs = []
     # index Raisenow rows for fast lookup + dropping
     rr = raisenow.set_index('idx_raisenow')
+    for _, s in stripe.iterrows():
+        # filter candidates by amount & name
+        cand = rr[
+            (rr['norm_amount'] == s['norm_amount']) &
+            (rr['norm_name']   == s['norm_name'])
+        ].copy()
+        if cand.empty:
+            continue
+        # compute absolute date difference (days only)
+        date_diff = (cand['norm_date'].dt.normalize() - s['norm_date'].normalize()).abs()
+        exact_cand = cand[date_diff == pd.Timedelta(0)]
+        if not exact_cand.empty:
+            # pick the first exact match
+            best = exact_cand.index[0]
+            pairs.append((int(s['idx_stripe']), int(best)))
+            rr = rr.drop(best)
+    # --- 1) Greedy exact same-day matches ---
+    pairs = []
+    # index Raisenow rows for fast lookup + dropping
+    rr = raisenow.set_index('idx_raisenow')
     for _, s in stripe.iterrows():
         # filter candidates by amount & name
         cand = rr[
@@ -137,6 +181,40 @@ def get_merged_df(table_name):
 
     combined = pd.DataFrame(merged_rows)
 
+    # --- slice out the requested view ---
+    # --- 2) Greedy fuzzy ±1-day matches on remaining rows ---
+    used_stripe = {s for s, _ in pairs}
+    stripe_left = stripe[~stripe['idx_stripe'].isin(used_stripe)].copy()
+    for _, s in stripe_left.iterrows():
+        cand = rr[
+            (rr['norm_amount'] == s['norm_amount']) &
+            (rr['norm_name']   == s['norm_name'])
+        ].copy()
+        if cand.empty:
+            continue
+        date_diff = (cand['norm_date'].dt.normalize() - s['norm_date'].normalize()).abs()
+        cand = cand[date_diff <= pd.Timedelta(days=1)]
+        if cand.empty:
+            continue
+        # pick the one with the smallest gap
+        best = date_diff.idxmin()
+        pairs.append((int(s['idx_stripe']), int(best)))
+        rr = rr.drop(best)
+
+    # --- build the merged DataFrame without suffixes ---
+    merged_rows = []
+    for s_idx, r_idx in pairs:
+        srow = stripe.loc[s_idx].to_dict()
+        rrow = raisenow.loc[r_idx].to_dict()
+        # drop any overlapping keys so we never get suffixes
+        for k in ['norm_amount','norm_name','norm_date','norm_email','idx_stripe']:
+            rrow.pop(k, None)
+        # now combine so stripe values win for those keys, and raisenow adds its own columns
+        merged = {**srow, **rrow}
+        merged_rows.append(merged)
+
+    combined = pd.DataFrame(merged_rows)
+
     # --- slice out the requested view ---
     if table_name == 'merged':
         result = combined
@@ -180,7 +258,7 @@ def upload():
         else:
             continue
 
-        existing = get_dataframe(key, [])
+        existing = get_dataframe(key)
         combined = pd.concat([existing, raw], ignore_index=True)
         deduped = combined.drop_duplicates(subset=[dedupe_col], keep='first').reset_index(drop=True)
 
@@ -215,6 +293,16 @@ def download():
             'raisenow_only'
         ]
     }
+    sheets = {
+        name: get_merged_df(name)
+        for name in [
+            'stripe_import',
+            'raiseNow_import',
+            'merged',
+            'stripe_only',
+            'raisenow_only'
+        ]
+    }
 
     output = BytesIO()
     with pd.ExcelWriter(output, engine='xlsxwriter') as writer:

app/templates/index.html

@@ -3,32 +3,46 @@
 <head>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
-  <title>Multi‐Table Excel Import</title>
+  <title>CDH Merger</title>
   <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
   <link href="https://unpkg.com/tabulator-tables@5.4.4/dist/css/tabulator.min.css" rel="stylesheet">
   <style>
     body { padding: 2rem; background: #f8f9fa; }
-    #table-container { height: 600px; margin-top: 1rem; }
+    #table-container { height: 800px; width: 100%; }
+    #table { width: 100%; }
+
+    /* Loading Overlay */
+    #loadingOverlay {
+      position: fixed;
+      top: 0;
+      left: 0;
+      width: 100vw;
+      height: 100vh;
+      background: rgba(0, 0, 0, 0.5);
+      display: none;
+      align-items: center;
+      justify-content: center;
+      z-index: 9999;
+    }
+
+    /* Ensure toolbar stretches full width */
+    #toolbar { width: 100%; }
   </style>
 </head>
 <body>
-  <div class="container">
+  <div class="container-fluid px-0">
-    <h1 class="mb-4">Excel Importer</h1>
+    <h1 class="mb-4 ps-3">CDH Merger</h1>
 
-    <div class="mb-3">
+    <!-- Toolbar -->
-      <form id="upload-form" class="mb-3">
+    <div class="d-flex flex-wrap align-items-center mb-3 px-3" id="toolbar">
-        <input type="file" name="files" … multiple>
+      <form id="upload-form" class="d-flex align-items-center me-3">
-        <button type="submit" class="btn btn-primary">Upload Files</button>
+        <input type="file" name="files" multiple class="form-control form-control-sm me-2">
+        <button type="submit" class="btn btn-primary btn-sm">Upload</button>
       </form>
-      <button id="download-excel" class="btn btn-success">Download All Tables</button>
+      <button id="download-excel" class="btn btn-success btn-sm me-3">Download All</button>
-    </div>
+      <div class="d-flex align-items-center">
-
+        <label for="table-select" class="form-label me-2 mb-0">Tabelle:</label>
-    <div class="row g-3 align-items-center">
+        <select id="table-select" class="form-select form-select-sm">
-      <div class="col-auto">
-        <label for="table-select" class="col-form-label">Select Table:</label>
-      </div>
-      <div class="col-auto">
-        <select id="table-select" class="form-select">
           <option value="stripe_import">Stripe Import</option>
           <option value="raiseNow_import">RaiseNow Import</option>
           <option value="merged">Merged</option>
@@ -38,70 +52,81 @@
       </div>
     </div>
 
-    <div id="table-container">
+    <!-- Table -->
+    <div id="table-container" class="border rounded shadow-sm px-3">
       <div id="table"></div>
     </div>
   </div>
 
+  <!-- Loading Overlay -->
+  <div id="loadingOverlay">
+    <div class="text-center text-white">
+      <div class="spinner-border" role="status" style="width: 4rem; height: 4rem;"><span class="visually-hidden">Loading...</span></div>
+      <p class="mt-3">Loading, please wait...</p>
+    </div>
+  </div>
+
   <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
   <script src="https://unpkg.com/tabulator-tables@5.4.4/dist/js/tabulator.min.js"></script>
   <script>
     const uploadForm = document.getElementById('upload-form');
     const tableSelect = document.getElementById('table-select');
+    const downloadBtn = document.getElementById('download-excel');
+    const loadingOverlay = document.getElementById('loadingOverlay');
     let table;
 
+    function showLoading() { loadingOverlay.style.display = 'flex'; }
+    function hideLoading() { loadingOverlay.style.display = 'none'; }
+
     uploadForm.addEventListener('submit', async e => {
-      e.preventDefault();
+      e.preventDefault(); showLoading();
+      try {
         const fd = new FormData(uploadForm);
-      const resp = await fetch('/upload', { method:'POST', body:fd });
+        const resp = await fetch('/upload', { method: 'POST', body: fd });
         const res = await resp.json();
-      if (resp.ok) {
+        if (resp.ok) await loadTable(tableSelect.value);
-        loadTable(tableSelect.value);
+        else alert(res.error || 'Upload failed');
-      } else alert(res.error || 'Upload failed');
+      } catch (err) {
+        console.error(err); alert('An error occurred');
+      } finally { hideLoading(); }
     });
 
     tableSelect.addEventListener('change', () => loadTable(tableSelect.value));
 
-    async function loadTable(name) {
+    downloadBtn.addEventListener('click', async () => {
-
+      showLoading();
-      // fetch data
+      try {
-      const resp = await fetch(`/get_table?table=${name}`);
+        const resp = await fetch('/download');
-      const json = await resp.json();
+        if (!resp.ok) throw new Error('Download failed');
-      
+        const blob = await resp.blob();
-      // error handling
+        const url = URL.createObjectURL(blob);
-      if (!resp.ok) {
+        const a = document.createElement('a');
-        return alert(json.error || 'Error loading');
+        a.href = url;
-      }
+        const disposition = resp.headers.get('Content-Disposition') || '';
-
+        const match = disposition.match(/filename="?([^";]+)"?/);
-      // column definitions
+        a.download = match ? match[1] : 'tables.xlsx';
-      const cols = json.columns.map(c => ({
+        document.body.appendChild(a); a.click(); a.remove(); URL.revokeObjectURL(url);
-        title: c,
+      } catch (err) {
-        field: c,
+        console.error(err); alert(err.message);
-        headerFilter: true
+      } finally { hideLoading(); }
-      }));
-
-      if (table) {
-        // update columns
-        table.setColumns(cols);
-        // returns a promise once render is done
-        table.replaceData(json.data);
-      } else {
-        // options for table
-        const opts = {
-          data: json.data,
-          layout: 'fitData',
-          height: '100%',
-          columns: cols
-        };
-        table = new Tabulator('#table', opts);
-      }
-    }
-
-    document.getElementById('download-excel')
-      .addEventListener('click', () => {
-        window.location = '/download';
     });
 
+    async function loadTable(name) {
+      showLoading();
+      try {
+        const resp = await fetch(`/get_table?table=${encodeURIComponent(name)}`);
+        const json = await resp.json();
+        if (!resp.ok) { alert(json.error || 'Error loading'); return; }
+        if (!Array.isArray(json.data) || !json.data.length) { alert('No data for this table'); return; }
+        const cols = json.columns.map(c => ({ title: c, field: c, headerFilter: true }));
+        const opts = { data: json.data, layout: 'fitData', height: '100%', columns: cols, responsiveLayout: 'hide' };
+        if (table) { table.setColumns(cols); await table.replaceData(json.data); }
+        else { table = new Tabulator('#table', opts); }
+      } catch (err) {
+        console.error(err); alert('Failed to load table data');
+      } finally { hideLoading(); }
+    }
+
     // initialize
     loadTable(tableSelect.value);
   </script>