Compare commits
No commits in common. "01152c5f82e818508328bfafa966a4d9f02db137" and "ee3f5c062c053ed3e1bce0d519177582371cd559" have entirely different histories.
01152c5f82
...
ee3f5c062c
100
app/app.py
100
app/app.py
@ -6,26 +6,23 @@ from flask_session import Session
|
||||
|
||||
app = Flask(__name__)
|
||||
app.secret_key = "gfbierpf934hftrntr45otgß45890tfh34gft45rw" # replace with a secure random key
|
||||
app.secret_key = "gfbierpf934hftrntr45otgß45890tfh34gft45rw" # replace with a secure random key
|
||||
app.config['SESSION_TYPE'] = 'filesystem'
|
||||
app.config['SESSION_FILE_DIR'] = './.flask_session/'
|
||||
Session(app)
|
||||
|
||||
STRIPE_COLS = ['Type', 'ID', 'Created', 'Description', 'Amount', 'Currency', 'Converted Amount', 'Fees', 'Net', 'Converted Currency', 'Details']
|
||||
RAISENOW_COLS = ['Identifikationsnummer', 'Erstellt', 'UTC-Offset', 'Status', 'Betrag', 'Währung', 'Übernommene Gebühren - Betrag', 'Übernommene Gebühren - Währung', 'Zahlungsmethode', 'Zahlungsanbieter', 'Nettobetrag', 'Auszahlungswährung']
|
||||
STRIPE_COLS = ['Type', 'ID', 'Created', 'Description', 'Amount', 'Currency', 'Converted Amount', 'Fees', 'Net', 'Converted Currency', 'Details']
|
||||
RAISENOW_COLS = ['Identifikationsnummer', 'Erstellt', 'UTC-Offset', 'Status', 'Betrag', 'Währung', 'Übernommene Gebühren - Betrag', 'Übernommene Gebühren - Währung', 'Zahlungsmethode', 'Zahlungsanbieter', 'Nettobetrag', 'Auszahlungswährung']
|
||||
|
||||
|
||||
def get_dataframe(key):
|
||||
def get_dataframe(key, cols):
|
||||
"""
|
||||
Load a DataFrame from session.
|
||||
Load a DataFrame from session or create an empty one with the given columns.
|
||||
"""
|
||||
records = session.get(key, [])
|
||||
if records:
|
||||
df = pd.DataFrame(records)
|
||||
else:
|
||||
df = pd.DataFrame()
|
||||
df = pd.DataFrame(columns=cols)
|
||||
return df
|
||||
|
||||
|
||||
@ -40,16 +37,11 @@ def get_merged_df(table_name):
|
||||
"""
|
||||
|
||||
# --- load & normalize Stripe ---
|
||||
stripe = get_dataframe('stripe_import')
|
||||
if not stripe.empty:
|
||||
stripe = (
|
||||
stripe
|
||||
get_dataframe('stripe_import', STRIPE_COLS)
|
||||
.query("Type == 'Charge'")
|
||||
.copy()
|
||||
)
|
||||
else:
|
||||
return stripe
|
||||
|
||||
stripe['idx_stripe'] = stripe.index
|
||||
stripe['norm_date'] = pd.to_datetime(stripe['Created'], format='%Y-%m-%d %H:%M')
|
||||
stripe['norm_amount'] = stripe['Amount'].astype(str).str.replace(',', '.').astype(float)
|
||||
@ -59,16 +51,12 @@ def get_merged_df(table_name):
|
||||
)
|
||||
|
||||
# --- load & normalize Raisenow ---
|
||||
raisenow = get_dataframe('raiseNow_import')
|
||||
if not raisenow.empty:
|
||||
raisenow = (
|
||||
raisenow
|
||||
get_dataframe('raiseNow_import', RAISENOW_COLS)
|
||||
.query("Zahlungsmethode != 'paypal'")
|
||||
.query("Status == 'succeeded'")
|
||||
.copy()
|
||||
)
|
||||
else:
|
||||
return raisenow
|
||||
|
||||
raisenow['idx_raisenow'] = raisenow.index
|
||||
raisenow['norm_date'] = pd.to_datetime(raisenow['Erstellt'], format='%Y-%m-%d %H:%M')
|
||||
@ -89,18 +77,6 @@ def get_merged_df(table_name):
|
||||
.str.extract(r'https?://[^/]+/([^/?#]+)')[0]
|
||||
)
|
||||
|
||||
# --- return raw tables if requested ---
|
||||
if table_name == 'stripe_import':
|
||||
return stripe.dropna(axis=1, how='all')
|
||||
if table_name == 'raiseNow_import':
|
||||
return raisenow.dropna(axis=1, how='all')
|
||||
# additional assignment: build a mask of rows where norm_zweck is still empty/NaN
|
||||
mask = raisenow['norm_zweck'].isna() | (raisenow['norm_zweck'] == '')
|
||||
raisenow.loc[mask, 'norm_zweck'] = (
|
||||
raisenow.loc[mask, 'raisenow_parameters.product.source_url']
|
||||
.str.extract(r'https?://[^/]+/([^/?#]+)')[0]
|
||||
)
|
||||
|
||||
# --- return raw tables if requested ---
|
||||
if table_name == 'stripe_import':
|
||||
return stripe.dropna(axis=1, how='all')
|
||||
@ -111,26 +87,6 @@ def get_merged_df(table_name):
|
||||
pairs = []
|
||||
# index Raisenow rows for fast lookup + dropping
|
||||
rr = raisenow.set_index('idx_raisenow')
|
||||
for _, s in stripe.iterrows():
|
||||
# filter candidates by amount & name
|
||||
cand = rr[
|
||||
(rr['norm_amount'] == s['norm_amount']) &
|
||||
(rr['norm_name'] == s['norm_name'])
|
||||
].copy()
|
||||
if cand.empty:
|
||||
continue
|
||||
# compute absolute date difference (days only)
|
||||
date_diff = (cand['norm_date'].dt.normalize() - s['norm_date'].normalize()).abs()
|
||||
exact_cand = cand[date_diff == pd.Timedelta(0)]
|
||||
if not exact_cand.empty:
|
||||
# pick the first exact match
|
||||
best = exact_cand.index[0]
|
||||
pairs.append((int(s['idx_stripe']), int(best)))
|
||||
rr = rr.drop(best)
|
||||
# --- 1) Greedy exact same-day matches ---
|
||||
pairs = []
|
||||
# index Raisenow rows for fast lookup + dropping
|
||||
rr = raisenow.set_index('idx_raisenow')
|
||||
for _, s in stripe.iterrows():
|
||||
# filter candidates by amount & name
|
||||
cand = rr[
|
||||
@ -181,40 +137,6 @@ def get_merged_df(table_name):
|
||||
|
||||
combined = pd.DataFrame(merged_rows)
|
||||
|
||||
# --- slice out the requested view ---
|
||||
# --- 2) Greedy fuzzy ±1-day matches on remaining rows ---
|
||||
used_stripe = {s for s, _ in pairs}
|
||||
stripe_left = stripe[~stripe['idx_stripe'].isin(used_stripe)].copy()
|
||||
for _, s in stripe_left.iterrows():
|
||||
cand = rr[
|
||||
(rr['norm_amount'] == s['norm_amount']) &
|
||||
(rr['norm_name'] == s['norm_name'])
|
||||
].copy()
|
||||
if cand.empty:
|
||||
continue
|
||||
date_diff = (cand['norm_date'].dt.normalize() - s['norm_date'].normalize()).abs()
|
||||
cand = cand[date_diff <= pd.Timedelta(days=1)]
|
||||
if cand.empty:
|
||||
continue
|
||||
# pick the one with the smallest gap
|
||||
best = date_diff.idxmin()
|
||||
pairs.append((int(s['idx_stripe']), int(best)))
|
||||
rr = rr.drop(best)
|
||||
|
||||
# --- build the merged DataFrame without suffixes ---
|
||||
merged_rows = []
|
||||
for s_idx, r_idx in pairs:
|
||||
srow = stripe.loc[s_idx].to_dict()
|
||||
rrow = raisenow.loc[r_idx].to_dict()
|
||||
# drop any overlapping keys so we never get suffixes
|
||||
for k in ['norm_amount','norm_name','norm_date','norm_email','idx_stripe']:
|
||||
rrow.pop(k, None)
|
||||
# now combine so stripe values win for those keys, and raisenow adds its own columns
|
||||
merged = {**srow, **rrow}
|
||||
merged_rows.append(merged)
|
||||
|
||||
combined = pd.DataFrame(merged_rows)
|
||||
|
||||
# --- slice out the requested view ---
|
||||
if table_name == 'merged':
|
||||
result = combined
|
||||
@ -258,7 +180,7 @@ def upload():
|
||||
else:
|
||||
continue
|
||||
|
||||
existing = get_dataframe(key)
|
||||
existing = get_dataframe(key, [])
|
||||
combined = pd.concat([existing, raw], ignore_index=True)
|
||||
deduped = combined.drop_duplicates(subset=[dedupe_col], keep='first').reset_index(drop=True)
|
||||
|
||||
@ -293,16 +215,6 @@ def download():
|
||||
'raisenow_only'
|
||||
]
|
||||
}
|
||||
sheets = {
|
||||
name: get_merged_df(name)
|
||||
for name in [
|
||||
'stripe_import',
|
||||
'raiseNow_import',
|
||||
'merged',
|
||||
'stripe_only',
|
||||
'raisenow_only'
|
||||
]
|
||||
}
|
||||
|
||||
output = BytesIO()
|
||||
with pd.ExcelWriter(output, engine='xlsxwriter') as writer:
|
||||
|
||||
@ -3,46 +3,32 @@
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>CDH Merger</title>
|
||||
<title>Multi‐Table Excel Import</title>
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link href="https://unpkg.com/tabulator-tables@5.4.4/dist/css/tabulator.min.css" rel="stylesheet">
|
||||
<style>
|
||||
body { padding: 2rem; background: #f8f9fa; }
|
||||
#table-container { height: 800px; width: 100%; }
|
||||
#table { width: 100%; }
|
||||
|
||||
/* Loading Overlay */
|
||||
#loadingOverlay {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 9999;
|
||||
}
|
||||
|
||||
/* Ensure toolbar stretches full width */
|
||||
#toolbar { width: 100%; }
|
||||
#table-container { height: 600px; margin-top: 1rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container-fluid px-0">
|
||||
<h1 class="mb-4 ps-3">CDH Merger</h1>
|
||||
<div class="container">
|
||||
<h1 class="mb-4">Excel Importer</h1>
|
||||
|
||||
<!-- Toolbar -->
|
||||
<div class="d-flex flex-wrap align-items-center mb-3 px-3" id="toolbar">
|
||||
<form id="upload-form" class="d-flex align-items-center me-3">
|
||||
<input type="file" name="files" multiple class="form-control form-control-sm me-2">
|
||||
<button type="submit" class="btn btn-primary btn-sm">Upload</button>
|
||||
<div class="mb-3">
|
||||
<form id="upload-form" class="mb-3">
|
||||
<input type="file" name="files" … multiple>
|
||||
<button type="submit" class="btn btn-primary">Upload Files</button>
|
||||
</form>
|
||||
<button id="download-excel" class="btn btn-success btn-sm me-3">Download All</button>
|
||||
<div class="d-flex align-items-center">
|
||||
<label for="table-select" class="form-label me-2 mb-0">Tabelle:</label>
|
||||
<select id="table-select" class="form-select form-select-sm">
|
||||
<button id="download-excel" class="btn btn-success">Download All Tables</button>
|
||||
</div>
|
||||
|
||||
<div class="row g-3 align-items-center">
|
||||
<div class="col-auto">
|
||||
<label for="table-select" class="col-form-label">Select Table:</label>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<select id="table-select" class="form-select">
|
||||
<option value="stripe_import">Stripe Import</option>
|
||||
<option value="raiseNow_import">RaiseNow Import</option>
|
||||
<option value="merged">Merged</option>
|
||||
@ -52,81 +38,70 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Table -->
|
||||
<div id="table-container" class="border rounded shadow-sm px-3">
|
||||
<div id="table-container">
|
||||
<div id="table"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Loading Overlay -->
|
||||
<div id="loadingOverlay">
|
||||
<div class="text-center text-white">
|
||||
<div class="spinner-border" role="status" style="width: 4rem; height: 4rem;"><span class="visually-hidden">Loading...</span></div>
|
||||
<p class="mt-3">Loading, please wait...</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="https://unpkg.com/tabulator-tables@5.4.4/dist/js/tabulator.min.js"></script>
|
||||
<script>
|
||||
const uploadForm = document.getElementById('upload-form');
|
||||
const tableSelect = document.getElementById('table-select');
|
||||
const downloadBtn = document.getElementById('download-excel');
|
||||
const loadingOverlay = document.getElementById('loadingOverlay');
|
||||
let table;
|
||||
|
||||
function showLoading() { loadingOverlay.style.display = 'flex'; }
|
||||
function hideLoading() { loadingOverlay.style.display = 'none'; }
|
||||
|
||||
uploadForm.addEventListener('submit', async e => {
|
||||
e.preventDefault(); showLoading();
|
||||
try {
|
||||
e.preventDefault();
|
||||
const fd = new FormData(uploadForm);
|
||||
const resp = await fetch('/upload', { method: 'POST', body: fd });
|
||||
const resp = await fetch('/upload', { method:'POST', body:fd });
|
||||
const res = await resp.json();
|
||||
if (resp.ok) await loadTable(tableSelect.value);
|
||||
else alert(res.error || 'Upload failed');
|
||||
} catch (err) {
|
||||
console.error(err); alert('An error occurred');
|
||||
} finally { hideLoading(); }
|
||||
if (resp.ok) {
|
||||
loadTable(tableSelect.value);
|
||||
} else alert(res.error || 'Upload failed');
|
||||
});
|
||||
|
||||
tableSelect.addEventListener('change', () => loadTable(tableSelect.value));
|
||||
|
||||
downloadBtn.addEventListener('click', async () => {
|
||||
showLoading();
|
||||
try {
|
||||
const resp = await fetch('/download');
|
||||
if (!resp.ok) throw new Error('Download failed');
|
||||
const blob = await resp.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
const disposition = resp.headers.get('Content-Disposition') || '';
|
||||
const match = disposition.match(/filename="?([^";]+)"?/);
|
||||
a.download = match ? match[1] : 'tables.xlsx';
|
||||
document.body.appendChild(a); a.click(); a.remove(); URL.revokeObjectURL(url);
|
||||
} catch (err) {
|
||||
console.error(err); alert(err.message);
|
||||
} finally { hideLoading(); }
|
||||
});
|
||||
|
||||
async function loadTable(name) {
|
||||
showLoading();
|
||||
try {
|
||||
const resp = await fetch(`/get_table?table=${encodeURIComponent(name)}`);
|
||||
|
||||
// fetch data
|
||||
const resp = await fetch(`/get_table?table=${name}`);
|
||||
const json = await resp.json();
|
||||
if (!resp.ok) { alert(json.error || 'Error loading'); return; }
|
||||
if (!Array.isArray(json.data) || !json.data.length) { alert('No data for this table'); return; }
|
||||
const cols = json.columns.map(c => ({ title: c, field: c, headerFilter: true }));
|
||||
const opts = { data: json.data, layout: 'fitData', height: '100%', columns: cols, responsiveLayout: 'hide' };
|
||||
if (table) { table.setColumns(cols); await table.replaceData(json.data); }
|
||||
else { table = new Tabulator('#table', opts); }
|
||||
} catch (err) {
|
||||
console.error(err); alert('Failed to load table data');
|
||||
} finally { hideLoading(); }
|
||||
|
||||
// error handling
|
||||
if (!resp.ok) {
|
||||
return alert(json.error || 'Error loading');
|
||||
}
|
||||
|
||||
// column definitions
|
||||
const cols = json.columns.map(c => ({
|
||||
title: c,
|
||||
field: c,
|
||||
headerFilter: true
|
||||
}));
|
||||
|
||||
if (table) {
|
||||
// update columns
|
||||
table.setColumns(cols);
|
||||
// returns a promise once render is done
|
||||
table.replaceData(json.data);
|
||||
} else {
|
||||
// options for table
|
||||
const opts = {
|
||||
data: json.data,
|
||||
layout: 'fitData',
|
||||
height: '100%',
|
||||
columns: cols
|
||||
};
|
||||
table = new Tabulator('#table', opts);
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('download-excel')
|
||||
.addEventListener('click', () => {
|
||||
window.location = '/download';
|
||||
});
|
||||
|
||||
// initialize
|
||||
loadTable(tableSelect.value);
|
||||
</script>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user