improve indexing
This commit is contained in:
parent
c7c52f0dc2
commit
a5930cb506
@ -2,14 +2,22 @@ import os
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from time import monotonic
|
||||
import re
|
||||
import helperfunctions as hf
|
||||
from collections import defaultdict
|
||||
|
||||
SEARCH_DB_NAME = 'search.db'
|
||||
ACCESS_LOG_DB_NAME = 'access_log.db'
|
||||
FOLDER_CONFIG = 'folder_secret_config.json'
|
||||
IGNORED_DIRS = {"Transkription", "@eaDir", ".app", "#recycle"}
|
||||
TRANSCRIPT_DIRNAME = "Transkription"
|
||||
TRANSCRIPT_EXT = ".md"
|
||||
IGNORED_DIRS = {TRANSCRIPT_DIRNAME, "@eaDir", ".app", "#recycle"}
|
||||
|
||||
# Logging/progress tuning (keep output light by default)
|
||||
LOG_STRUCTURE_DEPTH = int(os.getenv("INDEX_LOG_STRUCTURE_DEPTH", "0") or 0)
|
||||
PROGRESS_EVERY_SECS = float(os.getenv("INDEX_PROGRESS_SECS", "30"))
|
||||
PROGRESS_EVERY_FILES = int(os.getenv("INDEX_PROGRESS_FILES", "5000"))
|
||||
MAX_ERROR_LOGS = int(os.getenv("INDEX_MAX_ERROR_LOGS", "5"))
|
||||
|
||||
# Connect to the search database.
|
||||
search_db = sqlite3.connect(SEARCH_DB_NAME, check_same_thread=False)
|
||||
@ -24,10 +32,25 @@ def log(message: str):
|
||||
print(message, flush=True)
|
||||
|
||||
|
||||
def log_permission_error(path: str, stats: dict):
|
||||
"""Log permission errors sparingly to avoid noisy output."""
|
||||
stats["perm_errors"] += 1
|
||||
if stats["perm_errors"] <= MAX_ERROR_LOGS:
|
||||
log(f"Permission denied: {path}")
|
||||
elif stats["perm_errors"] == MAX_ERROR_LOGS + 1:
|
||||
log("Further permission errors suppressed.")
|
||||
|
||||
|
||||
def skip_dir(name: str) -> bool:
|
||||
"""Return True when a directory name should be skipped during traversal/logging."""
|
||||
return name.startswith('.') or name in IGNORED_DIRS
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
total_secs = int(seconds)
|
||||
minutes, secs = divmod(total_secs, 60)
|
||||
return f"{minutes}m {secs:02d}s"
|
||||
|
||||
def init_db():
|
||||
"""Initializes the database with the required schema."""
|
||||
cursor = search_db.cursor()
|
||||
@ -59,21 +82,28 @@ def init_db():
|
||||
|
||||
search_db.commit()
|
||||
|
||||
def scan_dir(directory):
|
||||
"""Recursively scan directories using os.scandir for improved performance."""
|
||||
def scan_dir(directory: str, stats: dict):
|
||||
"""Iteratively scan directories using os.scandir for improved performance."""
|
||||
stack = [directory]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
stats["dirs"] += 1
|
||||
try:
|
||||
with os.scandir(directory) as it:
|
||||
with os.scandir(current) as it:
|
||||
for entry in it:
|
||||
try:
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
# Skip unwanted directories immediately.
|
||||
if skip_dir(entry.name):
|
||||
stats["skipped_dirs"] += 1
|
||||
continue
|
||||
yield from scan_dir(entry.path)
|
||||
stack.append(entry.path)
|
||||
elif entry.is_file(follow_symlinks=False):
|
||||
yield entry
|
||||
except PermissionError:
|
||||
log(f"Permission denied: {directory}")
|
||||
return
|
||||
log_permission_error(entry.path, stats)
|
||||
except PermissionError:
|
||||
log_permission_error(current, stats)
|
||||
|
||||
def get_hit_count(relative_path):
|
||||
"""Returns the hit count for a given file from the access log database."""
|
||||
@ -94,6 +124,26 @@ def get_hit_counts_for_basefolder(basefolder: str) -> dict:
|
||||
return {row["rel_path"]: row["hit_count"] for row in cursor.fetchall()}
|
||||
|
||||
|
||||
def build_transcript_index(transcript_dir: str, stats: dict):
|
||||
"""Return a dict of basename -> transcript path for a transcript directory."""
|
||||
try:
|
||||
with os.scandir(transcript_dir) as it:
|
||||
index = {}
|
||||
for entry in it:
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
continue
|
||||
name = entry.name
|
||||
if not name.endswith(TRANSCRIPT_EXT):
|
||||
continue
|
||||
index[name[:-len(TRANSCRIPT_EXT)]] = entry.path
|
||||
return index
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except PermissionError:
|
||||
log_permission_error(transcript_dir, stats)
|
||||
return None
|
||||
|
||||
|
||||
def log_structure(root_path, max_depth=None, show_files=False):
|
||||
"""
|
||||
Log folder structure up to max_depth levels (root = depth 1).
|
||||
@ -130,14 +180,8 @@ def log_file(relative_path: str, filename: str):
|
||||
log(f" file: {relative_path} ({filename})")
|
||||
|
||||
|
||||
def log_directory_batch(directory: str, files: list[str]):
|
||||
"""Log file count for a directory without listing filenames."""
|
||||
if not files:
|
||||
return
|
||||
log(f" Dir {directory or '/'}: {len(files)} files")
|
||||
|
||||
|
||||
def updatefileindex():
|
||||
total_start = monotonic()
|
||||
cursor = search_db.cursor()
|
||||
totals = {"folders": 0, "scanned": 0, "deleted": 0}
|
||||
|
||||
@ -153,71 +197,102 @@ def updatefileindex():
|
||||
log(f"Processing folder: {foldername}")
|
||||
raw_folderpath = folder.get("folderpath")
|
||||
norm_folderpath = os.path.normpath(raw_folderpath)
|
||||
# Only log folder names up to 3 levels deep; suppress filenames
|
||||
log_structure(norm_folderpath, max_depth=1, show_files=False)
|
||||
# Optional shallow structure log (off by default)
|
||||
if LOG_STRUCTURE_DEPTH > 0:
|
||||
log_structure(norm_folderpath, max_depth=LOG_STRUCTURE_DEPTH, show_files=False)
|
||||
# Precompute the length of the base folder path (plus one for the separator)
|
||||
base_len = len(norm_folderpath) + 1
|
||||
base_prefix = norm_folderpath + os.sep
|
||||
base_len = len(base_prefix)
|
||||
# Prefetch hit counts for this basefolder to avoid per-file queries
|
||||
hitcount_map = get_hit_counts_for_basefolder(foldername)
|
||||
|
||||
# Accumulate scanned file data and keys for this base folder.
|
||||
scanned_files = [] # Each entry: (relative_path, basefolder, filename, filetype, transcript, hitcount)
|
||||
current_keys = set()
|
||||
dir_files = defaultdict(list) # map of directory -> list of filenames
|
||||
for entry in scan_dir(norm_folderpath):
|
||||
transcript = None
|
||||
entry_path = os.path.normpath(entry.path)
|
||||
# Get relative part by slicing if possible.
|
||||
if entry_path.startswith(norm_folderpath):
|
||||
rel_part = entry_path[base_len:]
|
||||
else:
|
||||
rel_part = os.path.relpath(entry_path, norm_folderpath)
|
||||
# Prepend the foldername so it becomes part of the stored relative path.
|
||||
relative_path = os.path.join(foldername, rel_part).replace(os.sep, '/')
|
||||
filetype = os.path.splitext(entry.name)[1].lower()
|
||||
scan_stats = {"dirs": 0, "skipped_dirs": 0, "perm_errors": 0}
|
||||
transcript_cache = {}
|
||||
transcripts_read = 0
|
||||
transcript_errors = 0
|
||||
|
||||
# Retrieve the hit count for this file from pre-fetched map.
|
||||
hit_count = hitcount_map.get(relative_path, 0)
|
||||
|
||||
# Determine the site
|
||||
site = None
|
||||
if foldername == 'Gottesdienste Speyer':
|
||||
site = 'Speyer'
|
||||
elif foldername == 'Gottesdienste Schwegenheim':
|
||||
site = 'Schwegenheim'
|
||||
|
||||
start_time = monotonic()
|
||||
last_log_time = start_time
|
||||
next_log_count = PROGRESS_EVERY_FILES
|
||||
scanned_count = 0
|
||||
|
||||
extract_structure = hf.extract_structure_from_string
|
||||
extract_date = hf.extract_date_from_string
|
||||
|
||||
for entry in scan_dir(norm_folderpath, scan_stats):
|
||||
transcript = None
|
||||
scanned_count += 1
|
||||
entry_path = entry.path
|
||||
# Get relative part by slicing if possible.
|
||||
if entry_path.startswith(base_prefix):
|
||||
rel_part = entry_path[base_len:]
|
||||
else:
|
||||
site = None
|
||||
rel_part = os.path.relpath(entry_path, norm_folderpath)
|
||||
# Prepend the foldername so it becomes part of the stored relative path.
|
||||
rel_part = rel_part.replace(os.sep, '/')
|
||||
relative_path = f"{foldername}/{rel_part}"
|
||||
name_root, name_ext = os.path.splitext(entry.name)
|
||||
filetype = name_ext.lower()
|
||||
|
||||
# Retrieve the hit count for this file from pre-fetched map.
|
||||
hit_count = hitcount_map.get(relative_path, 0)
|
||||
|
||||
# Check for a corresponding transcript file in a sibling "Transkription" folder.
|
||||
parent_dir = os.path.dirname(entry_path)
|
||||
transcript_dir = os.path.join(parent_dir, "Transkription")
|
||||
transcript_filename = os.path.splitext(entry.name)[0] + ".md"
|
||||
transcript_path = os.path.join(transcript_dir, transcript_filename)
|
||||
if os.path.exists(transcript_path):
|
||||
transcript_index = transcript_cache.get(parent_dir)
|
||||
if transcript_index is None and parent_dir not in transcript_cache:
|
||||
transcript_dir = os.path.join(parent_dir, TRANSCRIPT_DIRNAME)
|
||||
transcript_index = build_transcript_index(transcript_dir, scan_stats)
|
||||
transcript_cache[parent_dir] = transcript_index
|
||||
if transcript_index:
|
||||
transcript_path = transcript_index.get(name_root)
|
||||
if transcript_path:
|
||||
try:
|
||||
with open(transcript_path, 'r', encoding='utf-8') as tf:
|
||||
transcript = tf.read()
|
||||
transcripts_read += 1
|
||||
except Exception:
|
||||
transcript = None
|
||||
transcript_errors += 1
|
||||
|
||||
category, titel, name = hf.extract_structure_from_string(entry.name)
|
||||
category, titel, name = extract_structure(entry.name)
|
||||
|
||||
performance_date = hf.extract_date_from_string(relative_path)
|
||||
|
||||
# Debug: batch file logging per directory
|
||||
dir_files[os.path.dirname(relative_path)].append(entry.name)
|
||||
performance_date = extract_date(relative_path)
|
||||
|
||||
scanned_files.append((relative_path, foldername, entry.name, filetype, category, titel, name, performance_date, site, transcript, hit_count))
|
||||
current_keys.add((relative_path, entry.name))
|
||||
|
||||
# After scanning, log grouped files per directory
|
||||
for d, files in dir_files.items():
|
||||
log_directory_batch(d, files)
|
||||
# Light progress output
|
||||
now = monotonic()
|
||||
if scanned_count >= next_log_count or (now - last_log_time) >= PROGRESS_EVERY_SECS:
|
||||
elapsed = max(now - start_time, 0.0001)
|
||||
rate = scanned_count / elapsed
|
||||
log(f" progress: {scanned_count} files, {scan_stats['dirs']} dirs, {rate:.1f} files/s")
|
||||
last_log_time = now
|
||||
next_log_count = scanned_count + PROGRESS_EVERY_FILES
|
||||
|
||||
# Progress indicator
|
||||
dir_count = len(dir_files)
|
||||
file_count = len(scanned_files)
|
||||
log(f"Found {dir_count} folders and {file_count} files in '{foldername}'.")
|
||||
dir_count = scan_stats["dirs"]
|
||||
file_count = scanned_count
|
||||
elapsed = max(monotonic() - start_time, 0.0001)
|
||||
rate = file_count / elapsed
|
||||
log(f"Scan summary for '{foldername}': {dir_count} dirs, {file_count} files, {rate:.1f} files/s")
|
||||
if scan_stats["skipped_dirs"]:
|
||||
log(f" skipped dirs: {scan_stats['skipped_dirs']}")
|
||||
if scan_stats["perm_errors"]:
|
||||
log(f" permission errors: {scan_stats['perm_errors']}")
|
||||
if transcripts_read or transcript_errors:
|
||||
log(f" transcripts: {transcripts_read} read, {transcript_errors} errors")
|
||||
log("updating database...")
|
||||
scan_duration = format_duration(elapsed)
|
||||
|
||||
# Remove database entries for files under this base folder that are no longer on disk.
|
||||
pattern = foldername + os.sep + '%'
|
||||
@ -238,11 +313,14 @@ def updatefileindex():
|
||||
|
||||
# Commit changes after processing this base folder.
|
||||
search_db.commit()
|
||||
folder_scanned = len(scanned_files)
|
||||
folder_scanned = scanned_count
|
||||
totals["scanned"] += folder_scanned
|
||||
log(f"Indexed {folder_scanned} files (deleted {deleted_count}) in '{foldername}'")
|
||||
log(f"Scan duration for '{foldername}': {scan_duration}")
|
||||
|
||||
total_elapsed = max(monotonic() - total_start, 0.0001)
|
||||
log(f"Index update finished: folders={totals['folders']}, files indexed={totals['scanned']}, removed={totals['deleted']}")
|
||||
log(f"Total index duration: {format_duration(total_elapsed)}")
|
||||
return "File index updated successfully"
|
||||
|
||||
def convert_dates(search_db,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user