fix caching issue: no partial files for files other then audio/image
This commit is contained in:
parent
7fc513d404
commit
f8e705ab20
135
app.py
135
app.py
@ -15,7 +15,6 @@ import mimetypes
|
||||
from datetime import datetime, date, timedelta
|
||||
import diskcache
|
||||
import threading
|
||||
import time
|
||||
from flask_socketio import SocketIO, emit
|
||||
import geoip2.database
|
||||
from functools import lru_cache
|
||||
@ -939,87 +938,23 @@ def serve_file(subpath):
|
||||
_mark_request_logged(req_id)
|
||||
return response
|
||||
|
||||
# 5) Non-image branch: check if cached, otherwise create partial cache file
|
||||
# 5) Non-image branch: check if cached; if not, fill cache synchronously and then serve from cache
|
||||
try:
|
||||
with cache.read(subpath) as reader:
|
||||
file_path = reader.name
|
||||
cached_hit = True
|
||||
except KeyError:
|
||||
cached_hit = False
|
||||
|
||||
# Create a temporary cache file that we'll write to and serve simultaneously
|
||||
import hashlib
|
||||
import tempfile
|
||||
|
||||
# Generate cache key similar to diskcache
|
||||
cache_key = hashlib.md5(subpath.encode('utf-8')).hexdigest()
|
||||
cache_dir = os.path.join(cache.directory, cache_key[:2])
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
fd, cache_file_path = tempfile.mkstemp(
|
||||
prefix=f"{cache_key}_",
|
||||
suffix=".tmp",
|
||||
dir=cache_dir
|
||||
)
|
||||
os.close(fd)
|
||||
|
||||
# Write an initial chunk synchronously so the temp file exists with data
|
||||
initial_bytes = 0
|
||||
try:
|
||||
with open(full_path, 'rb') as source, open(cache_file_path, 'wb') as dest:
|
||||
chunk = source.read(1024 * 1024) # 1MB
|
||||
if chunk:
|
||||
dest.write(chunk)
|
||||
dest.flush()
|
||||
initial_bytes = len(chunk)
|
||||
# Copy full file into diskcache (blocks the request; avoids serving partial content)
|
||||
with open(full_path, 'rb') as source:
|
||||
cache.set(subpath, source, read=True)
|
||||
with cache.read(subpath) as reader:
|
||||
file_path = reader.name
|
||||
app.logger.info(f"Cached {subpath}")
|
||||
except Exception as e:
|
||||
app.logger.error(f"Failed to prime cache file for {subpath}: {e}")
|
||||
if os.path.exists(cache_file_path):
|
||||
try:
|
||||
os.remove(cache_file_path)
|
||||
except:
|
||||
pass
|
||||
abort(503, description="Service temporarily unavailable - cache initialization failed")
|
||||
|
||||
# Start copying to our cache file in chunks
|
||||
def copy_to_cache_chunked(start_offset):
|
||||
try:
|
||||
with open(full_path, 'rb') as source, open(cache_file_path, 'ab') as dest:
|
||||
source.seek(start_offset)
|
||||
while True:
|
||||
chunk = source.read(1024 * 1024) # 1MB chunks
|
||||
if not chunk:
|
||||
break
|
||||
dest.write(chunk)
|
||||
dest.flush() # Ensure data is written to disk immediately
|
||||
|
||||
# Once complete, register with diskcache for proper management
|
||||
try:
|
||||
if subpath in cache:
|
||||
if os.path.exists(cache_file_path):
|
||||
os.remove(cache_file_path)
|
||||
app.logger.info(f"Cache already populated for {subpath}, skipped duplicate registration")
|
||||
return
|
||||
with open(cache_file_path, 'rb') as f:
|
||||
cache.set(subpath, f, read=True)
|
||||
# Remove our temp file since diskcache now has it
|
||||
if os.path.exists(cache_file_path):
|
||||
os.remove(cache_file_path)
|
||||
app.logger.info(f"Finished caching {subpath}")
|
||||
except Exception as e:
|
||||
app.logger.error(f"Failed to register with diskcache: {e}")
|
||||
|
||||
except Exception as e:
|
||||
app.logger.error(f"Caching failed for {subpath}: {e}")
|
||||
if os.path.exists(cache_file_path):
|
||||
try:
|
||||
os.remove(cache_file_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Start the background copy
|
||||
cache_thread = threading.Thread(target=copy_to_cache_chunked, args=(initial_bytes,), daemon=True)
|
||||
cache_thread.start()
|
||||
file_path = cache_file_path
|
||||
app.logger.error(f"Cache fill failed for {subpath}: {e}")
|
||||
abort(503, description="Service temporarily unavailable - cache population failed")
|
||||
|
||||
# 6) Build response for non-image
|
||||
if as_attachment:
|
||||
@ -1031,49 +966,19 @@ def serve_file(subpath):
|
||||
|
||||
|
||||
# Single send_file call with proper attachment handling
|
||||
# For partial cache files, we need to handle this differently
|
||||
if not cached_hit:
|
||||
# Stream from the cache file as it's being written
|
||||
def generate():
|
||||
bytes_sent = 0
|
||||
with open(file_path, 'rb') as f:
|
||||
while bytes_sent < filesize:
|
||||
# Read what's available
|
||||
chunk = f.read(1024 * 1024) # 1MB chunks
|
||||
if chunk:
|
||||
bytes_sent += len(chunk)
|
||||
yield chunk
|
||||
else:
|
||||
# No data available yet, wait a bit
|
||||
time.sleep(0.1)
|
||||
response = send_file(
|
||||
file_path,
|
||||
mimetype=mimetype,
|
||||
conditional=True,
|
||||
as_attachment=as_attachment,
|
||||
download_name=filename if as_attachment else None
|
||||
)
|
||||
|
||||
if request.method == 'HEAD':
|
||||
response = make_response('', 200)
|
||||
else:
|
||||
response = make_response(generate())
|
||||
response.headers['Content-Type'] = mimetype
|
||||
response.headers['Content-Length'] = str(filesize)
|
||||
response.headers['Accept-Ranges'] = 'bytes'
|
||||
if as_attachment:
|
||||
response.headers['Content-Disposition'] = f'attachment; filename="{filename}"'
|
||||
response.headers['X-Content-Type-Options'] = 'nosniff'
|
||||
else:
|
||||
response.headers['Content-Disposition'] = 'inline'
|
||||
if as_attachment:
|
||||
response.headers['X-Content-Type-Options'] = 'nosniff'
|
||||
response.headers['Content-Disposition'] = 'attachment'
|
||||
else:
|
||||
# Cached file - use normal send_file
|
||||
response = send_file(
|
||||
file_path,
|
||||
mimetype=mimetype,
|
||||
conditional=True,
|
||||
as_attachment=as_attachment,
|
||||
download_name=filename if as_attachment else None
|
||||
)
|
||||
|
||||
if as_attachment:
|
||||
response.headers['X-Content-Type-Options'] = 'nosniff'
|
||||
response.headers['Content-Disposition'] = 'attachment'
|
||||
else:
|
||||
response.headers['Content-Disposition'] = 'inline'
|
||||
response.headers['Content-Disposition'] = 'inline'
|
||||
|
||||
response.headers['Cache-Control'] = 'public, max-age=86400'
|
||||
|
||||
|
||||
@ -109,6 +109,10 @@ search {
|
||||
opacity: 0.85;
|
||||
}
|
||||
|
||||
.search-query-wrap .input-group {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.input-clear-btn {
|
||||
border-top-left-radius: 0;
|
||||
border-bottom-left-radius: 0;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user