fix caching issue: no partial files for files other then audio/image
This commit is contained in:
parent
7fc513d404
commit
f8e705ab20
137
app.py
137
app.py
@ -15,7 +15,6 @@ import mimetypes
|
|||||||
from datetime import datetime, date, timedelta
|
from datetime import datetime, date, timedelta
|
||||||
import diskcache
|
import diskcache
|
||||||
import threading
|
import threading
|
||||||
import time
|
|
||||||
from flask_socketio import SocketIO, emit
|
from flask_socketio import SocketIO, emit
|
||||||
import geoip2.database
|
import geoip2.database
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
@ -939,87 +938,23 @@ def serve_file(subpath):
|
|||||||
_mark_request_logged(req_id)
|
_mark_request_logged(req_id)
|
||||||
return response
|
return response
|
||||||
|
|
||||||
# 5) Non-image branch: check if cached, otherwise create partial cache file
|
# 5) Non-image branch: check if cached; if not, fill cache synchronously and then serve from cache
|
||||||
try:
|
try:
|
||||||
with cache.read(subpath) as reader:
|
with cache.read(subpath) as reader:
|
||||||
file_path = reader.name
|
file_path = reader.name
|
||||||
cached_hit = True
|
cached_hit = True
|
||||||
except KeyError:
|
except KeyError:
|
||||||
cached_hit = False
|
cached_hit = False
|
||||||
|
|
||||||
# Create a temporary cache file that we'll write to and serve simultaneously
|
|
||||||
import hashlib
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
# Generate cache key similar to diskcache
|
|
||||||
cache_key = hashlib.md5(subpath.encode('utf-8')).hexdigest()
|
|
||||||
cache_dir = os.path.join(cache.directory, cache_key[:2])
|
|
||||||
os.makedirs(cache_dir, exist_ok=True)
|
|
||||||
fd, cache_file_path = tempfile.mkstemp(
|
|
||||||
prefix=f"{cache_key}_",
|
|
||||||
suffix=".tmp",
|
|
||||||
dir=cache_dir
|
|
||||||
)
|
|
||||||
os.close(fd)
|
|
||||||
|
|
||||||
# Write an initial chunk synchronously so the temp file exists with data
|
|
||||||
initial_bytes = 0
|
|
||||||
try:
|
try:
|
||||||
with open(full_path, 'rb') as source, open(cache_file_path, 'wb') as dest:
|
# Copy full file into diskcache (blocks the request; avoids serving partial content)
|
||||||
chunk = source.read(1024 * 1024) # 1MB
|
with open(full_path, 'rb') as source:
|
||||||
if chunk:
|
cache.set(subpath, source, read=True)
|
||||||
dest.write(chunk)
|
with cache.read(subpath) as reader:
|
||||||
dest.flush()
|
file_path = reader.name
|
||||||
initial_bytes = len(chunk)
|
app.logger.info(f"Cached {subpath}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
app.logger.error(f"Failed to prime cache file for {subpath}: {e}")
|
app.logger.error(f"Cache fill failed for {subpath}: {e}")
|
||||||
if os.path.exists(cache_file_path):
|
abort(503, description="Service temporarily unavailable - cache population failed")
|
||||||
try:
|
|
||||||
os.remove(cache_file_path)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
abort(503, description="Service temporarily unavailable - cache initialization failed")
|
|
||||||
|
|
||||||
# Start copying to our cache file in chunks
|
|
||||||
def copy_to_cache_chunked(start_offset):
|
|
||||||
try:
|
|
||||||
with open(full_path, 'rb') as source, open(cache_file_path, 'ab') as dest:
|
|
||||||
source.seek(start_offset)
|
|
||||||
while True:
|
|
||||||
chunk = source.read(1024 * 1024) # 1MB chunks
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
dest.write(chunk)
|
|
||||||
dest.flush() # Ensure data is written to disk immediately
|
|
||||||
|
|
||||||
# Once complete, register with diskcache for proper management
|
|
||||||
try:
|
|
||||||
if subpath in cache:
|
|
||||||
if os.path.exists(cache_file_path):
|
|
||||||
os.remove(cache_file_path)
|
|
||||||
app.logger.info(f"Cache already populated for {subpath}, skipped duplicate registration")
|
|
||||||
return
|
|
||||||
with open(cache_file_path, 'rb') as f:
|
|
||||||
cache.set(subpath, f, read=True)
|
|
||||||
# Remove our temp file since diskcache now has it
|
|
||||||
if os.path.exists(cache_file_path):
|
|
||||||
os.remove(cache_file_path)
|
|
||||||
app.logger.info(f"Finished caching {subpath}")
|
|
||||||
except Exception as e:
|
|
||||||
app.logger.error(f"Failed to register with diskcache: {e}")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
app.logger.error(f"Caching failed for {subpath}: {e}")
|
|
||||||
if os.path.exists(cache_file_path):
|
|
||||||
try:
|
|
||||||
os.remove(cache_file_path)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Start the background copy
|
|
||||||
cache_thread = threading.Thread(target=copy_to_cache_chunked, args=(initial_bytes,), daemon=True)
|
|
||||||
cache_thread.start()
|
|
||||||
file_path = cache_file_path
|
|
||||||
|
|
||||||
# 6) Build response for non-image
|
# 6) Build response for non-image
|
||||||
if as_attachment:
|
if as_attachment:
|
||||||
@ -1031,49 +966,19 @@ def serve_file(subpath):
|
|||||||
|
|
||||||
|
|
||||||
# Single send_file call with proper attachment handling
|
# Single send_file call with proper attachment handling
|
||||||
# For partial cache files, we need to handle this differently
|
response = send_file(
|
||||||
if not cached_hit:
|
file_path,
|
||||||
# Stream from the cache file as it's being written
|
mimetype=mimetype,
|
||||||
def generate():
|
conditional=True,
|
||||||
bytes_sent = 0
|
as_attachment=as_attachment,
|
||||||
with open(file_path, 'rb') as f:
|
download_name=filename if as_attachment else None
|
||||||
while bytes_sent < filesize:
|
)
|
||||||
# Read what's available
|
|
||||||
chunk = f.read(1024 * 1024) # 1MB chunks
|
if as_attachment:
|
||||||
if chunk:
|
response.headers['X-Content-Type-Options'] = 'nosniff'
|
||||||
bytes_sent += len(chunk)
|
response.headers['Content-Disposition'] = 'attachment'
|
||||||
yield chunk
|
|
||||||
else:
|
|
||||||
# No data available yet, wait a bit
|
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
if request.method == 'HEAD':
|
|
||||||
response = make_response('', 200)
|
|
||||||
else:
|
|
||||||
response = make_response(generate())
|
|
||||||
response.headers['Content-Type'] = mimetype
|
|
||||||
response.headers['Content-Length'] = str(filesize)
|
|
||||||
response.headers['Accept-Ranges'] = 'bytes'
|
|
||||||
if as_attachment:
|
|
||||||
response.headers['Content-Disposition'] = f'attachment; filename="{filename}"'
|
|
||||||
response.headers['X-Content-Type-Options'] = 'nosniff'
|
|
||||||
else:
|
|
||||||
response.headers['Content-Disposition'] = 'inline'
|
|
||||||
else:
|
else:
|
||||||
# Cached file - use normal send_file
|
response.headers['Content-Disposition'] = 'inline'
|
||||||
response = send_file(
|
|
||||||
file_path,
|
|
||||||
mimetype=mimetype,
|
|
||||||
conditional=True,
|
|
||||||
as_attachment=as_attachment,
|
|
||||||
download_name=filename if as_attachment else None
|
|
||||||
)
|
|
||||||
|
|
||||||
if as_attachment:
|
|
||||||
response.headers['X-Content-Type-Options'] = 'nosniff'
|
|
||||||
response.headers['Content-Disposition'] = 'attachment'
|
|
||||||
else:
|
|
||||||
response.headers['Content-Disposition'] = 'inline'
|
|
||||||
|
|
||||||
response.headers['Cache-Control'] = 'public, max-age=86400'
|
response.headers['Cache-Control'] = 'public, max-age=86400'
|
||||||
|
|
||||||
|
|||||||
@ -109,6 +109,10 @@ search {
|
|||||||
opacity: 0.85;
|
opacity: 0.85;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.search-query-wrap .input-group {
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
.input-clear-btn {
|
.input-clear-btn {
|
||||||
border-top-left-radius: 0;
|
border-top-left-radius: 0;
|
||||||
border-bottom-left-radius: 0;
|
border-bottom-left-radius: 0;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user