feat: 分块断点续传上传 — 20MB/块 + 分块级重试 + 断点查询

Edge 端新增 3 个端点: - POST /api/edge/video/chunk: 接收单块，保存到 task_{id}/chunk_{index:04d} - GET /api/edge/video/chunks: 查询已上传分块（断点续传） - POST /api/edge/video/assemble: 合并全部分块入队 NAS Dispatcher 重写: - 大文件(>50MB)自动分块上传(20MB/块) - 每块最多重试 3 次（分块级重试，非整文件级） - 上传前查询已上传分块，跳过已有的（断点续传） - 全部上传后调 /assemble 合并入队 - 小文件(<=50MB)走直接上传路径 - max_retries 3→5（文件级重试次数） - scheduler 切回生产目录解决: 360MB 视频跨公网单次上传超时/断连问题
2026-08-20 12:27:30 +08:00
parent fb4ccb64dc
commit b6c13a9047
3 changed files with 351 additions and 25 deletions
--- a/fam-edge/src/fam_edge/api_gateway/api_gateway.py
+++ b/fam-edge/src/fam_edge/api_gateway/api_gateway.py
@@ -129,6 +129,205 @@ def enqueue_task():
        return jsonify({"error": str(e)}), 500


+# ========== 分块上传（断点续传）==========
+
+CHUNK_SIZE = 20 * 1024 * 1024  # 20MB per chunk
+
+
+def _chunk_dir(task_id: int) -> str:
+    upload_dir = os.environ.get('FAM_UPLOAD_DIR', '/tmp/fam_uploads')
+    d = os.path.join(upload_dir, f"task_{task_id}")
+    os.makedirs(d, exist_ok=True)
+    return d
+
+
+@api_bp.route('/api/edge/video/chunk', methods=['POST'])
+def upload_chunk():
+    """接收单个分块，保存到 task_{id}/chunk_{index:04d}
+
+    断点续传：同一 task_id + chunk_index 重复上传会覆盖，
+    NAS 端可通过 /chunks 查询已上传分块，跳过已有的。
+    """
+    task_id_raw = request.form.get('task_id')
+    chunk_index_raw = request.form.get('chunk_index')
+    total_chunks_raw = request.form.get('total_chunks')
+    filename = request.form.get('filename', 'video.mp4')
+    chunk_file = request.files.get('chunk')
+
+    if not task_id_raw or not chunk_index_raw or not chunk_file:
+        return jsonify({"error": "缺少必填字段: task_id, chunk_index, chunk"}), 400
+
+    try:
+        task_id = int(task_id_raw)
+        chunk_index = int(chunk_index_raw)
+        total_chunks = int(total_chunks_raw) if total_chunks_raw else 0
+    except ValueError:
+        return jsonify({"error": "task_id/chunk_index 必须是整数"}), 400
+
+    d = _chunk_dir(task_id)
+    chunk_path = os.path.join(d, f"chunk_{chunk_index:04d}")
+
+    try:
+        chunk_file.save(chunk_path)
+        size_kb = os.path.getsize(chunk_path) / 1024
+
+        # 写元数据（首次上传时）
+        meta_path = os.path.join(d, "meta.json")
+        if not os.path.exists(meta_path):
+            import json
+            meta = {"filename": filename, "total_chunks": total_chunks}
+            with open(meta_path, 'w') as f:
+                json.dump(meta, f)
+
+        # 统计已上传分块
+        uploaded = sorted([
+            int(fn.split('_')[1]) for fn in os.listdir(d)
+            if fn.startswith('chunk_') and len(fn.split('_')) == 2
+        ])
+
+        logger.info(f"[task_id={task_id}] 分块 {chunk_index}/{total_chunks} 上传成功 "
+                     f"({size_kb:.0f}KB, 已上传 {len(uploaded)}/{total_chunks})")
+
+        return jsonify({
+            "status": "ok",
+            "task_id": task_id,
+            "chunk_index": chunk_index,
+            "uploaded_count": len(uploaded),
+            "total_chunks": total_chunks,
+        }), 200
+
+    except Exception as e:
+        logger.error(f"[task_id={task_id}] 分块上传失败: {e}", exc_info=True)
+        return jsonify({"error": str(e)}), 500
+
+
+@api_bp.route('/api/edge/video/chunks', methods=['GET'])
+def query_chunks():
+    """查询已上传分块列表（断点续传：NAS 重启后查询跳过已有分块）"""
+    task_id_raw = request.args.get('task_id')
+    if not task_id_raw:
+        return jsonify({"error": "缺少 task_id"}), 400
+
+    try:
+        task_id = int(task_id_raw)
+    except ValueError:
+        return jsonify({"error": "task_id 必须是整数"}), 400
+
+    d = _chunk_dir(task_id)
+    uploaded = sorted([
+        int(fn.split('_')[1]) for fn in os.listdir(d)
+        if fn.startswith('chunk_') and len(fn.split('_')) == 2
+    ]) if os.path.isdir(d) else []
+
+    total = 0
+    meta_path = os.path.join(d, "meta.json")
+    if os.path.exists(meta_path):
+        import json
+        try:
+            with open(meta_path) as f:
+                total = json.load(f).get('total_chunks', 0)
+        except (json.JSONDecodeError, IOError):
+            pass
+
+    return jsonify({
+        "task_id": task_id,
+        "uploaded_chunks": uploaded,
+        "uploaded_count": len(uploaded),
+        "total_chunks": total,
+    }), 200
+
+
+@api_bp.route('/api/edge/video/assemble', methods=['POST'])
+def assemble_chunks():
+    """合并所有分块为完整视频文件，入 SQLite 队列
+
+    NAS 上传完全部分块后调用此端点触发合并 + 入队。
+    """
+    task_id_raw = request.form.get('task_id')
+    if not task_id_raw:
+        return jsonify({"error": "缺少 task_id"}), 400
+
+    try:
+        task_id = int(task_id_raw)
+    except ValueError:
+        return jsonify({"error": "task_id 必须是整数"}), 400
+
+    camera_name = request.form.get('camera_name', '')
+    event_start_time = request.form.get('event_start_time', '')
+    known_members_context = request.form.get('known_members_context', '')
+
+    d = _chunk_dir(task_id)
+
+    # 读取元数据
+    import json
+    meta_path = os.path.join(d, "meta.json")
+    if not os.path.exists(meta_path):
+        return jsonify({"error": "元数据不存在，请先上传分块"}), 400
+
+    try:
+        with open(meta_path) as f:
+            meta = json.load(f)
+    except json.JSONDecodeError:
+        return jsonify({"error": "元数据损坏"}), 500
+
+    filename = meta.get('filename', 'video.mp4')
+    total_chunks = meta.get('total_chunks', 0)
+
+    # 检查分块完整性
+    chunk_files = sorted([
+        fn for fn in os.listdir(d)
+        if fn.startswith('chunk_') and len(fn.split('_')) == 2
+    ])
+
+    if total_chunks and len(chunk_files) < total_chunks:
+        missing = total_chunks - len(chunk_files)
+        return jsonify({
+            "error": f"分块不完整: {len(chunk_files)}/{total_chunks}，缺 {missing} 块",
+            "uploaded_count": len(chunk_files),
+            "total_chunks": total_chunks,
+        }), 400
+
+    # 合并分块
+    upload_dir = os.environ.get('FAM_UPLOAD_DIR', '/tmp/fam_uploads')
+    video_filename = f"task_{task_id}_{filename}"
+    video_path = os.path.join(upload_dir, video_filename)
+
+    try:
+        with open(video_path, 'wb') as out:
+            for cf in chunk_files:
+                chunk_path = os.path.join(d, cf)
+                with open(chunk_path, 'rb') as chunk_f:
+                    out.write(chunk_f.read())
+
+        size_mb = os.path.getsize(video_path) / 1024 / 1024
+        logger.info(f"[task_id={task_id}] 分块合并完成: {filename} ({size_mb:.1f}MB, {len(chunk_files)} 块)")
+
+        # 清理分块目录
+        import shutil
+        shutil.rmtree(d, ignore_errors=True)
+
+        # 入队
+        queue_id = queue_manager.enqueue(
+            nas_task_id=task_id,
+            video_filename=filename,
+            video_path=video_path,
+            camera_name=camera_name,
+            event_start_time=event_start_time,
+            known_members_context=known_members_context,
+        )
+
+        return jsonify({
+            "status": "queued",
+            "task_id": task_id,
+            "queue_id": queue_id,
+            "size_mb": round(size_mb, 1),
+        }), 202
+
+    except Exception as e:
+        logger.error(f"[task_id={task_id}] 合并失败: {e}", exc_info=True)
+        return jsonify({"error": str(e)}), 500
+
+
@api_bp.route('/api/edge/results', methods=['GET'])
 def get_results():
    """返回已完成但未拉取的结果，标记为已交付"""