diff --git a/AGENTS.md b/AGENTS.md index e161c6e..8bd17eb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -275,6 +275,7 @@ Agent 先不写代码，先交计划。人类明确批准后，才进入执行 - [ ] 路径清单逐条有结论 - [ ] 全局属性有运行时证据 - [ ] 测试代码未被弱化 +- [ ] 身份/续传类改动的验收覆盖配置别名与通配绑定，测试以内容或事件而非文件存在作为完成条件 **合并前** diff --git a/PyFlow/network_api/connect_tcp.py b/PyFlow/network_api/connect_tcp.py index 2aaf5a0..876f6c1 100644 --- a/PyFlow/network_api/connect_tcp.py +++ b/PyFlow/network_api/connect_tcp.py @@ -18,14 +18,13 @@ - an interrupted transfer is resumed after a disconnect: both sides persist their transfer stores under the protocol's ``.Flow`` directory and the receiver asks the sender to continue the file from the bytes already on - disk (``resume_transfers``; the wire lines are ``/resume_trans`` and - ``/forward_resume``). + disk (``_request_resumes``; every request uses ``/resume_trans`` with its + original sender and final target). The forwarding extensions use the module-level parsers `parse_forwarded_message`, `parse_forward_items_and_addrs`, `parse_forward_originator` and `forward_skip_message`; the reconnect logic -uses `parse_resume_command`, `parse_forward_resume_command` and -`_resume_offset_for`. +uses `parse_resume_command`, `format_resume_line` and `_resume_offset_for`. Concepts live in ``docs/source/Network_APIs/TCP_Server_APIs.rst`` and ``TCP_Client_APIs.rst``; argument, return and exception contracts live in the @@ -38,7 +37,6 @@ import ast import sys import time -import copy import shlex import socket import struct # SO_LINGER's (l_onoff, l_linger) pair, for abortive closes @@ -50,6 +48,10 @@ import errno import queue import json +import stat +import weakref +from contextlib import ExitStack, contextmanager +from functools import wraps from . import rsa_crypto from datetime import datetime from concurrent.futures import ThreadPoolExecutor @@ -136,6 +138,51 @@ def _is_closed_socket_error(exc): 10038, # WSAENOTSOCK (Windows) ) +def _is_local_port_conflict(exc): + """Return whether an OSError means a fixed local port is unavailable. + + Windows may report an occupied source port as WSAEADDRINUSE (10048) or + WSAEACCES (10013), depending on the socket state and reserved-port rules. + POSIX reports EADDRINUSE from bind. EACCES is accepted only on Windows; + callers must additionally confirm that the local port is fixed and nonzero. + """ + if not isinstance(exc, OSError): + return False + if exc.errno == errno.EADDRINUSE: + return True + if getattr(exc, "winerror", None) in (10048, 10013): + return True + return os.name == "nt" and exc.errno == errno.EACCES + + +def _should_fallback_local_port(port, exc): + """Return true only for a conflict on a fixed, nonzero local port.""" + return port not in (None, 0) and _is_local_port_conflict(exc) + + +def _bind_local_port_with_fallback(sock, local_address, log, port_label): + """Bind a listener port, falling back to an OS-assigned port once.""" + used_fallback = False + try: + try: + sock.bind(local_address) + except OSError as exc: + if not _should_fallback_local_port(local_address[1], exc): + raise + sock.bind((local_address[0], 0)) + used_fallback = True + port = sock.getsockname()[1] + except Exception: + sock.close() + raise + if used_fallback: + log( + f"{port_label} port {local_address[1]} is not available; " + f"using OS-assigned port {port}" + ) + return port + + MAX_DECODE_FAILURES = 3 # circuit breaker: close after N consecutive decode failures (re-exchange storm / garbage injection) SEND_WAIT_TIMEOUT = 30.0 # seconds a non-blocking client write waits for buffer room @@ -149,10 +196,10 @@ def _is_closed_socket_error(exc): # drops (see _persist_transfer_progress). After a reconnect the receiver asks # the sender to continue from the bytes it already holds (/resume_trans). RESUME_LINE = "/resume_trans" # receiver -> sender: continue this transfer from an offset -FORWARD_RESUME_LINE = "/forward_resume" # server -> forwarder: re-forward one item from an offset RESUME_ACCEPT_TIMEOUT = 20.0 # seconds a resumed transfer waits for the sender to connect RESUME_PORT_TIMEOUT = 20.0 # seconds a resumed sender waits for the receiver's transfer port RESUME_SECURE_TIMEOUT = 10.0 # seconds a resume drive waits for the framed control channel +FILE_COMMAND_TOKENS = 3 # a ``/file `` line carries this many tokens def _wait_writable(client_socket, timeout): @@ -315,30 +362,7 @@ def _debug_line(enabled, is_debug, parts): def _parse_destination_path(command_part): - """Extract the trailing destination directory from a received transfer - - command, or None when the command carries no destination. - - ```` is the file-client id appended by the sender; ```` is a - quoted client address tuple. The destination (when present) is the token - before ````, except for folder-creation commands, which carry no id - and put the destination last: - - /file - /file - /file - /file - /file_folder folder creation, default location - /file_folder folder creation with destination - /file_folder - /file_folder - - Args: - command_part (object): Value used by the operation. - - Returns: - NoneType | object: Result produced by the operation. - """ + """Extract the optional destination preceding a transfer's trailing FID.""" if len(command_part) < 3: return None if command_part[0].lower() == "/file_folder": @@ -413,6 +437,214 @@ def _peer_from_json(value): return None +def _read_transfer_progress(path, *, server_routes=False): + """Read scoped progress and, for servers, canonical relay route snapshots.""" + with open(path, "r", encoding="utf-8") as source: + data = json.load(source) + if isinstance(data, dict): + for text in data.get("upload") or {}: + key = json.loads(text) + if not ( + isinstance(key, list) and len(key) == 2 + and _peer_from_json(key[0]) is not None and isinstance(key[1], int) + ): + raise ValueError("unsupported upload progress key; archive the old progress file") + if server_routes: + try: + for fid, route in (data.get("relays") or {}).items(): + int(fid) + if _peer_from_json(route["originator"]) is None or not route["targets"]: + raise ValueError + seen = set() + for target, child in route["targets"]: + peer = _peer_from_json(target) + if peer is None or peer in seen or not isinstance(child, int): + raise ValueError + seen.add(peer) + completed = [_peer_from_json(target) for target in route.get("completed", [])] + if any(target is None or target not in seen for target in completed): + raise ValueError + if len(set(completed)) != len(completed): + raise ValueError + except (TypeError, ValueError, KeyError, AttributeError): + raise ValueError("unsupported server relay progress; archive the old file") from None + elif data.get("relays"): + raise ValueError("unsupported client relay progress; archive the old file") + return data + + +def _transfer_progress_view(snapshot, *, speed_enabled, runtime=None): + """Render an existing progress snapshot without merging scoped identities.""" + if not isinstance(snapshot, dict): + raise ValueError("transfer progress must be an object") + view = {"uploading": [], "downloading": [], "relays": []} + for source, destination in (("upload", "uploading"), ("download", "downloading")): + entries = snapshot.get(source, {}) + if not isinstance(entries, dict): + raise ValueError(f"{source} progress must be an object") + for text, entry in entries.items(): + if not isinstance(entry, list) or len(entry) != 5: + raise ValueError("transfer progress rows must have five fields") + if source == "upload": + receiver, fid = json.loads(text) + receiver = _peer_from_json(receiver) + if receiver is None or not isinstance(fid, int): + raise ValueError("unsupported upload progress key") + key = (receiver, fid) + else: + receiver, fid = None, int(text) + key = fid + peer = _peer_from_json(entry[4]) if entry[4] is not None else None + if entry[4] is not None and peer is None: + raise ValueError("invalid progress peer") + view[destination].append({ + "fid": fid, + "receiver": list(receiver) if receiver is not None else None, + "peer": list(peer) if peer is not None else None, + "name": os.path.basename(entry[3]), "path": entry[3], + "transferred": int(entry[0]), "total": int(entry[1]), + "speed": entry[2] if speed_enabled else None, + "active": key in runtime[0] if runtime is not None else None, + "socket_registered": key in runtime[1] if runtime is not None else None, + "paused": runtime[2].get(key, False) + if runtime is not None and receiver is not None else None, + }) + relays = snapshot.get("relays", {}) + if not isinstance(relays, dict) or any(not isinstance(row, dict) for row in relays.values()): + raise ValueError("relay progress must contain record objects") + view["relays"] = [{**row, "fid": int(fid)} for fid, row in relays.items()] + return view + + +def _get_transfer_progress(instance, *, include_persisted, server_routes=False): + """Observe local stores and an optional file; never load or persist state.""" + with instance._transfer_progress_lock: + snapshot = instance._transfer_progress_snapshot() + active = set(instance._active_transfer_fids) + sockets = set(instance._active_transfer_sockets) + with instance._transfer_pause_cond: + paused = dict(instance._transfer_pause) + speed_enabled = bool(instance.is_enable_real_time_network_speed) + persisted = { + "path": os.fspath(instance.file_trans_progress_file), + "status": "not_requested", "snapshot": None, "error": None, + } + result = { + "progress_enabled": bool(instance.is_enable_file_transfer_progress_bar), + "speed_enabled": speed_enabled, + "memory": _transfer_progress_view( + snapshot, speed_enabled=speed_enabled, runtime=(active, sockets, paused) + ), + "persisted": persisted, + } + if include_persisted: + try: + data = _read_transfer_progress(persisted["path"], server_routes=server_routes) + persisted["snapshot"] = _transfer_progress_view(data, speed_enabled=speed_enabled) + persisted["status"] = "ok" + except FileNotFoundError: + persisted["status"] = "missing" + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError) as exc: + persisted["status"] = "error" + persisted["error"] = str(exc) + return result + + +def _config_lifecycle(*, starting): + """Serialize deletion against startup and shutdown without locking their I/O.""" + def decorate(method): + @wraps(method) + def run(self, *args, **kwargs): + with self._transfer_progress_lock: + if starting: + if self._progress_discarded: + raise RuntimeError("transfer progress was discarded; create a new instance") + self._config_started = True + self._config_busy += 1 + self._config_closed = False + completed = False + try: + result = method(self, *args, **kwargs) + completed = True + return result + finally: + with self._transfer_progress_lock: + self._config_busy -= 1 + if not starting: + self._config_closed = completed + return run + return decorate + + +_CONFIG_FILES = ( + ("file_trans_progress_file", None), + ("messages_log_file", "messages"), + ("events_log_file", "events"), + ("sent_messages_log_file", "sent_messages"), + ("sent_events_log_file", "sent_events"), +) + + +def _delete_config(instance, config_name, *, is_list_config): + """List or unlink explicitly owned state files; never resolve user paths.""" + files = {} + for attr, store in _CONFIG_FILES: + path = os.path.abspath(os.fspath(getattr(instance, attr))) + files.setdefault(os.path.basename(path), {}).setdefault(path, []).append(store) + + def regular_file(path): + try: + return stat.S_ISREG(os.lstat(path).st_mode) + except FileNotFoundError: + return False + + if is_list_config: + names = [] + for name, paths in sorted(files.items()): + if len(paths) != 1: + raise ValueError(f"ambiguous configuration name: {name}") + if regular_file(next(iter(paths))): + names.append(name) + return names + if config_name is None: + raise ValueError("a configuration file name is required") + if not isinstance(config_name, str): + raise TypeError("config_name must be a file name string") + if (not config_name or config_name in (".", "..") + or any(char in config_name for char in "/\\\0*?[]") or config_name not in files): + raise ValueError("config_name must name an allowed local state file") + paths = files[config_name] + if len(paths) != 1: + raise ValueError(f"ambiguous configuration name: {config_name}") + path, stores = next(iter(paths.items())) + with instance._transfer_progress_lock: + if (instance.running or instance._config_busy + or (instance._config_started and not instance._config_closed) + or instance._active_transfer_fids or instance._active_transfer_sockets + or instance._resume_sends): + raise RuntimeError("stop the instance and finish active transfers before deletion") + if hasattr(instance, "_forward_relays"): + with instance._forward_relays_lock: + if instance._forward_relays: + raise RuntimeError("finish active relays before deletion") + with ExitStack() as locks: + for store in stores: + if store is not None: + locks.enter_context(getattr(instance, f"_{store}_dict_lock")) + if getattr(instance, f"{store}_dict"): + raise RuntimeError("flush pending records by closing before deletion") + try: + if not regular_file(path) and os.path.lexists(path): + raise ValueError("only regular configuration files may be deleted") + os.unlink(path) + deleted = True + except FileNotFoundError: + deleted = False + if None in stores: + instance._progress_discarded = True + return deleted + + def _resume_offset_for(entry): """Bytes of an interrupted transfer's file already on the receiver's disk. @@ -437,26 +669,82 @@ def _resume_offset_for(entry): def parse_resume_command(command): - """Split a ``/resume_trans `` line. + """Parse ``/resume_trans `` or return None. - Args: - command (str): Wire line in ``/resume_trans `` format; - ``fid`` and ``offset`` must be integers. - - Returns: - tuple | None: ``(fid, offset)``, or None when the line is malformed. - + Addresses are shell-quoted two-item tuple literals. Return the integer + fid/offset and two endpoint tuples; routing validates their actual owners. """ try: parts = shlex.split(command) - except ValueError: - return None - if len(parts) != 3 or parts[0].lower() != RESUME_LINE: + if len(parts) != 5 or parts[0].lower() != RESUME_LINE: + return None + fid, offset = int(parts[1]), int(parts[2]) + endpoints = [ast.literal_eval(text) for text in parts[3:]] + if fid < 0 or offset < 0 or any( + not isinstance(peer, tuple) or len(peer) != 2 + or not isinstance(peer[0], str) or not isinstance(peer[1], int) + for peer in endpoints + ): + return None + return fid, offset, *endpoints + except (ValueError, SyntaxError, TypeError): return None + + +def format_resume_line(fid, offset, origin, target): + """Format the only resume wire command, preserving both endpoint identities.""" + return (f"{RESUME_LINE} {fid} {offset} " + f"{shlex.quote(repr(tuple(origin)))} {shlex.quote(repr(tuple(target)))}") + + +@contextmanager +def _sending_transfer(instance, key): + """Serialize sends for one scoped fid; release after the last in-flight user.""" + with instance._transfer_progress_lock: + state = instance._resume_sends.get(key) + if state is None: + state = {"lock": threading.RLock(), "users": 0, "targets": set()} + instance._resume_sends[key] = state + state["users"] += 1 try: - return int(parts[1]), int(parts[2]) - except ValueError: - return None + with state["lock"]: + yield state + finally: + with instance._transfer_progress_lock: + state["users"] -= 1 + if not state["users"] and not state["targets"]: + instance._resume_sends.pop(key, None) + + +def _start_resume_send(instance, key, target, send): + """Reserve a target before starting its worker; duplicates join that request.""" + with instance._transfer_progress_lock: + if instance._progress_discarded or key not in instance.file_upload_progress: + return False + state = instance._resume_sends.get(key) + if state is None and key in instance._active_transfer_fids: + return True # an initial ordinary send already owns this upload + if state is None: + state = {"lock": threading.RLock(), "users": 0, "targets": set()} + instance._resume_sends[key] = state + if target in state["targets"]: + return True + state["targets"].add(target) + + def run(): + try: + with _sending_transfer(instance, key): + entry = instance.file_upload_progress.get(key) + if entry is not None: + send(entry) + finally: + with instance._transfer_progress_lock: + state["targets"].discard(target) + if not state["users"] and not state["targets"]: + instance._resume_sends.pop(key, None) + + threading.Thread(target=run, daemon=True).start() + return True def _pause_command_fid(command): @@ -497,35 +785,100 @@ def _open_transfer_target(path, offset): return target -def parse_forward_resume_command(command): - """Split a ``/forward_resume `` line. +def _receive_dir_for(base_dir, new_save_path, destination_path): + """Compose the receive directory of one folder command. - Args: - command (str): Wire line with a two-item tuple address, transfer ID, - and byte offset; the address host must be a string and both values - must be integers. + The single definition of where a folder command lands: an absolute + ``destination_path`` replaces ``base_dir``, a relative one nests under + it, and the command's ``/``-separated relative path nests below that. + ``setting_file_save_path`` and the folder-creation branch of + ``file_folder_transfer_*_recv_*_start_thread`` both use it, so a create + command and the file legs under it always agree on the directory. - Returns: - tuple | None: ``(target, fid, offset)``, or None when the line is - malformed. - + Args: + base_dir (str): Instance receive root (``file_transfer_dir``). + new_save_path (str | None): Command's relative folder path, ``/``-separated. + destination_path (str | None): Receiver-side destination directory. + + Returns: + str: Directory the command names, without creating it. """ - try: - parts = shlex.split(command) - except ValueError: + save_path = base_dir + if destination_path: + if os.path.isabs(destination_path): + save_path = destination_path + else: + save_path = os.path.join(save_path, destination_path) + if new_save_path: + path_list = new_save_path.split("/") + if path_list and path_list[0] == "": + del path_list[0] + for node in path_list: + save_path = os.path.join(save_path, node) + return save_path +def _rel_dirs_with_files(folder_path, base_path): + """Relative paths of the directories under ``folder_path`` that have files. + + A directory needs to exist at the receiver only when a file lands in it + or below it; an empty directory has nothing to receive. Each entry is in + ``"/data/sub"`` form, and every ancestor of a directory that has files is + included, because a file leg creates only its own leaf directory. + + Args: + folder_path (str): Folder being sent. + base_path (str): Parent directory the relative paths hang off. + + Returns: + set: Relative folder paths, ``""`` for the transfer root. + """ + needed = set() + base = os.path.normpath(base_path) + for root, _dirs, files in os.walk(folder_path): + if not files: + continue + rel = os.path.relpath(os.path.normpath(root), base) + rel = "" if rel == "." else "/" + rel.replace(os.sep, "/") + while True: + needed.add(rel) + trimmed = rel.rstrip("/") + if not trimmed: + break + rel = trimmed.rsplit("/", 1)[0] + return needed + +def _resumable_entry(progress, fid, peer, command_part): + """Return the interrupted download a file or folder-item command continues. + + The fid, original sender (``peer``) and file basename must all match; + anything else is a new transfer and never uses a path-only resume alias. + + Args: + progress (dict): ``file_recv_progress`` store, fid -> entry. + fid (int | None): Transfer id the line carries. + peer (tuple | None): Sender ``(ip, port)`` the line arrived from. + command_part (list): Shell-split command tokens. + + Returns: + list | None: The matching progress entry, or None. + """ + if fid is None or peer is None or len(command_part) < FILE_COMMAND_TOKENS: return None - if len(parts) != 4 or parts[0].lower() != FORWARD_RESUME_LINE: + kind = command_part[0].lower() + if kind == "/file": + filename = command_part[1] + elif kind == "/file_folder" and len(command_part) >= 4: + filename = command_part[2] + else: return None - try: - target = ast.literal_eval(parts[1]) - except (ValueError, SyntaxError): + entry = progress.get(fid) + if entry is None or entry[4] is None: return None - if not (isinstance(target, tuple) and len(target) == 2 and isinstance(target[0], str)): + if tuple(entry[4]) != tuple(peer): return None - try: - return target, int(parts[2]), int(parts[3]) - except ValueError: + if os.path.basename(str(entry[3])) != os.path.basename(filename): return None + return entry + def parse_forwarded_message(command): @@ -636,6 +989,122 @@ def parse_forward_originator(command, own_address=None): return None +# ---- transfer ids ---------------------------------------------------------- +# +# A transfer id keys the progress stores, the pause switches and the transfer +# socket registry, and it travels on the wire as the last token of ``/file`` +# and ``/file_folder`` lines. One allocator per instance: +# +# fid = (counter << 32) | (epoch << 1) +# +# ``epoch`` is random per instance, reducing restart collisions; scoped progress +# keys, not random uniqueness, isolate receivers and directions. ``counter`` +# only grows in the high bits. The low bit is reserved, always zero, and never +# selects a transfer's route or direction. +# +# The counters this replaced started at 0 in every instance and were also +# decremented when a leg ended, so an inbound fid from one peer and an outbound +# fid from this instance could be the same number: the shared registries +# (``_active_transfer_fids``, ``_active_transfer_sockets``, ``_transfer_pause``) +# then let one leg's cleanup tear down the other's socket. + +FID_LOW_BITS = 32 # low 32 bits: epoch (31) plus one reserved zero bit + + +def format_fid(fid): + """Render a transfer id for logs as ``e.``. + + Args: + fid (int): Transfer id. + + Returns: + str: Readable form; the inverse of the allocator's packing. + """ + low = fid & 0xFFFFFFFF + return f"e{low >> 1}.{fid >> FID_LOW_BITS}" + + +def _socket_local_endpoint(sock): + """Return the concrete local ``(ip, port)`` of a connected socket, or None. + + The kernel reports the address it actually chose for the connection, so a + wildcard bind (``0.0.0.0``) or a host alias (``localhost``) yields the + address the peer resolved and connected to. A socket that is already closed + or unbound yields None. + + Args: + sock (socket.socket | None): Control socket to inspect. + + Returns: + tuple | None: ``(ip, port)`` the resume identity should use. + """ + if sock is None: + return None + try: + address = sock.getsockname() + except OSError: + return None + return (address[0], address[1]) if address else None + + +def _socket_peer_endpoint(sock): + """Return the concrete remote ``(ip, port)`` of a connected socket, or None. + + Args: + sock (socket.socket | None): Control socket to inspect. + + Returns: + tuple | None: ``(ip, port)`` the peer is actually connected from. + """ + if sock is None: + return None + try: + address = sock.getpeername() + except OSError: + return None + return (address[0], address[1]) if address else None + + +class _FidAllocator: + """Hand out transfer ids that never collide with a live one. + + ``alloc`` skips a candidate that already keys one of the live tables. Those + tables are read under this allocator's own lock only; a caller must not + hold ``_transfer_progress_lock`` or ``_forward_relays_lock`` while + allocating, so the two lock families never nest. + """ + + def __init__(self, live_tables): + """Create an allocator for one instance. + + Args: + live_tables (tuple): Dicts whose keys are ids still in use — + progress stores and server routes/indexes. Held by reference, + never copied. + """ + self._epoch = int.from_bytes(os.urandom(4), "big") & 0x7FFFFFFF + self._counter = 0 + self._live_tables = live_tables + self._lock = threading.Lock() + + def alloc(self, *, receiver=None): + """Return an unused kindless transfer id. + + Args: + receiver (tuple | None): Receiver namespace of an outbound progress key. + + Returns: + int: Transfer id with the reserved low bit cleared. + """ + with self._lock: + while True: + self._counter += 1 + fid = (self._counter << FID_LOW_BITS) | (self._epoch << 1) + scoped = (tuple(receiver), fid) if receiver is not None else None + if not any(fid in table or scoped in table for table in self._live_tables): + return fid + + # ---- encrypted channel framing --------------------------------------------- # # On an encrypted socket every message travels as a frame: @@ -683,7 +1152,7 @@ def parse_forward_originator(command, own_address=None): # Plaintext lines still tolerated on an encrypted socket: the crypto # handshake/re-exchange lines, the frames-start announcement and the transfer -# port coordination messages. Anything else plaintext is dropped, so an +# port coordination message. Anything else plaintext is dropped, so an # injected line can neither run commands nor substitute key material # (``/crypto_ecdh_pub`` is only ever accepted inside an RSA frame). _PLAINTEXT_LINE_PREFIXES = ( @@ -694,11 +1163,10 @@ def parse_forward_originator(command, own_address=None): "/crypto_mode", CRYPTO_FRAMES_START, "/server_file_transfer_port", - "/forward_upload", ) # A plaintext handshake line means the peer reset its secure state # (re-exchange): drop framing so the dance runs in raw mode again. -_PLAINTEXT_RESET_PREFIXES = _PLAINTEXT_LINE_PREFIXES[:-3] +_PLAINTEXT_RESET_PREFIXES = _PLAINTEXT_LINE_PREFIXES[:-2] class _FrameError(Exception): @@ -720,6 +1188,81 @@ def _pem_body_of(pem): return body +TRANSFER_SIGN_BUFFER_LIMIT = 64 * 1024 + + +class _TransferSignBuffer: + """Bound protocol bytes and batch-scan complete records at the limit.""" + + def __init__(self, start_sign, ack_sign, error_sign): + self.start_sign = start_sign + self.ack_sign = ack_sign + self.error_sign = error_sign + self.lock = threading.Lock() + self.pending = [] + self.pending_bytes = 0 + self.partial = bytearray() + # Producer-side threshold scans and the caller consume under this lock. + self.start_seen = False + self.ack_seen = False + self.error_seen = False + + def feed(self, data): + """Queue complete lines and batch-scan when the byte quota is reached. + + Raises: + ValueError: A single unterminated line reaches the limit. + """ + offset = 0 + error_received = False + view = memoryview(data) + while offset < len(data): + with self.lock: + end = min(len(data), offset + TRANSFER_SIGN_BUFFER_LIMIT - len(self.partial)) + newline = data.find(b"\n", offset, end) + stop = newline + 1 if newline >= 0 else end + wire_size = len(self.partial) + stop - offset + if newline < 0 and wire_size >= TRANSFER_SIGN_BUFFER_LIMIT: + raise ValueError("unterminated transfer sign reaches 64 KiB") + if self.pending_bytes + wire_size > TRANSFER_SIGN_BUFFER_LIMIT: + self._scan_pending_locked() + self.partial.extend(view[offset:stop]) + offset = stop + if newline >= 0: + line = self.partial.decode("utf-8").strip() + if line: + self.pending.append((line, wire_size)) + self.pending_bytes += wire_size + error_received |= line == self.error_sign + self.partial.clear() + if self.pending_bytes + len(self.partial) >= TRANSFER_SIGN_BUFFER_LIMIT: + self._scan_pending_locked() + return error_received + + def _scan_pending_locked(self): + """Latch every pending marker and release the entire scanned prefix.""" + processed_count = 0 + consumed_bytes = 0 + for read_index in range(len(self.pending)): + line, wire_size = self.pending[read_index] + if line == self.start_sign: + self.start_seen = True + elif line == self.ack_sign: + self.ack_seen = True + elif line == self.error_sign: + self.error_seen = True + processed_count += 1 + consumed_bytes += wire_size + del self.pending[:processed_count] + self.pending_bytes -= consumed_bytes + + def consume(self): + """Scan and delete queued records, returning a thread-safe result snapshot.""" + with self.lock: + self._scan_pending_locked() + return self.start_seen, self.ack_seen, self.error_seen + + def _pem_text_of(body): """Rebuild a standard PEM text from its one-line base64 body. @@ -752,6 +1295,169 @@ def _stamp_nonce(stamp): return None +def _parse_transfer_port_announcement(command): + """Parse the four-token listener reply; legacy UUID replies are rejected.""" + try: + parts = shlex.split(command) + if len(parts) != 4 or parts[0].lower() != "/server_file_transfer_port": + return None + port, fid = int(parts[1]), int(parts[2]) + if not (0 < port < 65536 and fid >= 0): + return None + return port, fid, parts[3] + except (ValueError, TypeError): + return None + + +def _transfer_port_announcement(port, fid, stamp): + """Return one listener's port and encryption stamp for the sender's FID.""" + return f"/server_file_transfer_port {port} {fid} {stamp}\n" + + +class _TransferHandshakeRegistry: + """Own socket/FID negotiations and resources for each actual control session.""" + + def __init__(self): + self._changed = threading.Condition() + self._pending = {} + self._unresolved = set() + self._closed = weakref.WeakSet() + self._resources = {} + + def register(self, control_socket, fid): + key = (control_socket, fid) + with self._changed: + if (control_socket in self._closed or key in self._pending + or key in self._unresolved): + return None + record = {"key": key, "sent": False, "result": None, "cancelled": False} + self._pending[key] = record + return record + + def mark_sent(self, record): + """Reserve possible transmission before calling the control send path.""" + with self._changed: + if (record is None or self._pending.get(record["key"]) is not record + or record["sent"]): + return False + record["sent"] = True + return True + + def resolve_port(self, control_socket, fid, port, stamp): + return self._resolve((control_socket, fid), ("port", fid, port, stamp)) + + def resolve_error(self, control_socket, fid, message): + return self._resolve((control_socket, fid), ("error", message)) + + def pending(self, control_socket, fid): + with self._changed: + return self._pending.get((control_socket, fid)) + + def _resolve(self, key, result): + with self._changed: + record = self._pending.get(key) + if (record is None or not record["sent"] or record["cancelled"] + or record["result"] is not None or key in self._unresolved): + return False + record["result"] = result + self._changed.notify_all() + return True + + def _retire(self, record): + key = record["key"] + if self._pending.get(key) is record: + del self._pending[key] + if (record["sent"] and record["result"] is None + and key[0] not in self._closed): + self._unresolved.add(key) + + def wait(self, record, timeout): + results = self.wait_many((record,), timeout) + return results[record["key"]] if results is not None else None + + def wait_many(self, records, timeout): + with self._changed: + if any(record is None for record in records): + return None + self._changed.wait_for( + lambda: any(record["cancelled"] for record in records) + or all(record["result"] is not None for record in records), + timeout=max(0, timeout), + ) + # Decide result versus uncertainty under the same condition lock. + for record in records: + self._retire(record) + if any(record["cancelled"] or record["result"] is None for record in records): + return None + return {record["key"]: record["result"] for record in records} + + def cancel(self, record): + if record is None: + return + with self._changed: + self._retire(record) + record["cancelled"] = True + self._changed.notify_all() + + def track_socket(self, control_socket, transfer_socket): + """Register listeners before announcing them, and accepted/data sockets.""" + if control_socket is None: + return True + with self._changed: + if control_socket in self._closed: + self._close_socket(transfer_socket) + return False + self._resources.setdefault(control_socket, set()).add(transfer_socket) + return True + + def release_socket(self, control_socket, transfer_socket): + with self._changed: + resources = self._resources.get(control_socket) + if resources is not None: + resources.discard(transfer_socket) + if not resources: + del self._resources[control_socket] + + @staticmethod + def _close_socket(sock): + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + sock.close() + + def cancel_socket(self, control_socket): + with self._changed: + self._closed.add(control_socket) + for key, record in list(self._pending.items()): + if key[0] is control_socket: + record["cancelled"] = True + del self._pending[key] + self._unresolved.difference_update( + key for key in tuple(self._unresolved) if key[0] is control_socket + ) + resources = self._resources.pop(control_socket, ()) + self._changed.notify_all() + for sock in resources: + self._close_socket(sock) + + +@contextmanager +def _transfer_handshake(instance, control_socket, fid): + """Register before sending; an uncertain sent negotiation cannot be reused.""" + record = instance._transfer_handshakes.register(control_socket, fid) + try: + yield record + finally: + instance._transfer_handshakes.cancel(record) + + +def _send_transfer_request(instance, record, message, *, is_send_raw=False): + if not instance._transfer_handshakes.mark_sent(record): + return False + return instance.send_message(record["key"][0], message, is_send_raw=is_send_raw) + + def _crypto_is_protocol_line(message): """Return true for the protocol's own handshake/coordination lines. @@ -768,7 +1474,7 @@ def _crypto_is_protocol_line(message): """ if message.startswith("/crypto"): return True - return message.startswith(("/server_file_transfer_port", "/forward_upload")) + return message.startswith("/server_file_transfer_port") class _CryptoSession: @@ -1517,13 +2223,13 @@ class TCP_Server_Base: # TCP server class (the speed is None until the transfer has moved its first chunk, and while speed measurement is off; ``peer`` is the sender's ``(ip, port)``). - file_upload_progress (dict): Running uploads keyed by fid; each value is + file_upload_progress (dict): Uploads keyed by ``(receiver, fid)``; each value is ``[sent bytes, total bytes, current speed, file path, peer]``, with ``peer`` the receiver's ``(ip, port)``. A forwarded transfer stores the original peer on both sides, never the relay. file_trans_progress_file (str): Path under the protocol's ``.Flow`` directory where the two stores are written when a connection drops - and read back on the next reconnect (``resume_transfers``). + and read back on the next reconnect (``_request_resumes``). upload_network_speed (float | None): Bytes per second sent, or None when not measured. recv_network_speed (float | None): Bytes per second received, or None @@ -1621,9 +2327,9 @@ def __init__( # hold exactly the transfers running now; iterate over a copy while # transfers are in flight. A transfer interrupted by a disconnect # keeps its entry (and its partial file) so it can be resumed once - # the peer is back (see `resume_transfers`). + # the peer is back (see `_request_resumes`). self.file_recv_progress = {} # downloads: fid -> [received, total, speed, path, peer] - self.file_upload_progress = {} # uploads: fid -> [sent, total, speed, path, peer] + self.file_upload_progress = {} # (receiver, fid) -> [sent, total, speed, path, peer] self.upload_network_speed = None # bytes/s over the last NETWORK_SPEED_WINDOW seconds self.recv_network_speed = None self._network_speed_lock = threading.Lock() # serialises the speed sample windows @@ -1631,22 +2337,27 @@ def __init__( self._recv_speed_samples = collections.deque() self._upload_speed_total = 0 # bytes held in the sample windows self._recv_speed_total = 0 - self._forward_fid = 0 - self._forward_fid_lock = threading.Lock() self._forward_relays = {} self._forward_relays_lock = threading.Lock() - # Relay records kept after their relay died, so a disconnected - # forward can be resumed: relay fid and every target tfid -> record. - self._forward_resume_records = {} + # One route per local upstream fid; downstream keys point to that fid. + self._fid_routes = {} + self._fid_route_index = {} # (receiver, downstream fid) -> upstream fid # Resume requests that arrived before the originator reconnected: - # originator (ip, port) -> [(fid, target (ip, port), offset), ...]. - self._pending_relay_resumes = {} + # originator -> {(upstream fid, target): latest disk offset}. + self._pending_originator_resumes = {} + # Transfer ids for the legs this instance sends (see `_FidAllocator`). + # Inbound ids use local integer keys; outbound ids use (receiver, fid). + # Equal numbers cannot share progress, pause state or sockets. + self._fid_alloc = _FidAllocator( + (self.file_upload_progress, self.file_recv_progress, + self._fid_routes, self._fid_route_index) + ) # Outbound-transfer pause switches (see `pause_trans`/`start_trans`): - # fid -> bool. A leg waits while its own fid is marked paused. + # (receiver, fid) -> bool; a pause never affects a different receiver. self._transfer_pause = {} self._transfer_pause_cond = threading.Condition() self._active_transfer_fids = set() # fids of the legs running right now - self._active_transfer_sockets = {} # fid -> socket of the leg running right now + self._active_transfer_sockets = {} # receive fid / (receiver, fid) -> socket self.project_dir = os.path.dirname(os.path.abspath(__file__)) self.project_info_dir = os.path.join(self.project_dir, ".Flow") if os.path.exists(self.project_info_dir) == False: @@ -1663,6 +2374,11 @@ def __init__( self.project_info_dir, "server_file_trans_progress.json" ) self._transfer_progress_lock = threading.RLock() # serialises the persist/load pair + self._resume_sends = {} # transient locks and request sets; no file metadata + self._progress_discarded = False + self._config_started = False + self._config_closed = False + self._config_busy = 0 self.decode_command_table_file_path = os.path.join( self.project_dir, "decode_command_table.json" ) @@ -1680,10 +2396,7 @@ def __init__( self.alloc_port(port_add_step, port_range_num) self.server_socket = None self.clients = {} # store client info - self.file_client_id_lock = threading.Lock() - self.file_transfer_server_port_lock = threading.Lock() - self.file_server_port_list = [] - self.file_client_id = 0 + self._transfer_handshakes = _TransferHandshakeRegistry() self.client_lock = threading.Lock() # add the threading lock self.max_file_transfer_thread_num = max_file_transfer_thread_num MAX_CONCURRENT_FILES = self.max_file_transfer_thread_num @@ -1708,28 +2421,43 @@ def __init__( self._custom_handler_threaded = [{}, {}] self._custom_executor = ThreadPoolExecutor(max_workers=max_custom_workers) self._task_semaphore = threading.Semaphore(max_custom_workers) - # Inbound-event listeners (see add_message_listener/add_file_listener). - # They run on the receive thread, so a listener must not block and must - # never raise (exceptions are swallowed by the notify helpers). + # Inbound/outbound event listeners (see add_message_listener and + # add_file_listener). They run on the thread performing the relevant + # I/O; a listener must not block and its exceptions are swallowed by + # the notify helpers. self._message_listeners = [] self._file_listeners = [] + self._send_message_listeners = [] + # Set by stop(); a stopped instance never invokes a listener again. + self._listeners_stopped = False + # Sockets carrying the control protocol; only these are recorded. + self._control_sockets = set() self._event_listeners_lock = threading.Lock() - # Inbound message/event stores: external code reads these instead of - # registering listeners. Keyed by the sender's socket; each value is - # a list of [content, timestamp] pairs. When a store's total size - # reaches max_dict_size (64 KiB) it is flushed to its JSON log and - # cleared (see _record_message/_record_event/_flush_*_dict). + # Message/event stores: external code reads these instead of registering + # listeners. Keyed by the socket; each value is a list of [content, + # timestamp] pairs. messages_dict and sent_messages_dict hold plain text + # (inbound / outbound); events_dict and sent_events_dict hold "/" commands. + # When a store's total size reaches max_dict_size (64 KiB) it is flushed + # to its JSON log and cleared (see _append_record, _flush_record_stores). self.messages_dict = {} self.events_dict = {} + self.sent_messages_dict = {} + self.sent_events_dict = {} self._messages_dict_lock = threading.Lock() self._events_dict_lock = threading.Lock() + self._sent_messages_dict_lock = threading.Lock() + self._sent_events_dict_lock = threading.Lock() self._socket_keys = {} # record key -> "ip:port", captured while it was alive self._socket_keys_lock = threading.Lock() self._messages_dict_size = 0 self._events_dict_size = 0 + self._sent_messages_dict_size = 0 + self._sent_events_dict_size = 0 self.max_dict_size = 64 * 1024 self.messages_log_file = os.path.join(self.project_info_dir, "messages_log.json") self.events_log_file = os.path.join(self.project_info_dir, "events_log.json") + self.sent_messages_log_file = os.path.join(self.project_info_dir, "sent_messages_log.json") + self.sent_events_log_file = os.path.join(self.project_info_dir, "sent_events_log.json") self.is_extend_command = is_extend_command self.is_enable_encrypto = is_enable_encrypto self.is_custom_keys = is_custom_keys @@ -1835,15 +2563,15 @@ def _measure_network_speed(self, payload, is_upload): def _file_progress_start(self, progress_dict, fid, total_bytes, path, peer=None, initial=0): """Register one file transfer in a progress store. - + Args: progress_dict (dict): ``self.file_recv_progress`` for an inbound transfer, ``self.file_upload_progress`` for an outbound one. - fid (int | None): The transfer's fid; None tracks nothing (a leg without a protocol transfer id, such as a key push). + fid (int | tuple | None): Receive fid or upload ``(receiver, fid)`` key; None tracks no progress. total_bytes (int): Size of the file being transferred. path (str): Local path of the file: the source of an outbound transfer, the file being written for an inbound one. peer (tuple | None): ``(ip, port)`` of the transfer's original peer — the sender for an inbound transfer, the receiver for an outbound one. A forwarded transfer stores the original peer on both sides, never the relay; None marks a leg that cannot be resumed (there is no peer to continue it with). initial (int): Bytes already transferred when the leg starts; a resumed transfer continues a partially transferred file. - + Returns: list | None: The ``[transferred, total, speed, path, peer]`` entry the transfer updates, or None when tracking is off @@ -1873,15 +2601,13 @@ def _file_progress_update(self, entry, byte_count, is_upload): def _file_progress_finish(self, progress_dict, fid, entry, keep=False): """Drop a finished transfer's progress entry, or keep it for a resume. - A newer transfer may already hold the same fid (transfer ids are reused - as transfers end), so only this transfer's own entry is dropped. With - ``keep`` True an interrupted transfer keeps its entry — and the partial - file that entry points at — so the reconnect logic can continue it - (`resume_transfers`); the entry then lives until the transfer ends. + A resumed leg can replace an older session under the same scoped key. + Only the entry's owner is removed. ``keep`` preserves an interrupted + entry and its partial file for the next reconnect. Args: progress_dict (dict): Store the entry belongs to. - fid (int | None): Transfer id the entry is keyed by. + fid (int | tuple | None): Receive fid or upload ``(receiver, fid)`` key. entry (list | None): Entry to drop; None tracks nothing. keep (bool): Keep the entry (and persist the stores) instead of dropping it. Defaults to False. """ @@ -1903,6 +2629,80 @@ def _file_progress_finish(self, progress_dict, fid, entry, keep=False): # a transfer that ended must not be resumed on a later reconnect self._persist_transfer_progress() + def _register_fid_route(self, fid, originator, targets, completed=()): + """Register a route only when its upstream and target legs are unclaimed.""" + targets = {tuple(target): child for target, child in targets.items()} + completed = {tuple(target) for target in completed} + route = {"originator": tuple(originator), "targets": targets, "completed": completed} + upstream_leg_keys = {(target, fid) for target in targets} + with self._transfer_progress_lock: + active_collision = ( + fid in self._active_transfer_fids + or fid in self._active_transfer_sockets + or any(key in self._active_transfer_fids for key in upstream_leg_keys) + or any(key in self._active_transfer_sockets for key in upstream_leg_keys) + ) + with self._transfer_pause_cond: + paused_collision = any(key in self._transfer_pause for key in upstream_leg_keys) + with self._forward_relays_lock: + if ( + not targets + or not completed.issubset(targets) + or fid in self._fid_routes + or fid in self.file_recv_progress + or any(key in self.file_upload_progress for key in upstream_leg_keys) + or active_collision + or paused_collision + or any(key in self._forward_relays for key in upstream_leg_keys) + or any(key in self._fid_route_index for key in upstream_leg_keys) + or any( + (target, child) in self._fid_route_index + for target, child in targets.items() if target not in completed + ) + ): + return None + self._fid_routes[fid] = route + for target, child in targets.items(): + if target not in completed: + self._fid_route_index[(target, child)] = fid + return route + + def _lookup_fid_route(self, peer, fid): + """Find a local upstream or an exactly scoped downstream relay leg.""" + with self._forward_relays_lock: + upstream = fid if peer is None else self._fid_route_index.get((tuple(peer), fid)) + route = self._fid_routes.get(upstream) + return (upstream, route) if route is not None else None + + def _drop_fid_route(self, fid, route): + """Drop only the owning route and its derived downstream indexes.""" + with self._forward_relays_lock: + if self._fid_routes.get(fid) is not route: + return False + del self._fid_routes[fid] + for target, child in route["targets"].items(): + key = (target, child) + if self._fid_route_index.get(key) == fid: + del self._fid_route_index[key] + if os.path.exists(self.file_trans_progress_file): + self._persist_transfer_progress() + return True + + def _complete_fid_target(self, fid, route, target, child_fid): + """Accept one final ACK without releasing any other target's identity.""" + target = tuple(target) + with self._forward_relays_lock: + if (self._fid_routes.get(fid) is not route + or route["targets"].get(target) != child_fid + or target in route["completed"] + or self._fid_route_index.get((target, child_fid)) != fid): + return False + route["completed"].add(target) + del self._fid_route_index[(target, child_fid)] + self._persist_transfer_progress() + return True + + def _transfer_progress_snapshot(self): """Return the JSON-ready form of every transfer still in play. @@ -1921,7 +2721,8 @@ def entries(store): list | tuple: Result produced by the operation. """ return { - str(fid): [ + (json.dumps(fid, separators=(",", ":")) + if store is self.file_upload_progress else str(fid)): [ entry[0], entry[1], entry[2], @@ -1934,16 +2735,11 @@ def entries(store): with self._forward_relays_lock: relays = { str(key): { - "fid": record.get("fid"), - "originator": list(record["originator"]) if record.get("originator") else None, - "targets": [list(target) for target in record.get("targets") or []], - "tfids": list(record.get("tfids") or []), - "kind": record.get("kind"), - "rel_dir": record.get("rel_dir"), - "fname": record.get("fname"), - "destination": record.get("destination"), + "originator": list(route["originator"]), + "targets": [[list(target), child] for target, child in route["targets"].items()], + "completed": [list(target) for target in sorted(route["completed"])], } - for key, record in list(self._forward_resume_records.items()) + for key, route in self._fid_routes.items() } return { "upload": entries(self.file_upload_progress), @@ -1951,6 +2747,41 @@ def entries(store): "relays": relays, } + def get_transfer_progress(self, *, include_persisted=True) -> dict: + """Return a detached, local-only view of all retained transfer records. + + Args: + include_persisted (bool): Read this instance's progress file too. + Defaults to True; never loads it into the live stores. + + Returns: + dict: Flags, scoped memory rows and a separate persisted snapshot + with its path, read status and error. Disk runtime flags are + unknown (None). No completion or zombie status is inferred. + """ + return _get_transfer_progress(self, include_persisted=include_persisted, server_routes=True) + + def delete_config(self, config_name=None, *, is_list_config=False) -> bool | list[str]: + """List or delete a named local progress snapshot or message/event log. + + Args: + config_name (str | None): Allowed file basename, not a path. + is_list_config (bool): Only list existing regular files, ignoring + config_name. Defaults to False; listing is allowed while running. + + Returns: + bool | list[str]: Sorted file names in list mode; otherwise True + when deleted, False when absent. Discarding progress permanently + disables this object's reload/write/start; create a new instance. + + Raises: + ValueError: Invalid/ambiguous name, directory or symlink. + TypeError: A deletion name is not a string. + RuntimeError: Instance not closed, operations active or logs buffered. + OSError: The filesystem refuses inspection or deletion. + """ + return _delete_config(self, config_name, is_list_config=is_list_config) + def _persist_transfer_progress(self): """Write the transfer stores to the protocol's ``.Flow`` directory. @@ -1962,16 +2793,20 @@ def _persist_transfer_progress(self): a run without an interrupted transfer never creates it. A write failure is logged, never raised. """ - snapshot = self._transfer_progress_snapshot() with self._transfer_progress_lock: + if self._progress_discarded: + return + snapshot = self._transfer_progress_snapshot() if not any(snapshot.values()) and not os.path.exists(self.file_trans_progress_file): return tmp_path = f"{self.file_trans_progress_file}.{os.getpid()}.tmp" try: + if os.path.exists(self.file_trans_progress_file): + _read_transfer_progress(self.file_trans_progress_file, server_routes=True) with open(tmp_path, "w", encoding="utf-8") as f: json.dump(snapshot, f, ensure_ascii=False, indent=2) os.replace(tmp_path, self.file_trans_progress_file) - except OSError as e: + except (OSError, ValueError) as e: self._log(f"transfer progress file update failed: {e}") try: os.remove(tmp_path) @@ -1990,9 +2825,10 @@ def _load_transfer_progress(self): int: Number of transfer entries and relay records restored. """ with self._transfer_progress_lock: + if self._progress_discarded: + return 0 try: - with open(self.file_trans_progress_file, "r", encoding="utf-8") as f: - data = json.load(f) + data = _read_transfer_progress(self.file_trans_progress_file, server_routes=True) except FileNotFoundError: return 0 except (OSError, ValueError) as e: @@ -2001,22 +2837,15 @@ def _load_transfer_progress(self): if not isinstance(data, dict): return 0 restored = 0 - with self._forward_relays_lock: - for key_text, record in (data.get("relays") or {}).items(): - try: - key = int(key_text) - except (TypeError, ValueError): - continue - if not isinstance(record, dict): - continue - record = dict(record) - record["originator"] = _peer_from_json(record.get("originator")) - record["targets"] = [ - target - for target in (_peer_from_json(t) for t in record.get("targets") or []) - if target - ] - self._forward_resume_records.setdefault(key, record) + for key_text, record in (data.get("relays") or {}).items(): + try: + fid = int(key_text) + targets = {tuple(target): child for target, child in record["targets"]} + except (TypeError, ValueError, KeyError): + continue + if self._register_fid_route( + fid, record["originator"], targets, record.get("completed", ()) + ) is not None: restored += 1 for key, store in ( ("download", self.file_recv_progress), @@ -2024,12 +2853,18 @@ def _load_transfer_progress(self): ): for fid_text, row in (data.get(key) or {}).items(): try: - fid = int(fid_text) + fid = int(fid_text) if key == "download" else json.loads(fid_text) + if key == "upload": + fid = (_peer_from_json(fid[0]), int(fid[1])) + if fid[0] is None: + continue entry = [int(row[0]), int(row[1]), None, row[3], _peer_from_json(row[4])] except (TypeError, ValueError, IndexError, KeyError): continue - if fid in store or fid in self._forward_resume_records: - continue # a live transfer owns it, or a relay drives it + routed = self._lookup_fid_route(None if key == "download" else fid[0], + fid if key == "download" else fid[1]) + if fid in store or routed is not None: + continue # preserve live progress and route-owned legs store[fid] = entry restored += 1 return restored @@ -2046,17 +2881,15 @@ def _forward_relay_targets(self, fid, peer=None): writer matches. """ with self._forward_relays_lock: - relay = self._forward_relays.get(fid) + relay = self._forward_relays.get((tuple(peer), fid)) if peer is not None else None if relay is None or not relay.get("writer_pause"): return None, [] - if peer is None: - return relay, list(relay["writer_pause"]) for target in relay["writer_pause"]: if tuple(target) == tuple(peer): return relay, [target] return None, [] - def _mark_transfer_pause(self, fid, paused, peer=None): + def _mark_transfer_pause(self, fid, paused, peer): """Set or clear one transfer's pause switch (see `pause_trans`). Args: @@ -2067,6 +2900,9 @@ def _mark_transfer_pause(self, fid, paused, peer=None): Returns: bool: True when a live relay writer or outbound leg was found. """ + if peer is None: + raise ValueError("receiver peer is required") + transfer_key = (tuple(peer), fid) relay, targets = self._forward_relay_targets(fid, peer) if targets: with relay["cond"]: @@ -2076,17 +2912,13 @@ def _mark_transfer_pause(self, fid, paused, peer=None): relay["cond"].notify_all() return True with self._transfer_progress_lock: - live = fid in self._active_transfer_fids - if live and peer is not None: - entry = self.file_upload_progress.get(fid) - if entry is not None and entry[4] is not None and tuple(entry[4]) != tuple(peer): - return False # that leg belongs to another peer + live = transfer_key in self._active_transfer_fids with self._transfer_pause_cond: - self._transfer_pause[fid] = paused + self._transfer_pause[transfer_key] = paused self._transfer_pause_cond.notify_all() return live - def pause_trans(self, fid, peer=None): + def pause_trans(self, fid, peer): """Pause an outbound transfer of this instance. The sending leg with this fid stops at its next chunk and stays open @@ -2096,8 +2928,8 @@ def pause_trans(self, fid, peer=None): Args: fid (int): Transfer id the outbound leg is keyed by. - peer (tuple | None): Restrict the pause to the leg or relay writer - toward this ``(ip, port)``; None addresses every match. + peer (tuple): Receiver ``(ip, port)`` identifying the outbound leg + or relay writer; required even before the leg starts. Returns: bool: True when a running leg (or relay writer) was marked paused; @@ -2106,13 +2938,13 @@ def pause_trans(self, fid, peer=None): """ return self._mark_transfer_pause(fid, True, peer) - def start_trans(self, fid, peer=None): + def start_trans(self, fid, peer): """Resume an outbound transfer paused with `pause_trans`. Args: fid (int): Transfer id the outbound leg is keyed by. - peer (tuple | None): Restrict the resume to the leg or relay - writer toward this ``(ip, port)``; None addresses every match. + peer (tuple): Receiver ``(ip, port)`` identifying the outbound leg + or relay writer. Returns: bool: True when a live leg (or relay writer) was resumed. @@ -2123,7 +2955,7 @@ def _register_transfer_socket(self, fid, sock): """Record the socket of a leg, so a dead session can be cleared out. Args: - fid (int): Value used by the operation. + fid (int | tuple): Receive fid or outgoing ``(receiver, fid)`` key. sock (object): Value used by the operation. """ if fid is None: @@ -2135,7 +2967,7 @@ def _release_transfer_socket(self, fid, sock): """Forget a leg's socket once the leg is over (its own, not a newer one's). Args: - fid (int): Value used by the operation. + fid (int | tuple): Receive fid or outgoing ``(receiver, fid)`` key. sock (object): Value used by the operation. """ if fid is None: @@ -2152,15 +2984,17 @@ def _drop_stale_transfers(self, peer=None): resumed without two legs writing the same transfer. Args: - peer (tuple | None): Restrict to the legs whose progress entry names this ``(ip, port)``; None clears every leg of the instance (a fresh session replaces all of them). + peer (tuple | None): Match the outgoing receiver namespace or incoming progress peer; None clears all tracked legs. Returns: int: Number of legs closed. """ stale = [] + paused = [] for store in (self.file_upload_progress, self.file_recv_progress): for fid, entry in list(store.items()): - if peer is not None and (entry[4] is None or tuple(entry[4]) != tuple(peer)): + owner = fid[0] if store is self.file_upload_progress else entry[4] + if peer is not None and (owner is None or tuple(owner) != tuple(peer)): continue with self._transfer_progress_lock: sock = self._active_transfer_sockets.pop(fid, None) @@ -2168,6 +3002,8 @@ def _drop_stale_transfers(self, peer=None): self._active_transfer_fids.discard(fid) if sock is not None: stale.append(sock) + if store is self.file_upload_progress: + paused.append(fid) for sock in stale: try: sock.shutdown(socket.SHUT_RDWR) @@ -2177,6 +3013,14 @@ def _drop_stale_transfers(self, peer=None): sock.close() except OSError: pass + if paused: + # The leg may be waiting on this fid's pause switch. Clearing it here + # (outside the progress lock) lets the swept leg exit and release the + # per-key send lock a queued resume is waiting for. + with self._transfer_pause_cond: + for key in paused: + self._transfer_pause.pop(key, None) + self._transfer_pause_cond.notify_all() return len(stale) def pause_trans_remote(self, peer, fid): @@ -2216,61 +3060,31 @@ def start_trans_remote(self, peer, fid): return False return self.send_message(info["socket"], f"/start_trans {fid}") - def _wait_transfer_paused(self, fid): + def _wait_transfer_paused(self, fid, leg_socket=None): """Block a sending leg while its fid is marked paused. Runs before every chunk so ``/pause_trans`` stops the stream without closing the transfer; the wait ends when the leg is resumed, when the - instance stops, or when the fid is not paused at all. + instance stops, when the leg socket is closed, or when the fid is not + paused at all. ``leg_socket`` is the connection this leg writes to: + a reconnect sweep removes it from the registry before closing it, so + the registry alone cannot report the close. Args: fid (int): Value used by the operation. + leg_socket (socket.socket | None): The leg's own transfer socket. """ if fid is None: return with self._transfer_pause_cond: while self._transfer_pause.get(fid, False): - if not self.running: + sock = self._active_transfer_sockets.get(fid) + if (not self.running + or (sock is not None and sock.fileno() < 0) + or (leg_socket is not None and leg_socket.fileno() < 0)): return self._transfer_pause_cond.wait(timeout=0.5) - def _wait_transfer_server_port(self, fid, timeout=RESUME_PORT_TIMEOUT): - """Wait for the peer's ``/server_file_transfer_port`` of one fid. - - Args: - fid (int): Transfer id the peer announces its listener under. - timeout (float): Seconds to wait. - - Returns: - tuple | None: ``(port, stamp)`` announced for this fid, or None - when the announcement does not arrive in time. - """ - deadline = time.time() + timeout - while time.time() < deadline: - with self.file_transfer_server_port_lock: - for info in reversed(list(self.file_server_port_list)): - if info[1] == fid: - # the newest announcement wins: an older one may belong - # to a leg of a session that is already gone - self.file_server_port_list.remove(info) - stamp = info[2] if len(info) > 2 else "plain" - return info[0], stamp - time.sleep(0.1) - return None - - def _forget_transfer_port(self, fid): - """Drop pending port announcements of a fid an older session left behind. - - Args: - fid (int): Value used by the operation. - """ - if fid is None: - return - with self.file_transfer_server_port_lock: - self.file_server_port_list[:] = [ - info for info in self.file_server_port_list if info[1] != fid - ] - def _wait_control_secure(self, control_socket, timeout=RESUME_SECURE_TIMEOUT): """Wait until a control channel may carry framed protocol lines. @@ -2622,18 +3436,23 @@ def call_extension_functions(self, command): return handler(None, None, command) def add_message_listener(self, listener): - """Register ``listener(client_id, message)`` for every inbound line. + """Register ``listener(client_id, message)`` for inbound events. + + Complete control lines are reported as ``str`` (commands and crypto + handshake lines included); raw chunks from a non-control socket + (file/folder transfer, forwarding, temporary connection) are reported + as ``bytes`` without line or frame alignment. ``client_id`` is the peer + endpoint of the socket that produced the event, so a side-channel event + names that temporary TCP connection, not the control client. - Every line received from a client is reported, commands included - (``/help``, ``/file ...``, the crypto handshake lines, ...), so the - listener observes all inbound traffic; dispatch is unaffected, a - command still goes through the command handlers. + Control-line callbacks run on the receive thread (synchronous server) + or the ``asyncio.to_thread`` worker (asynchronous server); raw chunks + run on the thread reading that socket. Callbacks may run concurrently + and must be thread-safe, must not block, and exceptions raised inside + them are swallowed. ``stop()`` stops new delivery and clears the list. Args: - listener (callable): ``listener(client_id, message)`` where - ``client_id`` is the sender's ``"ip:port"``. It runs on the receive - thread, so it must not block, and exceptions raised inside it are - swallowed. + listener (callable): ``listener(client_id, message)``. """ with self._event_listeners_lock: if listener not in self._message_listeners: @@ -2651,19 +3470,59 @@ def remove_message_listener(self, listener): except ValueError: pass + def add_send_message_listener(self, listener): + """Register ``listener(target_id, message)`` for successful sends. + + Called synchronously on the sending thread after every payload that + `send_message` writes successfully, for ``str`` and ``bytes`` alike. + ``str`` payloads are stripped; ``bytes`` are reported unchanged. + ``target_id`` is the peer endpoint of the socket that wrote the + payload: the client's ``"ip:port"`` for a control socket, or the + temporary TCP connection's ``"ip:port"`` for a side channel. + + ``is_send_raw=True`` lines and internal protocol frames such as the + ECDH public-key publication are not reported. When several threads + send on one socket the writes stay serialized, but callback order is + not guaranteed to match wire order; this affects observation only. + The listener must not block, may run concurrently, and its exceptions + are swallowed. ``stop()`` stops new delivery. + + Args: + listener (callable): ``listener(target_id, message)``. + """ + with self._event_listeners_lock: + if listener not in self._send_message_listeners: + self._send_message_listeners.append(listener) + + def remove_send_message_listener(self, listener): + """Unregister a listener previously added by `add_send_message_listener`. + + Args: + listener (callable): Listener to remove; an unknown one is ignored. + """ + with self._event_listeners_lock: + try: + self._send_message_listeners.remove(listener) + except ValueError: + pass + def add_file_listener(self, listener): """Register ``listener(client_id, full_path, name, size, command)`` per saved file. - Fired after a file uploaded by a client (a direct send, or a forwarded - file/folder item staged on the server) has been fully written to - ``file_transfer_dir``. + Fired once after a file uploaded by a client (a direct send or a + forwarded item staged on the server) is fully written to its + destination. ``client_id`` is the uploader's control-connection + ``"ip:port"``; ``command`` is the wire command that triggered the + transfer, so pushes such as ``/crypto_pub_key`` can be recognised. The + callback runs on the transfer thread before the completion sign is sent + (it does not wait for an ACK); per-chunk data goes to the message + listeners as ``bytes`` instead. For ``/crypto_pub_key`` the key is not + registered yet, so the listener must not move or delete the file. It + must not block; exceptions are swallowed. ``stop()`` stops new delivery. Args: listener (callable): ``listener(client_id, full_path, name, size, - command)``; ``client_id`` is the uploader's ``"ip:port"`` and - ``command`` the wire command that triggered the transfer, so a - listener can recognise protocol pushes such as ``/crypto_pub_key``. - It runs on the transfer thread, so it must not block. + command)``. """ with self._event_listeners_lock: if listener not in self._file_listeners: @@ -2681,6 +3540,19 @@ def remove_file_listener(self, listener): except ValueError: pass + def _stop_listeners(self): + """Stop delivering listener events and release the callback lists.""" + with self._event_listeners_lock: + self._listeners_stopped = True + self._message_listeners.clear() + self._file_listeners.clear() + self._send_message_listeners.clear() + + def _listeners_are_stopped(self): + """Return True once stop() has disabled listener delivery.""" + with self._event_listeners_lock: + return self._listeners_stopped + def _notify_message_received(self, client_id, message): """Handle the operation. @@ -2689,14 +3561,58 @@ def _notify_message_received(self, client_id, message): message (object): Value used by the operation. """ with self._event_listeners_lock: + if self._listeners_stopped: + return listeners = list(self._message_listeners) for listener in listeners: + if self._listeners_are_stopped(): + return try: listener(client_id, message) except Exception as e: self._log(f"message listener error: {e}") self._log_exc() + def _notify_transfer_chunk(self, client_socket, data): + """Report one chunk read from a non-control socket to the message listeners. + + Side-channel sockets (file/folder transfers, forwards) carry raw bytes and + ``sign`` tokens, which belong to the transfer layer, so they are reported + to the listeners but never recorded in a store. + """ + if not data or client_socket in self._control_sockets: + return + with self._event_listeners_lock: + if self._listeners_stopped: + return + listeners = list(self._message_listeners) + for listener in listeners: + if self._listeners_are_stopped(): + return + try: + listener(self._socket_key(client_socket), data) + except Exception as e: + self._log(f"message listener error: {e}") + self._log_exc() + + def _notify_message_sent(self, client_socket, message): + """Report one written payload to the send listeners.""" + with self._event_listeners_lock: + if self._listeners_stopped: + return + listeners = list(self._send_message_listeners) + if not listeners: + return + target_id = self._socket_key(client_socket) + for listener in listeners: + if self._listeners_are_stopped(): + return + try: + listener(target_id, message) + except Exception as e: + self._log(f"send message listener error: {e}") + self._log_exc() + def _notify_file_received(self, client_id, full_path, name, size, command): """Handle the operation. @@ -2708,8 +3624,12 @@ def _notify_file_received(self, client_id, full_path, name, size, command): command (object): Value used by the operation. """ with self._event_listeners_lock: + if self._listeners_stopped: + return listeners = list(self._file_listeners) for listener in listeners: + if self._listeners_are_stopped(): + return try: listener(client_id, full_path, name, size, command) except Exception as e: @@ -2773,47 +3693,57 @@ def _record_key(self, sock): return key return self._socket_key(sock) - def _record_message(self, sock, content): - """Store one inbound plain-text message under the sender's socket. - - External code reads ``messages_dict`` (or the JSON log) instead of - registering a message listener. The entry is ``[content, timestamp]`` - with the timestamp of arrival. + def _append_record(self, store, log_path, sock, content): + """Append one ``[content, timestamp]`` pair under ``sock``; flush when full. Args: - sock (object): Value used by the operation. - content (object): Value used by the operation. + store (tuple): ``(dict, lock, size_attr)``; the dict maps socket to + a list of pairs, and ``size_attr`` names the byte-size attribute. + log_path (str): JSON log the store flushes into. + sock (object): Key socket. + content (str): Text to record. """ + d, lock, size_attr = store timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - with self._messages_dict_lock: + with lock: self._remember_socket_key(sock) - self.messages_dict.setdefault(sock, []).append([content, timestamp]) - self._messages_dict_size += len(content.encode("utf-8", "replace")) + len(timestamp) - if self._messages_dict_size >= self.max_dict_size: - self._flush_dict_locked( - self.messages_dict, "_messages_dict_size", self.messages_log_file - ) + d.setdefault(sock, []).append([content, timestamp]) + size = getattr(self, size_attr) + len(content.encode("utf-8", "replace")) + size += len(timestamp) + setattr(self, size_attr, size) + if size >= self.max_dict_size: + self._flush_dict_locked(d, size_attr, log_path) - def _record_event(self, sock, command): - """Store one inbound command as an event under the sender's socket. + def _record_message(self, sock, content): + """Store one inbound plain-text message under the sender's socket.""" + self._append_record( + (self.messages_dict, self._messages_dict_lock, "_messages_dict_size"), + self.messages_log_file, sock, content, + ) - The event content is the original wire command from the peer (file - transfers, folder transfers, extension commands, ...). External code - reads ``events_dict`` (or the JSON log) instead of registering a - listener. File-transfer events get their completion timestamp via - ``_update_event_timestamp``. + def _record_event(self, sock, command): + """Store one inbound "/" command under the sender's socket.""" + self._append_record( + (self.events_dict, self._events_dict_lock, "_events_dict_size"), + self.events_log_file, sock, command, + ) - Args: - sock (object): Value used by the operation. - command (object): Value used by the operation. - """ - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - with self._events_dict_lock: - self._remember_socket_key(sock) - self.events_dict.setdefault(sock, []).append([command, timestamp]) - self._events_dict_size += len(command.encode("utf-8", "replace")) + len(timestamp) - if self._events_dict_size >= self.max_dict_size: - self._flush_dict_locked(self.events_dict, "_events_dict_size", self.events_log_file) + def _record_sent(self, sock, content): + """Store one outbound text: a "/" line as a sent event, other text as a sent message.""" + if content.startswith("/"): + self._append_record( + (self.sent_events_dict, self._sent_events_dict_lock, "_sent_events_dict_size"), + self.sent_events_log_file, sock, content, + ) + else: + self._append_record( + ( + self.sent_messages_dict, + self._sent_messages_dict_lock, + "_sent_messages_dict_size", + ), + self.sent_messages_log_file, sock, content, + ) def _update_event_timestamp(self, sock, command, timestamp): """Stamp the completion time onto the recorded event for ``command``. @@ -2880,24 +3810,49 @@ def _flush_dict_locked(self, d, size_attr, path): d.clear() setattr(self, size_attr, 0) self._merge_json_log(path, snapshot) - other = self.events_dict if d is self.messages_dict else self.messages_dict - with self._socket_keys_lock: # a key is only kept while it is buffered + with self._socket_keys_lock: # a key is only kept while some store buffers it for sock in snapshot: - if sock not in d and sock not in other: + if not any(sock in store for store in self._record_dicts()): self._socket_keys.pop(sock, None) + def _record_dicts(self): + """Return the four record stores, in flush order.""" + return ( + self.messages_dict, + self.events_dict, + self.sent_messages_dict, + self.sent_events_dict, + ) + def _flush_messages_dict(self): - """Handle the operation.""" + """Flush the inbound messages store into its JSON log.""" with self._messages_dict_lock: self._flush_dict_locked( self.messages_dict, "_messages_dict_size", self.messages_log_file ) def _flush_events_dict(self): - """Handle the operation.""" + """Flush the inbound events store into its JSON log.""" with self._events_dict_lock: self._flush_dict_locked(self.events_dict, "_events_dict_size", self.events_log_file) + def _flush_record_stores(self): + """Flush every record store into its JSON log (called on stop).""" + with self._messages_dict_lock: + self._flush_dict_locked( + self.messages_dict, "_messages_dict_size", self.messages_log_file + ) + with self._events_dict_lock: + self._flush_dict_locked(self.events_dict, "_events_dict_size", self.events_log_file) + with self._sent_messages_dict_lock: + self._flush_dict_locked( + self.sent_messages_dict, "_sent_messages_dict_size", self.sent_messages_log_file + ) + with self._sent_events_dict_lock: + self._flush_dict_locked( + self.sent_events_dict, "_sent_events_dict_size", self.sent_events_log_file + ) + def _merge_json_log(self, path, snapshot): """Merge ``snapshot`` into the JSON log at ``path`` (append per socket). @@ -3033,19 +3988,26 @@ def broadcast( the client the message came from. """ with self.client_lock: - disconnected_clients = [] - for addr, client_info in self.clients.items(): - if exclude_client and addr == exclude_client: - continue - try: - self.send_message(client_info["socket"], message) - except: - disconnected_clients.append(addr) - self._log_exc() - for addr in disconnected_clients: # del disconnected clients - if addr in self.clients: + targets = [ + (addr, client_info["socket"]) + for addr, client_info in self.clients.items() + if not (exclude_client and addr == exclude_client) + ] + disconnected_clients = [] + for addr, client_socket in targets: + try: + self.send_message(client_socket, message) + except: + disconnected_clients.append((addr, client_socket)) + self._log_exc() + if disconnected_clients: + with self.client_lock: + for addr, client_socket in disconnected_clients: + client_info = self.clients.get(addr) + if client_info is None or client_info.get("socket") is not client_socket: + continue self._debug(f"deleting the disconnected client: {addr}") - self.clients[addr]["socket"].close() + client_socket.close() del self.clients[addr] def send_msg_to_specific_client( @@ -3119,13 +4081,16 @@ def _crypto_get_send_lock(self, client_socket): self._crypto_send_locks[client_socket] = lock return lock - def send_message(self, client_socket, message): # send message to specific client + def send_message(self, client_socket, message, is_send_raw=False): + # send message to specific client """Write one message to a client socket, framing it when the channel is up. Args: client_socket (socket.socket): Target connection. message (str | bytes): Payload; a str is stripped and newline terminated, bytes are sent as they are. + is_send_raw (bool): Write the payload as plaintext even when the channel + is encrypted; only for the crypto handshake and transfer coordination lines. Returns: bool: True when the payload was written, False for an unsupported @@ -3135,6 +4100,25 @@ def send_message(self, client_socket, message): # send message to specific clie RuntimeError: If the server is not running or no socket was passed. OSError: If the socket write fails (the original error is re-raised). """ + if is_send_raw: + deal_msg = message.strip() + if not deal_msg.endswith("\n"): + deal_msg += "\n" + encoded = deal_msg.encode("utf-8") + self._sendall(client_socket, encoded) + self._measure_network_speed(encoded, is_upload=True) + return True + written = self._write_message(client_socket, message) + if written: + self._after_send(client_socket, message) + return written + + def _write_message(self, client_socket, message): + """Write one payload: encrypted when the channel is up, plaintext otherwise. + + Returns: + bool: True when written, False for an unsupported payload type. + """ if not self.running or not client_socket: self._log("disable the connect to server") raise RuntimeError("connection error") @@ -3177,6 +4161,14 @@ def send_message(self, client_socket, message): # send message to specific clie self._log_exc() raise + def _after_send(self, client_socket, message): + """Record and report one payload that `_write_message` has written.""" + if isinstance(message, str): + message = message.strip() + if message and client_socket in self._control_sockets: + self._record_sent(client_socket, message) + self._notify_message_sent(client_socket, message) + def _sendall(self, client_socket, data): """Write ``data`` in full, waiting for room on a non-blocking socket. @@ -3204,20 +4196,6 @@ def _sendall(self, client_socket, data): raise BrokenPipeError(errno.EPIPE, "socket closed while sending") view = view[sent:] - def _send_raw(self, client_socket, text): - """Send a plaintext crypto-protocol message, bypassing encryption. - - Args: - client_socket (socket.socket): Value used by the operation. - text (object): Value used by the operation. - """ - data = text.strip() - if not data.endswith("\n"): - data += "\n" - encoded = data.encode("utf-8") - self._sendall(client_socket, encoded) - self._measure_network_speed(encoded, is_upload=True) - def _crypto_mark_encrypted(self, client_socket, peer_role, peer_pem_path): """Handle the operation. @@ -3276,7 +4254,7 @@ def _crypto_after_flip(self, client_socket): return try: # plaintext by design: the peer needs this line before it may frame - self._send_raw(client_socket, CRYPTO_FRAMES_START) + self.send_message(client_socket, CRYPTO_FRAMES_START, is_send_raw=True) except Exception: self._log_exc() self._crypto_publish_ecdh_pub(client_socket) @@ -3386,7 +4364,8 @@ def receive_message(self, client_socket, msg_length): # receive message plaintext, served from the channel's buffer; a socket without one is read raw. A read that started while the channel was raw is classified by the channel when it returns, so bytes of a flip that landed during - the read never land in the line reader. + the read never land in the line reader. Reads from a non-control socket + are also reported to the message listeners (see `_notify_transfer_chunk`). Args: client_socket (socket.socket): Connection to read from. @@ -3401,10 +4380,12 @@ def receive_message(self, client_socket, msg_length): # receive message if channel is None: data = client_socket.recv(msg_length) self._measure_network_speed(data, is_upload=False) + self._notify_transfer_chunk(client_socket, data) return data data = channel.take(msg_length) if data: self._measure_network_speed(data, is_upload=False) + self._notify_transfer_chunk(client_socket, data) return data # buffered plaintext is served even after a disable raw = client_socket.recv(self.recv_messages_chunk_len if channel.secure else msg_length) if not raw: @@ -3417,10 +4398,12 @@ def receive_message(self, client_socket, msg_length): # receive message leftover = channel.drain_plaintext() if leftover: self._measure_network_speed(leftover, is_upload=False) + self._notify_transfer_chunk(client_socket, leftover) return leftover # hand back whatever was already read continue # framing is off now: keep serving in raw mode if plain: self._measure_network_speed(plain, is_upload=False) + self._notify_transfer_chunk(client_socket, plain) return plain # plaintext mode: the line reader gets the bytes def _crypto_channel_error(self, client_socket): @@ -3520,7 +4503,9 @@ def _crypto_on_decode_failure(self, client_socket): try: with self._crypto_lock: nonce = state.get("my_nonce") or self._crypto_fresh_nonce() - self._send_raw(client_socket, f"/crypto_key_exchange {nonce} 1") + self.send_message( + client_socket, f"/crypto_key_exchange {nonce} 1", is_send_raw=True + ) except Exception: self._log_exc() @@ -3568,7 +4553,11 @@ def _crypto_on_client_hello(self, client_socket, client_address, client_nonce, f session = self._crypto_sessions.get(client_address) if session is not None: session.reset_ecdh() # a fresh handshake is a fresh session key - self._send_raw(client_socket, f"/crypto_key_exchange_ack {my_nonce} 1 {1 if force else 0}") + self.send_message( + client_socket, + f"/crypto_key_exchange_ack {my_nonce} 1 {1 if force else 0}", + is_send_raw=True, + ) threading.Thread( target=self._crypto_wait_server_ready, args=(client_socket, client_address, state), @@ -3629,7 +4618,7 @@ def _crypto_wait_server_ready(self, client_socket, client_address, state): with self._crypto_lock: state["ready_sent"] = True try: - self._send_raw(client_socket, "/crypto_ready") + self.send_message(client_socket, "/crypto_ready", is_send_raw=True) except Exception: self._log_exc() self._crypto_try_server_flip(client_socket, state) @@ -3653,37 +4642,28 @@ def _crypto_push_pub_to_client(self, client_socket, client_address): break time.sleep(0.05) try: - with self.file_client_id_lock: - client_id = copy.copy(self.file_client_id) - self.file_client_id += 1 - send_msg = f"/crypto_pub_key {client_id}" - self._send_raw(client_socket, send_msg) - file_transfer_client_port = self.palloc() - file_server_port = None - waiting_time = 0 - while True: - time.sleep(0.05) - with self.file_transfer_server_port_lock: - for port_info in list(self.file_server_port_list): - if port_info[1] == client_id: - file_server_port = port_info[0] - self.file_server_port_list.remove(port_info) - break - if file_server_port is not None: - break - waiting_time += 1 - if waiting_time >= 200: + client_id = self._fid_alloc.alloc(receiver=client_address) + with _transfer_handshake(self, client_socket, client_id) as handshake: + send_msg = f"/crypto_pub_key {client_id}" + if not _send_transfer_request(self, handshake, send_msg, is_send_raw=True): + return False + result = self._transfer_handshakes.wait(handshake, 10.0) + if result is None or result[0] != "port" or result[1] != client_id: self._log("crypto: transfer port waiting timeout, public key push failed") return False - self.file_transfer_mode( # a key push carries the keys themselves: always plaintext - self.crypto.pub_path, - client_address[0], - file_server_port, - file_transfer_client_port, - control_socket=client_socket, - crypto="plain", - ) - self.pfree(file_transfer_client_port) + _, _, file_server_port, _stamp = result + file_transfer_client_port = self.palloc() + try: + self.file_transfer_mode( # public-key bytes always travel plaintext + self.crypto.pub_path, + client_address[0], + file_server_port, + file_transfer_client_port, + control_socket=client_socket, + crypto="plain", + ) + finally: + self.pfree(file_transfer_client_port) except Exception: self._log_exc() return False @@ -3737,7 +4717,7 @@ def _crypto_store_received_pub(self, full_path, peer_role, client_address): peer_socket = self.clients.get(client_address, {}).get("socket") if not ok and peer_socket is not None: try: - self._send_raw(peer_socket, f"/crypto_reject {reason}") + self.send_message(peer_socket, f"/crypto_reject {reason}", is_send_raw=True) except Exception: self._log_exc() try: @@ -3842,9 +4822,12 @@ async def _async_receive(self, client_socket, loop): with self._crypto_lock: channel = self._channels.get(client_socket) if channel is None: - return await loop.sock_recv(client_socket, self.recv_messages_chunk_len) + data = await loop.sock_recv(client_socket, self.recv_messages_chunk_len) + self._notify_transfer_chunk(client_socket, data) + return data data = channel.take(self.recv_messages_chunk_len) if data: + self._notify_transfer_chunk(client_socket, data) return data # buffered plaintext is served even after a disable raw = await loop.sock_recv(client_socket, self.recv_messages_chunk_len) if not raw: @@ -3856,9 +4839,11 @@ async def _async_receive(self, client_socket, loop): await asyncio.to_thread(self._crypto_channel_error, client_socket) leftover = channel.drain_plaintext() if leftover: + self._notify_transfer_chunk(client_socket, leftover) return leftover continue # framing is off now: keep serving in raw mode if plain: + self._notify_transfer_chunk(client_socket, plain) return plain # plaintext mode: the line reader gets the bytes async def _handle_client_async(self, client_socket, client_address): @@ -3924,6 +4909,7 @@ def _register_client(self, client_socket, client_address): """ client_id = f"{client_address[0]}:{client_address[1]}" with self.client_lock: # add new client + self._control_sockets.add(client_socket) self.clients[client_address] = { "socket": client_socket, "address": client_address, @@ -3976,7 +4962,11 @@ def _announce_client(self, client_socket, client_id): return False # announce our encryption mode; a mismatched peer is disconnected in handle_command try: - self._send_raw(client_socket, f"/crypto_mode {1 if self.is_enable_encrypto else 0}") + self.send_message( + client_socket, + f"/crypto_mode {1 if self.is_enable_encrypto else 0}", + is_send_raw=True, + ) except Exception as e: # the peer vanished right after the welcome: the caller's cleanup # closes the connection; never let this escape @@ -4037,21 +5027,25 @@ def _unregister_client(self, client_socket, client_address, client_id): client_address (tuple): Peer ``(ip, port)``. client_id (str): Client id used in the console log. """ + self._transfer_handshakes.cancel_socket(client_socket) with self.client_lock: - if client_address in self.clients: + owns_peer = self.clients.get(client_address, {}).get("socket") is client_socket + if owns_peer: del self.clients[client_address] with self._crypto_lock: - self._crypto_state.pop(client_address, None) - self._crypto_sessions.pop(client_address, None) + if owns_peer: + self._crypto_state.pop(client_address, None) + self._crypto_sessions.pop(client_address, None) self._crypto_sock_addr.pop(client_socket, None) self._crypto_peer.pop(client_socket, None) self._crypto_push_active.discard(client_socket) self._encrypted_sockets.discard(client_socket) + self._control_sockets.discard(client_socket) self._channels.pop(client_socket, None) self._crypto_send_locks.pop(client_socket, None) client_socket.close() # the transfers that died with this connection are written to the - # progress file so a reconnect can continue them (resume_transfers) + # progress file so a reconnect can continue them (_request_resumes) self._persist_transfer_progress() self._debug(f"client disconnected: {client_id}") self._debug(f"current connection count: {len(self.clients)}") @@ -4143,18 +5137,10 @@ def handle_command( command, client_id, client_socket ) elif shlex.split(command.lower())[0] == "/server_file_transfer_port": - parts = command.split(" ") - with self.file_transfer_server_port_lock: - self.file_transfer_server_port = int(parts[1]) - try: - file_client_id = int(parts[2]) - stamp = parts[3] if len(parts) > 3 else "plain" # listener mode+nonce - self.file_server_port_list.append( - [self.file_transfer_server_port, file_client_id, stamp] - ) - except: - self._log_exc() - pass + announcement = _parse_transfer_port_announcement(command) + if announcement is not None: + port, fid, stamp = announcement + self._transfer_handshakes.resolve_port(client_socket, fid, port, stamp) elif shlex.split(command.lower())[0] == "/forward_item": threading.Thread( target=self._forward_item_handler, @@ -4183,7 +5169,7 @@ def handle_command( if parsed is not None: threading.Thread( target=self._resume_request, - args=(client_address, parsed[0], parsed[1]), + args=(client_socket, client_address, *parsed), daemon=True, ).start() return None @@ -4376,25 +5362,10 @@ def forward_message_to(self, target, message, originator_addr): return True def forward_target_command( - self, kind, rel_dir, fname, originator_addr, tfid, destination_path=None + self, kind, rel_dir, fname, originator_addr, tfid, + destination_path=None, ): - """Build the tagged wire command that pushes one forwarded item. - - Public API for forward extensions. The originator tuple sits before the - trailing transfer id, where the receiver's existing parsers ignore it and - `parse_forward_originator` recovers it for attribution. - - Args: - kind (str): "file" or "file_folder". - rel_dir (str): Relative folder path (folders only). - fname (str): File or folder name. - originator_addr (tuple): Originating client ``(ip, port)``. - tfid (int): Transfer id shared by the pushed item. - destination_path (str | None): Receiver-side destination directory. - - Returns: - str: Command line to hand to `send_message`. - """ + """Build the receiver command using its sender-allocated child FID.""" originator = shlex.quote(repr(originator_addr)) if kind == "file": if destination_path: @@ -4408,41 +5379,28 @@ def forward_target_command( f"/file_folder {shlex.quote(rel_dir)} {shlex.quote(fname)} " f"{originator} {shlex.quote(destination_path)} {tfid}" ) - return f"/file_folder {shlex.quote(rel_dir)} {shlex.quote(fname)} {originator} {tfid}" + return ( + f"/file_folder {shlex.quote(rel_dir)} {shlex.quote(fname)} " + f"{originator} {tfid}" + ) def forward_item_to( - self, target, kind, rel_dir, fname, originator_addr, tfid, destination_path=None + self, target, kind, rel_dir, fname, originator_addr, tfid, + destination_path=None, ): - """Push one forwarded file or folder item to a connected target. - - Public API for forward extensions: sends the line built by - `forward_target_command`, which the receiver attributes with - `parse_forward_originator`. - - Args: - target (tuple): Destination ``(ip, port)``. - kind (str): "file" or "file_folder". - rel_dir (str): Relative folder path (folders only). - fname (str): File or folder name. - originator_addr (tuple): Originating client ``(ip, port)``. - tfid (int): Transfer id shared by the pushed item. - destination_path (str | None): Receiver-side destination directory. - - Returns: - bool: False when ``target`` is not connected (a console notice is - printed); True when the command was sent. - """ + """Push one forwarded item to its target's receive listener.""" client_info = self.clients.get(target) if client_info is None: self._log(forward_skip_message(target)) return False - self.send_message( - client_info["socket"], + control = client_info["socket"] + handshake = self._transfer_handshakes.pending(control, tfid) + return _send_transfer_request( + self, handshake, self.forward_target_command( - kind, rel_dir, fname, originator_addr, tfid, destination_path + kind, rel_dir, fname, originator_addr, tfid, destination_path, ), ) - return True def _execute_custom_handler(self, handler, command, client_socket=None, client_address=None): """Handle the operation. @@ -4479,7 +5437,13 @@ def _execute_custom_handler(self, handler, command, client_socket=None, client_a def file_folder_transfer_server_recv_server_start_thread( # start a file folder server thread on server self, command, client_id, client_socket ): - """Start a server-side folder receive operation. + """Create a folder, or start a server-side folder receive operation. + + A command without a client id (``/file_folder `` or + ``/file_folder ``) is folder creation: the directory is + created here, on the control line, and no transfer leg is opened. A + command with an id carries one file of the folder and starts the usual + receive leg. Args: command (object): Value used by the operation. @@ -4491,6 +5455,18 @@ def file_folder_transfer_server_recv_server_start_thread( # start a file folder # >= 4 tokens means a file transfer (/file_folder [] ); # 2-3 tokens is folder creation (optional destination, no client id) file_name = command_part[2] if len(command_part) >= 4 else None + if file_name is None: + created = _receive_dir_for( + self.file_transfer_dir, + relative_folder_path, + _parse_destination_path(command_part), + ) + try: + os.makedirs(created, exist_ok=True) + self._log(f"folder {created} created for {client_id}") + except OSError as e: + self._log(f"folder {created} could not be created for {client_id}: {e}") + return folder_transfer_server_recv_server_start_thread = threading.Thread( target=self.file_transfer_server_recv_server_start, args=(client_id, client_socket, command, relative_folder_path, file_name), @@ -4550,7 +5526,6 @@ def file_transfer_mode_recv( file_name, command, peer=None, - resume_entry=None, ): """Receive one file over a transfer sub-connection. @@ -4563,11 +5538,9 @@ def file_transfer_mode_recv( file_name (str | None): Destination filename, when specified. command (str): Wire command that initiated the transfer. peer (tuple | None): Original sender ``(ip, port)`` for resuming. - resume_entry (list | None): Saved progress entry for a resumed file. Returns: - None | bool: None after starting or finishing; False when a resume request - cannot be sent. + None: after the leg starts or finishes. """ file_running = True client_file_socket = None @@ -4581,27 +5554,22 @@ def close_socket(): nonlocal server_file_socket file_running = False self._crypto_drop_channel(client_file_socket) - client_file_socket.close() - server_file_socket.close() + if client_file_socket is not None: + client_file_socket.close() + self._transfer_handshakes.release_socket(client_socket, client_file_socket) + if server_file_socket is not None: + server_file_socket.close() + self._transfer_handshakes.release_socket(client_socket, server_file_socket) def setting_file_save_path(): - """Handle the operation.""" + """Resolve, and create, the directory this leg writes into.""" nonlocal save_path - save_path = self.file_transfer_dir + save_path = _receive_dir_for( + self.file_transfer_dir, new_save_path, destination_path + ) if new_save_path or destination_path: - if destination_path: - if os.path.isabs(destination_path): - save_path = destination_path - else: - save_path = os.path.join(save_path, destination_path) - if new_save_path: - path_list = new_save_path.split("/") - if path_list and path_list[0] == "": - del path_list[0] - for node in path_list: - save_path = os.path.join(save_path, node) os.makedirs(save_path, exist_ok=True) - if file_name or new_save_path == None: + if file_name or new_save_path is None: return save_path close_socket() return None @@ -4692,6 +5660,11 @@ def file_transfer_client_recv(client_id): "ErrorSignReceivedWhileReceivingFileSize: client reported error and disconnected" ) file_size = int.from_bytes(size_bytes, "big") + if resume_entry is not None and resume_entry[1] != file_size: + raise ConnectionError( + f"resume rejected: {filename} is {file_size} bytes, " + f"the partial file expects {resume_entry[1]}" + ) self.send_message(client_file_socket, self.server_received_file_header_sign) original_filename = filename if resume_entry is not None: @@ -4742,7 +5715,6 @@ def file_transfer_client_recv(client_id): remaining -= len(chunk) self._file_progress_update(progress, len(chunk), is_upload=False) f.truncate() # a longer partial file drops its stale tail - finished_ok = True # TOFU first: the ack is only a notification, and a failed # ack send must not skip the key registration (the server # would never announce readiness and the handshake hangs) @@ -4766,7 +5738,9 @@ def file_transfer_client_recv(client_id): except OSError: pass try: - self.send_message(client_file_socket, self.server_received_file_data_sign) + finished_ok = bool(self.send_message( + client_file_socket, self.server_received_file_data_sign + )) except Exception: self._log_exc() close_socket() @@ -4802,38 +5776,41 @@ def file_transfer_client_recv(client_id): server_file_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_file_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_file_socket.bind((server_file_address, server_file_port)) - file_transfer_server_port = server_file_socket.getsockname()[1] + file_transfer_server_port = _bind_local_port_with_fallback( + server_file_socket, + (server_file_address, server_file_port), + self._log, + "file transfer listener", + ) command_part = shlex.split(command) - file_client_id = command_part[len(command_part) - 1] + try: + file_client_id = int(command_part[-1]) + if file_client_id < 0: + raise ValueError + except (IndexError, ValueError): + server_file_socket.close() + return progress_fid = _transfer_fid(command_part) # keys this leg in file_recv_progress + resume_entry = _resumable_entry(self.file_recv_progress, progress_fid, peer, command_part) destination_path = _parse_destination_path(command_part) stamp = self._crypto_transfer_stamp( client_socket, force_plain=command_part[0] == "/crypto_pub_key" ) - self._forget_transfer_port(progress_fid) # an older session's entry is stale - transfer_server_port_msg = "/server_file_transfer_port {} {} {}\n".format( + transfer_server_port_msg = _transfer_port_announcement( file_transfer_server_port, file_client_id, stamp ) - self._send_raw( - client_socket, transfer_server_port_msg - ) # coordination line: always plaintext (like the handshake) server_file_socket.listen(1) if resume_entry is not None: - # the receiver drives a resumed transfer: it asks the sender to - # continue the file from the bytes already on disk - offset = _resume_offset_for(resume_entry) - if progress_fid is None or not self.send_message( - client_socket, f"{RESUME_LINE} {progress_fid} {offset}" - ): - self._log( - f"resume: cannot ask {client_id} to continue transfer {progress_fid}" - ) - server_file_socket.close() - return False + # the sender answers this resume with /file; its connection is the continued leg server_file_socket.settimeout(RESUME_ACCEPT_TIMEOUT) try: + if not self._transfer_handshakes.track_socket(client_socket, server_file_socket): + return + if not self.send_message(client_socket, transfer_server_port_msg, is_send_raw=True): + return client_file_socket, client_file_address = server_file_socket.accept() + if not self._transfer_handshakes.track_socket(client_socket, client_file_socket): + return if resume_entry is not None: client_file_socket.settimeout(None) # the accept timeout is the listener's only self._crypto_attach_transfer_in(client_file_socket, client_socket, stamp) @@ -4861,6 +5838,7 @@ def file_transfer_client_recv(client_id): close_socket() finally: server_file_socket.close() + self._transfer_handshakes.release_socket(client_socket, server_file_socket) def diff_multiple_file_diff_multiple_client_transfer_server_recv_client_start(self, message): """Send grouped files and folders to their listed clients. @@ -5109,17 +6087,19 @@ def limited_transfer(): self._log(f"start to send file: {cmd} (limit {self.max_file_transfer_thread_num})") def get_all_files_in_folder(): - """Handle the operation.""" - for root, dirs, files in os.walk(folder_path): + """Announce the directories that need it, then send every file.""" + needed = _rel_dirs_with_files(folder_path, base_path) + if transfer_path in needed: + send_folder_transfer_command(transfer_path) + for root, _dirs, files in os.walk(folder_path): rel_dir = get_relative_path(base_path, root) - if root != folder_path: + if root != folder_path and rel_dir in needed: send_folder_transfer_command(rel_dir) for file in files: start_file_transfer_with_limit(rel_dir, file, root) self._log(f"finished sending all files in folder {folder_path}") transfer_path = get_relative_path(base_path, folder_path) - send_folder_transfer_command(transfer_path) get_all_files_in_folder() def file_transfer_server_recv_client_start_thread(self, message, file_folder_abspath=None): @@ -5160,50 +6140,31 @@ def file_transfer_server_recv_client_start(self, message, file_folder_abspath): self._log_exc() return False self._debug(client_socket, client_address, message) - with self.file_client_id_lock: - client_id = copy.copy(self.file_client_id) - send_msg = message.strip() + " " + str(self.file_client_id) + "\n" - self.file_client_id += 1 + client_id = self._fid_alloc.alloc(receiver=client_ip) try: - waiting_time = 0 if shlex.split(message.lower())[0] == "/file_folder": filename = os.path.join(file_folder_abspath, shlex.split(message)[2]) else: filename = shlex.split(message)[1] - self.send_message(client_socket, send_msg) - file_transfer_client_port = self.palloc() - file_server_port = None - stamp = "plain" - is_find_port = True - while is_find_port: - time.sleep(1) - if len(self.file_server_port_list) > 0: - with self.file_transfer_server_port_lock: - for port_info in self.file_server_port_list: - if port_info[1] == client_id: - file_server_port = port_info[0] - stamp = port_info[2] # the listener's mode+nonce - self.file_server_port_list.remove(port_info) - is_find_port = False - break - pass - waiting_time += 1 - if waiting_time >= 20: - self._log( - "ErrorWhileReceiveFileServerPort: transfer port waiting timeout, file sending failed" - ) + with _sending_transfer(self, (tuple(client_ip), client_id)), \ + _transfer_handshake(self, client_socket, client_id) as handshake: + send_msg = f"{message.strip()} {client_id}\n" + if not _send_transfer_request(self, handshake, send_msg): return False - self.file_transfer_mode( - filename, - client_address, - file_server_port, - file_transfer_client_port, - control_socket=client_socket, - crypto=stamp, - fid=client_id, - peer=client_ip, # the receiver a resume is asked to continue from - ) - self.pfree(file_transfer_client_port) + result = self._transfer_handshakes.wait(handshake, RESUME_PORT_TIMEOUT) + if result is None or result[0] != "port" or result[1] != client_id: + self._log("ErrorWhileReceiveFileServerPort: transfer port waiting timeout, file sending failed") + return False + _, _, file_server_port, stamp = result + file_transfer_client_port = self.palloc() + try: + self.file_transfer_mode( + filename, client_address, file_server_port, file_transfer_client_port, + control_socket=client_socket, crypto=stamp, fid=client_id, + peer=client_ip, + ) + finally: + self.pfree(file_transfer_client_port) except IndexError: self._log("invalid command, please use '/file '") self._log_exc() @@ -5237,6 +6198,10 @@ def file_transfer_mode( # noqa: PLR0911 - peer-close and timeout exits are dist bool: True when the receiver acknowledges the file; otherwise False. """ self._log(f"start to send file: {filename}") + receiver = tuple(peer) if peer is not None else ( + control_socket.getpeername() if control_socket is not None else (server_address, server_port) + ) + transfer_key = (receiver, fid) if fid is not None else None client_file_socket = None reset_time = 0 @@ -5246,17 +6211,27 @@ def close_socket(): nonlocal client_file_socket file_running = False self._crypto_drop_channel(client_file_socket) + self._transfer_handshakes.release_socket(control_socket, client_file_socket) client_file_socket.close() - with self.file_client_id_lock: - self.file_client_id -= 1 while True: try: client_file_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if not self._transfer_handshakes.track_socket(control_socket, client_file_socket): + return False client_file_socket.bind((self.host, client_port)) client_file_socket.connect((server_address, server_port)) break except Exception as e: + self._transfer_handshakes.release_socket(control_socket, client_file_socket) + client_file_socket.close() + if _should_fallback_local_port(client_port, e): + self._log( + f"file transfer source port {client_port} is not available ({e}); " + "retrying from an OS-assigned port" + ) + client_port = 0 + continue self._log(f"file transfer connection error: {e}") self._log_exc() if reset_time >= 20: @@ -5273,14 +6248,17 @@ def close_socket(): close_socket() return False file_running = True - self._register_transfer_socket(fid, client_file_socket) - file_receive_data_from_server = "" + self._register_transfer_socket(transfer_key, client_file_socket) + transfer_signs = _TransferSignBuffer( + self.server_start_file_transfer_sign, + self.server_received_file_data_sign, + self.error_sign, + ) def receive_file_transfer_messages(): """Handle the operation.""" nonlocal file_running nonlocal client_file_socket - nonlocal file_receive_data_from_server while file_running: try: data = self.receive_message(client_file_socket, self.recv_messages_chunk_len) @@ -5292,8 +6270,7 @@ def receive_file_transfer_messages(): pass # send_message already logged real errors; a dead peer is expected close_socket() break - file_receive_data_from_server = data.decode("utf-8").strip() - if file_receive_data_from_server == self.error_sign: + if transfer_signs.feed(data): self._log( "\nError sign received from server, file transfer may have failed" ) @@ -5318,12 +6295,16 @@ def receive_file_transfer_messages(): source_missing = False # True when the source file no longer exists try: while True: - if file_receive_data_from_server == self.server_start_file_transfer_sign: + # Snapshot closure before consuming so ACK + EOF cannot race + # a stale empty-queue check into a false failure. + running = file_running + start_seen, _, error_seen = transfer_signs.consume() + if start_seen: break - if file_receive_data_from_server == self.error_sign: + if error_seen: close_socket() - break - if not file_running: + return False + if not running: # peer closed the file connection: the receive thread # already cleaned up; nothing more to wait for return False @@ -5351,32 +6332,40 @@ def receive_file_transfer_messages(): if resume_offset < 0 or resume_offset > file_size: resume_offset = 0 # not this file's offset any more: send it whole progress = self._file_progress_start( - self.file_upload_progress, - fid, - file_size, - filename, - peer=peer, - initial=resume_offset, + self.file_upload_progress, transfer_key, file_size, filename, + peer=peer, initial=resume_offset, ) with open(filename, "rb") as f: if resume_offset: f.seek(resume_offset) while True: - self._wait_transfer_paused(fid) + transfer_signs.consume() + self._wait_transfer_paused(transfer_key, client_file_socket) file_data = f.read(self.file_transfer_chunk_len) if not file_data: break - self.send_message(client_file_socket, file_data) + try: + written = self.send_message(client_file_socket, file_data) + except OSError as e: + self._log(f"send error: {e}") + close_socket() # the peer is gone; stop instead of reading the rest + return False + if not written: + self._log("send error: the transfer connection is closed") + close_socket() + return False self._file_progress_update(progress, len(file_data), is_upload=True) extra_time = (file_size // (100 * 1024 * 1024)) * 10 timeout = int(30 + extra_time) while True: - if file_receive_data_from_server == self.server_received_file_data_sign: + running = file_running + _, ack_seen, error_seen = transfer_signs.consume() + if ack_seen: break - if file_receive_data_from_server == self.error_sign: + if error_seen: close_socket() - break - if not file_running: + return False + if not running: # peer closed the file connection: the receive thread # already cleaned up; nothing more to wait for return False @@ -5421,272 +6410,218 @@ def receive_file_transfer_messages(): return False finally: self._file_progress_finish( - self.file_upload_progress, - fid, - progress, + self.file_upload_progress, transfer_key, progress, keep=( not file_sent and not source_missing and progress is not None - and progress[4] is not None # resumable: it has a peer + and progress[4] is not None ), ) - self._release_transfer_socket(fid, client_file_socket) + self._release_transfer_socket(transfer_key, client_file_socket) with self._transfer_pause_cond: - self._transfer_pause.pop(fid, None) + self._transfer_pause.pop(transfer_key, None) # ---- transfer reconnect (server side) ---------------------------------- - def resume_transfers(self, peer=None): - """Continue the interrupted transfers of a peer that is back. + def _request_resumes(self, peer): + """Ask a peer that just connected to continue this side's interrupted downloads. - Reads the persisted stores back (`_load_transfer_progress`) and, for - every download this instance was receiving from ``peer``, opens a - fresh transfer listener and asks the sender to continue the file from - the bytes already on disk. Restored uploads need no action here: the - peer drives their resume from its own side. + Reads the persisted stores back (`_load_transfer_progress`). Every download + this instance was receiving from ``peer`` gets its offset set to the bytes + already on disk, and the unified origin/target request is sent; the sender + answers with ``/file `` and the continued leg is received by + `file_transfer_mode_recv`. Uploads need no action here. Args: - peer (tuple | None): Peer ``(ip, port)`` that just connected; None - resumes every peer that is connected right now. + peer (tuple): Peer ``(ip, port)`` that just connected. Returns: - int: Number of resumed receive legs started. + int: Number of resume requests sent. """ restored = self._load_transfer_progress() if restored: self._log(f"transfer reconnect: restored {restored} interrupted transfer(s)") + peer = tuple(peer) with self.client_lock: - connected = {tuple(address) for address in self.clients} - candidates = [] + info = self.clients.get(peer) + if info is None or not self._wait_control_secure(info["socket"]): + return 0 + target = self._control_local_endpoint(info["socket"]) + started = 0 for fid, entry in list(self.file_recv_progress.items()): - if entry[4] is None: - continue # without a peer there is nobody to continue with - with self._forward_relays_lock: - if fid in self._forward_resume_records: - continue # a forwarded transfer: its target drives the resume + if entry[4] is None or tuple(entry[4]) != peer: + continue # not a download from this peer + if self._lookup_fid_route(None, fid) is not None: + continue # a forwarded transfer: its target drives the resume with self._transfer_progress_lock: if fid in self._active_transfer_fids: continue # a leg is already running for this fid - peer_addr = tuple(entry[4]) - if peer is not None and peer_addr != tuple(peer): - continue - if peer is None and peer_addr not in connected: - continue - candidates.append((fid, entry, peer_addr)) - started = 0 - for fid, entry, peer_addr in candidates: - with self.client_lock: - info = self.clients.get(peer_addr) - if info is None: - continue - if not self._wait_control_secure(info["socket"]): - continue - threading.Thread( - target=self._resume_receive_leg, - args=( - info["socket"], - peer_addr, - fid, - entry, - f"/file {shlex.quote(os.path.basename(str(entry[3])))} {fid}", - ), - daemon=True, - ).start() - started += 1 + entry[0] = _resume_offset_for(entry) + if self.send_message( + info["socket"], format_resume_line(fid, entry[0], peer, target) + ): + started += 1 return started - def _resume_receive_leg(self, control_socket, peer_addr, fid, entry, command): - """Receive the rest of one interrupted transfer, appending to its file. - - Args: - control_socket (socket.socket): Value used by the operation. - peer_addr (tuple): Value used by the operation. - fid (int): Value used by the operation. - entry (object): Value used by the operation. - command (object): Value used by the operation. - - Returns: - bool: True when the sender was asked to continue, False when the - request could not be sent. - """ - self._log(f"resume: continuing transfer {fid} with {peer_addr} into {entry[3]}") - entry[0] = _resume_offset_for(entry) # the offset the sender is asked for - port = self.palloc() - try: - return self.file_transfer_mode_recv( - self.host, - port, - control_socket, - f"{peer_addr[0]}:{peer_addr[1]}", - None, - None, - command, - peer=peer_addr, - resume_entry=entry, - ) is not False - finally: - self.pfree(port) - def resume_trans(self, peer_addr, fid, offset): - """Continue one interrupted transfer this instance was sending. + """Queue direct or forwarded recovery from the existing upload row. - Runs when a receiver asks to continue a transfer (``/resume_trans``, - see `resume_transfers` on the receiving side): the leg is - re-established against the receiver's fresh listener and the source - file is read from the byte offset the receiver already holds. Also the - public API for resuming one transfer on demand. + The unified request has validated origin, target and route before this + method. Direct recovery sends ``/file``; forwarded replay keeps the + stable upstream fid and uses ``/forward_item`` with ``fid=`` and + ``resume=``. Args: - peer_addr (tuple | None): Receiver ``(ip, port)`` the request came - from; None skips the peer check. + peer_addr (tuple | None): Receiver ``(ip, port)`` for this upload. fid (int): Transfer id of the interrupted transfer. offset (int): Byte offset the receiver already holds. Returns: - bool: True when the resume leg was started. + bool: True when the request was queued or joined in flight. """ - entry = self.file_upload_progress.get(fid) - if entry is None: - self._log(f"resume: no outgoing transfer with fid {fid} to continue") + if peer_addr is None: return False - if peer_addr is not None and entry[4] is not None and tuple(entry[4]) != tuple(peer_addr): - self._log(f"resume: transfer {fid} is for {entry[4]}, not {peer_addr}, ignored") + peer_addr = tuple(peer_addr) + key = (peer_addr, fid) + entry = self.file_upload_progress.get(key) + if entry is None or entry[4] is None or tuple(entry[4]) != peer_addr: + self._log(f"resume: no outgoing transfer {fid} for {peer_addr}") return False with self.client_lock: - info = self.clients.get(tuple(peer_addr)) if peer_addr is not None else None + info = self.clients.get(peer_addr) if info is None: - self._log(f"resume: {peer_addr} is not connected, transfer {fid} kept") - return False - port_info = self._wait_transfer_server_port(fid) - if port_info is None: - self._log(f"resume: {peer_addr} did not announce a transfer port for {fid}") return False - port, stamp = port_info - threading.Thread( - target=self._run_resume_send_leg, - args=(entry[3], tuple(peer_addr), port, stamp, info["socket"], fid, offset), - daemon=True, - ).start() - return True - def _run_resume_send_leg(self, path, peer_addr, port, stamp, control_socket, fid, offset): - """Run one resumed sending leg on its own thread. + def send(entry): + with _transfer_handshake(self, info["socket"], fid) as handshake: + command = f"/file {shlex.quote(os.path.basename(str(entry[3])))} {fid}" + if not _send_transfer_request(self, handshake, command): + return + result = self._transfer_handshakes.wait(handshake, RESUME_PORT_TIMEOUT) + if result is None or result[0] != "port" or result[1] != fid: + self._log(f"resume: {peer_addr} did not announce a transfer port for {fid}") + return + _, _, port, stamp = result + client_port = self.palloc() + try: + self.file_transfer_mode( + entry[3], peer_addr[0], port, client_port, + control_socket=info["socket"], crypto=stamp, fid=fid, + peer=peer_addr, resume_offset=offset, + ) + finally: + self.pfree(client_port) - Args: - path (str): Value used by the operation. - peer_addr (tuple): Value used by the operation. - port (int): Value used by the operation. - stamp (object): Value used by the operation. - control_socket (socket.socket): Value used by the operation. - fid (int): Value used by the operation. - offset (int): Value used by the operation. - """ - client_port = self.palloc() - try: - self.file_transfer_mode( - path, - peer_addr[0], - port, - client_port, - control_socket=control_socket, - crypto=stamp, - fid=fid, - peer=peer_addr, - resume_offset=offset, - ) - finally: - self.pfree(client_port) + return _start_resume_send(self, key, peer_addr, send) - def _resume_request(self, client_address, fid, offset): - """Route one client's ``/resume_trans`` to the transfer it names. - The fid identifies either this server's own outbound leg to that - client, or a target leg of a forward relay — the relay record then - names the originator, which is asked to re-forward to this client. + def _is_control_local_endpoint(self, endpoint, sock): + """Report whether ``endpoint`` names this control connection's local side. + + The concrete address the kernel chose (a wildcard bind or a host alias + resolves to it) and the configured address both identify this instance; + either is accepted so a peer that learned the name from configuration and + one that learned it from the connection agree. Args: - client_address (tuple): Value used by the operation. - fid (int): Value used by the operation. - offset (int): Value used by the operation. + endpoint (tuple): ``(ip, port)`` sent on the wire. + sock (socket.socket | tuple | None): Control socket the line arrived + on, or the peer ``(ip, port)`` when it is not available. Returns: - object: Result produced by the operation. + bool: True when the endpoint is this side of that connection. """ - with self._forward_relays_lock: - record = self._forward_resume_records.get(fid) - if record is not None: - originator = record.get("originator") - targets = [tuple(target) for target in record.get("targets") or []] - if ( - originator is not None - and tuple(client_address) in targets - and tuple(originator) != tuple(client_address) - ): - return self._route_forward_resume(record, client_address, offset) - return self.resume_trans(client_address, fid, offset) - - def _route_forward_resume(self, record, target_addr, offset): - """Ask a forwarded transfer's originator to continue it for one target. + endpoint = tuple(endpoint) + if endpoint == (self.host, self.port): + return True + if isinstance(sock, tuple): + return endpoint == tuple(sock) + return endpoint == _socket_local_endpoint(sock) - The server only relays: the record of the dead relay names the - originating client, which re-drives the item to this target from the - offset the target already holds (``/forward_resume``). An originator - that is not back yet gets the request as soon as it reconnects. + def _control_local_endpoint(self, sock): + """Return the endpoint this instance advertises on a control connection. Args: - record (object): Value used by the operation. - target_addr (object): Value used by the operation. - offset (int): Value used by the operation. + sock (socket.socket | tuple | None): Control socket of the + reconnected peer, or its ``(ip, port)``. Returns: - bool | object: Result produced by the operation. + tuple: The concrete local ``(ip, port)``, or the configured address + when the socket is already gone. """ - originator = record.get("originator") - originator_fid = record.get("fid") - if originator is None or originator_fid is None: - return False - request = ( - f"{FORWARD_RESUME_LINE} {shlex.quote(repr(tuple(target_addr)))} " - f"{originator_fid} {offset}" - ) - with self.client_lock: - info = self.clients.get(tuple(originator)) - if info is None: - with self._forward_relays_lock: - self._pending_relay_resumes.setdefault(tuple(originator), []).append( - (originator_fid, tuple(target_addr), offset) - ) - self._log( - f"resume: originator {tuple(originator)} is not connected, " - f"request for {tuple(target_addr)} kept" - ) - return False - self._log( - f"resume: asking {tuple(originator)} to continue forwarded transfer " - f"{originator_fid} for {tuple(target_addr)}" - ) - return self.send_message(info["socket"], request) + if isinstance(sock, tuple): + return tuple(sock) + return _socket_local_endpoint(sock) or (self.host, self.port) - def _flush_pending_relay_resumes(self, client_address): - """Send the forwarded-resume requests that waited for this originator. + def _resume_request(self, client_socket, client_address, fid, offset, origin, target): + """Validate requester, origin, target and scoped route before recovery. - Args: - client_address (tuple): Value used by the operation. + ``client_socket`` is the requester's control socket (or its peer address + when a caller drives the method directly); it names the local side the + origin must match. """ + client_address, origin, target = tuple(client_address), tuple(origin), tuple(target) + if target != client_address: + return False + if self._is_control_local_endpoint(origin, client_socket): + return self.resume_trans(client_address, fid, offset) + found = self._lookup_fid_route(client_address, fid) + if found is not None: + upstream, route = found + if route["originator"] == origin: + return self._queue_resume_to_originator(upstream, route, target, offset) + return False + self._log(f"resume: transfer {format_fid(fid)} has no matching origin/target route") + return False + + def _queue_resume_to_originator(self, originator_fid, route, target_addr, offset): + """Keep the latest target offset until its originator accepts the request.""" + originator, target = route["originator"], tuple(target_addr) with self._forward_relays_lock: - pending = self._pending_relay_resumes.pop(tuple(client_address), None) - if not pending: - return + if (self._fid_routes.get(originator_fid) is not route + or target not in route["targets"] or target in route["completed"]): + return False + self._pending_originator_resumes.setdefault(originator, {})[ + (originator_fid, target) + ] = offset + return self._flush_pending_originator_resumes(originator) > 0 + + def _flush_pending_originator_resumes(self, client_address): + """Send queued requests through the secure control path without early removal.""" + client_address = tuple(client_address) + with self._forward_relays_lock: + pending = list(self._pending_originator_resumes.get(client_address, {}).items()) with self.client_lock: - info = self.clients.get(tuple(client_address)) - if info is None: - return - for fid, target, offset in pending: - self.send_message( - info["socket"], - f"{FORWARD_RESUME_LINE} {shlex.quote(repr(tuple(target)))} {fid} {offset}", - ) + info = self.clients.get(client_address) + if not pending or info is None or not self._wait_control_secure(info["socket"]): + return 0 + sent = 0 + for (fid, target), offset in pending: + with _sending_transfer(self, (client_address, fid)): + with self._forward_relays_lock: + waiting = self._pending_originator_resumes.get(client_address, {}) + if waiting.get((fid, target)) != offset: + continue # another flush sent it, or a newer offset replaced it + route = self._fid_routes.get(fid) + valid = (route is not None and route["originator"] == client_address + and target in route["targets"] and target not in route["completed"]) + if valid: + try: + if not self.send_message( + info["socket"], format_resume_line(fid, offset, client_address, target) + ): + continue + except OSError: + continue + sent += 1 + with self._forward_relays_lock: + waiting = self._pending_originator_resumes.get(client_address, {}) + if waiting.get((fid, target)) == offset: + waiting.pop((fid, target), None) + if not waiting: + self._pending_originator_resumes.pop(client_address, None) + return sent def _resume_after_connect(self, client_address): """Continue this peer's interrupted transfers after it (re)connected. @@ -5695,30 +6630,17 @@ def _resume_after_connect(self, client_address): client_address (tuple): Value used by the operation. """ self._drop_stale_transfers(peer=tuple(client_address)) - self._flush_pending_relay_resumes(tuple(client_address)) - self.resume_transfers(peer=tuple(client_address)) + self._request_resumes(tuple(client_address)) + self._flush_pending_originator_resumes(tuple(client_address)) # ---- native in-memory forward relay (server side) ---------------------- - def _forward_alloc_fid(self): - """Handle the operation. - - Returns: - object: Result produced by the operation. - """ - with self._forward_fid_lock: - self._forward_fid += 1 - return self._forward_fid - - def _forward_notify_error(self, sock, msg): - """Handle the operation. - - Args: - sock (object): Value used by the operation. - msg (object): Value used by the operation. - """ + def _forward_notify_error(self, sock, msg, fid): + """Return one forwarding setup failure to its socket/FID negotiation.""" + if fid is None: + return try: - self.send_message(sock, f"/forward_error {msg}") + self.send_message(sock, f"/forward_error {fid} {shlex.quote(str(msg))}") except Exception: pass @@ -5749,39 +6671,42 @@ def _forward_parse_items_addrs(self, tokens): return items, addrs def _forward_item_handler(self, sock, addr, cmd): - """Server-side entry for /forward_item [resume=] <...> . - - The server never touches disk: it only relays the uploaded byte - stream to every reachable target client from memory. ``resume=`` - marks a repeated forward of a file the target already holds partially: - the uploader starts at that offset, so the relay expects - ``size - offset`` bytes of stream. - - Args: - sock (object): Value used by the operation. - addr (tuple): Value used by the operation. - cmd (object): Value used by the operation. - """ + """Start a relay whose upstream FID was allocated by the sender.""" try: parts = shlex.split(cmd) - except Exception: - self._forward_notify_error(sock, "invalid forward command") + except ValueError: return - if len(parts) < 3: - self._forward_notify_error(sock, "invalid forward command") + if len(parts) < 4: + self._log("forward: rejected command without an upstream FID") return kind = parts[1].lower() rest = parts[2:] + upstream_fid = None + resume_requested = False resume_offset = 0 - if rest and rest[0].startswith("resume="): + while rest and rest[0].startswith(("resume=", "fid=")): + option, value = rest.pop(0).split("=", 1) try: - resume_offset = max(0, int(rest[0][len("resume=") :])) + number = int(value) + if number < 0: + raise ValueError except ValueError: - self._forward_notify_error(sock, "invalid resume offset") + self._forward_notify_error(sock, f"invalid {option}", upstream_fid) return - rest = rest[1:] - # optional destination directory is the very last argument, after the - # client address tuples + if option == "resume": + if resume_requested: + self._forward_notify_error(sock, "duplicate resume", upstream_fid) + return + resume_offset = number + resume_requested = True + else: + if upstream_fid is not None: + self._forward_notify_error(sock, "duplicate fid", upstream_fid) + return + upstream_fid = number + if upstream_fid is None: + self._log("forward: rejected command without an upstream FID") + return destination_path = None if rest and not (rest[-1].startswith("(") and rest[-1].endswith(")")): destination_path = rest[-1] @@ -5797,26 +6722,28 @@ def _forward_item_handler(self, sock, addr, cmd): continue valid_targets.append(target) if not valid_targets: - self._forward_notify_error(sock, "no reachable targets to forward to") + self._forward_notify_error(sock, "no reachable targets to forward to", upstream_fid) return if kind == "file": if not items: - self._forward_notify_error(sock, "missing file name") + self._forward_notify_error(sock, "missing file name", upstream_fid) return - rel_dir = "" - fname = items[0] + rel_dir, fname = "", items[0] elif kind == "folder": if len(items) < 2: - self._forward_notify_error(sock, "missing folder file") + self._forward_notify_error(sock, "missing folder file", upstream_fid) return - rel_dir = items[0] - fname = items[1] + rel_dir, fname = items[0], items[1] else: - self._forward_notify_error(sock, f"unknown forward kind: {kind}") + self._forward_notify_error(sock, f"unknown forward kind: {kind}", upstream_fid) return threading.Thread( target=self._forward_relay, args=(sock, addr, kind, rel_dir, fname, valid_targets, destination_path, resume_offset), + kwargs={ + "upstream_fid": upstream_fid, + "resume_requested": resume_requested, + }, daemon=True, ).start() @@ -5832,25 +6759,43 @@ def _forward_read_sign(self, sock, sign, timeout=15): timeout (float): Value used by the operation. Returns: - bool: Result produced by the operation. + bytes | None: The decoded requested marker, or None on failure. """ buf = b"" target = sign.encode("utf-8") err = self.error_sign.encode("utf-8") - deadline = time.time() + timeout - while time.time() < deadline: - try: - data = self.receive_message(sock, self.recv_messages_chunk_len) - except Exception: - return False - if not data: - return False - buf += data - if target in buf: - return True - if err in buf: - return False - return False + overlap = max(len(target), len(err)) - 1 + if overlap >= TRANSFER_SIGN_BUFFER_LIMIT: + return None + deadline = time.monotonic() + timeout + previous_timeout = sock.gettimeout() + try: + while time.monotonic() < deadline: + sock.settimeout(max(0.001, deadline - time.monotonic())) + try: + data = self.receive_message(sock, self.recv_messages_chunk_len) + except Exception: + return None + if not data: + return None + offset = 0 + matched = False + while offset < len(data): + end = min(len(data), offset + TRANSFER_SIGN_BUFFER_LIMIT - len(buf)) + window = buf + data[offset:end] + offset = end + if err in window: + return None + matched |= target in window + buf = window[-overlap:] if overlap else b"" + # Scan the whole read before accepting a match: a later error + # in that same read still takes precedence, as before. + if matched: + return target + return None + finally: + if sock.fileno() != -1: + sock.settimeout(previous_timeout) def _forward_recv_exact(self, sock, n): """Handle the operation. @@ -5903,6 +6848,8 @@ def _forward_relay( targets, destination_path=None, resume_offset=0, + upstream_fid=None, + resume_requested=False, ): """Relay one file/folder item to every target, streaming from the @@ -5920,58 +6867,105 @@ def _forward_relay( targets (object): Value used by the operation. destination_path (str): Value used by the operation. resume_offset (object): Value used by the operation. + upstream_fid (int): Upstream ID allocated by the source sender. """ - fid = self._forward_alloc_fid() - tfids = [self._forward_alloc_fid() for _ in targets] + targets = list(dict.fromkeys(tuple(target) for target in targets)) + fid = upstream_fid + if fid is None: + self._log("forward: rejected relay without an upstream FID") + return + if resume_requested: + with self._forward_relays_lock: + route = self._fid_routes.get(fid) + if (route is None or route["originator"] != tuple(originator_addr) + or not targets or any(target not in route["targets"] for target in targets) + or any(target in route["completed"] for target in targets)): + self._forward_notify_error( + forwarder_sock, "no pending relay target for this sender", fid + ) + return + tfids = [route["targets"][target] for target in targets] + else: + tfids = [self._fid_alloc.alloc(receiver=target) for target in targets] + route = self._register_fid_route( + fid, originator_addr, dict(zip(targets, tfids)) + ) + if route is None: + self._forward_notify_error( + forwarder_sock, "relay fid is already in use", fid + ) + return relay = { "cond": threading.Condition(), "writer_pause": {target: False for target in targets}, + "route": route, } + relay_keys = [(target, leg_fid) for target, tfid in zip(targets, tfids) for leg_fid in (fid, tfid)] with self._forward_relays_lock: - for key in [fid] + tfids: + if any(key in self._forward_relays for key in relay_keys): + self._forward_notify_error( + forwarder_sock, "relay target is already active", fid + ) + return + for key in relay_keys: self._forward_relays[key] = relay - record = { - "fid": fid, - "originator": [originator_addr[0], originator_addr[1]], - "targets": [[target[0], target[1]] for target in targets], - "tfids": list(tfids), - "kind": kind, - "rel_dir": rel_dir, - "fname": fname, - "destination": destination_path, - } - # the record survives the relay so an interrupted forward can be - # routed back to its originator (see `_route_forward_resume`) - with self._forward_relays_lock: - for key in [fid] + tfids: - self._forward_resume_records[key] = record + if os.path.exists(self.file_trans_progress_file): + self._persist_transfer_progress() + target_conns = [] + up_sock = up_client = None + sport = None + allocated_sport = None + target_handshakes = {} + owned_sockets = [] + terminal_sent = False + + def fail(message): + nonlocal terminal_sent + if not terminal_sent: + terminal_sent = True + self._forward_notify_error(forwarder_sock, message, fid) + + def own(control, transfer): + if not self._transfer_handshakes.track_socket(control, transfer): + return False + owned_sockets.append((control, transfer)) + return True + try: - # ask every target to open a receive transfer socket; the - # destination directory (if any) goes before the target id so - # the receiver's own destination parsing sees it for target, tfid in zip(targets, tfids): - self.forward_item_to( - target, kind, rel_dir, fname, originator_addr, tfid, destination_path - ) - # collect every target's advertised transfer port and mode stamp - ports = {} - deadline = time.time() + 20 - while len(ports) < len(tfids) and time.time() < deadline: - with self.file_transfer_server_port_lock: - for info in list(self.file_server_port_list): - if info[1] in tfids and info[1] not in ports: - ports[info[1]] = (info[0], info[2]) - self.file_server_port_list.remove(info) - if len(ports) < len(tfids): - time.sleep(0.05) - if len(ports) < len(tfids): - self._forward_notify_error(forwarder_sock, "target port setup timed out") + target_control = self.clients.get(target, {}).get("socket") + handshake = (self._transfer_handshakes.register(target_control, tfid) + if target_control is not None else None) + if handshake is None: + fail(f"target {target} has no available listener negotiation") + return + target_handshakes[(target, tfid)] = handshake + for target, tfid in zip(targets, tfids): + if not self.forward_item_to( + target, kind, rel_dir, fname, originator_addr, tfid, destination_path, + ): + fail(f"could not request a listener from {target}") + return + results = self._transfer_handshakes.wait_many( + tuple(target_handshakes.values()), 20.0 + ) + if results is None: + fail("target port setup timed out") return + ports = {} + for target, tfid in zip(targets, tfids): + result = results[target_handshakes[(target, tfid)]["key"]] + if result[0] != "port": + fail(f"target {target} rejected its listener") + return + ports[(target, tfid)] = (result[2], result[3]) # connect to each target and wait for its start sign - target_conns = [] for target, tfid in zip(targets, tfids): - tport, stamp = ports[tfid] + tport, stamp = ports[(target, tfid)] sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + target_control = target_handshakes[(target, tfid)]["key"][0] + if not own(target_control, sock) or not own(forwarder_sock, sock): + return try: sock.connect((target[0], tport)) except Exception as e: @@ -5979,7 +6973,6 @@ def _forward_relay( sock.close() continue try: - target_control = self.clients.get(target, {}).get("socket") self._crypto_attach_transfer_out(sock, target_control, stamp) except _CryptoUnavailable as exc: self._log(f"forward: target {target} transfer aborted: {exc}") @@ -5991,37 +6984,56 @@ def _forward_relay( continue target_conns.append((tfid, target, sock)) if not target_conns: - self._forward_notify_error(forwarder_sock, "no target accepted the transfer") + fail("no target accepted the transfer") return # open the upload listener and hand it to the forwarder up_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if not own(forwarder_sock, up_sock): + return up_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - up_sock.bind((self.host, self.palloc())) - sport = up_sock.getsockname()[1] + allocated_sport = self.palloc() + sport = _bind_local_port_with_fallback( + up_sock, + (self.host, allocated_sport), + self._log, + "forward listener", + ) up_stamp = self._crypto_transfer_stamp(forwarder_sock) up_sock.listen(1) # coordination line: always plaintext, so it also reaches a peer # that has not finished flipping to framed mode - self._send_raw(forwarder_sock, f"/forward_upload {fid} {sport} {up_stamp}") + terminal_sent = True + if not self.send_message( + forwarder_sock, _transfer_port_announcement(sport, fid, up_stamp), + is_send_raw=True, + ): + return up_client, _ = up_sock.accept() + if not own(forwarder_sock, up_client): + return self._crypto_attach_transfer_in(up_client, forwarder_sock, up_stamp) self.send_message(up_client, self.server_start_file_transfer_sign) - delivered = self._forward_pump( + self._forward_pump( up_client, target_conns, forwarder_sock, fid, fname, relay, resume_offset ) - self._crypto_drop_channel(up_client) - up_client.close() - up_sock.close() - self.pfree(sport) - if delivered >= len(target_conns): - # every target got the whole item: nothing is left to resume - with self._forward_relays_lock: - for key in [fid] + tfids: - self._forward_resume_records.pop(key, None) + except Exception as exc: + self._log(f"forward: relay {fid} interrupted: {exc}") + fail(f"relay setup interrupted: {exc}") finally: + for control, transfer in owned_sockets: + self._transfer_handshakes.release_socket(control, transfer) + for transfer_socket in [up_client, up_sock, *(sock for _, _, sock in target_conns)]: + if transfer_socket is not None: + self._crypto_drop_channel(transfer_socket) + transfer_socket.close() + if allocated_sport is not None: + self.pfree(allocated_sport) + for handshake in target_handshakes.values(): + self._transfer_handshakes.cancel(handshake) with self._forward_relays_lock: - for key in [fid] + tfids: - self._forward_relays.pop(key, None) + for key in relay_keys: + if self._forward_relays.get(key) is relay: + del self._forward_relays[key] def _forward_pump( self, up_sock, target_conns, forwarder_sock, fid, fname, relay, resume_offset=0 @@ -6051,98 +7063,102 @@ def _forward_pump( low_water = max(1, max_bytes // 2) cond = relay["cond"] writer_pause = relay["writer_pause"] - state = {"buffered": 0, "paused": False} - queues = {tfid: queue.Queue() for tfid, _, _ in target_conns} + state = {"buffered": 0, "paused": False, "complete": False, "failed": False} + queues = {(target, tfid): queue.Queue() for tfid, target, _ in target_conns} END = object() - result = {"ok": 0} + result = {"ok": 0, "ack": None} - def enqueue(chunk): - """Handle the operation. + def release_buffered(byte_count): + """Called under cond; failed writers also release upstream backpressure.""" + state["buffered"] -= byte_count + if state["paused"] and state["buffered"] <= low_water: + state["paused"] = False + try: + self.send_message(forwarder_sock, f"/start_trans {fid}") + except Exception: + pass + cond.notify_all() - Args: - chunk (bytes | str): Value used by the operation. - """ + def enqueue(chunk): with cond: - state["buffered"] += len(chunk) * len(target_conns) - for q in queues.values(): - q.put(chunk) + state["buffered"] += len(chunk) * len(queues) + for pending in queues.values(): + pending.put(chunk) if not state["paused"] and state["buffered"] > max_bytes: state["paused"] = True - try: - self.send_message(forwarder_sock, f"/pause_trans {fid}") - except Exception: - pass + self.send_message(forwarder_sock, f"/pause_trans {fid}") def writer(tfid, target, sock): - """Handle the operation. - - Args: - tfid (int): Value used by the operation. - target (tuple): Value used by the operation. - sock (object): Value used by the operation. - """ - q = queues[tfid] - while True: - with cond: - while writer_pause.get(target, False): - cond.wait() - item = q.get() - if item is END: - break - try: - self.send_message(sock, item) - except Exception: + key = (target, tfid) + pending = queues[key] + try: + while True: with cond: - while not q.empty(): - state["buffered"] -= len(q.get()) - break + while writer_pause.get(target, False) and not state["failed"]: + if sock.fileno() < 0: + return + cond.wait(timeout=0.5) + item = pending.get() + if item is END: + break + try: + if not self.send_message(sock, item): + return + finally: + with cond: + release_buffered(len(item)) + if state["complete"]: + ack = self._forward_read_sign(sock, self.server_received_file_data_sign) + if ack is not None and self._complete_fid_target( + fid, relay["route"], target, tfid + ): + with cond: + result["ok"] += 1 + result["ack"] = ack + except Exception as exc: + self._log(f"forward: target {target} interrupted: {exc}") + finally: with cond: - state["buffered"] -= len(item) - if state["paused"] and state["buffered"] <= low_water: - state["paused"] = False - try: - self.send_message(forwarder_sock, f"/start_trans {fid}") - except Exception: - pass - cond.notify_all() - if self._forward_read_sign(sock, self.server_received_file_data_sign): - result["ok"] += 1 - self._crypto_drop_channel(sock) - try: + queues.pop(key, None) + while not pending.empty(): + item = pending.get_nowait() + if item is not END: + release_buffered(len(item)) + self._crypto_drop_channel(sock) sock.close() - except Exception: - pass def reader(): - """Handle the operation.""" try: name_len_b = self._forward_recv_exact(up_sock, 4) name_len = int.from_bytes(name_len_b, "big") name_b = self._forward_recv_exact(up_sock, name_len) size_b = self._forward_recv_exact(up_sock, 8) file_size = int.from_bytes(size_b, "big") - except Exception as e: - self._log(f"forward: uploader header read failed: {e}") - for q in queues.values(): - q.put(END) - return - for chunk in (name_len_b, name_b, size_b): - enqueue(chunk) - # a repeated forward uploads only the remainder of the file: the - # offset the command carried says how much of the header's size - # does not travel any more - remaining = max(0, file_size - resume_offset) - while remaining > 0: + if not 0 <= resume_offset <= file_size: + raise ValueError("resume offset exceeds the forwarded file") + for chunk in (name_len_b, name_b, size_b): + enqueue(chunk) + remaining = file_size - resume_offset + while remaining > 0: + with cond: + while state["paused"]: + if up_sock.fileno() < 0: + raise ConnectionError("uploader control session closed") + cond.wait(timeout=0.5) + chunk = self._forward_recv_exact( + up_sock, min(self.file_transfer_chunk_len, remaining) + ) + remaining -= len(chunk) + enqueue(chunk) + state["complete"] = True + except Exception as exc: + self._log(f"forward: uploader stream interrupted: {exc}") + finally: with cond: - while state["paused"]: - cond.wait() - chunk = self._forward_recv_exact( - up_sock, min(self.file_transfer_chunk_len, remaining) - ) - remaining -= len(chunk) - enqueue(chunk) - for q in queues.values(): - q.put(END) + state["failed"] = not state["complete"] + for pending in queues.values(): + pending.put(END) + cond.notify_all() threads = [ threading.Thread(target=writer, args=(tfid, target, sock), daemon=True) @@ -6152,14 +7168,31 @@ def reader(): t.start() reader() for t in threads: - t.join(timeout=120) + t.join() + route = relay["route"] + with self._forward_relays_lock: + complete = ( + state["complete"] + and self._fid_routes.get(fid) is route + and route["completed"] == route["targets"].keys() + and result["ack"] is not None + ) try: - self.send_message(up_sock, self.server_received_file_data_sign) - except Exception: - pass + # A target's existing ACK is the only success marker. Never confirm + # upstream receipt independently of the full accepted target set. + # Send it as a protocol line: send_message adds no newline to bytes. + written = self.send_message( + up_sock, + result["ack"].decode("ascii") if complete else self.error_sign, + ) + if complete and written: + self._drop_fid_route(fid, route) + except Exception as exc: + self._log(f"forward: final ACK for {fid} could not be sent: {exc}") self._log(f"forward: relayed {fname} to {result['ok']}/{len(target_conns)} targets") return result["ok"] + @_config_lifecycle(starting=True) def start_TCP_Server(self): # set up server socket """Bind the server socket, then accept clients until `stop` runs. @@ -6273,7 +7306,7 @@ def console_input(self): # deal consule input """Read console commands until the server stops. Handles ``/stop``, ``/status``, ``/clients``, ``/send_msg``, ``/file``, - ``/file_folder``, ``/multiple_file_multiple_client``, + ``/file_folder``, ``/multiple_file_multiple_client``, ``/transfers``, ``/diff_multiple_file_diff_multiple_client`` and ``/help``; the forward commands are client-only and are refused here. Any other name goes to the handlers registered with ``where_to_run="client"``. Ctrl-C and EOF stop the @@ -6287,6 +7320,11 @@ def console_input(self): # deal consule input self._log("shutting down...") self.running = False self.stop() + elif deal_cmd.lower().split(maxsplit=1)[:1] == ["/transfers"]: + if deal_cmd.lower() != "/transfers": + print("Usage: /transfers") + else: + print(json.dumps(self.get_transfer_progress(), ensure_ascii=False, indent=2)) elif deal_cmd.lower() == "/status": self._log(f"current connection count: {len(self.clients)}") self._log(f"server running: {self.running}") @@ -6321,6 +7359,7 @@ def console_input(self): # deal consule input "/stop - stop the server\n", "\t/status - display server status\n", "\t/clients - display connected clients\n", + "\t/transfers - inspect local memory and persisted transfer progress\n", "\t/send_msg ... ", " ... ", " ... ... - send message or ", @@ -6372,6 +7411,7 @@ def console_input(self): # deal consule input self._log_exc() pass + @_config_lifecycle(starting=False) def stop(self): # shutting down the server """Stop the server and release everything it owns. @@ -6382,13 +7422,14 @@ def stop(self): # shutting down the server """ self.running = False self._stopped = True # also cancels a start_TCP_Server() still binding + self._stop_listeners() self.free_port() - self._flush_messages_dict() - self._flush_events_dict() + self._flush_record_stores() # in-flight transfers die with the server: persist them for the next run self._persist_transfer_progress() with self.client_lock: # close all clients connections for client_info in self.clients.values(): + self._transfer_handshakes.cancel_socket(client_info["socket"]) try: # a client thread blocked in recv() keeps the connection half # open past close(): the shutdown sends its FIN and wakes it @@ -6464,7 +7505,7 @@ class TCP_Client_Base: # TCP client class the original peer on both sides, never the relay. file_trans_progress_file (str): Path under the protocol's ``.Flow`` directory where the two stores are written when the connection - drops and read back on the next reconnect (`resume_transfers`). + drops and read back on the next reconnect (`_request_resumes`). upload_network_speed (float | None): Bytes per second sent, or None when not measured. recv_network_speed (float | None): Bytes per second received, or None @@ -6548,7 +7589,7 @@ def __init__( False nothing is measured and every speed value is None. Defaults to True. is_auto_reconnect (bool): Reconnect (and resume the interrupted - transfers, see `resume_transfers`) when the connection drops + transfers, see `_request_resumes`) when the connection drops while `start_TCP_client` runs. The transfers that died with the connection are persisted to the protocol's ``.Flow`` directory and read back on the next connect. Defaults to @@ -6568,9 +7609,9 @@ def __init__( # current speed, local file path, peer (ip, port)]. An entry appears # when a transfer starts and is removed when it ends; a transfer # interrupted by a disconnect keeps its entry (and its partial file) - # so it can be resumed once the peer is back (see `resume_transfers`). + # so it can be resumed once the peer is back (see `_request_resumes`). self.file_recv_progress = {} # downloads: fid -> [received, total, speed, path, peer] - self.file_upload_progress = {} # uploads: fid -> [sent, total, speed, path, peer] + self.file_upload_progress = {} # (receiver, fid) -> [sent, total, speed, path, peer] self.upload_network_speed = None # bytes/s over the last NETWORK_SPEED_WINDOW seconds self.recv_network_speed = None self._network_speed_lock = threading.Lock() # serialises the speed sample windows @@ -6578,24 +7619,12 @@ def __init__( self._recv_speed_samples = collections.deque() self._upload_speed_total = 0 # bytes held in the sample windows self._recv_speed_total = 0 - self._forward_upload_queue = queue.Queue() - # Relay records of the forwards this client started, kept after their - # leg died so a target's resume request can re-drive them: relay fid - # and every target tfid -> record (see `_resume_forward`). - self._forward_resume_records = {} - # Resumes this client asked a forward's originator for, keyed by the - # local path of the partial file: the fresh relay leg lands on that - # path and must append to what is already there (see - # `_take_pending_resume`). - self._resume_pending_paths = {} # path -> (old fid, byte offset) - self._forward_resume_lock = threading.Lock() - self._forward_error = None # Outbound-transfer pause switches (see `pause_trans`/`start_trans`): - # fid -> bool. A leg waits while its own fid is marked paused. + # (receiver, fid) -> bool, including forwarded uploads to the server. self._transfer_pause = {} self._transfer_pause_cond = threading.Condition() self._active_transfer_fids = set() # fids of the legs running right now - self._active_transfer_sockets = {} # fid -> socket of the leg running right now + self._active_transfer_sockets = {} # receive fid / (receiver, fid) -> socket self.project_dir = os.path.dirname(os.path.abspath(__file__)) self.file_transfer_dir = os.path.join(self.project_dir, "received_files") self.decode_command_table_file_path = os.path.join( @@ -6614,13 +7643,16 @@ def __init__( self.project_info_dir, "client_file_trans_progress.json" ) self._transfer_progress_lock = threading.RLock() # serialises the persist/load pair + self._resume_sends = {} # transient locks and request sets; no file metadata + self._progress_discarded = False + self._config_started = False + self._config_closed = False + self._config_busy = 0 if os.path.exists(self.project_temp_info_dir) == False: os.mkdir(self.project_temp_info_dir) self.all_allocated_ports_list = [] self.is_hand_alloc_port = None - self.file_transfer_server_port = None - self.file_server_port_list = [] - self.file_transfer_server_port_lock = threading.Lock() + self._transfer_handshakes = _TransferHandshakeRegistry() self.client_ports_list = [] self.client_port = client_port self.host = host @@ -6644,8 +7676,12 @@ def __init__( "please set wait_server_timeout to None or set is_wait_server to False", ] raise ValueError(" ".join(error_msg)) - self.file_client_id = 0 - self.file_client_id_lock = threading.Lock() + # Transfer ids for the legs this instance sends (see `_FidAllocator`). + # Inbound keys are local integers; outbound keys include the server. + # Forward uploads keep their logical target in the progress value. + self._fid_alloc = _FidAllocator( + (self.file_upload_progress, self.file_recv_progress) + ) MAX_CONCURRENT_FILES = self.max_thread_num self.file_semaphore = threading.Semaphore(MAX_CONCURRENT_FILES) with open(self.decode_command_table_file_path, "r", encoding="utf-8") as f: @@ -6667,28 +7703,42 @@ def __init__( self._custom_handler_threaded = [{}, {}] self._custom_executor = ThreadPoolExecutor(max_workers=max_custom_workers) self._task_semaphore = threading.Semaphore(max_custom_workers) - # Inbound-event listeners (see add_message_listener/add_file_listener). - # They run on the receive thread, so a listener must not block and must - # never raise (exceptions are swallowed by the notify helpers). + # Inbound/outbound event listeners (see add_message_listener and + # add_file_listener). They run on the thread performing the relevant + # I/O; a listener must not block and its exceptions are swallowed by + # the notify helpers. self._message_listeners = [] self._file_listeners = [] + self._send_message_listeners = [] + # Set by close(); a closed instance never invokes a listener again. + self._listeners_stopped = False + self._control_sockets = set() # the control link to the server; only it is recorded self._event_listeners_lock = threading.Lock() - # Inbound message/event stores: external code reads these instead of - # registering listeners. Keyed by the sender's socket; each value is - # a list of [content, timestamp] pairs. When a store's total size - # reaches max_dict_size (64 KiB) it is flushed to its JSON log and - # cleared (see _record_message/_record_event/_flush_*_dict). + # Message/event stores: external code reads these instead of registering + # listeners. Keyed by the socket; each value is a list of [content, + # timestamp] pairs. messages_dict and sent_messages_dict hold plain text + # (inbound / outbound); events_dict and sent_events_dict hold "/" commands. + # When a store's total size reaches max_dict_size (64 KiB) it is flushed + # to its JSON log and cleared (see _append_record, _flush_record_stores). self.messages_dict = {} self.events_dict = {} + self.sent_messages_dict = {} + self.sent_events_dict = {} self._messages_dict_lock = threading.Lock() self._events_dict_lock = threading.Lock() + self._sent_messages_dict_lock = threading.Lock() + self._sent_events_dict_lock = threading.Lock() self._socket_keys = {} # record key -> "ip:port", captured while it was alive self._socket_keys_lock = threading.Lock() self._messages_dict_size = 0 self._events_dict_size = 0 + self._sent_messages_dict_size = 0 + self._sent_events_dict_size = 0 self.max_dict_size = 64 * 1024 self.messages_log_file = os.path.join(self.project_info_dir, "messages_log.json") self.events_log_file = os.path.join(self.project_info_dir, "events_log.json") + self.sent_messages_log_file = os.path.join(self.project_info_dir, "sent_messages_log.json") + self.sent_events_log_file = os.path.join(self.project_info_dir, "sent_events_log.json") self.is_extend_command = is_extend_command self.is_enable_encrypto = is_enable_encrypto self.is_custom_keys = is_custom_keys @@ -6786,19 +7836,23 @@ def call_extension_functions(self, command): return handler(self.client_socket, client_id, command) def add_message_listener(self, listener): - """Register ``listener(sender_id, message)`` for every inbound line. + """Register ``listener(sender_id, message)`` for inbound events. + + Complete control lines are reported as ``str`` (commands included); a + message forwarded through ``/send_msg_from`` is reported as its + payload. Raw chunks from a non-control socket (file transfer, + forwarding, temporary connection) are reported as ``bytes`` without + line or frame alignment. ``sender_id`` is the forwarding/origin + client's ``"ip:port"``, ``None`` for a direct server push, or the + temporary TCP peer endpoint for a side-channel chunk. - Every line received from the server is reported, commands included, so - the listener observes all inbound traffic; a message another client - forwarded here is reported as its payload, attributed to the forwarder. + Control-line callbacks run on the client receive thread; raw chunks run + on the thread reading that socket. Callbacks may run concurrently and + must be thread-safe, must not block, and exceptions raised inside them + are swallowed. ``close()`` stops new delivery and clears the list. Args: - listener (callable): ``listener(sender_id, message)``; ``sender_id`` is - the author's ``"ip:port"`` — the forwarding client for a message - another client forwarded here (``/send_msg_from`` envelope), or None - for a direct push from the server, which names no client author. It - runs on the receive thread, so it must not block, and exceptions - raised inside it are swallowed. + listener (callable): ``listener(sender_id, message)``. """ with self._event_listeners_lock: if listener not in self._message_listeners: @@ -6816,17 +7870,57 @@ def remove_message_listener(self, listener): except ValueError: pass + def add_send_message_listener(self, listener): + """Register ``listener(target_id, message)`` for successful sends. + + Called synchronously on the sending thread after every payload that + `send_message` writes successfully, for ``str`` and ``bytes`` alike. + ``str`` payloads are stripped; ``bytes`` are reported unchanged. + ``target_id`` is the peer endpoint of the socket that wrote the + payload: the server's ``"ip:port"`` for the control socket, or the + temporary TCP connection's ``"ip:port"`` for a side channel. + + ``is_send_raw=True`` lines and internal protocol frames such as the + ECDH public-key publication are not reported. When several threads + send on one socket the writes stay serialized, but callback order is + not guaranteed to match wire order; this affects observation only. + The listener must not block, may run concurrently, and its exceptions + are swallowed. ``close()`` stops new delivery. + + Args: + listener (callable): ``listener(target_id, message)``. + """ + with self._event_listeners_lock: + if listener not in self._send_message_listeners: + self._send_message_listeners.append(listener) + + def remove_send_message_listener(self, listener): + """Unregister a listener previously added by `add_send_message_listener`. + + Args: + listener (callable): Listener to remove; an unknown one is ignored. + """ + with self._event_listeners_lock: + try: + self._send_message_listeners.remove(listener) + except ValueError: + pass + def add_file_listener(self, listener): """Register ``listener(full_path, name, size, command)`` per saved inbound file. - Fired after a file pushed by the server (a direct send, or a forwarded - file/folder item) has been fully written to ``file_transfer_dir``. + Fired once after a file pushed by the server (a direct send or a + forwarded item) is fully written to disk. ``command`` is the wire + command that triggered the transfer, so pushes such as + ``/crypto_pub_key`` can be recognised. The callback runs on the transfer + thread before the completion sign is sent (it does not wait for an + ACK); per-chunk data goes to the message listeners as ``bytes`` + instead. For ``/crypto_pub_key`` the server key is not registered yet, + so the listener must not move or delete the file. It must not block; + exceptions are swallowed. ``close()`` stops new delivery. Args: - listener (callable): ``listener(full_path, name, size, command)``; - ``command`` is the wire command that triggered the transfer, so a - listener can recognise protocol pushes such as ``/crypto_pub_key``. - It runs on the transfer thread, so it must not block. + listener (callable): ``listener(full_path, name, size, command)``. """ with self._event_listeners_lock: if listener not in self._file_listeners: @@ -6844,6 +7938,19 @@ def remove_file_listener(self, listener): except ValueError: pass + def _stop_listeners(self): + """Stop delivering listener events and release the callback lists.""" + with self._event_listeners_lock: + self._listeners_stopped = True + self._message_listeners.clear() + self._file_listeners.clear() + self._send_message_listeners.clear() + + def _listeners_are_stopped(self): + """Return True once close() has disabled listener delivery.""" + with self._event_listeners_lock: + return self._listeners_stopped + def _notify_message_received(self, sender, message): """Handle the operation. @@ -6852,14 +7959,58 @@ def _notify_message_received(self, sender, message): message (object): Value used by the operation. """ with self._event_listeners_lock: + if self._listeners_stopped: + return listeners = list(self._message_listeners) for listener in listeners: + if self._listeners_are_stopped(): + return try: listener(sender, message) except Exception as e: self._log(f"message listener error: {e}") self._log_exc() + def _notify_transfer_chunk(self, client_socket, data): + """Report one chunk read from a non-control socket to the message listeners. + + Side-channel sockets (file/folder transfers, forwards) carry raw bytes and + ``sign`` tokens, which belong to the transfer layer, so they are reported + to the listeners but never recorded in a store. + """ + if not data or client_socket in self._control_sockets: + return + with self._event_listeners_lock: + if self._listeners_stopped: + return + listeners = list(self._message_listeners) + for listener in listeners: + if self._listeners_are_stopped(): + return + try: + listener(self._socket_key(client_socket), data) + except Exception as e: + self._log(f"message listener error: {e}") + self._log_exc() + + def _notify_message_sent(self, client_socket, message): + """Report one written payload to the send listeners.""" + with self._event_listeners_lock: + if self._listeners_stopped: + return + listeners = list(self._send_message_listeners) + if not listeners: + return + target_id = self._socket_key(client_socket) + for listener in listeners: + if self._listeners_are_stopped(): + return + try: + listener(target_id, message) + except Exception as e: + self._log(f"send message listener error: {e}") + self._log_exc() + def _notify_file_received(self, full_path, name, size, command): """Handle the operation. @@ -6870,8 +8021,12 @@ def _notify_file_received(self, full_path, name, size, command): command (object): Value used by the operation. """ with self._event_listeners_lock: + if self._listeners_stopped: + return listeners = list(self._file_listeners) for listener in listeners: + if self._listeners_are_stopped(): + return try: listener(full_path, name, size, command) except Exception as e: @@ -6935,47 +8090,95 @@ def _record_key(self, sock): return key return self._socket_key(sock) - def _record_message(self, sock, content): - """Store one inbound plain-text message under the sender's socket. - - External code reads ``messages_dict`` (or the JSON log) instead of - registering a message listener. The entry is ``[content, timestamp]`` - with the timestamp of arrival. + def _append_record(self, store, log_path, sock, content): + """Append one ``[content, timestamp]`` pair under ``sock``; flush when full. Args: - sock (object): Value used by the operation. - content (object): Value used by the operation. + store (tuple): ``(dict, lock, size_attr)``; the dict maps socket to + a list of pairs, and ``size_attr`` names the byte-size attribute. + log_path (str): JSON log the store flushes into. + sock (object): Key socket. + content (str): Text to record. """ + d, lock, size_attr = store timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - with self._messages_dict_lock: + with lock: self._remember_socket_key(sock) - self.messages_dict.setdefault(sock, []).append([content, timestamp]) - self._messages_dict_size += len(content.encode("utf-8", "replace")) + len(timestamp) - if self._messages_dict_size >= self.max_dict_size: - self._flush_dict_locked( - self.messages_dict, "_messages_dict_size", self.messages_log_file - ) + d.setdefault(sock, []).append([content, timestamp]) + size = getattr(self, size_attr) + len(content.encode("utf-8", "replace")) + size += len(timestamp) + setattr(self, size_attr, size) + if size >= self.max_dict_size: + self._flush_dict_locked(d, size_attr, log_path) + + def _record_message(self, sock, content): + """Store one inbound plain-text message under the sender's socket.""" + self._append_record( + (self.messages_dict, self._messages_dict_lock, "_messages_dict_size"), + self.messages_log_file, sock, content, + ) def _record_event(self, sock, command): - """Store one inbound command as an event under the sender's socket. + """Store one inbound "/" command under the sender's socket.""" + self._append_record( + (self.events_dict, self._events_dict_lock, "_events_dict_size"), + self.events_log_file, sock, command, + ) - The event content is the original wire command from the peer (file - transfers, folder transfers, extension commands, ...). External code - reads ``events_dict`` (or the JSON log) instead of registering a - listener. File-transfer events get their completion timestamp via - ``_update_event_timestamp``. + def _record_sent(self, sock, content): + """Store one outbound text: a "/" line as a sent event, other text as a sent message.""" + if content.startswith("/"): + self._append_record( + (self.sent_events_dict, self._sent_events_dict_lock, "_sent_events_dict_size"), + self.sent_events_log_file, sock, content, + ) + else: + self._append_record( + ( + self.sent_messages_dict, + self._sent_messages_dict_lock, + "_sent_messages_dict_size", + ), + self.sent_messages_log_file, sock, content, + ) - Args: - sock (object): Value used by the operation. - command (object): Value used by the operation. - """ - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + def _record_dicts(self): + """Return the four record stores, in flush order.""" + return ( + self.messages_dict, + self.events_dict, + self.sent_messages_dict, + self.sent_events_dict, + ) + + def _flush_messages_dict(self): + """Flush the inbound messages store into its JSON log.""" + with self._messages_dict_lock: + self._flush_dict_locked( + self.messages_dict, "_messages_dict_size", self.messages_log_file + ) + + def _flush_events_dict(self): + """Flush the inbound events store into its JSON log.""" with self._events_dict_lock: - self._remember_socket_key(sock) - self.events_dict.setdefault(sock, []).append([command, timestamp]) - self._events_dict_size += len(command.encode("utf-8", "replace")) + len(timestamp) - if self._events_dict_size >= self.max_dict_size: - self._flush_dict_locked(self.events_dict, "_events_dict_size", self.events_log_file) + self._flush_dict_locked(self.events_dict, "_events_dict_size", self.events_log_file) + + def _flush_record_stores(self): + """Flush every record store into its JSON log (called on stop).""" + with self._messages_dict_lock: + self._flush_dict_locked( + self.messages_dict, "_messages_dict_size", self.messages_log_file + ) + with self._events_dict_lock: + self._flush_dict_locked(self.events_dict, "_events_dict_size", self.events_log_file) + with self._sent_messages_dict_lock: + self._flush_dict_locked( + self.sent_messages_dict, "_sent_messages_dict_size", self.sent_messages_log_file + ) + with self._sent_events_dict_lock: + self._flush_dict_locked( + self.sent_events_dict, "_sent_events_dict_size", self.sent_events_log_file + ) def _update_event_timestamp(self, sock, command, timestamp): """Stamp the completion time onto the recorded event for ``command``. @@ -7042,24 +8245,11 @@ def _flush_dict_locked(self, d, size_attr, path): d.clear() setattr(self, size_attr, 0) self._merge_json_log(path, snapshot) - other = self.events_dict if d is self.messages_dict else self.messages_dict - with self._socket_keys_lock: # a key is only kept while it is buffered + with self._socket_keys_lock: # a key is only kept while some store buffers it for sock in snapshot: - if sock not in d and sock not in other: + if not any(sock in store for store in self._record_dicts()): self._socket_keys.pop(sock, None) - def _flush_messages_dict(self): - """Handle the operation.""" - with self._messages_dict_lock: - self._flush_dict_locked( - self.messages_dict, "_messages_dict_size", self.messages_log_file - ) - - def _flush_events_dict(self): - """Handle the operation.""" - with self._events_dict_lock: - self._flush_dict_locked(self.events_dict, "_events_dict_size", self.events_log_file) - def _merge_json_log(self, path, snapshot): """Merge ``snapshot`` into the JSON log at ``path`` (append per socket). @@ -7174,15 +8364,15 @@ def _measure_network_speed(self, payload, is_upload): def _file_progress_start(self, progress_dict, fid, total_bytes, path, peer=None, initial=0): """Register one file transfer in a progress store. - + Args: progress_dict (dict): ``self.file_recv_progress`` for an inbound transfer, ``self.file_upload_progress`` for an outbound one. - fid (int | None): The transfer's fid; None tracks nothing (a leg without a protocol transfer id, such as a key push). + fid (int | tuple | None): Receive fid or upload ``(receiver, fid)`` key; None tracks no progress. total_bytes (int): Size of the file being transferred. path (str): Local path of the file: the source of an outbound transfer, the file being written for an inbound one. peer (tuple | None): ``(ip, port)`` of the transfer's original peer — the sender for an inbound transfer, the receiver for an outbound one. A forwarded transfer stores the original peer on both sides, never the relay; None marks a leg that cannot be resumed (there is no peer to continue it with). initial (int): Bytes already transferred when the leg starts; a resumed transfer continues a partially transferred file. - + Returns: list | None: The ``[transferred, total, speed, path, peer]`` entry the transfer updates, or None when tracking is off @@ -7212,15 +8402,13 @@ def _file_progress_update(self, entry, byte_count, is_upload): def _file_progress_finish(self, progress_dict, fid, entry, keep=False): """Drop a finished transfer's progress entry, or keep it for a resume. - A newer transfer may already hold the same fid (transfer ids are reused - as transfers end), so only this transfer's own entry is dropped. With - ``keep`` True an interrupted transfer keeps its entry — and the partial - file that entry points at — so the reconnect logic can continue it - (`resume_transfers`); the entry then lives until the transfer ends. + A resumed leg can replace an older session under the same scoped key. + Only the entry's owner is removed. ``keep`` preserves an interrupted + entry and its partial file for the next reconnect. Args: progress_dict (dict): Store the entry belongs to. - fid (int | None): Transfer id the entry is keyed by. + fid (int | tuple | None): Receive fid or upload ``(receiver, fid)`` key. entry (list | None): Entry to drop; None tracks nothing. keep (bool): Keep the entry (and persist the stores) instead of dropping it. Defaults to False. """ @@ -7260,7 +8448,8 @@ def entries(store): list | tuple: Result produced by the operation. """ return { - str(fid): [ + (json.dumps(fid, separators=(",", ":")) + if store is self.file_upload_progress else str(fid)): [ entry[0], entry[1], entry[2], @@ -7270,46 +8459,70 @@ def entries(store): for fid, entry in list(store.items()) } - with self._forward_resume_lock: - relays = { - str(key): { - "fid": record.get("fid"), - "kind": record.get("kind"), - "rel_dir": record.get("rel_dir"), - "fname": record.get("fname"), - "path": record.get("path"), - "destination": record.get("destination"), - "targets": [list(target) for target in record.get("targets") or []], - } - for key, record in list(self._forward_resume_records.items()) - } return { "upload": entries(self.file_upload_progress), "download": entries(self.file_recv_progress), - "relays": relays, + "relays": {}, } + def get_transfer_progress(self, *, include_persisted=True) -> dict: + """Return a detached, local-only view of all retained transfer records. + + Args: + include_persisted (bool): Read this instance's progress file too. + Defaults to True; never loads it into the live stores. + + Returns: + dict: Flags, scoped memory rows and a separate persisted snapshot + with its path, read status and error. Disk runtime flags are + unknown (None). No completion or zombie status is inferred. + """ + return _get_transfer_progress(self, include_persisted=include_persisted) + + def delete_config(self, config_name=None, *, is_list_config=False) -> bool | list[str]: + """List or delete a named local progress snapshot or message/event log. + + Args: + config_name (str | None): Allowed file basename, not a path. + is_list_config (bool): Only list existing regular files, ignoring + config_name. Defaults to False; listing is allowed while running. + + Returns: + bool | list[str]: Sorted file names in list mode; otherwise True + when deleted, False when absent. Discarding progress permanently + disables this object's reload/write/start; create a new instance. + + Raises: + ValueError: Invalid/ambiguous name, directory or symlink. + TypeError: A deletion name is not a string. + RuntimeError: Instance not closed, operations active or logs buffered. + OSError: The filesystem refuses inspection or deletion. + """ + return _delete_config(self, config_name, is_list_config=is_list_config) + def _persist_transfer_progress(self): """Write the transfer stores to the protocol's ``.Flow`` directory. - Every in-flight transfer — uploads, downloads, and the relay records - of the forwards this client started — is written to - ``client_file_trans_progress.json`` when a connection drops, and the - file is what the reconnect logic reads back. Once nothing is left to - transfer the file is emptied; it is only rewritten while it exists, so - a run without an interrupted transfer never creates it. A write - failure is logged, never raised. + Upload and download rows are written to ``client_file_trans_progress.json`` + when a connection drops. Forwarded uploads use those same five-item rows; + the ``relays`` area is empty. The reconnect logic reads the rows back. + Once nothing is left the existing file is emptied; an uninterrupted run + creates no progress file. Write failures are logged, never raised. """ - snapshot = self._transfer_progress_snapshot() with self._transfer_progress_lock: + if self._progress_discarded: + return + snapshot = self._transfer_progress_snapshot() if not any(snapshot.values()) and not os.path.exists(self.file_trans_progress_file): return tmp_path = f"{self.file_trans_progress_file}.{os.getpid()}.tmp" try: + if os.path.exists(self.file_trans_progress_file): + _read_transfer_progress(self.file_trans_progress_file) with open(tmp_path, "w", encoding="utf-8") as f: json.dump(snapshot, f, ensure_ascii=False, indent=2) os.replace(tmp_path, self.file_trans_progress_file) - except OSError as e: + except (OSError, ValueError) as e: self._log(f"transfer progress file update failed: {e}") try: os.remove(tmp_path) @@ -7319,18 +8532,18 @@ def _persist_transfer_progress(self): def _load_transfer_progress(self): """Read the persisted interrupted transfers back into the stores. - Every entry read back keeps the bytes it had already transferred, so - the reconnect logic can ask the sender to continue from there; every - relay record comes back so an interrupted forward can be re-driven. - Entries a live transfer (or a relay) already owns are left alone. + Existing five-item rows retain bytes and full paths for reconnect. + Live scoped keys are not overwritten. Nonempty legacy client relay + metadata is rejected without modifying the old snapshot. Returns: - int: Number of transfer entries and relay records restored. + int: Number of upload and download rows restored. """ with self._transfer_progress_lock: + if self._progress_discarded: + return 0 try: - with open(self.file_trans_progress_file, "r", encoding="utf-8") as f: - data = json.load(f) + data = _read_transfer_progress(self.file_trans_progress_file) except FileNotFoundError: return 0 except (OSError, ValueError) as e: @@ -7339,34 +8552,22 @@ def _load_transfer_progress(self): if not isinstance(data, dict): return 0 restored = 0 - with self._forward_resume_lock: - for key_text, record in (data.get("relays") or {}).items(): - try: - key = int(key_text) - except (TypeError, ValueError): - continue - if not isinstance(record, dict): - continue - record = dict(record) - record["targets"] = [ - target - for target in (_peer_from_json(t) for t in record.get("targets") or []) - if target - ] - self._forward_resume_records.setdefault(key, record) - restored += 1 for key, store in ( ("download", self.file_recv_progress), ("upload", self.file_upload_progress), ): for fid_text, row in (data.get(key) or {}).items(): try: - fid = int(fid_text) + fid = int(fid_text) if key == "download" else json.loads(fid_text) + if key == "upload": + fid = (_peer_from_json(fid[0]), int(fid[1])) + if fid[0] is None: + continue entry = [int(row[0]), int(row[1]), None, row[3], _peer_from_json(row[4])] except (TypeError, ValueError, IndexError, KeyError): continue - if fid in store or fid in self._forward_resume_records: - continue # a live transfer owns it, or a relay drives it + if fid in store: + continue # a live transfer owns this exact scoped key store[fid] = entry restored += 1 return restored @@ -7382,14 +8583,14 @@ def _mark_transfer_pause(self, fid, paused, peer=None): Returns: bool: True when a running outbound leg was found. """ + receiver = (self.host, self.port) + if peer is not None and tuple(peer) != receiver: + return False + transfer_key = (receiver, fid) with self._transfer_progress_lock: - live = fid in self._active_transfer_fids - if live and peer is not None: - entry = self.file_upload_progress.get(fid) - if entry is not None and entry[4] is not None and tuple(entry[4]) != tuple(peer): - return False # that leg belongs to another peer + live = transfer_key in self._active_transfer_fids with self._transfer_pause_cond: - self._transfer_pause[fid] = paused + self._transfer_pause[transfer_key] = paused self._transfer_pause_cond.notify_all() return live @@ -7404,9 +8605,8 @@ def pause_trans(self, fid, peer=None): Args: fid (int): Transfer id the outbound leg is keyed by. - peer (tuple | None): Restrict the pause to the leg whose original - receiver is this ``(ip, port)``; None pauses whichever leg - holds the fid. + peer (tuple | None): Actual receiving server ``(ip, port)``; + None uses ``(self.host, self.port)``. A different peer is rejected. Returns: bool: True when a running leg was marked paused; False when @@ -7420,8 +8620,8 @@ def start_trans(self, fid, peer=None): Args: fid (int): Transfer id the outbound leg is keyed by. - peer (tuple | None): Restrict the resume to the leg whose original - receiver is this ``(ip, port)``; None addresses every match. + peer (tuple | None): Actual receiving server ``(ip, port)``; + None uses ``(self.host, self.port)``. Returns: bool: True when a running leg was resumed. @@ -7467,7 +8667,7 @@ def _register_transfer_socket(self, fid, sock): """Record the socket of a leg, so a dead session can be cleared out. Args: - fid (int): Value used by the operation. + fid (int | tuple): Receive fid or outgoing ``(receiver, fid)`` key. sock (object): Value used by the operation. """ if fid is None: @@ -7479,7 +8679,7 @@ def _release_transfer_socket(self, fid, sock): """Forget a leg's socket once the leg is over (its own, not a newer one's). Args: - fid (int): Value used by the operation. + fid (int | tuple): Receive fid or outgoing ``(receiver, fid)`` key. sock (object): Value used by the operation. """ if fid is None: @@ -7496,15 +8696,17 @@ def _drop_stale_transfers(self, peer=None): resumed without two legs writing the same transfer. Args: - peer (tuple | None): Restrict to the legs whose progress entry names this ``(ip, port)``; None clears every leg of the instance (a fresh session replaces all of them). + peer (tuple | None): Match the outgoing receiver namespace or incoming progress peer; None clears all tracked legs. Returns: int: Number of legs closed. """ stale = [] + paused = [] for store in (self.file_upload_progress, self.file_recv_progress): for fid, entry in list(store.items()): - if peer is not None and (entry[4] is None or tuple(entry[4]) != tuple(peer)): + owner = fid[0] if store is self.file_upload_progress else entry[4] + if peer is not None and (owner is None or tuple(owner) != tuple(peer)): continue with self._transfer_progress_lock: sock = self._active_transfer_sockets.pop(fid, None) @@ -7512,6 +8714,8 @@ def _drop_stale_transfers(self, peer=None): self._active_transfer_fids.discard(fid) if sock is not None: stale.append(sock) + if store is self.file_upload_progress: + paused.append(fid) for sock in stale: try: sock.shutdown(socket.SHUT_RDWR) @@ -7521,63 +8725,41 @@ def _drop_stale_transfers(self, peer=None): sock.close() except OSError: pass + if paused: + # The leg may be waiting on this fid's pause switch. Clearing it here + # (outside the progress lock) lets the swept leg exit and release the + # per-key send lock a queued resume is waiting for. + with self._transfer_pause_cond: + for key in paused: + self._transfer_pause.pop(key, None) + self._transfer_pause_cond.notify_all() return len(stale) - def _wait_transfer_paused(self, fid): + def _wait_transfer_paused(self, fid, leg_socket=None): """Block a sending leg while its fid is marked paused. Runs before every chunk so ``/pause_trans`` stops the stream without closing the transfer; the wait ends when the leg is resumed, when the - instance stops, or when the fid is not paused at all. + instance stops, when the leg socket is closed, or when the fid is not + paused at all. ``leg_socket`` is the connection this leg writes to: + a reconnect sweep removes it from the registry before closing it, so + the registry alone cannot report the close. Args: fid (int): Value used by the operation. + leg_socket (socket.socket | None): The leg's own transfer socket. """ if fid is None: return with self._transfer_pause_cond: while self._transfer_pause.get(fid, False): - if not self.running: + sock = self._active_transfer_sockets.get(fid) + if (not self.running + or (sock is not None and sock.fileno() < 0) + or (leg_socket is not None and leg_socket.fileno() < 0)): return self._transfer_pause_cond.wait(timeout=0.5) - def _wait_transfer_server_port(self, fid, timeout=RESUME_PORT_TIMEOUT): - """Wait for the peer's ``/server_file_transfer_port`` of one fid. - - Args: - fid (int): Transfer id the peer announces its listener under. - timeout (float): Seconds to wait. - - Returns: - tuple | None: ``(port, stamp)`` announced for this fid, or None - when the announcement does not arrive in time. - """ - deadline = time.time() + timeout - while time.time() < deadline: - with self.file_transfer_server_port_lock: - for info in reversed(list(self.file_server_port_list)): - if info[1] == fid: - # the newest announcement wins: an older one may belong - # to a leg of a session that is already gone - self.file_server_port_list.remove(info) - stamp = info[2] if len(info) > 2 else "plain" - return info[0], stamp - time.sleep(0.1) - return None - - def _forget_transfer_port(self, fid): - """Drop pending port announcements of a fid an older session left behind. - - Args: - fid (int): Value used by the operation. - """ - if fid is None: - return - with self.file_transfer_server_port_lock: - self.file_server_port_list[:] = [ - info for info in self.file_server_port_list if info[1] != fid - ] - def _wait_control_secure(self, control_socket, timeout=RESUME_SECURE_TIMEOUT): """Wait until a control channel may carry framed protocol lines. @@ -7970,7 +9152,11 @@ def _crypto_negotiate_mode(self): Returns: bool: Result produced by the operation. """ - self._send_raw(self.client_socket, f"/crypto_mode {1 if self.is_enable_encrypto else 0}") + self.send_message( + self.client_socket, + f"/crypto_mode {1 if self.is_enable_encrypto else 0}", + is_send_raw=True, + ) if not self._crypto_mode_event.wait(timeout=self._crypto_mode_timeout): self._log("crypto: server did not announce its encryption mode, disconnecting") self.close() @@ -7980,6 +9166,7 @@ def _crypto_negotiate_mode(self): return False return True + @_config_lifecycle(starting=True) def connect(self): # connect to server """Connect to the server and start reading from it. @@ -7993,34 +9180,42 @@ def connect(self): # connect to server enabled, the key exchange has been started); False when the attempt failed or the mode negotiation closed the connection. """ - while True: - try: - self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.timeout != None: - self.client_socket.settimeout(self.timeout) # connect over 5 seconds timeout - else: - self.client_socket.settimeout(5) - self._log(f"connecting to {self.host}:{self.port}...") - if self.client_port == None: - pass - else: - self.local_address = (self.client_host, self.client_port) + while True: + try: + local_address = ( + (self.client_host, self.client_port) + if self.client_port is not None + else None + ) + while True: + self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if self.timeout != None: + self.client_socket.settimeout(self.timeout) # connect over 5 seconds timeout + else: + self.client_socket.settimeout(5) + self._log(f"connecting to {self.host}:{self.port}...") + self.local_address = local_address try: - self.client_socket.bind(self.local_address) + if local_address is not None: + self.client_socket.bind(local_address) + self.client_socket.connect((self.host, self.port)) + break except OSError as e: - # the configured port is not ours to take right now (a - # session that just ended can keep it, in TIME_WAIT on - # Windows): connect anyway, from an OS-assigned port. - # The interrupted transfers of the previous session - # are matched by address, so they stay stored until a - # reconnect that can use the configured port again. + # The configured source port may be unavailable because + # of TIME_WAIT or a Windows reserved-port denial. The + # previous session can then no longer match transfers by + # local address; as before, reconnect from an OS port. + self.client_socket.close() + if local_address is None or not _should_fallback_local_port( + local_address[1], e + ): + raise self._log( f"client port {self.client_port} is not available ({e}); " f"connecting from an OS-assigned port instead" ) - self.local_address = None - self.client_socket.connect((self.host, self.port)) + local_address = None if self._stopped: # close() arrived while the socket was connecting: stay # closed instead of reviving the connection @@ -8028,6 +9223,7 @@ def connect(self): # connect to server self.client_socket = None return False self.running = True + self._control_sockets.add(self.client_socket) # this client's only control link is the server: the socket is # up, so every leg still registered belongs to the previous # session and is over. The sweep runs before the receive thread @@ -8064,7 +9260,7 @@ def connect(self): # connect to server # a (re)connected session carries the transfers that died with # the previous one: continue them from what is already on disk threading.Thread( - target=self.resume_transfers, kwargs={"peer": (self.host, self.port)}, daemon=True + target=self._request_resumes, daemon=True ).start() return True except socket.timeout: @@ -8098,16 +9294,19 @@ def receive_messages(self): # get server msg lines additionally go to `handle_server_command`. Any end of the connection clears ``running`` and releases the port range. """ + control_socket = self.client_socket buffer = "" - while self.running: + while self.running and self.client_socket is control_socket: try: - data = self.receive_message(self.client_socket, self.recv_messages_chunk_len) + data = self.receive_message(control_socket, self.recv_messages_chunk_len) + if self.client_socket is not control_socket: + break if not data: self._log("\nbreak the connection from server") self.running = False self.free_port() # the transfers running on this session died with it: keep - # them on disk for the next connect (resume_transfers) + # them on disk for the next connect (_request_resumes) self._persist_transfer_progress() break buffer += data.decode("utf-8") @@ -8137,9 +9336,9 @@ def receive_messages(self): # get server msg else: # a command from the server: reported to the listeners # like every inbound line, then handled - self._record_event(self.client_socket, message) + self._record_event(control_socket, message) self._notify_message_received(sender, message) - self.handle_server_command(message) + self.handle_server_command(message, control_socket=control_socket) self._log(f"\n[server] {message}") continue self._notify_message_received(sender, message) @@ -8149,6 +9348,8 @@ def receive_messages(self): # get server msg except socket.timeout: continue except ConnectionResetError: + if self.client_socket is not control_socket: + break self._log("\nReset by server, connection closed") self._log_exc() self.running = False @@ -8156,6 +9357,8 @@ def receive_messages(self): # get server msg self._persist_transfer_progress() break except Exception as e: + if self.client_socket is not control_socket: + break if not _is_closed_socket_error(e): self._log(f"\nget msg error: {e}") self._log_exc() @@ -8163,8 +9366,10 @@ def receive_messages(self): # get server msg self.free_port() self._persist_transfer_progress() break + self._transfer_handshakes.cancel_socket(control_socket) - def send_message(self, client_socket, message): # send msg to server + def send_message(self, client_socket, message, is_send_raw=False): + # send msg to server """Write one line to a socket, encrypting when the channel is up. Args: @@ -8172,11 +9377,33 @@ def send_message(self, client_socket, message): # send msg to server ``self.client_socket``. message (str | bytes): Payload; a str is stripped and newline terminated, bytes are sent as they are. + is_send_raw (bool): Write the payload as plaintext even when the channel + is encrypted; only for the crypto handshake and transfer coordination lines. Returns: bool: True when the payload was written; False when the client is not running, no socket was passed, or the payload type is unsupported. """ + if is_send_raw: + deal_msg = message.strip() + if not deal_msg.endswith("\n"): + deal_msg += "\n" + encoded = deal_msg.encode("utf-8") + client_socket.sendall(encoded) + self._measure_network_speed(encoded, is_upload=True) + return True + written = self._write_message(client_socket, message) + if written: + self._after_send(client_socket, message) + return written + + def _write_message(self, client_socket, message): # noqa: PLR0911 - same exits as the old send_message body + """Write one line: encrypted when the channel is up, plaintext otherwise. + + Returns: + bool: True when written; False when the client is not running, no + socket was passed, or the payload type is unsupported. + """ if not self.running or not self.client_socket: self._log("disable the connect to server") return False @@ -8217,6 +9444,14 @@ def send_message(self, client_socket, message): # send msg to server self._log_exc() return False + def _after_send(self, client_socket, message): + """Record and report one payload that `_write_message` has written.""" + if isinstance(message, str): + message = message.strip() + if message and client_socket in self._control_sockets: + self._record_sent(client_socket, message) + self._notify_message_sent(client_socket, message) + def send_message_to_server(self, message): """Send the payload of a console line to the server. @@ -8251,10 +9486,12 @@ def receive_message(self, client_socket, msg_length): # receive msg if channel is None: data = client_socket.recv(msg_length) self._measure_network_speed(data, is_upload=False) + self._notify_transfer_chunk(client_socket, data) return data data = channel.take(msg_length) if data: self._measure_network_speed(data, is_upload=False) + self._notify_transfer_chunk(client_socket, data) return data # buffered plaintext is served even after a disable raw = client_socket.recv(self.recv_messages_chunk_len if channel.secure else msg_length) if not raw: @@ -8267,26 +9504,14 @@ def receive_message(self, client_socket, msg_length): # receive msg leftover = channel.drain_plaintext() if leftover: self._measure_network_speed(leftover, is_upload=False) + self._notify_transfer_chunk(client_socket, leftover) return leftover # hand back whatever was already read continue # framing is off now: keep serving in raw mode if plain: self._measure_network_speed(plain, is_upload=False) + self._notify_transfer_chunk(client_socket, plain) return plain # plaintext mode: the line reader gets the bytes - def _send_raw(self, client_socket, text): - """Send a plaintext crypto-protocol message, bypassing encryption. - - Args: - client_socket (socket.socket): Value used by the operation. - text (object): Value used by the operation. - """ - data = text.strip() - if not data.endswith("\n"): - data += "\n" - encoded = data.encode("utf-8") - client_socket.sendall(encoded) - self._measure_network_speed(encoded, is_upload=True) - def _crypto_mark_encrypted(self, client_socket, peer_role, peer_pem_path): """Handle the operation. @@ -8356,7 +9581,7 @@ def _crypto_after_flip(self, client_socket): return try: # plaintext by design: the peer needs this line before it may frame - self._send_raw(client_socket, CRYPTO_FRAMES_START) + self.send_message(client_socket, CRYPTO_FRAMES_START, is_send_raw=True) except Exception: self._log_exc() self._crypto_publish_ecdh_pub(client_socket) @@ -8555,7 +9780,7 @@ def _crypto_exchange_thread(self, force=False): hello = f"/crypto_key_exchange {self._crypto_my_nonce}" if force: hello += " 1" - self._send_raw(self.client_socket, hello) + self.send_message(self.client_socket, hello, is_send_raw=True) if not ack_event.wait(timeout=30): if not self._crypto_still_current(gen): return # superseded by a newer handshake; that flow owns the connection @@ -8603,7 +9828,7 @@ def _crypto_on_server_hello(self, server_nonce, force): if session is not None: session.reset_ecdh() # a re-exchange is a fresh session key ack = f"/crypto_key_exchange_ack {my_nonce} 1 {1 if force else 0}" - self._send_raw(self.client_socket, ack) + self.send_message(self.client_socket, ack, is_send_raw=True) self._crypto_after_ack("1" if force else "0", force) def _crypto_try_flip(self): @@ -8668,7 +9893,7 @@ def _crypto_after_ack(self, server_needs, force): else: request_pub = False if request_pub: - self._send_raw(self.client_socket, "/crypto_pub_key_request") + self.send_message(self.client_socket, "/crypto_pub_key_request", is_send_raw=True) def _wait_done(): """Handle the operation.""" @@ -8690,7 +9915,7 @@ def _wait_done(): if stale: return try: - self._send_raw(self.client_socket, "/crypto_ready") + self.send_message(self.client_socket, "/crypto_ready", is_send_raw=True) except OSError: pass # connection already closed (reject/breaker): nothing to announce with self._crypto_lock: @@ -8708,52 +9933,44 @@ def _crypto_push_pub_key(self): Returns: bool: Result produced by the operation. """ + control_socket = self.client_socket while ( True ): # serialise pushes: wait for a concurrent push (double handshake) instead of skipping it with self._crypto_lock: - if self.client_socket not in self._crypto_push_active: - self._crypto_push_active.add(self.client_socket) + if control_socket not in self._crypto_push_active: + self._crypto_push_active.add(control_socket) break time.sleep(0.05) try: - with self.file_client_id_lock: - client_id = copy.copy(self.file_client_id) - self.file_client_id += 1 - send_msg = f"/crypto_pub_key {client_id}" - self._send_raw(self.client_socket, send_msg) - file_transfer_client_port = self.palloc() - file_server_port = None - waiting_time = 0 - while True: - time.sleep(0.05) - with self.file_transfer_server_port_lock: - for port_info in list(self.file_server_port_list): - if port_info[1] == client_id: - file_server_port = port_info[0] - self.file_server_port_list.remove(port_info) - break - if file_server_port is not None: - break - waiting_time += 1 - if waiting_time >= 200: + client_id = self._fid_alloc.alloc(receiver=(self.host, self.port)) + with _transfer_handshake(self, control_socket, client_id) as handshake: + send_msg = f"/crypto_pub_key {client_id}" + if not _send_transfer_request(self, handshake, send_msg, is_send_raw=True): + return False + result = self._transfer_handshakes.wait(handshake, 10.0) + if result is None or result[0] != "port" or result[1] != client_id: self._log("crypto: transfer port waiting timeout, public key push failed") return False - self.file_transfer_mode( # a key push carries the keys themselves: always plaintext - self.crypto.pub_path, - self.host, - file_server_port, - file_transfer_client_port, - control_socket=self.client_socket, - crypto="plain", - ) - self.pfree(file_transfer_client_port) + _, _, file_server_port, _stamp = result + file_transfer_client_port = self.palloc() + try: + self.file_transfer_mode( # public-key bytes always travel plaintext + self.crypto.pub_path, + self.host, + file_server_port, + file_transfer_client_port, + control_socket=control_socket, + crypto="plain", + ) + finally: + self.pfree(file_transfer_client_port) except Exception: self._log_exc() return False finally: with self._crypto_lock: - self._crypto_push_active.discard(self.client_socket) + self._crypto_push_active.discard(control_socket) return True def _crypto_store_received_pub(self, full_path, peer_role, server_addr): @@ -8794,7 +10011,7 @@ def _crypto_store_received_pub(self, full_path, peer_role, server_addr): except OSError: pass try: - self._send_raw(self.client_socket, f"/crypto_reject {reason}") + self.send_message(self.client_socket, f"/crypto_reject {reason}", is_send_raw=True) except Exception: self._log_exc() try: @@ -8805,21 +10022,26 @@ def _crypto_store_received_pub(self, full_path, peer_role, server_addr): with self._crypto_lock: self._crypto_server_pub_event.set() - def handle_server_command(self, command): # deal with special command from server + def handle_server_command(self, command, control_socket=None): """Dispatch one command line pushed by the server. Handles the protocol's own lines: ``/crypto_mode`` (a mismatch closes the connection), ``/client_alloc_port_range``, the ``/crypto_*`` exchange lines, and the transfer lines ``/file``, ``/file_folder``, - ``/server_file_transfer_port``, ``/forward_upload``, ``/pause_trans``, - ``/start_trans``, ``/resume_trans``, ``/forward_resume`` and - ``/forward_error``. Any other name goes to the handlers registered for + ``/server_file_transfer_port``, ``/pause_trans``, ``/start_trans``, + ``/resume_trans`` and FID-scoped ``/forward_error``. + Any other name goes to the handlers registered for the "server" side via `register_command`; an unknown command is only reported on the console. Args: command (str): Line including its leading ``/``. + control_socket (socket.socket | None): Actual source socket of the line. """ + if control_socket is None: + control_socket = self.client_socket + if control_socket is not self.client_socket: + return # a replaced control session cannot dispatch into the new one client_id = f"{self.client_host}:{self.client_port}" if command.lower().split(" ")[0] == "/crypto_mode": try: @@ -8849,18 +10071,10 @@ def handle_server_command(self, command): # deal with special command from serv f"server allocated port range for each client: {self.each_client_port_range}" ) elif shlex.split(command.lower())[0] == "/server_file_transfer_port": - parts = command.split(" ") - with self.file_transfer_server_port_lock: - self.file_transfer_server_port = int(parts[1]) - try: - file_client_id = int(parts[2]) - stamp = parts[3] if len(parts) > 3 else "plain" # listener mode+nonce - self.file_server_port_list.append( - [self.file_transfer_server_port, file_client_id, stamp] - ) - except: - self._log_exc() - pass + announcement = _parse_transfer_port_announcement(command) + if announcement is not None: + port, fid, stamp = announcement + self._transfer_handshakes.resolve_port(control_socket, fid, port, stamp) elif ( self.is_enable_encrypto and self.crypto is not None @@ -8940,15 +10154,6 @@ def handle_server_command(self, command): # deal with special command from serv self.file_folder_transfer_client_recv_server_start_thread( command, client_id, self.client_socket ) - elif shlex.split(command.lower())[0] == "/forward_upload": - try: - parts = shlex.split(command) - fid = int(parts[1]) - sport = int(parts[2]) - except (IndexError, ValueError): - return - stamp = parts[3] if len(parts) > 3 else "plain" # upload leg's mode+nonce - self._forward_upload_queue.put((fid, sport, stamp)) elif shlex.split(command.lower())[0] == "/pause_trans": fid = _pause_command_fid(command) if fid is not None: @@ -8959,18 +10164,21 @@ def handle_server_command(self, command): # deal with special command from serv self.start_trans(fid) elif shlex.split(command.lower())[0] == RESUME_LINE: parsed = parse_resume_command(command) - if parsed is not None: - threading.Thread( - target=self.resume_trans, - args=((self.host, self.port), parsed[0], parsed[1]), - daemon=True, - ).start() - elif shlex.split(command.lower())[0] == FORWARD_RESUME_LINE: - parsed = parse_forward_resume_command(command) - if parsed is not None: - self._resume_forward(parsed[0], parsed[1], parsed[2]) + if parsed is not None and self.client_socket is not None: + fid, offset, origin, target = parsed + if origin == self.client_socket.getsockname(): + self.resume_trans(target, fid, offset) elif shlex.split(command.lower())[0] == "/forward_error": - self._forward_error = command + try: + parts = shlex.split(command) + except ValueError: + return + try: + fid = int(parts[1]) if len(parts) == 3 else -1 + except ValueError: + return + if fid >= 0: + self._transfer_handshakes.resolve_error(control_socket, fid, parts[2]) else: cmd_parts = shlex.split(command.strip()) if not cmd_parts: @@ -9071,8 +10279,8 @@ def interactive_mode(self): # Interactive mode ``/quit`` closes the connection; ``/send_msg``, ``/file``, ``/multiple_file``, ``/file_folder``, ``/multiple_file_folder``, - ``/forward_file``, ``/forward_folder`` and ``/forward_send_msg`` are handled - locally; any other name goes to the handlers registered with + ``/forward_file``, ``/forward_folder``, ``/forward_send_msg`` and + ``/transfers`` are handled locally; other names go to handlers registered with ``where_to_run="client"``, and anything left is sent to the server as it stands. Ctrl-C and EOF close the connection. """ @@ -9088,6 +10296,13 @@ def interactive_mode(self): # Interactive mode self.send_message(self.client_socket, "/quit") time.sleep(0.5) break + elif message.split(maxsplit=1)[0].lower() == "/transfers": + if message.strip().lower() != "/transfers": + print("Usage: /transfers") + else: + print(json.dumps( + self.get_transfer_progress(), ensure_ascii=False, indent=2 + )) elif shlex.split(message.lower())[0] == "/send_msg": self.send_message_to_server(message) elif shlex.split(message.lower())[0] == "/file": @@ -9255,17 +10470,19 @@ def limited_transfer(): self._log(f"start to send file: {cmd} (limit {self.max_thread_num})") def get_all_files_in_folder(): - """Handle the operation.""" - for root, dirs, files in os.walk(folder_path): + """Announce the directories that need it, then send every file.""" + needed = _rel_dirs_with_files(folder_path, base_path) + if transfer_path in needed: + send_folder_transfer_command(transfer_path) + for root, _dirs, files in os.walk(folder_path): rel_dir = get_relative_path(base_path, root) - if root != folder_path: + if root != folder_path and rel_dir in needed: send_folder_transfer_command(rel_dir) for file in files: start_file_transfer_with_limit(rel_dir, file, root) self._log(f"finished sending all files in folder {folder_path}") transfer_path = get_relative_path(base_path, folder_path) - send_folder_transfer_command(transfer_path) get_all_files_in_folder() def multiple_file_transfer_client_recv_client_start(self, message): @@ -9316,51 +10533,33 @@ def file_transfer_client_recv_client_start(self, message, file_folder_abspath): message (object): Value used by the operation. file_folder_abspath (str): Value used by the operation. """ - client_id = None - with self.file_client_id_lock: - client_id = copy.copy(self.file_client_id) - send_msg = message.strip() + " " + str(self.file_client_id) - self.file_client_id += 1 + client_id = self._fid_alloc.alloc(receiver=(self.host, self.port)) + peer = (self.host, self.port) + control_socket = self.client_socket try: - waiting_time = 0 if shlex.split(message.lower())[0] == "/file_folder": filename = os.path.join(file_folder_abspath, shlex.split(message)[2]) else: filename = shlex.split(message)[1] - self.send_message(self.client_socket, send_msg) - file_transfer_client_port = self.palloc() - file_server_port = None - stamp = "plain" - is_find_port = True - while is_find_port: - time.sleep(1) - if len(self.file_server_port_list) > 0: - with self.file_transfer_server_port_lock: - for port_info in self.file_server_port_list: - if port_info[1] == client_id: - file_server_port = port_info[0] - stamp = port_info[2] # the listener's mode+nonce - self.file_server_port_list.remove(port_info) - is_find_port = False - break - pass - waiting_time += 1 - if waiting_time >= 20: - self._log( - "ErrorWhileReceiveFileServerPort: transfer port waiting timeout, file sending failed" - ) + with _sending_transfer(self, (peer, client_id)), \ + _transfer_handshake(self, control_socket, client_id) as handshake: + send_msg = f"{message.strip()} {client_id}" + if not _send_transfer_request(self, handshake, send_msg): return False - self.file_transfer_mode( - filename, - self.host, - file_server_port, - file_transfer_client_port, - control_socket=self.client_socket, - crypto=stamp, - fid=client_id, - peer=(self.host, self.port), # the receiver a resume is asked to continue from - ) - self.pfree(file_transfer_client_port) + result = self._transfer_handshakes.wait(handshake, RESUME_PORT_TIMEOUT) + if result is None or result[0] != "port" or result[1] != client_id: + self._log("ErrorWhileReceiveFileServerPort: transfer port waiting timeout, file sending failed") + return False + _, _, file_server_port, stamp = result + file_transfer_client_port = self.palloc() + try: + self.file_transfer_mode( + filename, self.host, file_server_port, file_transfer_client_port, + control_socket=control_socket, crypto=stamp, fid=client_id, + peer=peer, + ) + finally: + self.pfree(file_transfer_client_port) except IndexError: self._log_exc() self._log("invalid command, please use '/file '") @@ -9391,9 +10590,10 @@ def file_transfer_mode( # noqa: PLR0911 - peer-close and timeout exits are dist resume_offset (int): Byte offset to resume from; defaults to zero. Returns: - bool: True when the receiver acknowledges the file; otherwise False. + bool: True only after the receiver's final data ACK; otherwise False. """ self._log(f"start to send file: {filename}") + transfer_key = ((self.host, self.port), fid) if fid is not None else None client_file_socket = None reset_time = 0 @@ -9403,17 +10603,27 @@ def close_socket(): nonlocal client_file_socket file_running = False self._crypto_drop_channel(client_file_socket) + self._transfer_handshakes.release_socket(control_socket, client_file_socket) client_file_socket.close() - with self.file_client_id_lock: - self.file_client_id -= 1 while True: try: client_file_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if not self._transfer_handshakes.track_socket(control_socket, client_file_socket): + return False client_file_socket.bind((self.client_host, client_port)) client_file_socket.connect((server_address, server_port)) break except Exception as e: + self._transfer_handshakes.release_socket(control_socket, client_file_socket) + client_file_socket.close() + if _should_fallback_local_port(client_port, e): + self._log( + f"file transfer source port {client_port} is not available ({e}); " + "retrying from an OS-assigned port" + ) + client_port = 0 + continue self._log(f"file transfer connection error: {e}") self._log_exc() if reset_time >= 20: @@ -9430,14 +10640,17 @@ def close_socket(): close_socket() return False file_running = True - self._register_transfer_socket(fid, client_file_socket) - file_receive_data_from_server = "" + self._register_transfer_socket(transfer_key, client_file_socket) + transfer_signs = _TransferSignBuffer( + self.server_start_file_transfer_sign, + self.server_received_file_data_sign, + self.error_sign, + ) def receive_file_transfer_messages(): """Handle the operation.""" nonlocal file_running nonlocal client_file_socket - nonlocal file_receive_data_from_server while file_running: try: data = self.receive_message(client_file_socket, self.recv_messages_chunk_len) @@ -9449,8 +10662,7 @@ def receive_file_transfer_messages(): pass # send_message already logged real errors; a dead peer is expected close_socket() break - file_receive_data_from_server = data.decode("utf-8").strip() - if file_receive_data_from_server == self.error_sign: + if transfer_signs.feed(data): self._log( "\nError sign received from server, file transfer may have failed" ) @@ -9475,12 +10687,14 @@ def receive_file_transfer_messages(): source_missing = False # True when the source file no longer exists try: while True: - if file_receive_data_from_server == self.server_start_file_transfer_sign: + running = file_running + start_seen, _, error_seen = transfer_signs.consume() + if start_seen: break - if file_receive_data_from_server == self.error_sign: + if error_seen: close_socket() - break - if not file_running: + return False + if not running: # peer closed the file connection: the receive thread # already cleaned up; nothing more to wait for return False @@ -9508,32 +10722,40 @@ def receive_file_transfer_messages(): if resume_offset < 0 or resume_offset > file_size: resume_offset = 0 # not this file's offset any more: send it whole progress = self._file_progress_start( - self.file_upload_progress, - fid, - file_size, - filename, - peer=peer, - initial=resume_offset, + self.file_upload_progress, transfer_key, file_size, filename, + peer=peer, initial=resume_offset, ) with open(filename, "rb") as f: if resume_offset: f.seek(resume_offset) while True: - self._wait_transfer_paused(fid) + transfer_signs.consume() + self._wait_transfer_paused(transfer_key, client_file_socket) file_data = f.read(self.file_transfer_chunk_len) if not file_data: break - self.send_message(client_file_socket, file_data) + try: + written = self.send_message(client_file_socket, file_data) + except OSError as e: + self._log(f"send error: {e}") + close_socket() # the peer is gone; stop instead of reading the rest + return False + if not written: + self._log("send error: the transfer connection is closed") + close_socket() + return False self._file_progress_update(progress, len(file_data), is_upload=True) extra_time = (file_size // (100 * 1024 * 1024)) * 10 timeout = int(30 + extra_time) while True: - if file_receive_data_from_server == self.server_received_file_data_sign: + running = file_running + _, ack_seen, error_seen = transfer_signs.consume() + if ack_seen: break - if file_receive_data_from_server == self.error_sign: + if error_seen: close_socket() - break - if not file_running: + return False + if not running: # peer closed the file connection: the receive thread # already cleaned up; nothing more to wait for return False @@ -9578,283 +10800,126 @@ def receive_file_transfer_messages(): return False finally: self._file_progress_finish( - self.file_upload_progress, - fid, - progress, + self.file_upload_progress, transfer_key, progress, keep=( not file_sent and not source_missing and progress is not None - and progress[4] is not None # resumable: it has a peer + and progress[4] is not None ), ) - self._release_transfer_socket(fid, client_file_socket) + self._release_transfer_socket(transfer_key, client_file_socket) with self._transfer_pause_cond: - self._transfer_pause.pop(fid, None) + self._transfer_pause.pop(transfer_key, None) # ---- transfer reconnect (client side) ---------------------------------- - def resume_transfers(self, peer=None): - """Continue the interrupted transfers of a peer that is back. + def _request_resumes(self): + """Continue this client's interrupted downloads after it (re)connected. - Reads the persisted stores back (`_load_transfer_progress`) and drives - every download that can be continued: a file the server was sending is - - Args: - peer (tuple | None): Peer ``(ip, port)`` that just connected; None - resumes every peer the client can reach. + Loads the existing receive rows and sends their stable fid, disk offset, + original sender and this connection's target identity through one wire + command. Upload rows are resumed only when their receivers ask. Returns: - int: Number of resumed transfers started or requested. + int: Number of resume requests sent. """ restored = self._load_transfer_progress() if restored: self._log(f"transfer reconnect: restored {restored} interrupted transfer(s)") if not self.running or self.client_socket is None: return 0 - server_peer = (self.host, self.port) - if peer is not None and tuple(peer) != server_peer: - # every resume this client drives goes through the server: it is - # the sender of a pushed file and the relay of a forwarded one - return 0 if not self._wait_control_secure(self.client_socket): return 0 - candidates = [] + # A direct row recorded before the identity change names the configured + # server address; the concrete peer of this connection is what the + # server recognises as itself, so migrate the row in memory (the + # progress file keeps its format). + configured_peer = (self.host, self.port) + connected_peer = _socket_peer_endpoint(self.client_socket) + started = 0 for fid, entry in list(self.file_recv_progress.items()): if entry[4] is None: continue # without a peer there is nobody to continue with - with self._forward_resume_lock: - if fid in self._forward_resume_records: - continue # this client relays that transfer: its target drives it with self._transfer_progress_lock: if fid in self._active_transfer_fids: continue # a leg is already running for this fid - candidates.append((fid, entry, tuple(entry[4]))) - started = 0 - for fid, entry, peer_addr in candidates: - if peer_addr == server_peer: - # the server is the sender: ask it directly - threading.Thread( - target=self._resume_receive_leg, - args=(self.client_socket, server_peer, fid, entry), - daemon=True, - ).start() - started += 1 - elif self._request_forward_resume(fid, entry, peer_addr): - # a forwarded transfer: the server routes the request to the - # originating client, which re-forwards it to this one + if connected_peer is not None and tuple(entry[4]) == configured_peer: + entry[4] = connected_peer + entry[0] = _resume_offset_for(entry) + if self.send_message( + self.client_socket, + format_resume_line(fid, entry[0], entry[4], self.client_socket.getsockname()), + ): started += 1 return started - def _request_forward_resume(self, fid, entry, originator): - """Ask the server to have a forward's originator continue a transfer. - - The local path is registered before the request goes out: the fresh - relay leg arrives under a new fid but writes the same path, and - `_take_pending_resume` hands its offset to that leg. - - Args: - fid (int): Value used by the operation. - entry (object): Value used by the operation. - originator (object): Value used by the operation. - - Returns: - bool: Result produced by the operation. - """ - offset = _resume_offset_for(entry) - with self._forward_resume_lock: - self._resume_pending_paths[entry[3]] = (fid, offset) - if not self.send_message(self.client_socket, f"{RESUME_LINE} {fid} {offset}"): - with self._forward_resume_lock: - self._resume_pending_paths.pop(entry[3], None) - return False - self._log( - f"resume: asked the server to continue transfer {fid} from {originator} " - f"at byte {offset}" - ) - return True - - def _take_pending_resume(self, path): - """Return the offset a pending resume of this path starts at, or None. - - A forward target that lost a transfer asks its originator to re-send - it; the fresh leg the server then pushes carries a new transfer id but - lands on the same path, so the state registered by the request is what - the leg appends from — and the interrupted entry it replaces is - dropped here. - - Args: - path (str): Value used by the operation. - - Returns: - NoneType | object: Result produced by the operation. - """ - with self._forward_resume_lock: - pending = self._resume_pending_paths.pop(path, None) - if pending is None: - return None - old_fid, offset = pending - old_entry = self.file_recv_progress.pop(old_fid, None) - if old_entry is not None: - with self._transfer_progress_lock: - self._active_transfer_fids.discard(old_fid) - return offset - - def _resume_receive_leg(self, control_socket, peer_addr, fid, entry): - """Receive the rest of one interrupted transfer, appending to its file. - - Args: - control_socket (socket.socket): Value used by the operation. - peer_addr (tuple): Value used by the operation. - fid (int): Value used by the operation. - entry (object): Value used by the operation. - - Returns: - bool: True when the sender was asked to continue, False when the - request could not be sent. - """ - entry[0] = _resume_offset_for(entry) # the offset the sender is asked for - self._log(f"resume: continuing transfer {fid} with {peer_addr} into {entry[3]}") - port = self.palloc() - try: - return self.file_transfer_mode_recv( - self.host, - port, - control_socket, - f"{peer_addr[0]}:{peer_addr[1]}", - None, - None, - f"/file {shlex.quote(os.path.basename(str(entry[3])))} {fid}", - peer=peer_addr, - resume_entry=entry, - ) is not False - finally: - self.pfree(port) def resume_trans(self, peer_addr, fid, offset): - """Continue one interrupted transfer this instance was sending. + """Queue direct or forwarded recovery from the existing upload row. - Runs when a receiver asks to continue a transfer (``/resume_trans``, - see `resume_transfers` on the receiving side): the leg is - re-established against the receiver's fresh listener and the source - file is read from the byte offset the receiver already holds. Also the - public API for resuming one transfer on demand. + Server-bound uploads answer with ``/file``; other final targets replay + through the server with the stable upstream fid. The retained upload + path is sufficient because the receiver owns its full destination path. + Runtime locks coalesce duplicate targets and serialize different + targets without persisting file metadata. Args: - peer_addr (tuple | None): Receiver ``(ip, port)`` the request came - from; None skips the peer check. + peer_addr (tuple | None): Final receiver; None selects the server. fid (int): Transfer id of the interrupted transfer. offset (int): Byte offset the receiver already holds. Returns: - bool: True when the resume leg was started. + bool: True when the request was queued or joined in flight. """ - entry = self.file_upload_progress.get(fid) - if entry is None: - self._log(f"resume: no outgoing transfer with fid {fid} to continue") - return False - if peer_addr is not None and entry[4] is not None and tuple(entry[4]) != tuple(peer_addr): - self._log(f"resume: transfer {fid} is for {entry[4]}, not {peer_addr}, ignored") - return False - if not self.running or self.client_socket is None: - self._log(f"resume: not connected to the server, transfer {fid} kept") - return False - port_info = self._wait_transfer_server_port(fid) - if port_info is None: - self._log(f"resume: {peer_addr} did not announce a transfer port for {fid}") + server_peer = (self.host, self.port) + control_socket = self.client_socket + if not self.running or control_socket is None: return False - port, stamp = port_info - threading.Thread( - target=self._run_resume_send_leg, - args=(entry[3], peer_addr, port, stamp, self.client_socket, fid, offset), - daemon=True, - ).start() - return True - - def _run_resume_send_leg(self, path, peer_addr, port, stamp, control_socket, fid, offset): - """Run one resumed sending leg on its own thread. - - Args: - path (str): Value used by the operation. - peer_addr (tuple): Value used by the operation. - port (int): Value used by the operation. - stamp (object): Value used by the operation. - control_socket (socket.socket): Value used by the operation. - fid (int): Value used by the operation. - offset (int): Value used by the operation. - """ - client_port = self.palloc() - try: - self.file_transfer_mode( - path, - peer_addr[0] if peer_addr else self.host, - port, - client_port, - control_socket=control_socket, - crypto=stamp, - fid=fid, - peer=peer_addr, - resume_offset=offset, - ) - finally: - self.pfree(client_port) - - def _forward_resume_record(self, fid): - """Return the relay record a forwarded resume for ``fid`` belongs to. - - Args: - fid (int): Value used by the operation. - - Returns: - object: Result produced by the operation. - """ - with self._forward_resume_lock: - return self._forward_resume_records.get(fid) - - def _resume_forward(self, target, fid, offset): - """Re-drive a forwarded transfer to one target, from ``offset``. - - Runs when the server relays a target's resume request - (``/forward_resume``): the recorded relay says what the transfer was, - so the item is forwarded again — only to the target that asked, and - only the remainder of the file it already holds partially. + # The server names itself with the concrete endpoint of this control + # connection; the configured address is accepted too so a peer using + # either convention is recognised. The upload row key keeps the + # configured namespace, so persisted rows are unchanged. + server_endpoints = {server_peer} + connected_endpoint = _socket_peer_endpoint(control_socket) + if connected_endpoint is not None: + server_endpoints.add(connected_endpoint) + target = tuple(peer_addr) if peer_addr is not None else ( + connected_endpoint or server_peer + ) + key = (server_peer, fid) + is_direct = target in server_endpoints + + def send(entry): + if not is_direct: + self._forward_one( + "file", "", os.path.basename(str(entry[3])), entry[3], [target], + resume_offset=offset, upstream_fid=fid, + ) + return + if entry[4] is None or tuple(entry[4]) != server_peer: + return + with _transfer_handshake(self, control_socket, fid) as handshake: + command = f"/file {shlex.quote(os.path.basename(str(entry[3])))} {fid}" + if not _send_transfer_request(self, handshake, command): + return + result = self._transfer_handshakes.wait(handshake, RESUME_PORT_TIMEOUT) + if result is None or result[0] != "port" or result[1] != fid: + self._log(f"resume: server did not announce a transfer port for {fid}") + return + _, _, port, stamp = result + client_port = self.palloc() + try: + self.file_transfer_mode( + entry[3], self.host, port, client_port, + control_socket=control_socket, crypto=stamp, fid=fid, + peer=server_peer, resume_offset=offset, + ) + finally: + self.pfree(client_port) - Args: - target (tuple): Target client ``(ip, port)`` that lost the file. - fid (int): The relay's transfer id (this client's own namespace). - offset (int): Byte offset the target already holds. + return _start_resume_send(self, key, target, send) - Returns: - bool: True when the re-forward was started. - """ - record = self._forward_resume_record(fid) - with self._forward_resume_lock: - self._forward_resume_records.pop(fid, None) - entry = self.file_upload_progress.pop(fid, None) - if record is None and entry is None: - self._log(f"resume: no forwarded transfer with fid {fid} to continue") - return False - record = record or {} - targets = [tuple(item) for item in record.get("targets") or []] - kind = record.get("kind") or "file" - rel_dir = record.get("rel_dir") or "" - fname = record.get("fname") or os.path.basename(str((entry or [None, None, None, ""])[3])) - path = record.get("path") or (entry[3] if entry else None) - destination = record.get("destination") - if path is None or not os.path.isfile(path): - self._log(f"resume: the source of forwarded transfer {fid} is gone, giving up") - return False - if targets and tuple(target) not in targets: - self._log(f"resume: {target} is not a target of forwarded transfer {fid}, ignored") - return False - self._log(f"resume: forwarding {path} to {target} again, from byte {offset}") - threading.Thread( - target=self._forward_one, - args=(kind, rel_dir, fname, path, [tuple(target)], destination), - kwargs={"resume_offset": offset}, - daemon=True, - ).start() - return True # ---- native in-memory forward (client side) ---------------------------- @@ -10005,91 +11070,82 @@ def _forward_rel_path(self, base_path, abs_path): return "/" + rel.replace(os.sep, "/") def _forward_one( - self, kind, rel_dir, fname, abspath, addrs, destination_path=None, resume_offset=0 + self, + kind, + rel_dir, + fname, + abspath, + addrs, + destination_path=None, + resume_offset=0, + upstream_fid=None, ): - """Forward one item to every target through the server's relay. - - The relay the server answers with is recorded under its fid so a - target that lost the transfer can have it re-driven after a reconnect - (`_resume_forward`); ``resume_offset`` re-sends only the remainder of - a file the target already holds partially. - - Args: - kind (object): Value used by the operation. - rel_dir (str): Value used by the operation. - fname (object): Value used by the operation. - abspath (str): Value used by the operation. - addrs (list | dict): Value used by the operation. - destination_path (str): Value used by the operation. - resume_offset (object): Value used by the operation. - - Returns: - bool | object: Result produced by the operation. - """ + """Forward one item using a source-owned upstream FID.""" if not os.path.isfile(abspath): self._log(f"forward: {abspath} is not a valid file, skipped") return False - msg = f"/forward_item {kind}" - if resume_offset: - # the relay must know how much of the file does not travel again - msg += f" resume={int(resume_offset)}" - if kind == "folder": - msg += f" {shlex.quote(rel_dir)}" - msg += f" {shlex.quote(fname)} " + " ".join(shlex.quote(str(a)) for a in addrs) - if destination_path: - msg += f" {shlex.quote(destination_path)}" - self._forward_error = None - self.send_message(self.client_socket, msg) - upload = None - deadline = time.time() + 30 - while time.time() < deadline: - if self._forward_error is not None: - self._log(f"forward: server error: {self._forward_error}") - self._forward_error = None + server_peer = (self.host, self.port) + control_socket = self.client_socket + is_resume = upstream_fid is not None + if is_resume: + key = (server_peer, upstream_fid) + if key not in self.file_upload_progress: + return False + else: + upstream_fid = self._fid_alloc.alloc(receiver=server_peer) + key = (server_peer, upstream_fid) + if self.is_enable_file_transfer_progress_bar: + self.file_upload_progress[key] = [ + 0, os.path.getsize(abspath), None, str(abspath), + (_peer_from_json(addrs[0]) or tuple(addrs[0])) if addrs else None, + ] + self._persist_transfer_progress() + with _sending_transfer(self, key): + with _transfer_handshake(self, control_socket, upstream_fid) as handshake: + msg = f"/forward_item {kind} fid={int(upstream_fid)}" + if is_resume: + msg += f" resume={int(resume_offset)}" + if kind == "folder": + msg += f" {shlex.quote(rel_dir)}" + msg += f" {shlex.quote(fname)} " + " ".join( + shlex.quote(str(address)) for address in addrs + ) + if destination_path: + msg += f" {shlex.quote(destination_path)}" + if not _send_transfer_request(self, handshake, msg): + return False + result = self._transfer_handshakes.wait(handshake, 30.0) + if result is None: + self._log(f"forward: timed out waiting for an upload slot for {fname}") + return False + if result[0] == "error": + self._log(f"forward: server error: {result[1]}") return False + if result[0] != "port" or result[1] != upstream_fid: + self._log(f"forward: server changed upstream fid for {fname}") + return False + _, fid, sport, stamp = result + targets = [ + (_peer_from_json(target) or (target[0], int(target[1]))) + for target in addrs + ] + client_port = self.palloc() try: - upload = self._forward_upload_queue.get(timeout=0.2) - break - except queue.Empty: - continue - if upload is None: - self._log(f"forward: timed out waiting for an upload slot for {fname}") - return False - fid, sport, stamp = upload - targets = [(_peer_from_json(target) or (target[0], int(target[1]))) for target in addrs] - record = { - "fid": fid, - "kind": kind, - "rel_dir": rel_dir, - "fname": fname, - "path": abspath, - "destination": destination_path, - "targets": [list(target) for target in targets], - } - with self._forward_resume_lock: - self._forward_resume_records[fid] = record - client_port = self.palloc() - try: - sent = self.file_transfer_mode( - abspath, - self.host, - sport, - client_port, - control_socket=self.client_socket, - crypto=stamp, - fid=fid, - peer=targets[0] if targets else None, # a forwarded leg stores its original target - resume_offset=resume_offset, - ) - finally: - self.pfree(client_port) - with self._transfer_pause_cond: - self._transfer_pause.pop(fid, None) - if sent: - # the whole item went through: nothing is left to resume for this relay - with self._forward_resume_lock: - self._forward_resume_records.pop(fid, None) - return sent + return self.file_transfer_mode( + abspath, + self.host, + sport, + client_port, + control_socket=control_socket, + crypto=stamp, + fid=fid, + peer=targets[0] if targets else None, + resume_offset=resume_offset, + ) + finally: + self.pfree(client_port) + with self._transfer_pause_cond: + self._transfer_pause.pop((server_peer, fid), None) def file_folder_transfer_client_recv_server_start_thread( self, command, client_id, client_socket @@ -10106,6 +11162,18 @@ def file_folder_transfer_client_recv_server_start_thread( # >= 4 tokens means a file transfer (/file_folder [] ); # 2-3 tokens is folder creation (optional destination, no client id) file_name = command_part[2] if len(command_part) >= 4 else None + if file_name is None: + created = _receive_dir_for( + self.file_transfer_dir, + relative_folder_path, + _parse_destination_path(command_part), + ) + try: + os.makedirs(created, exist_ok=True) + self._log(f"folder {created} created for {client_id}") + except OSError as e: + self._log(f"folder {created} could not be created for {client_id}: {e}") + return folder_transfer_client_recv_server_start_thread = threading.Thread( target=self.file_transfer_client_recv_server_start, args=(client_id, client_socket, command, relative_folder_path, file_name), @@ -10174,8 +11242,10 @@ def _command_peer(self, command): A file pushed by the server itself carries this client's own address, which `parse_forward_originator` filters out: the sender is then the - server. A forwarded item carries the originating client's address — - the original peer a resume asks to continue the transfer. + server, identified by the concrete endpoint of the control connection + (a wildcard bind or a host alias resolves to it). A forwarded item + carries the originating client's address — the original peer a resume + asks to continue the transfer. Args: command (object): Value used by the operation. @@ -10185,7 +11255,9 @@ def _command_peer(self, command): """ originator = parse_forward_originator(command, own_address=self._own_address_text()) peer = _peer_from_text(originator) if originator else None - return peer if peer is not None else (self.host, self.port) + if peer is not None: + return peer + return _socket_peer_endpoint(self.client_socket) or (self.host, self.port) def file_transfer_mode_recv( self, @@ -10197,7 +11269,6 @@ def file_transfer_mode_recv( file_name, command, peer=None, - resume_entry=None, ): """Receive one file over a transfer sub-connection. @@ -10210,11 +11281,9 @@ def file_transfer_mode_recv( file_name (str | None): Destination filename, when specified. command (str): Wire command that initiated the transfer. peer (tuple | None): Original sender ``(ip, port)`` for resuming. - resume_entry (list | None): Saved progress entry for a resumed file. Returns: - None | bool: None after starting or finishing; False when a resume request - cannot be sent. + None: after the leg starts or finishes. """ file_running = True client_file_socket = None @@ -10228,27 +11297,22 @@ def close_socket(): nonlocal server_file_socket file_running = False self._crypto_drop_channel(client_file_socket) - client_file_socket.close() - server_file_socket.close() + if client_file_socket is not None: + client_file_socket.close() + self._transfer_handshakes.release_socket(client_socket, client_file_socket) + if server_file_socket is not None: + server_file_socket.close() + self._transfer_handshakes.release_socket(client_socket, server_file_socket) def setting_file_save_path(): - """Handle the operation.""" + """Resolve, and create, the directory this leg writes into.""" nonlocal save_path - save_path = self.file_transfer_dir + save_path = _receive_dir_for( + self.file_transfer_dir, new_save_path, destination_path + ) if new_save_path or destination_path: - if destination_path: - if os.path.isabs(destination_path): - save_path = destination_path - else: - save_path = os.path.join(save_path, destination_path) - if new_save_path: - path_list = new_save_path.split("/") - if path_list and path_list[0] == "": - del path_list[0] - for node in path_list: - save_path = os.path.join(save_path, node) os.makedirs(save_path, exist_ok=True) - if file_name or new_save_path == None: + if file_name or new_save_path is None: return save_path close_socket() return None @@ -10337,6 +11401,11 @@ def file_transfer_client_recv(client_id): "ErrorSignReceivedWhileReceivingFileSize: client reported error and disconnected" ) file_size = int.from_bytes(size_bytes, "big") + if resume_entry is not None and resume_entry[1] != file_size: + raise ConnectionError( + f"resume rejected: {filename} is {file_size} bytes, " + f"the partial file expects {resume_entry[1]}" + ) self.send_message(client_file_socket, self.server_received_file_header_sign) original_filename = filename if resume_entry is not None: @@ -10355,9 +11424,7 @@ def file_transfer_client_recv(client_id): else: final_filename = os.path.basename(original_filename) full_path = os.path.join(save_path, final_filename).strip() - # a forwarded transfer this client asked to continue arrives - # as a fresh leg on the same path: keep the bytes on disk - resume_from = self._take_pending_resume(full_path) or 0 + resume_from = 0 if resume_from <= 0 or resume_from > file_size: resume_from = 0 # not this file's offset any more: start it over os.makedirs(os.path.dirname(full_path), exist_ok=True) @@ -10389,7 +11456,6 @@ def file_transfer_client_recv(client_id): remaining -= len(chunk) self._file_progress_update(progress, len(chunk), is_upload=False) f.truncate() # a longer partial file drops its stale tail - finished_ok = True # TOFU first: the ack is only a notification, and a failed # ack send must not skip the key registration (readiness # would never be announced and the handshake hangs) @@ -10403,7 +11469,9 @@ def file_transfer_client_recv(client_id): if command_part[0] == "/crypto_pub_key": self._crypto_store_received_pub(full_path, "server", (self.host, self.port)) try: - self.send_message(client_file_socket, self.server_received_file_data_sign) + finished_ok = bool(self.send_message( + client_file_socket, self.server_received_file_data_sign + )) except Exception: self._log_exc() close_socket() @@ -10439,38 +11507,41 @@ def file_transfer_client_recv(client_id): server_file_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_file_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_file_socket.bind((server_file_address, server_file_port)) - file_transfer_server_port = server_file_socket.getsockname()[1] + file_transfer_server_port = _bind_local_port_with_fallback( + server_file_socket, + (server_file_address, server_file_port), + self._log, + "file transfer listener", + ) command_part = shlex.split(command) - file_client_id = command_part[len(command_part) - 1] + try: + file_client_id = int(command_part[-1]) + if file_client_id < 0: + raise ValueError + except (IndexError, ValueError): + server_file_socket.close() + return progress_fid = _transfer_fid(command_part) # keys this leg in file_recv_progress + resume_entry = _resumable_entry(self.file_recv_progress, progress_fid, peer, command_part) destination_path = _parse_destination_path(command_part) stamp = self._crypto_transfer_stamp( client_socket, force_plain=command_part[0] == "/crypto_pub_key" ) - self._forget_transfer_port(progress_fid) # an older session's entry is stale - transfer_server_port_msg = "/server_file_transfer_port {} {} {}\n".format( + transfer_server_port_msg = _transfer_port_announcement( file_transfer_server_port, file_client_id, stamp ) - self._send_raw( - client_socket, transfer_server_port_msg - ) # coordination line: always plaintext (like the handshake) server_file_socket.listen(1) if resume_entry is not None: - # the receiver drives a resumed transfer: it asks the sender to - # continue the file from the bytes already on disk - offset = _resume_offset_for(resume_entry) - if progress_fid is None or not self.send_message( - client_socket, f"{RESUME_LINE} {progress_fid} {offset}" - ): - self._log( - f"resume: cannot ask {client_id} to continue transfer {progress_fid}" - ) - server_file_socket.close() - return False + # the sender answers this resume with /file; its connection is the continued leg server_file_socket.settimeout(RESUME_ACCEPT_TIMEOUT) try: + if not self._transfer_handshakes.track_socket(client_socket, server_file_socket): + return + if not self.send_message(client_socket, transfer_server_port_msg, is_send_raw=True): + return client_file_socket, client_file_address = server_file_socket.accept() + if not self._transfer_handshakes.track_socket(client_socket, client_file_socket): + return if resume_entry is not None: client_file_socket.settimeout(None) # the accept timeout is the listener's only self._crypto_attach_transfer_in(client_file_socket, client_socket, stamp) @@ -10498,7 +11569,9 @@ def file_transfer_client_recv(client_id): close_socket() finally: server_file_socket.close() + self._transfer_handshakes.release_socket(client_socket, server_file_socket) + @_config_lifecycle(starting=False) def close(self): # close connection """Close the connection and release everything the client owns. @@ -10508,24 +11581,36 @@ def close(self): # close connection """ self.running = False self._stopped = True # also cancels a connect() still in flight + self._stop_listeners() + if self.client_socket: + self._transfer_handshakes.cancel_socket(self.client_socket) + try: + # the receive thread may be blocked in recv() on this socket: + # close() alone sends no FIN and leaves that read waiting for its + # timeout, so the server would not see the disconnect + self.client_socket.shutdown(socket.SHUT_RDWR) + except OSError: + pass # peer already gone + receive_thread = self.receive_thread + if ( + receive_thread is not None + and receive_thread is not threading.current_thread() + and receive_thread.ident is not None + ): + # close() returns only after the receive loop has stopped: no + # in-flight chunk may reach a listener after this point + receive_thread.join() self.free_port() - self._flush_messages_dict() - self._flush_events_dict() + self._flush_record_stores() # transfers still in flight are kept on disk for the next connect self._persist_transfer_progress() with self._crypto_lock: if self.client_socket is not None: self._channels.pop(self.client_socket, None) self._encrypted_sockets.discard(self.client_socket) + self._control_sockets.discard(self.client_socket) self._crypto_session = None if self.client_socket: - try: - # the receive thread may be blocked in recv() on this socket: - # close() alone sends no FIN and leaves that read waiting for its - # timeout, so the server would not see the disconnect - self.client_socket.shutdown(socket.SHUT_RDWR) - except OSError: - pass # peer already gone self.client_socket.close() self._log("connection closed") @@ -10546,6 +11631,7 @@ def _release_client_socket(self): sock = self.client_socket if sock is None: return + self._transfer_handshakes.cancel_socket(sock) try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) except OSError: @@ -10557,6 +11643,7 @@ def _release_client_socket(self): with self._crypto_lock: self._channels.pop(sock, None) self._encrypted_sockets.discard(sock) + self._control_sockets.discard(sock) def start_TCP_client(self): # start client """Connect to the server and start the client loop. @@ -10570,7 +11657,7 @@ def start_TCP_client(self): # start client With ``is_auto_reconnect`` True a dropped connection does not end the client: it reconnects (waiting for the server when ``is_wait_server`` is set) and every session carries the interrupted transfers of the - previous one (`resume_transfers`). A close or ``/quit`` still stops + previous one (`_request_resumes`). A close or ``/quit`` still stops the client. """ while True: diff --git a/PyFlow/transfer_web/web_backend/server_backend.py b/PyFlow/transfer_web/web_backend/server_backend.py index e47b117..cc301e5 100644 --- a/PyFlow/transfer_web/web_backend/server_backend.py +++ b/PyFlow/transfer_web/web_backend/server_backend.py @@ -212,14 +212,15 @@ FTP_ERROR_COMMAND = "/ftp_error" -def _transfer_rows(progress): - """JSON rows for one progress store (``fid -> [transferred, total, speed, path, peer]``). +def _transfer_rows(progress, *, uploading): + """JSON rows for uploads keyed by ``(receiver, fid)`` or downloads by ``fid``. Returns: list[dict]: One row per running transfer, ordered by file name. """ rows = [] - for fid, entry in list(progress.items()): + for key, entry in list(progress.items()): + fid = key[1] if uploading else key rows.append( { "fid": fid, @@ -683,8 +684,8 @@ def _transfers_payload(self): "speed_enabled": bool(server.is_enable_real_time_network_speed), "upload_network_speed": server.upload_network_speed, "recv_network_speed": server.recv_network_speed, - "uploading": _transfer_rows(server.file_upload_progress), - "downloading": _transfer_rows(server.file_recv_progress), + "uploading": _transfer_rows(server.file_upload_progress, uploading=True), + "downloading": _transfer_rows(server.file_recv_progress, uploading=False), } def _client_list_for(self, addr): diff --git a/PyFlow/transfer_web/web_front/client_backend.py b/PyFlow/transfer_web/web_front/client_backend.py index c6da8a8..216ac1c 100644 --- a/PyFlow/transfer_web/web_front/client_backend.py +++ b/PyFlow/transfer_web/web_front/client_backend.py @@ -260,14 +260,15 @@ def _channel_ready(client): return client.client_socket in client._encrypted_sockets -def _transfer_rows(progress): - """JSON rows for one progress store (``fid -> [transferred, total, speed, path, peer]``). +def _transfer_rows(progress, *, uploading): + """JSON rows for uploads keyed by ``(receiver, fid)`` or downloads by ``fid``. Returns: list[dict]: One row per running transfer, ordered by file name. """ rows = [] - for fid, entry in list(progress.items()): + for key, entry in list(progress.items()): + fid = key[1] if uploading else key rows.append( { "fid": fid, @@ -1585,8 +1586,8 @@ def _transfers_payload(self): "speed_enabled": bool(client.is_enable_real_time_network_speed), "upload_network_speed": client.upload_network_speed, "recv_network_speed": client.recv_network_speed, - "uploading": _transfer_rows(client.file_upload_progress), - "downloading": _transfer_rows(client.file_recv_progress), + "uploading": _transfer_rows(client.file_upload_progress, uploading=True), + "downloading": _transfer_rows(client.file_recv_progress, uploading=False), } # ------------------------------------------------------------------- run diff --git a/docs/plans/connect-port-fallback-report.md b/docs/plans/connect-port-fallback-report.md new file mode 100644 index 0000000..004d777 --- /dev/null +++ b/docs/plans/connect-port-fallback-report.md @@ -0,0 +1,72 @@ +# 轻量增量报告：固定 client_port 的 connect 阶段端口冲突降级 + +状态：已实施，待人工审查；未创建 git 提交、未合并、未推送。 + +## 1. 提交意图 + +配置了固定 `client_port` 的 `TCP_Client_Base.connect()` 在 `bind()` 成功、`connect()` 因本端端口冲突失败时（Windows `WSAEADDRINUSE` 10048），改为用新 socket 从 OS 分配端口重试一次，而不是直接返回 False。 + +## 2. 改动内容 + +- `PyFlow/network_api/connect_tcp.py` + - 新增模块级 `_is_local_port_conflict(exc)`：识别 `errno.EADDRINUSE` 与 Windows `winerror == 10048`。与既有 `_is_closed_socket_error` 同层、同风格。 + - `TCP_Client_Base.connect()`：`bind` + `connect` 收进一个固定端口重试循环。任一步失败且命中端口冲突时，关闭当前 socket、记录日志、丢弃 `local_address`，下一轮从 OS 分配端口重连；非端口冲突或已无固定端口时原样抛出。 + +## 3. 为什么这样改 + +对应失败用例 `test_reconnected_handshake_rejects_reply_from_old_control_socket` 在 Windows CI 的 `assert client.connect()`： + +``` +client_port 固定 -> SO_REUSEADDR 允许 bind 已被 TIME_WAIT 占用的端口 + -> connect 才报 WinError 10048 + -> 原实现只降级 bind 失败（8803-8814），connect 失败落入通用 except -> return False +``` + +日志中没有 "client port ... is not available" 一行，证明 `bind` 已通过，缺陷在 `connect` 这一侧。POSIX 上同一冲突由 `bind` 报出，原逻辑已覆盖，所以这是 Windows 独有的缺口。 + +## 4. 数据流 + +```text +connect() + local_address = (client_host, client_port) # client_port 为 None 时跳过 bind + loop: + new socket + SO_REUSEADDR + timeout + bind(local_address)? -> OSError 且 _is_local_port_conflict -> close, local_address=None, 重试 + connect((host, port)) -> OSError 且 _is_local_port_conflict -> close, local_address=None, 重试 + -> 其它 OSError -> 抛出，沿用外层 timeout/refused/通用分支 + break + 之后: _stopped 检查 -> running=True -> 控制 socket 登记 -> receive 线程 -> 模式协商 +``` + +固定端口可用时行为完全不变（首轮 bind 成功、connect 成功，循环只跑一次）。 + +## 5. 局部验证 + +1. **新增分支冒烟**（临时脚本，已删除）：真实 server + 真实 client，`client_port` 固定；monkeypatch `socket.socket.connect` 使**首次** connect 抛 `OSError(EADDRINUSE)`。 + +``` +connect(): True +attempts: [('127.0.0.1', 50970), ('0.0.0.0', 0)] +final local: ('127.0.0.1', 39008) +fallback used: True +bound fixed port on 1st attempt: True +``` + +第一次尝试绑定了固定端口 50970 并在 connect 处失败；第二次不 bind、由内核分配 39008 后连接成功。 + +2. **既有回归**： + - `test/integration/network_api/test_tcp_transfer_resume.py -k "configured_port_is_taken or upload_resumes_after_a_reconnect or download_resumes_after_connect"` → 3 passed。 + - `test/unit/network_api/test_TCP_client.py` + 整个 `test_tcp_transfer_resume.py` → 50 passed in 107s。 + - 修复前同两套件在本机 3.14 也已通过（该缺陷只在 Windows 触发），故本轮新增的唯一证据是上面的 1。 + +## 6. 明确未覆盖 + +- **Windows 真机未验证**：本机是 Linux，10048 由 monkeypatch 模拟，不是真实内核行为。合入前建议在 Windows CI 上跑 `test_reconnected_handshake_*` 确认。 +- 只降级一次。若 OS 分配端口后仍冲突（极不可能），原错误照抛。 +- 未改动 `_release_client_socket()`。它依赖 "socket 尚未 shutdown" 才能靠 `SO_LINGER(1,0)` 触发 RST；测试用 `shutdown(SHUT_RDWR)` 前置，那条路径仍会留下 TIME_WAIT，本修复让客户端能绕过去，但没有消除根因。 +- 从 OS 分配端口后，本端地址变化，按地址匹配的断点续传不会命中旧记录（与既有 bind 失败降级语义一致，已有注释说明）。 + +## 7. 风险与待确认 + +- 风险：低。单函数、单路径、~40 行，固定端口可用时零行为变化。 +- 待确认：是否接受 "只降级一次、不重试固定端口"。另一种做法是固定端口冲突时先短退避再重试固定端口，但那会把重连延迟拉长到 TIME_WAIT 量级（Windows 约 2 分钟），不合适。 diff --git a/docs/plans/connect_tcp-listener-lifecycle-plan.md b/docs/plans/connect_tcp-listener-lifecycle-plan.md new file mode 100644 index 0000000..34c4336 --- /dev/null +++ b/docs/plans/connect_tcp-listener-lifecycle-plan.md @@ -0,0 +1,261 @@ +# connect_tcp 监听器生命周期与锁范围修复计划（最终版） + +状态：待人类批准；本文只定计划，不写代码。 + +## 1. 需求卡（Agent 复述） + +### 目标 + +1. **P2-1**：`TCP_Client_Base.close()` 内 join 接收线程；`close()` 返回后本实例不再投递新监听器事件；`close()/stop()` 被调用时停止所有监听器投递（置 stopped、清空消息/发送/文件监听器列表）。 +2. **P1-1**：`broadcast()` 不在 `client_lock` 内同步调用发送监听器，消除监听器重入 `broadcast()/stop()` 的确定性死锁。 +3. **P2-4**：更新监听器契约文档：文本行 / 原始 `bytes` / 发送 / 文件事件；线程、并发、close/stop、ID 语义。 +4. **P2-2（仅文档）**：明确发送监听器回调顺序不保证等于线序。该问题只影响监听器的观察顺序，不影响 socket 写线顺序、接收顺序、文件数据完整性或任何协议功能，因此本轮不改代码。 +5. **P1-2（最后执行）**：Web 端两个消息监听器显式忽略/处理侧信道 `bytes`，消除每块一次 `TypeError` 和日志放大。 + +### 非目标 + +- 不实现发送事件 notifier，不改变发送监听器的执行线程/同步语义。 +- 不修改 `_SecureChannel.send` 的发送锁范围，不修改加密/传输协议。 +- 不改变 P2-2 的任何运行时行为，只在文档中声明不保证发送回调线序。 +- 不改变 P2-3、P2-5、P3-1 的现有行为（均已撤回）。 +- 不修改公共方法签名、依赖、CI、分支保护。 +- 不让 `TCP_Server_Base.stop()` join 每客户端线程；本轮只停止新监听器投递。 +- 不改变 `is_send_raw=True` 与 `publish_pub` 等内部协议行不通知监听器的设计。 + +### 受影响范围（文件） + +修改： + +- `PyFlow/network_api/connect_tcp.py` + - `TCP_Server_Base`：`__init__`、`stop`、`broadcast`、`_notify_*` + - `TCP_Client_Base`：`__init__`、`close`、`_notify_*` + - `add_message_listener` / `add_send_message_listener` / `add_file_listener` docstring +- `docs/source/Network_APIs/TCP_Server_APIs.rst` +- `docs/source/Network_APIs/TCP_Client_APIs.rst` +- `PyFlow/transfer_web/web_backend/server_backend.py`（仅最后提交） +- `PyFlow/transfer_web/web_front/client_backend.py`（仅最后提交） + +新增/扩充测试： + +- 新增 `test/integration/network_api/test_listener_lifecycle.py` +- 扩充 `test/integration/network_api/test_tcp_broadcast_alloc.py` +- 扩充 `test/unit/transfer_web/` 下的 Web 监听器类型测试（最后提交） + +### 执行路径清单（本轮必须覆盖） + +关闭/停止： + +1. `close()` 时接收线程正阻塞在 `recv`：shutdown 唤醒，join 成功，返回后无新事件。 +2. `close()` 时接收线程已 `recv` 到一块数据但尚未通知：join 等待本轮处理结束；返回后不得出现该块的 `bytes` 或 `str` 事件。 +3. `close()` 从消息监听器内调用（接收线程自身）：跳过 self-join，不得抛 `RuntimeError`；当前回调返回后接收循环退出。 +4. `close()` 未连接 / 接收线程为 `None`：幂等返回。 +5. `close()` 连续调用两次：幂等。 +6. `is_auto_reconnect` 的 `_release_client_socket()` 路径：不清空监听器、不 join；重连后监听器仍有效。 +7. `close()` 时发送监听器已在其他线程执行：停止标志阻止新投递；当前回调允许结束。 +8. `TCP_Server_Base.stop()`：停止新监听器投递并清空列表；不 join 客户端线程。 +9. `stop()` 从发送监听器内调用：不得因 `client_lock` 重入死锁。 + +广播： + +10. `broadcast()` 正常发送全部客户端。 +11. `broadcast(exclude_client=...)` 排除指定客户端。 +12. `broadcast()` 发送失败：关闭并删除对应客户端。 +13. 发送监听器内再次调用 `broadcast()`：不得死锁；监听器自身需防递归。 +14. 广播快照期间客户端被并发移除 / 同地址新客户端加入：删除时按 socket 身份校验，不误删新客户端。 +15. `is_asynic_clients_io=True` 时广播行为不变。 + +文档： + +16. 消息监听器：控制行 `str` 与侧信道 `bytes` 的区别、线程、并发。 +17. 发送监听器：只覆盖非 raw `send_message`；内部协议行不投递；**回调顺序不保证等于线序**；该顺序不影响数据正确性。 +18. 文件监听器：完整文件落盘后触发；早于完成 sign；逐块事件由消息监听器投递；不等待 ACK。 +19. ID 语义：按实际执行 I/O 的 socket 对端记录；文件转发/传输时是临时 TCP 的 `ip:port`，不是控制连接 ID。 +20. close/stop：停止新投递、清空监听器、自动重连不受影响。 + +Web： + +21. 服务端 `_on_incoming_message` 收到 `bytes`：不再抛 `TypeError`，不产生聊天事件。 +22. 客户端 `_on_incoming_message` 收到 `bytes`：同上。 +23. 真实文件传输时不再每块产生一条监听器错误日志。 + +### 安全与数据边界 + +- 监听器仍然看到解密后的明文；本轮不改变可见数据范围。 +- 发送监听器仍在原发送线程同步执行；本轮不改变回调线程，也不改变同步语义。 +- 不在 `_event_listeners_lock` 内调用用户回调。 +- 不在 `client_lock` 内调用 `send_message` 或通知监听器。 +- `close()/stop()` 后不再发起新投递；已进入回调的线程允许自然结束。 +- 内部协议行（`is_send_raw`、`publish_pub`）仍不进入发送监听器。 +- 侧信道事件 ID 继续按实际 socket 对端记录，不改成控制连接 ID。 +- P2-2 只是观察顺序问题：TCP 写线由发送锁串行化，接收顺序由接收线程串行化，文件数据块不会因监听器回调顺序不同而乱序或损坏。 + +### 验收证据（运行时 / 机械证据） + +1. **关闭生命周期**： + - `close()` 返回后 `receive_thread.is_alive()` 为 False； + - `_listeners_stopped` 为 True，三个监听器列表为空； + - 关闭后直接调用四个 `_notify_*`，回调计数不增加； + - 强制在 `recv` 与 `_notify_transfer_chunk` 之间暂停接收线程，`close()` 必须等待；释放后返回，且该块不再产生 `bytes` 或 `str` 事件。 +2. **广播**： + - 发送监听器内调用 `broadcast()`，`broadcast("trigger")` 在超时前返回； + - 现有死客户端删除测试继续通过。 +3. **Web 类型守卫**： + - 直接以 `bytes` 调用两个 `_on_incoming_message` 不抛异常、不产生事件； + - 真实文件传输期间无监听器 `TypeError` 日志。 +4. **回归**： + - `test_message_listeners.py` + - `test_side_channel_records.py` + - `test_send_message_store.py` + - `test_tcp_broadcast_alloc.py` + - `test_tcp_async_server.py` + - 加密/文件传输相关测试 +5. **文档**：`make -C docs html` 成功。 + +### 禁止事项 + +- 不得删除、跳过或弱化测试；不得加 `|| true`；不得降低 coverage。 +- 不得改 CI、依赖、分支保护。 +- 不得改公共方法签名。 +- 不得在 `_event_listeners_lock` 内调用用户回调。 +- 不得在 `client_lock` 内调用 `send_message` 或通知监听器。 +- 不得把 P2-2 的运行时行为修改混入本任务；只允许文档声明。 +- 不得修改加密/传输协议逻辑。 +- 不得在 `is_auto_reconnect` 的 `_release_client_socket()` 路径清空监听器或 join。 +- `close()` 的 join 不设静默超时；用 self-join guard 处理在接收线程内调用 `close()` 的情况。 +- Web 修复必须最后执行，不得混入前面的提交。 + +### 风险等级 + +中。高风险片段：`close()` 的 shutdown/join 顺序、监听器停止标志、`broadcast()` 的锁边界与失败客户端删除。 + +## 2. 数据流 + +### 2.1 close + +```text +调用方 + -> running = False + -> _stopped = True + -> _stop_listeners(): 置 stopped，清空 message/send/file 列表 + -> client_socket.shutdown(SHUT_RDWR) # 唤醒 recv + -> join(receive_thread) # 非当前线程才 join + -> free_port / flush stores / persist progress + -> 丢弃 channel、encrypted、control socket 标记 + -> client_socket.close() + -> 返回 +``` + +返回后： + +- 接收线程已退出，不再产生消息 / 原始块 / 文件事件。 +- 新发送不会触发发送监听器。 +- 其他线程中已经进入的发送回调可以继续执行到返回。 + +### 2.2 stop + +```text +调用方 + -> running = False + -> _stopped = True + -> _stop_listeners() + -> free_port / flush / persist + -> close 所有 client sockets、server socket + -> 唤醒 async accept loop + -> 返回 +``` + +不 join 客户端线程；只保证不再投递新事件。 + +### 2.3 broadcast + +```text +broadcast + -> with client_lock: 快照 [(addr, socket), ...] + -> 释放 client_lock + -> 逐目标 send_message() # 监听器在此同步触发，但不在 client_lock 内 + -> 收集失败 (addr, socket) + -> with client_lock: + 若 clients[addr]["socket"] is socket: close + del + -> 返回 +``` + +## 3. 原子提交拆分 + +### 提交 1：broadcast 不在 client_lock 内通知发送监听器（P1-1） + +- 意图：消除发送监听器重入 `broadcast()/stop()` 的确定性死锁。 +- 文件：`connect_tcp.py`，`test_tcp_broadcast_alloc.py`。 +- 变更： + - `client_lock` 下只做 `(addr, socket)` 快照； + - 释放锁后逐个 `send_message`； + - 失败目标重新加锁，按 socket 身份校验后关闭/删除； + - 保持 `exclude_client` 与“失败即断开删除”的现有语义。 +- 预估：代码约 20 行，测试约 50 行。 +- 验证：新增重入测试；现有 broadcast / async server 测试。 + +### 提交 2：close/stop 停止监听器投递，close join 接收线程（P2-1） + +- 意图：兑现“close 被调用时停止所有监听器”，消除关闭窗口内的在途事件和 `bytes`/`str` 重复。 +- 文件：`connect_tcp.py`，新增 `test_listener_lifecycle.py`。 +- 变更： + - 两个类新增 `_listeners_stopped` 与 `_stop_listeners()`； + - 四个 `_notify_*` 在取快照前检查 stopped； + - `TCP_Client_Base.close()` 按 2.1 顺序 shutdown + join； + - `TCP_Server_Base.stop()` 在关闭客户端前调用 `_stop_listeners()`； + - `_release_client_socket()` 不调用 `_stop_listeners()`。 +- 预估：代码约 50 行，测试约 120 行。 +- 验证：关闭生命周期测试；现有监听器/广播测试。 + +### 提交 3：监听器契约文档修正（P2-4 + P2-2 声明） + +- 意图：文档与实现一致；明确发送回调不保证线序且不影响功能。 +- 文件：两份 RST + `connect_tcp.py` 三个 `add_*` docstring。 +- 内容： + - 消息监听器：控制行 `str` / 侧信道 `bytes`、线程、并发； + - 发送监听器：只覆盖非 raw `send_message`；内部协议行不投递；**并发发送同一 socket 时，回调顺序不保证等于线序；该顺序仅影响观察顺序，不影响 socket 写线、接收顺序或文件数据完整性**；需要严格顺序的调用方应自行串行化发送； + - 文件监听器：完整落盘后、完成 sign 前、逐块由消息监听器负责； + - ID：实际 I/O socket 对端；文件转发/传输是临时 TCP `ip:port`； + - close/stop：停止新投递、清空列表、自动重连不受影响。 +- 预估：文档约 70 行。 +- 验证：`make -C docs html`。 + +### 提交 4：Web 监听器忽略原始 bytes（P1-2，最后执行） + +- 意图：消除 Web 端每个侧信道块一次 `TypeError` 和日志放大。 +- 文件：`server_backend.py`、`client_backend.py`、Web 单元测试。 +- 变更：两个 `_on_incoming_message` 入口判断 `isinstance(text, bytes)`，原始块直接返回（或进入独立处理分支，不生成聊天事件）。 +- 预估：代码约 8 行，测试约 30 行。 +- 验证：直接 `bytes` 调用不抛异常；真实文件传输无监听器错误日志；现有 Web 测试。 + +## 4. 路径到提交映射 + +| 执行路径 | 提交 | 验证 | +|---|---|---| +| close 阻塞在 recv | 2 | `test_close_joins_receive_thread_and_clears_listeners` | +| close 时有在途 recv 块 | 2 | `test_close_does_not_deliver_inflight_chunk_after_return` | +| close 从接收监听器内调用 | 2 | `test_close_from_receive_listener_does_not_self_join` | +| close 未连接 / 重复调用 | 2 | 生命周期测试幂等用例 | +| 自动重连路径 | 2 | 现有 auto-reconnect 测试 + 监听器仍在断言 | +| close 时发送回调已在执行 | 2 | 停止标志测试 + 文档声明 | +| stop 停止新投递 | 2 | `test_stop_clears_listeners_and_blocks_new_delivery` | +| stop 从监听器内调用 | 1 + 2 | 广播重入测试变体 | +| broadcast 正常 / exclude / 死客户端 | 1 | `test_tcp_broadcast_alloc.py` | +| async server broadcast | 1 | `test_tcp_async_server.py` | +| 发送回调线序 | 3 | 文档声明；不写行为测试 | +| Web bytes 类型守卫 | 4 | Web 单元测试 + 文件传输日志检查 | +| 文档契约 | 3 | `make -C docs html` | + +## 5. 明确未覆盖 + +- P2-2 运行时顺序：不修；文档声明。原因：只影响监听器观察顺序，不改变 socket 写线、接收顺序、文件数据完整性或协议功能。 +- P2-3、P2-5、P3-1：已撤回，不改行为。 +- `TCP_Server_Base.stop()` 不 join 客户端线程；若要求 stop 返回后零回调，需先跟踪并 join 所有客户端线程，属独立任务。 +- 已进入的监听器回调无法被 Python 强制终止；close/stop 只保证不再发起新投递。 +- 发送监听器内的 `close()/stop()` 仍可能在业务锁上重入；本轮只修复 `broadcast` 这条已知锁路径，其他锁路径不在范围内。 + +## 6. 合并前检查 + +- 红旗复查：无删测试、无弱化断言、无 CI/依赖改动、无公共签名改动。 +- 规则沉淀：新增关闭生命周期、广播重入回归测试；文档明确发送回调线序边界。 +- 文档收尾：两份 RST 与三个 `add_*` docstring 同步更新。 +- 人工批准后才开始提交 1；每个提交附提交报告并等待审查。 diff --git a/docs/plans/final-ack-delivery-report.md b/docs/plans/final-ack-delivery-report.md new file mode 100644 index 0000000..b319afb --- /dev/null +++ b/docs/plans/final-ack-delivery-report.md @@ -0,0 +1,94 @@ +# 终态 ACK 交付修复报告 + +状态：实现、回归与本地运行验收完成，待人工审查。未创建 git 提交、未合并、未推送，未触发远端 CI。 + +后续修正：本报告记录的是先前粘包修复阶段，当时完整 sign 历史及未终止尾部尚无容量上限。需求方随后完成本地方案审查并要求有界化；原 `_take_wire_lines` 和历史列表已由消费后删除的有界队列替代，实施与当前验收见 [transfer-sign-buffer-bounds-report.md](transfer-sign-buffer-bounds-report.md)。下文测试数字为该历史阶段证据，不冒称本次新增限制的验证结果。 + +## 1. 恢复点与提交意图 + +恢复原会话 `01a1aa1d-bfac-75d9-9634-ed1bae67d734` 中的最后一项实际需求：取消 ACK 发送时机重构，继续增量修复 ACK 交付问题。原会话的较早 recap 仍写“等待确认/另行讨论”，但后续用户消息与已执行工具记录表明，代码修复、最初的粘包回归和部分验证已经完成；中断在边界单测与报告收尾。 + +单一提交意图：两端发送腿按完整协议行识别 ACK，不再把一次 socket read 当成一条 sign；中继回传同一个已确认 ACK 时保留换行终止。没有新增 ACK-of-ACK、结果通道、重试或延迟关闭。 + +本报告是未提交工作区的审查材料，不表示人工批准或合并前闸门已通过。原会话按轻量增量实施；本次没有补写虚构的审批记录。 + +## 2. 改动文件与关键决策 + +| 文件 | 变更 | +|---|---| +| `PyFlow/network_api/connect_tcp.py` | `_take_wire_lines` 缓存 bytes 尾部，完整行才解码；两端 `file_transfer_mode` 累积所有完整 sign；`_forward_pump` 将已确认 marker 作为文本交给既有 `send_message`，由其追加换行 | +| `test/integration/network_api/test_tcp_transfer_resume.py` | 原粘包回归扩展为上传/下载 × 初传/EOF 重放四种真实 socket 场景；注入只合并真实收到的字节，不制造 ACK；断言最终返回值、内容和进度清理 | +| `test/unit/network_api/test_wire_lines.py`（新增） | 枚举每个读取切分点；覆盖多行、空行、CRLF、未终止尾部与逐字节 ACK，未遇换行不得提前交付 | +| `docs/source/File_Transfer/File_Transfer.rst` | 明确 sign 的换行与流式解析契约，以及 ACK 时机、超时和关闭行为不变 | +| `docs/plans/final-ack-delivery-requirement.md` | 用已证实的字节流解析根因替换 RST/FIN 假说；更新实际修复边界 | +| `docs/plans/test-flake-fixes-report.md` | 保留其历史范围，链接后续 ACK 修复，避免继续把本问题记成未处理 | +| 本报告（新增） | 路径逐项验收、历史证据与本次验证分开记录 | + +公共 API、FID 分配、五项进度模型、依赖、CI、覆盖率、加密握手均未改动。没有删除/跳过测试、放宽断言或新增 lint 忽略规则；原有失败必须返回 `False` 的要求仍保留。 + +## 3. 根因与数据流 + +原实现把 `data.decode("utf-8").strip()` 的全部结果与单个 sign 做 `==` 比较。header sign 与最终 ACK 落在同一次读时，结果包含两行，两次单行比较都失败；随后 EOF 使发送端返回 `False`。它不是“只保留最后一行”，也没有证据支持通过新增 drain 或 ACK 握手修复本缺陷。 + +现有数据流：真实 socket 字节（加密腿先走原有解密入口）→ `receive_message` → `_take_wire_lines` → 所有完整 sign 加入该腿的已收集合 → 原有 start/ACK/error 等待逻辑 → 真实终态 ACK 才成功并清理上传进度。 + +未终止尾部以 bytes 保留，下一次读继续拼接；不会把 EOF 当作换行，也不会丢掉 EOF 之前已经解析的 ACK。完整行才 UTF-8 解码，避免跨读字符被替换。原有 ACK/error 判定顺序未重新定义。 + +中继：目标 writer 读到真实最终 ACK → 保留原有全目标完成门槛 → 同一 `send_message` 发送该 marker 文本并追加换行 → 写成功才删除路由。原 `_forward_read_sign` 返回不带换行的 bytes，而 `send_message` 对 bytes 不加换行；不修正此点会让新分行器一直缓存上游 ACK。没有新增成功信号或绕过统一发送/加密入口。 + +## 4. 需求卡路径验收 + +以下为本次独立 Python 3.10 loopback 烟测的实际结果，不是只依据“测试通过”推断覆盖。实例进度、日志及密钥目录隔离到临时目录；观察包装只记录实际发送/返回结果。 + +| 路径 | 实际执行与证据 | 结论 | +|---|---|---| +| P1 服务端接收普通上传 | 33,568 字节；接收侧 ACK 写成功，发送侧返回 `True`，文件逐字节相同，双侧进度清理 | 覆盖 | +| P2 文件夹项上传 | `bundle/nested` 下 33,569 字节；ACK 写成功、发送侧 `True`、目标内容相同、进度清理 | 覆盖 | +| P3 上传 EOF 重放 | 接收记录和文件已完整，offset=total=33,570；无正文重发，ACK 写成功、发送侧 `True`、记录清理 | 覆盖 | +| P4 服务端下发 | 33,571 字节；客户端 ACK 写成功、服务端发送 `True`、内容相同、进度清理 | 覆盖 | +| P5 下载 EOF 重放 | offset=total=33,572；客户端 ACK 写成功、服务端 `True`、记录清理 | 覆盖 | +| P6 中继 ACK 聚合 | 实际 `_forward_one` 转发 49,159 字节；观察到服务端经文本发送上游 ACK 并写成功；源端 `True`、目标内容相同、源上传记录及中继路由清理 | 覆盖 | +| P7 公钥推送 | 真实加密控制连接建立；双向公钥发送腿均返回 `True`，双向接收 ACK 写成功，控制连接进入 framed 态 | 覆盖 | +| P8 错误路径 | size 不符返回 `False` 且保留原 partial；声明 8,192 字节却只发 7 字节，保留真实 partial 和 `[7,8192]` 进度；header 前断开，不产生成功接收记录或目标文件 | 覆盖 | + +另在同一真实加密连接上完成了 66,416 字节上传和 66,417 字节下载：监听 stamp 均为 `enc:`，两端发送调用返回 `True`、落盘内容相同、上传进度清理。本项不等同于全局加密覆盖审计，未据此声称所有连接均已抓包验收。 + +逐字节读取的独立运行探针还对两种发送端各执行了三种终态：完整 ACK 返回 `True` 并清理上传记录；缺少末尾换行的 ACK 返回 `False` 并保留记录；error sign 返回 `False` 并保留记录。六种结果全部符合契约。该临时探针最初误用临时数据端口查询客户端上传键；产品客户端键使用控制端点 `(self.host, self.port)`，修正探针断言后通过，没有为此改动产品逻辑。 + +## 5. 机械验证与修复前证据 + +### 本次恢复后实际执行 + +```sh +uv run --python 3.10 --with pytest --with flask pytest -q +``` + +结果：**663 passed in 389.46s**，无失败/跳过。随后只为消除新增 PLR0915，将确定性传输注入逻辑抽成同文件辅助函数；产品代码未再变化。 + +最终辅助函数抽取后的定向验证： + +```sh +uv run --python 3.10 --with pytest --with flask pytest \ + test/unit/network_api/test_wire_lines.py \ + test/integration/network_api/test_tcp_transfer_resume.py::test_coalesced_receiver_signs_keep_the_final_ack -q +``` + +结果：**6 passed in 8.57s**（两条边界契约与四个真实传输场景）。 + +最终 Ruff 检查：`connect_tcp.py` 242 条、既有 integration 文件 44 条、新增 unit 文件 0 条；新增 parser/传输注入/回归块没有诊断。没有把全文件 Ruff 声称为通过，也没有处理或屏蔽无关存量问题。 + +### 从原会话工具记录恢复的历史证据（不是本次重跑） + +- 最初粘包回归在未修复产品代码上：`1 failed in 2.18s`，失败断言为实际 `file_transfer_mode=False`，期望 `True`；恢复修复后同一用例通过。 +- 修复后的 Python 3.10 原有 ACK 失败/恢复八个用例循环：15 轮全部通过，每轮 `8 passed`。该循环早于本次 bytes 尾部实现与四场景扩展，因此只作历史证据，不冒称最终版本又重复了 15 轮。 +- 本次没有通过 stash、回滚工作区或修改既有检查策略来重复证明历史故障。 + +## 6. 明确未覆盖、风险与交付边界 + +- Windows、macOS 未在本次实机运行，也未触发远端 CI；不声称跨平台已验收。历史 socketpair 与加密测试修复的 Windows 限制仍见原报告。 +- TCP 写成功但 ACK 真正未送达的网络故障仍按既有失败/续传规则处理；本次解决解析导致的“已收到却未识别”，不承诺端到端绝对交付。 +- 未增加 ACK 时机重构、shutdown/drain、额外超时、EOF 尾部兜底或新的兼容入口。公钥推送仍使用既有明文文件腿及 TOFU 流程。 +- 本地 smoke 为临时进程/临时目录执行；没有留下 smoke 脚本或测试数据文件。 +- 人工审查、远端平台验证与提交/合并授权未发生。当前交付是完成实现及本地验收的工作区，不是已合并或已发布版本。 + +规则沉淀：永久边界契约与四种确定性粘包回归防止重新引入“一次 recv 等于一条消息”；运行文档明确中继 ACK 也必须是换行终止的协议行。 diff --git a/docs/plans/final-ack-delivery-requirement.md b/docs/plans/final-ack-delivery-requirement.md new file mode 100644 index 0000000..87915cd --- /dev/null +++ b/docs/plans/final-ack-delivery-requirement.md @@ -0,0 +1,82 @@ +# 终态 ACK 交付可靠性：需求卡与机制结论 + +状态：代码与确定性回归已实施，运行证据及审查报告见 [final-ack-delivery-report.md](final-ack-delivery-report.md)。未创建 git 提交、未合并、未推送，人工审查仍未放行。原会话将修复按轻量增量执行；本次恢复其未完成的边界测试、验收与文档收尾，不重构 ACK 发送时机。 + +## 1. 目标 + +接收端仍在原先的时机发送终态数据 ACK（`server_received_file_data_sign`）并关闭传输连接；发送端必须正确解析 TCP 字节流中的 ACK，不能因粘包、拆包或紧随其后的 EOF 而误判失败、遗留上传进度记录。不保证 TCP 写成功但实际传输丢失的 ACK 交付。 + +## 2. 非目标 + +- 不改终态 ACK 的报文格式、不去重协议（仍是同一行 sign）。 +- 不改错误 sign 的语义与发送时机。 +- 不改断点续传的匹配规则、进度记录的五项格式、FID 分配。 +- 不改加密算法、鉴权、CI、依赖、覆盖率阈值。 +- 不引入 ACK-of-ACK、新结果通道、延迟关闭、drain 或新的超时。 + +## 3. 受影响的执行路径清单 + +终态 ACK 的既有发送点（不改变发送时机）： + +1. `TCP_Server_Base.file_transfer_mode_recv`：服务端收完上传后，通过 `send_message` 发送 ACK 并关闭传输 socket。 +2. `TCP_Client_Base.file_transfer_mode_recv`：客户端下载接收侧，同一模式。 +3. `TCP_Server_Base._forward_pump`：每个目标的 writer 读到真实 ACK，全部已接受目标完成后，上游仍通过 `send_message` 回传该 ACK；本次修正此处的换行终止。 + +必须逐条给出结论的路径（下表保留最初的故障证据；本次验收见报告）： + +| # | 路径 | 触发场景 | 当前证据 | +|---|---|---|---| +| P1 | 服务端接收·普通文件上传 | `/file` 直传 | 3.10 CI 失败 | +| P2 | 服务端接收·文件夹内文件 | `/file_folder` | 3.10 CI 失败（`[exception-folder]`） | +| P3 | 服务端接收·EOF 重放（`resume_offset == file_size`） | 只发 header、`remaining == 0` | 3.10 CI 失败（第三次尝试） | +| P4 | 客户端接收·服务端下发文件 | 下载方向 | 未直接失败，同一代码点 | +| P5 | 客户端接收·下载的 EOF 重放 | 下载续传 | 未直接失败，同一代码点 | +| P6 | 转发中继·目标腿 ACK 聚合 | `/forward_item` | 未直接失败，同一语义 | +| P7 | 服务端接收·`/crypto_pub_key` 推送 | 公钥文件传输 | 未直接失败，走 P1 同一函数 | +| P8 | 错误路径（收不全、size 不符、对端断开） | — | 不涉及终态 ACK，但改 `close_socket()` 时必须回归 | + +## 4. 需求方观察到的行为 + +Linux 3.10 CI： + +- `test_server_ack_failure_keeps_received_file_until_eof_confirmation[false-file]`、`[exception-folder]`：第三次 `file_transfer_mode` 期望 `(606, 65573, True)`，实际 `False`。 +- `test_server_final_error_keeps_upload_until_successful_resume[initial-file]`：`server.file_upload_progress` 中的行永不被清理。 +- `test_reconnected_handshake_rejects_reply_from_old_control_socket[upload]/[download]`：服务端日志显示 `file new_session.bin received ... size 32781 bytes`（已收全），客户端仍以 `break the file transfer connection from server` 收场并返回 False。 + +三次尝试的服务端日志都打印了 `file ack_failure.bin received ... size 65573 bytes`，即 **ACK 的发送点都被走到了**，失败发生在 ACK 写出之后。 + +## 5. 已确认的根因与修复决策 + +原会话用真实传输探针对照确认：不合并读时 `file_transfer_mode=True`，把 header sign 与 final ACK 合并到一次读后为 `False`。原实现用 `data.decode("utf-8").strip()` 的全部结果与单个 sign 做 `==` 比较；两行一起到达时匹配失败，随后 EOF 导致返回 `False`。这不是“只保留最后一行”，而是把整次读误当一条消息。 + +此前的 RST、ACK/FIN 顺序假说并未证明是本缺陷根因，不能据此增加 shutdown/drain 或改变发送时机。原会话回归测试还记录了修复前必失败、修复后通过；本次扩展为上传/下载与初传/EOF 重放四种确定性粘包场景。 + +决策：两端发送腿均累积完整 sign 行；未遇换行的尾部以 bytes 保留到下次读，完整行才 UTF-8 解码。错误 sign 仍走既有失败路径，真实 ACK 才能成功；不以 EOF 刷出未终止的尾部。 + +中继的 `_forward_read_sign` 返回不带换行的 marker bytes，而 `send_message` 只给 str 加换行。上游改为发送该已确认 marker 的文本形式，继续走同一发送/加密入口；不生成第二个确认信号，不改变全目标完成门槛。 + +## 6. 实施范围与审查边界 + +- 产品实现只改 `PyFlow/network_api/connect_tcp.py` 内的分行器、两处发送腿接收循环和中继 ACK 的文本发送。 +- 验证文件：`test/integration/network_api/test_tcp_transfer_resume.py` 的真实 socket 粘包回归；`test/unit/network_api/test_wire_lines.py` 的所有读边界、空行及未终止 ACK 契约。 +- 公共 API、FID/进度数据模型、依赖、CI、加密握手、接收端 ACK 时机均不改动。 +- 人工最终审查与平台 CI 仍是交付闸门；不能把本地通过写成 Windows 已验收。 + +## 7. 验收证据要求 + +- 每条路径 P1–P7 各跑一次真实传输，且在**收尾阶段**抓取两侧行为：接收侧确认 ACK 已写出；发送侧确认 `file_transfer_mode` 返回 True。 +- 必须能在修复前稳定复现、修复后稳定通过。若无法稳定复现，需先给出"复现手段"作为前置任务，不允许仅凭"看起来修好了"放行。 +- 错误路径 P8 回归：收不全、size 不符、对端提前断开，确认不会因为改动变成"假成功"。 +- 测试代码不得弱化断言（不得把 `is True` 改成 `in (True, False)`）。 + +## 8. 禁止事项 + +- 不删/跳过测试、不弱化断言、不改 CI/依赖/覆盖率。 +- 不引入第二套 ACK 机制或第二个结果通道。 +- 不绕过既有 `send_message` / 传输入口。 +- 不顺手改 `_release_client_socket` 或加密握手（属于其他任务）。 +- 不自行提交、合并、推送。 + +## 9. 风险等级 + +中。影响两端发送腿的 sign 识别与中继 ACK 的换行终止，不改变接收端收尾时序。若把未完成/错误传输误识别为成功，会清理进度记录并破坏恢复；必须保留错误路径与未终止 ACK 的失败证据。 diff --git a/docs/plans/forward-resume-identity-and-pause-plan.md b/docs/plans/forward-resume-identity-and-pause-plan.md new file mode 100644 index 0000000..2bffdf9 --- /dev/null +++ b/docs/plans/forward-resume-identity-and-pause-plan.md @@ -0,0 +1,369 @@ +# 文件转发/resume 身份与暂停清扫修复 — 需求卡、计划与报告 + +状态：**Commit 1（R1）、Commit 2（R3）与 Commit 3（文档收尾）已实现，等待人类审查**。审计证据来自只读复核与 `/tmp/pyflow_audit/` 下的真实 socket 复现脚本；提交结果见 `docs/plans/forward-resume-identity-and-pause-report.md`。 + +--- + +## 0. 审计基线（已复现） + +### R1：resume 用配置地址比较，`localhost` / `0.0.0.0` 下双向失效 + +实测（server 绑 `0.0.0.0`，client 配 `localhost`）： + +| 来源 | 值 | +|---|---| +| `server.host, server.port`（配置） | `('0.0.0.0', 47397)` | +| server 已接受控制 socket 的 `getsockname()` | `('127.0.0.1', 47397)` | +| `client.host, client.port`（配置） | `('localhost', 47397)` | +| client 的 `getpeername()` | `('127.0.0.1', 47397)` | + +端口相同，host 字符串不同。失败点： + +- server `_request_resumes`（`connect_tcp.py:6336`）把 `target=(self.host, self.port)` 写上线； +- client `resume_trans`（`10623-10635`）用 `target != (self.host, self.port)` 判定“转发”，误走 `/forward_item`； +- server `_resume_request`（`6399`）用 `origin == (self.host, self.port)` 判定“直传”，失配； +- client `_command_peer`（`10979-10996`）把直传 peer 记为 `(self.host, self.port)`。 + +**不是回环专属**：server 在 `192.168.1.10` 上绑 `0.0.0.0`、client 在另一台机器配 `192.168.1.10` 时，server 通告 `('0.0.0.0', P)`，client 的 `(self.host, self.port)` 是 `('192.168.1.10', P)`，同样失配。DNS 别名（`nas.lan` vs `192.168.1.10`）、多网卡、IPv6 `localhost=::1` 同理。 + +### R3：清扫关闭 socket 但不解除 pause，暂停 leg 永久阻塞并占住 per-fid 发送锁 + +server 侧端到端复现（`/tmp/pyflow_audit/repro_r3_server.py`）： + +``` +A ready sockets: True row: True pause: True leg_alive: True +B swept active: False sockets: False pause: True state_users: 1 leg_alive: True +C resume_queued: True send_called: 0 leg_alive: True targets: {('127.0.0.1', 59594)} +D after_release leg_alive: False result: False send_called: 1 +``` + +- `_drop_stale_transfers`（`2935-2970`）只 `pop` socket、`discard` active fid，不删 `_transfer_pause`、不通知； +- `_wait_transfer_paused`（`3008-3026`）重新查登记表，pop 后 `sock is None`，`fileno()<0` 退出条件永不成立；server 的 `self.running` 不随单客户端断开而变 False； +- leg 不退出 ⇒ `_sending_transfer` 不释放 state 锁 ⇒ 已“接受”的 resume 在 `run()` 里永久等待；只有人工 `start_trans` 或实例 stop/close 才解除。 + +### R2（不纳入本次） + +`_forward_one` 预注册进度行在 leg 启动前失败时残留。现有 `delete_config` 可清理，且复核确认不会自动重发；按评审决定作为已接受风险，不改。 + +--- + +## 1. 需求卡 + +### 目标 + +- **R1**：断线重连后的 resume 在“配置地址与实际连接地址不同名”的部署下仍能完成，且**不改变** `/resume_trans` 线上格式、进度文件 schema、命令语义、公共方法签名。 +- **R3**：断线清扫遇到已暂停的发送 leg 时，leg 必须在有限时间内结束并释放 per-fid 发送锁；排队中的 resume 不得依赖人工 `start_trans`。 + +### 非目标 + +- 不解决 NAT 下的端点身份（记录为已知限制，见 §7）。 +- 不解决 `client_port` 变化导致的地址不可匹配（既有约束）。 +- 不新增命令、字段、状态文件；不做旧 progress 文件格式迁移（仅内存归一化旧行，见 §2.1）。 +- 不改 CI、依赖、覆盖率策略、公共方法签名。 +- 不改 R2 的现有行为。 + +### 受影响范围 + +- `PyFlow/network_api/connect_tcp.py` + - server：`_request_resumes`、`_resume_request`、`handle_command` 的 `RESUME_LINE` 分派、`_drop_stale_transfers`、`_wait_transfer_paused`、`file_transfer_mode` 的 pause 调用； + - client：`_request_resumes`、`resume_trans`、`_command_peer`、`_drop_stale_transfers`、`_wait_transfer_paused`、`file_transfer_mode` 的 pause 调用。 +- 测试：`test/unit/network_api/` 与 `test/integration/network_api/test_tcp_transfer_resume.py`。 +- 文档：`docs/source/File_Transfer/File_Transfer.rst`、`docs/source/changes/`。 + +### 执行路径清单（全集） + +| # | 路径 | 期望 | +|---|---|---| +| 1 | 直传 client→server 上传 resume（server 为接收端发起） | 完成，行清理 | +| 2 | 直传 server→client 下载 resume（client 为接收端发起） | 完成，行清理 | +| 3 | 转发 target resume（target→server→source，source 在线） | 完成，route 按现有语义清理 | +| 4 | 转发 target resume（source 离线，pending 保留后重连 flush） | 保留后执行 | +| 5 | 转发零偏移 / EOF 重放（`resume=0`、`offset==size`） | 复用 route，不新分配 fid | +| 6 | `localhost` vs `127.0.0.1` 双向 | 完成 | +| 7 | server 绑 `0.0.0.0`、client 用具体 IP 双向 | 完成 | +| 8 | 旧持久化行（`entry[4]` 为配置地址） | 内存归一化后完成 | +| 9 | 控制 socket 已消失 / `getsockname()` 抛 `OSError` | 不崩溃，保留进度 | +| 10 | 非 resume 的新传输 | 行为不变 | +| 11 | 外部调用 `resume_trans(peer_addr=None)` | 行为不变 | +| 12 | paused + 清扫 + 排队 resume（server 发送端） | leg 结束，resume 自动执行 | +| 13 | paused + 清扫 + 排队 resume（client 发送端） | 同上 | +| 14 | 清扫后 pause 被重新 `/pause_trans` 置位 | leg 仍能退出 | +| 15 | 普通 `pause_trans`/`start_trans`（无断线） | 行为不变 | +| 16 | 转发回压暂停 + 清扫（`_forward_pump` 路径） | leg 结束，不阻塞后续 resume | + +### 安全与数据边界 + +- `/resume_trans` 的 `origin`/`target` 仍是线上自报字段；server 仍以**实际控制 peer** 做 route 作用域校验，不放宽跨 peer 访问。 +- 直传判定从“单一配置字符串”扩展为“实际端点或配置端点”，两者都必须指向当前已连接 peer；不引入“0.0.0.0 等于任意主机”这类宽规则。 +- 不在日志中新增密钥/明文文件内容；地址本就在线上与日志中出现。 +- 不改进度文件 JSON schema；旧行只在内存中替换直传 peer。 + +### 验收证据 + +- 每条路径有真实 socket 测试或复现脚本；R1 至少覆盖 #6、#7、#8；R3 至少覆盖 #12、#13。 +- 改前失败、改后通过的同一命令输出；线程/锁释放有断言。 +- 既有 resume/forward 套件不回退；不删除、不弱化断言。 + +### 禁止事项 + +- 禁止改线格式、进度 schema、公共签名、CI、依赖。 +- 禁止用硬编码 `{'localhost','0.0.0.0','127.0.0.1'}` 表替代端点判定（理由见 §2.1）。 +- 禁止删除/弱化测试、加 `|| true`、降低覆盖率。 +- 禁止绕过控制 peer 作用域。 +- 禁止把 R2 混入本次提交。 + +### 风险等级 + +中。R1 触及身份判定（跨 peer 误判风险）；R3 触及线程/锁生命周期（并发回归风险）。 + +--- + +## 2. 方案 + +### 2.1 R1：实际端点优先，配置端点作为同一 peer 的别名 + +判定规则（两侧同构）： + +- 直传 server 端点（client 视角）= `control_socket.getpeername()` **或** `(self.host, self.port)`。 +- 直传 client 端点（server 视角）= `control_socket.getsockname()` **或** `(self.host, self.port)`。 +- 转发端点（target/origin）继续使用线上实际地址，路由作用域不变。 + +具体改动： + +| 位置 | 现状 | 改为 | +|---|---|---| +| server `_request_resumes`（`6336`） | `target = (self.host, self.port)` | `target = info["socket"].getsockname()`；`OSError` 时回退配置值并记录 | +| client `resume_trans`（`10623-10635`） | `target != (self.host, self.port)` 判转发 | `target` 命中 `getpeername()` 或配置值时判直传；upload 行 key 仍为 `(self.host, self.port)`，不改 schema | +| client `_command_peer`（`10979-10996`） | 直传返回 `(self.host, self.port)` | 直传返回 `control_socket.getpeername()`；socket 已消失时回退配置值 | +| client `_request_resumes`（`10575-10607`） | `origin = entry[4]` | 直传旧行（`entry[4] == (self.host, self.port)`）先内存归一化为 `getpeername()`，再发送 | +| server `_resume_request`（`6395-6409`） | `origin == (self.host, self.port)` | `origin` 命中 `client_socket.getsockname()` 或配置值判直传；`handle_command` 把实际 `client_socket` 传入 | +| `format_resume_line` / `parse_resume_command` | — | 不改 | +| 进度 JSON | — | 不改；新行 `peer` 记录实际端点，旧行读入后按上述规则归一化 | + +**为什么不用三字符串表**（评审提案的复核结论）： + +- LAN 反例：server 在 `192.168.1.10` 绑 `0.0.0.0`，client 配 `192.168.1.10`。表里只有 `0.0.0.0`，无法知道应等同 `192.168.1.10`；若把 `0.0.0.0` 视为“任意主机”，则是比现有行为更宽的身份规则。 +- DNS 别名：`nas.lan` vs `192.168.1.10` 不在表内。 +- 多网卡：同一 `0.0.0.0` 对应多个本地 IP，不能归一成单个回环地址。 +- `localhost` 可能解析为 `::1`，`127.0.0.1` 不覆盖。 +- 实际端点由内核给出，天然覆盖上述非 NAT 场景；配置端点作为回退只用于 socket 不可用/旧行兼容。 + +NAT 明确不在本次范围：client 的 `getpeername()` 是公网地址，server 的 `getsockname()` 是私网地址；二者不可比。现有实现同样不支持，记录为已知限制。 + +### 2.2 R3：清扫时解除 pause，并让暂停等待持有自己的 socket + +| 位置 | 改动 | +|---|---| +| `_drop_stale_transfers`（server `2935-2970` / client `8475-8510`） | 收集被清扫的上传 key；循环结束后 `with _transfer_pause_cond: pop(key); notify_all()`。放在 `_transfer_progress_lock` 之外，避免锁序变化 | +| `_wait_transfer_paused`（server `3008-3026` / client `8511-8529`） | 增加可选参数 `leg_socket=None`；退出条件增加 `leg_socket is not None and leg_socket.fileno() < 0`；两处 `file_transfer_mode` 传入 `client_file_socket` | +| `file_transfer_mode`（两处） | 调用改为 `_wait_transfer_paused(transfer_key, client_file_socket)` | +| 公共 API | 无变化；`_wait_transfer_paused` 为内部方法 | + +双保险：即使 pause 标志被后续 `/pause_trans` 重新置位，已关闭的 leg socket 也能让等待退出；清扫同时清掉遗留 pause，避免同一 fid 的后续 leg 无端暂停。 + +--- + +## 3. 数据流 + +### 3.1 R1 直传上传（server 接收，client 发送） + +``` +client 断开 → server _unregister_client 持久化 download 行（peer=client 实际地址） +client 重连 → server _register_client → _resume_after_connect + → _request_resumes(client_addr) + target = accepted_control.getsockname() # 实际本地端点 + /resume_trans + → client handle_server_command + origin == client_socket.getsockname()? # 现有校验，实际地址 + resume_trans(target=server_actual) + target == getpeername() or configured? # 判直传 + /file → file_transfer_mode(resume_offset=offset) + → server file_transfer_mode_recv：_resumable_entry(fid, peer=client_addr) 命中 +``` + +### 3.2 R1 直传下载（client 接收，server 发送） + +``` +client 断开 → client _persist_transfer_progress（download 行 peer=实际 server 端点） +client 重连 → connect() → _request_resumes() + origin = entry[4]（旧行先归一化为 getpeername） + target = client_socket.getsockname() + /resume_trans → server _resume_request + target == client_address ✓ + origin == client_socket.getsockname() or configured? → 直传 + resume_trans(client_addr, fid, offset) → /file → file_transfer_mode +``` + +### 3.3 转发 resume（不变） + +``` +target → /resume_trans +server _resume_request：target==client_address，_lookup_fid_route(target, child) + route.originator == origin → _queue_resume_to_originator + → /resume_trans → source +source resume_trans：target != server → _forward_one(resume_offset, upstream_fid) +``` + +### 3.4 R3 暂停清扫 + +``` +leg 在 _sending_transfer(key) 内，停在 _wait_transfer_paused(key, leg_sock) +重连/清扫 → _drop_stale_transfers(peer) + pop socket / discard active / close + with pause_cond: pop(key); notify_all() +leg 醒来：leg_sock.fileno() < 0 或 pause 已清 → 返回 + → send 失败 → finally：_file_progress_finish(keep=True) + → _sending_transfer 释放 state 锁 +排队 resume 的 run() 取得锁 → send(entry) 执行 +``` + +--- + +## 4. 路径映射 + +| # | 路径 | 生产改动 | 测试 | +|---|---|---|---| +| 1 | 直传上传 resume | `_request_resumes`、`_resume_request`、`_command_peer` | `test_tcp_transfer_resume.py` 新增/现有 | +| 2 | 直传下载 resume | `resume_trans`、`_request_resumes`、`_command_peer` | 同上 | +| 3 | 转发在线 resume | 无（仅判定辅助） | 现有 `test_forwarded_file_resumes_through_the_server` | +| 4 | 转发离线 pending | 无 | 现有 `test_pending_resume_survives_failed_send...` | +| 5 | 零偏移/EOF | 无 | 现有零偏移与 EOF 测试 | +| 6 | localhost 别名 | 全部 R1 改动 | 新增 `test_resume_across_localhost_alias` | +| 7 | 0.0.0.0 绑定 | 全部 R1 改动 | 新增 `test_resume_across_wildcard_bind` | +| 8 | 旧行归一化 | `_request_resumes` | 新增 `test_resume_normalizes_legacy_configured_peer` | +| 9 | socket 已消失 | `getsockname/getpeername` 回退 | 新增 OSError 注入 | +| 10 | 新传输 | 无 | 现有文件/文件夹套件 | +| 11 | `resume_trans(None)` | 默认 target = `getpeername()` | 单测 | +| 12 | server paused + sweep | `_drop_stale_transfers`、`_wait_transfer_paused` | 新增 `test_paused_server_leg_released_by_sweep` | +| 13 | client paused + sweep | 同上 | 新增 `test_paused_client_leg_released_by_connect` | +| 14 | 清扫后重新 pause | socket 捕获退出 | 上述测试内加断言 | +| 15 | 普通 pause/start | 无 | 现有 pause 测试 | +| 16 | 转发回压 + 清扫 | 同 #12 | 转发 pause 测试扩展 | + +--- + +## 5. 原子提交拆分 + +### Commit 1：R1 — resume 身份改用实际控制端点（别名/通配绑定） + +- **意图**：断线重连后，直传与转发的 resume 不再因配置 host 字符串差异失败；线格式、schema、公共签名不变。 +- **文件**：`PyFlow/network_api/connect_tcp.py`、`test/integration/network_api/test_tcp_transfer_resume.py`、`docs/source/File_Transfer/File_Transfer.rst`、`docs/source/changes/transfer-fid-routes.md`（或新变更记录）、`docs/plans/forward-resume-identity-and-pause-report.md`。 +- **预估**：生产 60–120 行；测试 180–300 行；文档 30–60 行。 +- **局部验证**： + ```bash + .venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -k "resume_across or legacy_configured or download_resumes or upload_resumes" -q + .venv/bin/python -m pytest test/unit/network_api/test_resume_matching.py test/unit/network_api/test_fid_routes.py -q + ``` + 以及 `/tmp/pyflow_audit/repro_hostname.py`、`repro_bindaddr.py` 的改前/改后对照（改前失败、改后完成）。 +- **明确未覆盖**：NAT；`client_port` 变化；跨版本旧 server 通告。 +- **报告**：见 §7.1。 + +### Commit 2：R3 — 清扫暂停 leg 的退出与锁释放 + +- **意图**：断线清扫后，暂停中的发送 leg 在有限时间内结束；同一 fid 的 resume 自动执行，无需人工 `start_trans`。 +- **文件**：`PyFlow/network_api/connect_tcp.py`、`test/integration/network_api/test_tcp_transfer_resume.py`、`docs/source/File_Transfer/File_Transfer.rst`、`docs/plans/forward-resume-identity-and-pause-report.md`。 +- **预估**：生产 25–50 行；测试 100–180 行；文档 20–40 行。 +- **局部验证**： + ```bash + .venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -k "paused and sweep" -q + .venv/bin/python -m pytest test/integration/network_api/test_tcp_file_transfer.py -k "pause" -q + ``` + `/tmp/pyflow_audit/repro_r3_server.py` 改前（C 处 `send_called=0`、leg 存活）改后（resume 自动执行、leg 结束）。 +- **明确未覆盖**：暂停持久化（进程重启后 pause 不恢复）；NAT。 +- **报告**：见 §7.2。 + +### Commit 3：文档收尾与规则沉淀（仅文档） + +- **意图**：总报告、变更记录、`get_transfer_progress` 中 `peer` 字段语义说明（新行记录实际端点）、已知限制（NAT、client_port 变化）。 +- **文件**：`docs/plans/forward-resume-identity-and-pause-report.md`、`docs/source/changes/`、必要时 `AGENTS.md` 检查项。 +- **局部验证**：`git diff --stat` 仅文档；链接/标题检查。 +- **报告**：总报告 §8。 + +> 拆分依据是单一意图，不是行数。两个代码提交均低于 300 行生产改动；若实现超出 300 行，退回重拆而不是合并意图。 + +--- + +## 6. 验收命令 + +```bash +# 编译/静态检查（不新增告警） +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py + +# 定向回归 +.venv/bin/python -m pytest test/unit/network_api/test_resume_matching.py test/unit/network_api/test_fid_routes.py -q +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -q +.venv/bin/python -m pytest test/integration/network_api/test_tcp_file_transfer.py -k "pause or forward or folder" -q + +# 相关全量（提交前） +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py \ + test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py \ + test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py \ + test/integration/network_api/test_tcp_async_server.py test/unit/transfer_web/test_web_transfers.py -q +``` + +运行时证据（两条，必须改前/改后各跑一次）： + +```bash +.venv/bin/python /tmp/pyflow_audit/repro_hostname.py +.venv/bin/python /tmp/pyflow_audit/repro_bindaddr.py +.venv/bin/python /tmp/pyflow_audit/repro_r3_server.py +``` + +--- + +## 7. 提交报告模板 + +### 7.1 Commit 1（R1） + +```text +提交意图：resume 身份判定改用实际控制端点，配置端点仅作同一 peer 的别名；线格式/schema 不变。 +改动内容：<文件与关键行> +为什么这样改：对应需求卡路径 #1/#2/#6/#7/#8/#9/#11；修复 R1。 +局部验证：<命令 + 实际输出，含改前失败/改后通过> +明确未覆盖：NAT；client_port 变化；跨版本旧 server。 +风险与待确认：身份判定放宽为“实际或配置”两值；route 作用域未放宽；旧行仅在内存归一化。 +``` + +### 7.2 Commit 2（R3） + +```text +提交意图：断线清扫时解除 pause，并让暂停等待以 leg 自身 socket 的关闭为退出条件；排队 resume 自动执行。 +改动内容：<文件与关键行> +为什么这样改：对应需求卡路径 #12/#13/#14/#16；修复 R3。 +局部验证：<命令 + 实际输出，含改前 leg 卡死/resume 不执行、改后自动执行> +明确未覆盖：暂停持久化；进程重启后的 pause 恢复。 +风险与待确认：pause 清理与 notify 的锁序；leg socket 捕获的调用点覆盖。 +``` + +--- + +## 8. 总体报告模板 + +```text +需求卡路径清单逐条对照：1–16 覆盖/未覆盖/部分覆盖 +总体数据流与架构影响：仅身份判定与暂停生命周期；无新命令/字段 +所有原子提交汇总：Commit 1 / 2 / 3 +明确未覆盖项与已知风险：NAT、client_port 变化、暂停持久化、跨版本兼容 +对 CI、测试、依赖、公共接口的影响：无 CI/依赖；公共签名不变；get_transfer_progress 的 peer 字段对新行记录实际端点 +建议的验收命令与证据清单：§6 +``` + +--- + +## 9. 合并前闸门 + +1. 红旗复查：无删测试/弱化断言/CI 改动/依赖改动/绕过控制 peer 作用域。 +2. 规则沉淀：新增 R1/R3 回归测试；在 `docs/source/changes/` 记录“身份用实际端点、配置端点仅别名”“清扫必须解除 pause”两条约束。 +3. 文档收尾：`File_Transfer.rst` 的 resume 段说明身份来源与已知限制；变更记录更新。 +4. 人工最终确认后才允许提交远端。 + +--- + +## 10. 明确未覆盖与已知限制 + +- **NAT**：client `getpeername()` 与 server `getsockname()` 不在同一地址空间；本次不解决。 +- **client_port 变化**：地址变化后 server 无法把持久化行匹配到新 peer；既有约束。 +- **多网卡 + 通配绑定**：实际端点能匹配“本次连接使用的本地 IP”；若同一客户端跨会话换网卡，仍按新地址匹配，旧行可能无法续传。 +- **跨版本**：旧 server 仍通告配置地址时，新 client 的“实际或配置”判定在配置字符串一致时可兼容；不一致时不保证。 +- **已运行**：Commit 1/2 的定向回归与相关套件，结果见提交报告；本轮未额外做跨平台 CI 验证。 diff --git a/docs/plans/forward-resume-identity-and-pause-report.md b/docs/plans/forward-resume-identity-and-pause-report.md new file mode 100644 index 0000000..7392f1e --- /dev/null +++ b/docs/plans/forward-resume-identity-and-pause-report.md @@ -0,0 +1,274 @@ +# 文件转发/resume 身份与暂停清扫修复 — 提交报告 + +状态：Commit 1（R1）、Commit 2（R3）与 Linux 3.10 CI 测试修复已实现；Commit 3 为文档收尾与规则沉淀。等待人类审查。 +计划与需求卡见 `docs/plans/forward-resume-identity-and-pause-plan.md`。 + +--- + +## Commit 1（R1）：resume 身份改用实际控制端点 + +### 提交意图 + +断线重连后的 resume 不再因“配置 host 字符串与实际连接地址不同名”而失败；`/resume_trans` 线格式、进度 JSON schema、命令语义、公共方法签名均不变。 + +### 改动内容 + +| 文件 | 关键变更 | +|---|---| +| `PyFlow/network_api/connect_tcp.py` | 新增 `_socket_local_endpoint` / `_socket_peer_endpoint`；server `_request_resumes` 用控制 socket 的 `getsockname()` 作为 `target`；server `_resume_request` 改为接收实际 `client_socket`，`origin` 命中实际本地端点或配置端点即判直传；client `_request_resumes` 将旧直传行的 `entry[4] == (host, port)` 在内存中归一化为 `getpeername()`；client `resume_trans` 用 `getpeername()` 或配置端点判定直传，upload 行 key 仍为 `(host, port)`；client `_command_peer` 直传时记录 `getpeername()` | +| `test/integration/network_api/test_tcp_transfer_resume.py` | 新增参数化回归：`localhost` 别名、`0.0.0.0` 绑定 × 旧行/新行，双向 resume；新增关闭 socket 回退测试 | +| `test/unit/network_api/test_fid_routes.py` | `_resume_request` 新签名的调用点适配（传入服务端本地端点） | +| `docs/source/File_Transfer/File_Transfer.rst` | resume 身份来源、别名/通配绑定行为、旧行内存迁移、NAT/client_port 限制 | +| `docs/source/changes/transfer-fid-routes.md` | 追加本次变更记录 | + +### 为什么这样改 + +对应需求卡路径 #1、#2、#6、#7、#8、#9、#11。旧实现比较的是两侧各自的**配置字符串**，server 绑 `0.0.0.0` 或 client 配 `localhost` 时，端口相同但 host 不同，导致直传被误判为转发或 server 拒绝 origin，resume 静默失效。实际端点由内核给出，天然覆盖通配绑定、host 别名、LAN 地址；配置端点作为同一 peer 的别名保留，兼容旧持久化行与 socket 已消失的回退。 + +### 局部验证 + +```bash +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py +# 通过；ruff --select F,E9 仍只有既有 console `response` F841（无关存量） + +.venv/bin/python -m pytest test/unit/network_api/test_fid_routes.py -q +# 28 passed in 0.27s + +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -q +# 57 passed in 150.33s（含新增 5 条参数化/回退用例） + +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py \ + -k "resume_identity_uses_the_connected_endpoint or closed_control_socket_falls_back" -q +# 5 passed, 52 deselected in 15.33s + +.venv/bin/python -m pytest test/unit/network_api \ + test/integration/network_api/test_tcp_file_transfer.py \ + test/integration/network_api/test_crypto_tcp.py \ + test/integration/test_forward_extension.py \ + test/integration/test_forward_msg_attribution.py -q +# 339 passed in 179.61s +``` + +运行时对照（真实 socket；`/tmp/pyflow_audit/`）： + +- 改前：server 绑 `0.0.0.0` + client 配 `127.0.0.1`，以及 server `127.0.0.1` + client `localhost`，双向 resume 均失败，partial 停在 4096 字节； +- 改后：同一脚本两种配置双向均完成，内容与源文件一致，接收行与发送行清理。 + +```text +localhost upload_started: 1 row_cleared: True size: 8192 content_ok: True +localhost download_started: 1 row_cleared: True size: 8192 content_ok: True +wildcard upload_started: 1 row_cleared: True size: 8192 content_ok: True +wildcard download_started: 1 row_cleared: True size: 8192 content_ok: True +``` + +（`sender_row_cleared` 在脚本采样时刻尚未完成最终清理，属采样时序；完整套件中的对应断言已通过。） + +### 明确未覆盖 + +- NAT：client 的 `getpeername()` 与 server 的 `getsockname()` 不在同一地址空间，本次不解决。 +- `client_port` 变化：地址变化后 server 无法把持久化行匹配到新 peer，既有约束。 +- 跨版本：旧 server 仍通告配置地址时，新 client 的“实际或配置”判定在配置字符串一致时兼容；不一致不保证。 +- 本轮未改 Commit 2（R3）的任何行为。 + +### 风险与待确认 + +- 身份判定从单一配置值扩展为“实际端点或配置端点”，两者都指向当前已连接 peer；转发 route 仍以实际控制 peer 作用域，未放宽跨 peer 访问。 +- 新直传下载行的 `peer` 字段记录实际 server 端点（`get_transfer_progress` 可见）；旧行读入后按需原地归一化，不写回文件。 +- `_resume_request` 的内部签名增加 `client_socket`；已适配仓库内全部调用点与测试 monkeypatch。 + +--- + +## Commit 2（R3）：清扫释放暂停 leg + +### 提交意图 + +断线重连的清扫遇到已暂停的发送 leg 时，leg 必须在有限时间内结束并释放 per-fid 发送锁；排队中的 resume 不再需要人工 `start_trans`。 + +### 改动内容 + +| 文件 | 关键变更 | +|---|---| +| `PyFlow/network_api/connect_tcp.py` | `_drop_stale_transfers`（server/client）收集被清扫的上传 key，在关闭 socket 后于 `_transfer_pause_cond` 下 `pop` 并 `notify_all`；`_wait_transfer_paused` 增加 `leg_socket` 参数，leg 自己的 socket 已关闭即退出；两处 `file_transfer_mode` 传入 `client_file_socket`，并在某次 chunk 发送失败（`send_message` 返回 False 或抛 `OSError`）时立即 `close_socket()` 并返回 False，避免 peer 消失后继续读取剩余文件 | +| `test/integration/network_api/test_tcp_transfer_resume.py` | 新增 `test_paused_leg_is_released_by_stale_sweep`（server/client 两种发送端）；适配 `_watch_sign_capacity` 的 monkeypatch 签名 | +| `docs/source/File_Transfer/File_Transfer.rst` | 说明清扫会解除暂停并让排队 resume 继续 | +| `docs/source/changes/transfer-fid-routes.md` | 追加本次变更记录 | + +### 为什么这样改 + +清扫只把 socket 从 `_active_transfer_sockets` 移除并关闭，不移除 `_transfer_pause`；暂停循环重新查登记表得到 `None`，`fileno()<0` 的退出条件不成立，而 server 的 `running` 仍为 True，于是 leg 永久阻塞并持有 `_sending_transfer` 的 per-fid 锁。同 fid 的 resume 已返回“已排队”，但 `run()` 永远拿不到锁。修复同时处理两侧：清扫解除暂停并唤醒；暂停等待以 leg 自己的 socket 关闭为准，即使 pause 被重新置位也能退出。 + +### 局部验证 + +```bash +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py +# 通过；ruff --select F,E9 仍只有既有 console `response` F841 + +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py \ + -k "paused_leg_is_released_by_stale_sweep" -q +# 2 passed, 57 deselected in 3.49s + +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -q +# 59 passed in 157.35s + +.venv/bin/python -m pytest test/unit/network_api -q +# 261 passed in 18.14s + +.venv/bin/python -m pytest test/integration/network_api/test_tcp_file_transfer.py -k "pause" -q +# 1 passed, 33 deselected + +.venv/bin/python -m pytest test/unit/network_api \ + test/integration/network_api/test_tcp_file_transfer.py \ + test/integration/network_api/test_crypto_tcp.py \ + test/integration/test_forward_extension.py \ + test/integration/test_forward_msg_attribution.py -q +# 339 passed in 173.37s +``` + +改前复现（`/tmp/pyflow_audit/repro_r3_server.py`）：清扫后 1.5s `pause=True, state_users=1, leg_alive=True`；queued resume 2s 内 `send_called=0`。改后同一测试路径：leg 在有限时间内结束，`pause` 键消失，queued resume 自动执行（无需 `start_trans`）。 + +### 明确未覆盖 + +- 暂停状态不持久化：进程重启后 pause 不恢复（既有约束）。 +- 转发回压暂停与普通 pause 共用同一路径，已由代码覆盖；未单独构造 relay 背压 + 断线时序。 +- 未改变 `pause_trans`/`start_trans` 在活会话中的语义。 + +### 风险与待确认 + +- 只清除“确实被清扫（有已注册 socket）”的上传 key，预启动但未注册 socket 的暂停开关不会被误清。 +- `notify_all` 在 `_transfer_progress_lock` 之外获取 `_transfer_pause_cond`，不引入新的锁嵌套。 +- `file_transfer_mode` 在 leg socket 已关闭时直接返回 False，进度行按既有 `keep=True` 规则保留，后续 resume 仍可继续。 + +--- + +## CI 修复（Linux Python 3.10）：广播测试的共享目的路径竞态 + +### 现象 + +`test_multiple_file_multiple_client_console_command` 在 Linux 3.10 上读到 +`dest/broadcast.bin` 为空（`b''`），断言在 `wait_until(exists())` 之后立即执行。 + +### 根因 + +接收 leg 在开始时就会以 `wb` 创建并截断目标文件；该测试把同一个绝对目的路径 +交给两个客户端，两个 leg 又都写这个文件。`exists()` 只证明某个 leg 已经打开文件， +后续 leg 的截断/写入窗口会让紧随其后的读取看到空文件。这是测试等待条件不足， +不是传输内容错误；本地同一 Python 3.10 连续 30 次未复现，CI 调度暴露了窗口。 + +### 修复 + +在两个客户端上注册 `add_file_listener`，等两个接收方都报告“完整写入”后再读文件； +断言仍校验完整 payload，并额外要求两个 client 都收到，强度高于原来的 `exists()`。 +修复后本地连续 50 次通过。 + +### 附带的产品修正 + +Commit 2 最初在暂停等待返回后检查 `client_file_socket.fileno() < 0` 并立即中止。 +接收方发完最终 ACK 后会关闭自己的连接，发送端的接收线程可能先看到 EOF 并关闭 +本端 socket；此时 ACK 已在 sign buffer 中，但该检查会把一次已完成的传输误判为失败。 +现改为只在 **某次 chunk 发送失败**（`send_message` 返回 False 或抛 `OSError`）时中止： +被清扫的暂停 leg 会在首个失败 chunk 立刻停止，已完成传输的最终 ACK 仍由原 ACK +等待路径消费。 + +### 验证 + +```bash +.venv/bin/python -m pytest test/integration/network_api/test_tcp_file_transfer.py::test_multiple_file_multiple_client_console_command -q +# 连续 50 次通过（修复后）；修复前 CI Linux 3.10 失败 + +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py \ + -k "paused_leg_is_released_by_stale_sweep or sign_capacity" -q +# 6 passed + +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py \ + test/integration/network_api/test_tcp_file_transfer.py -q +# 93 passed in 216.71s（59 resume + 34 file transfer） + +.venv/bin/python -m pytest test/unit/network_api \ + test/integration/network_api/test_tcp_file_transfer.py \ + test/integration/network_api/test_crypto_tcp.py \ + test/integration/test_forward_extension.py \ + test/integration/test_forward_msg_attribution.py -q +# 339 passed in 184.80s +``` + +--- + +## 总体报告（Commit 1–3） + +### 需求卡路径清单逐条对照 + +| # | 路径 | 结论 | 证据 | +|---|---|---|---| +| 1 | 直传 client→server 上传 resume | 覆盖 | 新参数化用例 + `test_upload_resumes_after_a_reconnect` | +| 2 | 直传 server→client 下载 resume | 覆盖 | 新参数化用例 + `test_download_resumes_from_the_partial_file` | +| 3 | 转发 target resume（source 在线） | 覆盖 | 既有 `test_forwarded_file_resumes_through_the_server`，未改语义 | +| 4 | 转发 source 离线 pending | 覆盖 | 既有 `test_pending_resume_survives_failed_send...` | +| 5 | 零偏移 / EOF 重放 | 覆盖 | 既有零偏移与 EOF 测试 | +| 6 | `localhost` vs `127.0.0.1` 双向 | 覆盖 | 新 `test_resume_identity_uses_the_connected_endpoint[localhost-alias-*]` | +| 7 | server 绑 `0.0.0.0` 双向 | 覆盖 | 同上 `[wildcard-bind-*]` | +| 8 | 旧持久化行（配置地址） | 覆盖 | 同上 `[legacy-row]`，内存归一化 | +| 9 | socket 消失 / `OSError` 回退 | 覆盖 | `test_closed_control_socket_falls_back_to_the_configured_endpoint` | +| 10 | 非 resume 新传输 | 覆盖 | 339 相关套件 | +| 11 | `resume_trans(peer_addr=None)` | 部分覆盖 | 默认 target 改为 `getpeername()`；直传分类由新用例覆盖，未单独构造 None 入参用例 | +| 12 | server 发送端 paused + 清扫 | 覆盖 | `test_paused_leg_is_released_by_stale_sweep[server]` | +| 13 | client 发送端 paused + 清扫 | 覆盖 | `test_paused_leg_is_released_by_stale_sweep[client]` | +| 14 | 清扫后重新 `/pause_trans` | 部分覆盖 | `leg_socket` 退出条件存在；未单独注入重新 pause 时序 | +| 15 | 普通 pause/start（无断线） | 覆盖 | 既有 pause 测试 | +| 16 | relay 背压暂停 + 清扫 | 部分覆盖 | 与普通 pause 共用 `_wait_transfer_paused`；未单独构造背压后断线时序 | + +### 总体数据流与架构影响 + +- `/resume_trans` 线格式、进度 JSON schema、公共方法签名未变；身份字段仍是 + `(ip, port)` 元组，只是写入实际控制端点、校验时接受配置端点作为同一 peer 的别名。 +- 转发 resume 仍以实际控制 peer 做 route 作用域，未放宽跨 peer 访问。 +- 清扫现在是“关闭 socket + 清除该上传 key 的 pause + 唤醒”；暂停等待同时观察 + leg 自身 socket；某次 chunk 发送失败即停止 leg，避免 peer 消失后继续读文件。 +- 接收方发完最终 ACK 后关闭连接的正常路径仍由既有 ACK 等待消费，不会因 + socket 关闭被误判为失败。 +- 测试侧把广播测试从“文件存在”改为“两个接收方都报告完整写入”，属于强化而非弱化。 + +### 所有原子提交汇总 + +| 提交 | 意图 | 状态 | +|---|---|---| +| Commit 1 | resume 身份改用实际控制端点，配置端点仅作别名 | 已实现，待提交 | +| Commit 2 | 清扫释放暂停 leg，失败写入立即停止 | 已实现，待提交 | +| 测试修复 | 广播测试等待两个接收方完成，消除共享目的路径竞态 | 已实现，待提交 | +| Commit 3 | 本文档收尾、总报告、`get_transfer_progress` 语义、规则沉淀 | 本次 | + +### 明确未覆盖项与已知风险 + +- NAT 端点身份；`client_port` 变化；跨版本旧 server 通告（见计划 §10）。 +- 暂停状态不持久化，进程重启后不恢复。 +- 路径 11、14、16 为部分覆盖，原因见上表。 +- 未在 Windows CI 上实跑本轮改动；本地为 Linux Python 3.10.20。 +- CI 需在提交后重新触发上游 PR run 才能得到最终平台矩阵结果。 + +### 对 CI、测试、依赖、公共接口的影响 + +- CI 配置、依赖、覆盖率策略未改。 +- 测试新增/强化：R1 参数化用例、R3 双发送端用例、广播测试监听器等待；未删除或弱化断言。 +- 公共方法签名未变；内部 `_resume_request`、`_wait_transfer_paused` 签名调整，调用点与测试 monkeypatch 已适配。 +- `get_transfer_progress` 的 `peer` 字段语义已写入 Server/Client API 文档：直传下载记录实际 server 端点，旧行在内存中归一化。 + +### 建议的验收命令与证据清单 + +```bash +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py +.venv/bin/python -m pytest test/unit/network_api -q +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -q +.venv/bin/python -m pytest test/integration/network_api/test_tcp_file_transfer.py -q +.venv/bin/python -m pytest test/integration/network_api/test_crypto_tcp.py -q +.venv/bin/python -m pytest test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py -q +``` + +运行时证据：`/tmp/pyflow_audit/verify_r1_v2.py`（localhost/0.0.0.0 双向）、 +`/tmp/pyflow_audit/repro_r3_server.py`（paused + sweep + queued resume）、 +广播测试连续 50 次。 + +### 规则沉淀 + +- 新增 R1/R3 回归与广播测试的“等待完整写入/事件”规则，防止同类问题回归。 +- `AGENTS.md` 快速检查清单新增：身份/续传类改动必须覆盖配置别名与通配绑定，测试不得以文件存在作为完成条件。 +- 文档收尾：`File_Transfer.rst`、Server/Client API 文档、变更记录均已更新。 diff --git a/docs/plans/ordinary-transfer-progress-inspection-o1-report.md b/docs/plans/ordinary-transfer-progress-inspection-o1-report.md new file mode 100644 index 0000000..9c6521c --- /dev/null +++ b/docs/plans/ordinary-transfer-progress-inspection-o1-report.md @@ -0,0 +1,116 @@ +# O1 报告：Server 普通发送错误符号不得成功收尾 + +状态：实现、定向回归、独立运行时烟测与完整相关套件均完成，提交人类审查，不自动推进 O2–O4。未创建 git 提交、未合并、未推送；未进入原 FID 大计划的更改 4–6。 + +## 提交意图与批准范围 + +人类明确“审计完毕，开始01的更改”。本项只修复 `TCP_Server_Base.file_transfer_mode` 在等待开始符号、等待最终数据 ACK 时收到错误符号的退出行为，不改变接收端的 ACK 生命周期。 + +## 改动内容 + +| 文件 | 关键变更 | +|---|---| +| `PyFlow/network_api/connect_tcp.py` | 两处错误符号分支的 `break` 改成 `return False`；生产逻辑仅此两行变化 | +| `test/integration/network_api/test_tcp_transfer_resume.py` | 新增 `test_server_final_error_keeps_upload_until_successful_resume`，文件/目录项 × 初次/从 4096 字节恢复，验证真实错误符号、失败返回、上传及磁盘记录保留、原 ID EOF 再确认与原路径内容 | +| `docs/source/File_Transfer/File_Transfer.rst` | 写明普通发送的最终 ACK 成功条件、开始阶段拒绝与失败保留边界 | +| `docs/source/changes/transfer-fid-routes.md` | 增加 O1 变更记录，不覆写更改 3 的历史范围 | +| `docs/plans/ordinary-transfer-progress-inspection-plan.md`、`docs/plans/transfer-fid-and-resume-unify-plan.md` | 记录 O1 授权、当前状态与后续人工闸门 | +| 本报告 | 改前/改后命令、真实运行时证据、未覆盖与风险 | + +Client 相应两个分支原本已经返回 False，本次未重复改写。未修改其他调用点、测试断言、CI、依赖、覆盖率策略、公共签名、持久化五项值或线路格式。 + +## 数据流与为什么这样改 + +入口：Server 普通文件或目录项发送，或既有恢复入口，最终调用同一个 `file_transfer_mode`。 + +1. 建立传输连接，继续使用既有加密附着和 `send_message` / `receive_message`。 +2. 等待开始符号：错误符号立即 False，不再读取并发送后续文件元数据；首次尝试不新建进度，已有中断记录保持。 +3. 开始符号成功后发送文件头、从 `resume_offset` 读取并发送内容，按既有方式记录进度。 +4. 等待最终数据确认：错误符号立即 False，不能执行 `file_sent=True`、成功日志或成功清理；只有既有最终成功 ACK 才进入成功分支。 +5. `finally` 不变：当前 owning-entry 的失败上传具备 peer 时保留并持久化；活动 socket、活动 fid 及本次暂停项仍按原范围释放。源文件不存在的既有不可恢复处理不变。 + +“发送字节数已等于总大小”不是接收成功证据。该修复兑现计划 2.1 的普通发送契约，不新增完成通道或状态副本。 + +## 改前复现 + +在生产修复前新增并运行真实 socket 回归： + +```bash +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -k server_final_error_keeps_upload_until_successful_resume -q +``` + +结果：**4 failed，23 deselected，9.30s，退出码 1**。四种组合均在接收端已写入 65573 字节后，由既有 `send_message` 真正发出错误符号替代最终 ACK；Server 收到错误符号，却返回 True。捕获输出先有 `Error sign received...`，随后仍有 `Success: file ... sent successfully`。失败断言为 `assert result is False`，不是测试装配或超时失败。 + +## 改后永久回归 + +修复后同一命令结果：**4 passed，23 deselected，17.72s，退出码 0**。 + +回归没有 mock 整条传输。仅在 Client 最终 ACK 边界写出真实错误符号并抛出指定异常，使 Client 按既有失败路径保留接收行；通过 Event 分离文件头确认和最终错误，避免无帧 TCP 合并两种符号造成错误覆盖不到分支。 + +验证内容：Server 返回 False；完整接收内容保持；上传完整 scoped key 和五项磁盘记录保留；Client 沿既有恢复驱动以原 fid 在 EOF 再确认；最终两端 owning 记录清理，目录项不变成接收根目录下的新文件。全部 socket worker 在 teardown 中 join，未留下活动线程。 + +## 独立真实运行时证据 + +另建一次性 `/tmp/pyflow-o1-smoke.py`，不调用 pytest 测试函数，运行真实 Server/Client、普通文件/目录项接收入口和 `file_transfer_mode`。每种场景都覆盖明文/加密、文件/目录项、offset=0/4096。仅在接收端开始符号或最终 ACK 边界注入真实错误符号，不替换文件传输实现。 + +```bash +.venv/bin/python /tmp/pyflow-o1-smoke.py +``` + +结果：**退出码 0，62.32s，输出 `O1_SMOKE_COMPLETE`**。 + +| 路径 | 场景数 | 观察结果 | +|---|---:|---| +| 开始阶段错误 | 8 | 返回 False；Server 二进制发送尝试为 0；首次不新建上传/目标文件，恢复尝试不破坏原记录与原部分文件；无成功日志 | +| 文件体完成后错误 | 8 | 返回 False；65573 字节内容相同；原上传 key 与磁盘五项记录保留；无错误尝试的成功日志 | +| 上述失败后的 EOF 再确认 | 8 | `resume_trans` 使用同一 fid、offset=65573；实际发送腿返回 True；两端记录清理，原文件内容不变 | +| 正常最终 ACK | 8 | 初次/恢复发送返回 True；上传/接收 owning 记录清理，文件内容相同 | + +这对应 24 个独立初始场景以及其中的 EOF 恢复，不将恢复次数再算成新增初始场景。 + +加密场景等待两端真实握手完成，确认每次传输 stamp 为 `enc:`。记录接收端实际送入帧解码器的 wire bytes，验证所发送区间的 128 字节明文片段不出现在原始帧流中；body/error 及其恢复均经过既有安全入口。这是被执行场景的观察，不是全局加密覆盖审计。 + +输出同时记录内容 SHA-256。例如加密目录项从 4096 字节恢复、最终被拒绝后再确认，原 fid 恢复成功，最终文件 SHA-256 为： + +```text +8566a6558862ad8797dcf42e6a2b53f71d6d40f9c57c8e8eb50b61e185554fc7 +``` + +该行的 `result=false`、`error_sent=true`、`eof_recovered_original_fid=true`，实际捕获加密帧共 62164 字节（含失败尝试和 EOF 再确认）；脚本内部比较完整接收内容与源 payload，而非仅比较长度或非空。 + +## 编译与静态检查 + +```bash +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py +ruff check --select F,E9 test/integration/network_api/test_tcp_transfer_resume.py +``` + +- 编译：退出码 0。 +- 生产与测试联合 Ruff：退出码 1，仅既有 `connect_tcp.py:6728` 的 `response` 未使用（F841）；前序原子报告已记录。本次不修改无关控制台分支，不宣称全量 lint 通过。 +- 测试文件单独 F/E9：`All checks passed!`，退出码 0。 + +## 明确未覆盖与风险 + +- O2 未实施：Server 接收端在最终 ACK 明确写出失败后仍可能清理接收记录；Client 对应修复保持，但不以此声称 Server 已修复。 +- O3/O4 未实施：没有新增 `get_transfer_progress`、`/transfers` 或 `delete_config`。 +- 不解决 ACK 成功写入 TCP 而对方应用没收到的不确定窗口，不新增 ACK 收据/去重/重试协议，不扩展现有最终 ACK 等待时限。 +- 不改固定 client_port 依赖、端点迁移、源文件外部修改/删除、旧断点迁移、kind/tfid、现行转发恢复线路或 ID 分配权。 +- O1 没有新增 UI、HTTP、远程管理入口。没有全仓测试、完整 lint、文档构建、跨平台或负载验证；完整相关套件仅覆盖所列命令。 + +## 规则沉淀与人工闸门 + +已补永久行为回归：“错误符号不得成功返回或丢失可恢复上传；完整文件依然可用原 ID/原路径在 EOF 再确认。”没有删除用例、弱化断言、增加跳过或修改 coverage/CI。 + +本项只提交审查；未获人类明确放行，不进入 O2。O4 是补充计划的第四项，不是原 FID 大计划的原子更改 4。 + +## 完整相关套件与证据收尾 + +```bash +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py test/unit/transfer_web/test_web_transfers.py -q +``` + +结果：**237 passed，254.28s，退出码 0**。覆盖相关网络单测、普通两向文件/目录、恢复、加密/公钥、原生/扩展转发、异步和现有 Web 消费者。新增定向回归已包含在该总数中，不相加为唯一用例数量；本轮结果不挪用更改 3 的历史数量。 + +一次性 smoke 脚本在记录证据后删除，脚本自建的临时目录由 `TemporaryDirectory` 在成功退出时清理。以上结果仅证明 O1 已实施范围；Server 接收 ACK 缺口、查询和清理 API 仍待 O2–O4。 + diff --git a/docs/plans/ordinary-transfer-progress-inspection-o2-report.md b/docs/plans/ordinary-transfer-progress-inspection-o2-report.md new file mode 100644 index 0000000..6f2a3a7 --- /dev/null +++ b/docs/plans/ordinary-transfer-progress-inspection-o2-report.md @@ -0,0 +1,102 @@ +# O2 报告：Server 最终 ACK 明确失败后保留接收进度 + +状态：最小实现、定向回归、独立真实运行与完整相关套件均完成，待人类审查。只提交 O2，不自动进入 O3/O4 或原 FID 大计划更改 4–6；未创建 git 提交、未合并、未推送。 + +## 提交意图与批准范围 + +人类明确“审计通过，开始02”。O1 已获审计通过，本项仅修复 Server 普通接收端在最终 ACK 明确写出失败后仍清理接收行的问题，不增加 ACK 收据、自动重试或额外等待协议。 + +## 改动内容与数据流 + +- `PyFlow/network_api/connect_tcp.py`：删除文件写完后的提前 `finished_ok=True`；最终 ACK 改为 `finished_ok = bool(self.send_message(...))`，与 Client 接收端既有模式一致。 +- `test/integration/network_api/test_tcp_transfer_resume.py`：新增 `test_server_ack_failure_keeps_received_file_until_eof_confirmation`，覆盖文件/目录项 × ACK 返回 False/抛异常；每例验证初次完整接收失败、EOF 再确认仍失败、下一次 EOF 再确认成功及落盘清理。 +- 文件传输说明、变更记录、补充计划、原 FID 计划和本报告同步 O2 范围与证据。 + +数据流：Client 文件/目录项上传或既有恢复 → Server 写入完整目标文件并截断旧尾部 → 接收通知、事件更新及公钥 TOFU 注册 → 发送既有最终 ACK → 返回真才成功清理；False/异常时沿用原 `finally` 保留并持久化可恢复接收行。 + +保留的是原 fid 对应的 `[bytes, total, speed, full_path, peer]`，其中停止后的 speed 为 None。完整文件不删除；原路径、来源和总大小继续用于后续稳定 fid 恢复。已有 active fid/socket 释放、owning-entry 判断及 peer/进度开关边界不改。接收通知仍表示本地文件已写完，不升级为对端已收到 ACK。 + +无新公共签名、持久化字段、协议标志、发送旁路或重复工具函数；O1 的两处发送错误分支保持。未修改 Client 接收逻辑、CI、依赖或覆盖率策略。 + +## 改前复现与改后回归 + +```bash +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py -k server_ack_failure_keeps_received_file_until_eof_confirmation -q +``` + +| 阶段 | 实际结果 | +|---|---| +| 生产修复前 | 4 failed，27 deselected，9.45s，退出码 1 | +| 最小修复后 | 4 passed，27 deselected，26.03s，退出码 0 | + +改前四种组合都已实际写完 65573 字节，Client 发送腿返回 False，Server 接收 worker 已收尾；但 `file_recv_progress[606]` 不存在，回归在读取原接收行时出现 `KeyError: 606`，不是连接或装配超时。 + +修复后，第一次和 EOF 第二次 ACK 明确失败均保留原接收行及磁盘 `download` 行，原文件内容相同；第三次 ACK 成功后两端记录清理，持久化 `download` 为空。发送结果顺序为 `(原 fid, 0, False)`、`(原 fid, 65573, False)`、`(原 fid, 65573, True)`。目录项继续使用嵌套完整路径，没有在接收根目录创建同名文件。所有受追踪 socket worker 在 teardown 中结束。 + +测试仅在最终 ACK 的发送边界注入 False/异常，不 mock 掉收发核心、磁盘或恢复分派；没有删除或弱化已有测试。 + +## 编译与静态检查 + +```bash +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py +ruff check --select F,E9 test/integration/network_api/test_tcp_transfer_resume.py +``` + +- 编译退出码 0。 +- 联合 Ruff 退出码 1，仅既有控制台分支 `response` 未使用（F841），当前行号 6729；本次不修改无关代码，不宣称完整 lint 通过。 +- 测试文件单独 F/E9：`All checks passed!`，退出码 0。 + +## 明确未覆盖与风险 + +- O3/O4 未实现：没有新增进度查询、`/transfers` 或 `delete_config`。原 FID 大计划更改 4–6 仍未进入。 +- ACK 成功写入 TCP、但对端应用没有收到的窗口仍存在；本项只保守处理明确 False/异常，不证明应用层收到，不改变最终 ACK 等待时限。 +- 进度关闭或无可恢复 peer 时不会凭空创建接收行，不升级为支持自动恢复；接收文件与进度记录的生命周期仍按原边界区分。 +- 没有跨端点身份迁移、旧断点迁移、源文件外部修改处理、全仓测试、完整 lint、文档构建、跨平台或负载审计。 +- 没有 UI 或远程管理入口变化；公钥注册与握手顺序必须在实际运行中验收，不能仅凭代码顺序声称通过。 + +## 规则沉淀与人工闸门 + +永久行为回归固化“文件完整不等于最终 ACK 写出成功”“EOF 再确认再次失败也不能丢掉恢复记录”“成功后清理持久化接收行”。不改变默认持久化策略，不新增僵尸判定或自动清理。 + +O2 局部验收已完成，现交付人类审查；未获明确放行不进入 O3。 + +## 独立真实运行与新进程读回 + +```bash +.venv/bin/python /tmp/pyflow-o2-smoke.py +``` + +脚本不调用 pytest 测试函数。实际启动 Server/Client，以既有接收入口、文件发送和恢复分派完成全部路径；仅在最终 ACK 边界注入指定 False/异常，真实读写文件。输出 `O2_SMOKE_COMPLETE`，正常退出，143.26s。 + +| 运行路径 | 实际数量 | 验收观察 | +|---|---:|---| +| 明文/加密 × 文件/目录项 × 初次/从 4096 字节恢复 × False/异常 | 16 组 | 每组发送结果为 False、EOF False、EOF True；原路径、fid、peer、总大小和内容不变；成功后清理内存及磁盘接收行 | +| 失败状态新进程读回 | 32 次 | 每组两次失败后均另起 Python 进程，从真实产生的快照调用 `_load_transfer_progress`，恢复数量为 1，五项接收行与真实文件完全对应 | +| 关闭进度记录时 ACK 失败 | 4 组 | 明文/加密 × False/异常，完整文件保留，不虚构上传/接收行，不生成可恢复进度文件 | +| 公钥最终 ACK 失败 | 2 组 | 在连接前注入 False/异常；实际 ACK 尝试前已有接收通知、TOFU 记录和 `peer_pub_ok=True`；两端最终进入 framed 状态，公钥任务没有普通进度行 | + +每组文件失败都比较完整 payload，EOF 再确认后的文件 inode 与完整路径保持；目录项没有在接收根目录出现新同名文件。本地通知发生在每次 ACK 尝试之前，没有被 ACK 失败跳过，也不被解释成对端确认完成。 + +加密文件场景等待真实握手完成并确认 `enc:` stamp，记录实际传入帧解码器的 wire bytes，检查当前发送区间的 128 字节明文片段未出现在原始帧流中。这是具体运行路径的观察，不是全局加密审计。 + +例：加密目录项从 4096 字节恢复、最终 ACK 抛异常，fid 为 40917317460，实际三次发送结果： + +```text +[[40917317460,4096,false],[40917317460,65573,false],[40917317460,65573,true]] +``` + +该组新进程读回两次，原路径及 inode 保持，最终 SHA-256 为 `6a141060e2ab02e4b57c6a84c97958d9e1cc3b7d54f2c76fa0823d142cafa2ea`。同组实际捕获加密帧 62458 字节；其恢复不是重新构造一份期望 JSON 来代替失败快照。 + +两次公钥失败均输出 `both_framed=true`、`registered_before_ack=true`、`tracked_key_rows=0`；钩子在 `connect()` 前安装且实际命中一次，不依赖“注入时握手可能已经完成”的竞态。 + +## 完整相关套件与证据收尾 + +```bash +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py test/unit/transfer_web/test_web_transfers.py -q +``` + +结果：**241 passed，286.12s，退出码 0**。包含新增 O2 回归、O1 普通发送回归、双向普通文件/目录、恢复、加密和公钥、转发、异步及现有 Web 消费者。定向回归属于该总数，不重复相加；不挪用 O1 的历史 237 passed 作为本次结果。 + +一次性 smoke 脚本在记录证据后删除，其自建临时目录由 `TemporaryDirectory` 成功退出时清理；已检查 `/tmp/pyflow-o2-smoke.py` 与 `/tmp/pyflow-o2-smoke-*` 均不存在。保留永久行为回归及本报告，不保留临时测试装配。 + diff --git a/docs/plans/ordinary-transfer-progress-inspection-o3-report.md b/docs/plans/ordinary-transfer-progress-inspection-o3-report.md new file mode 100644 index 0000000..0bc67ca --- /dev/null +++ b/docs/plans/ordinary-transfer-progress-inspection-o3-report.md @@ -0,0 +1,152 @@ +# O3 报告：Server/Client 本地只读进度查询 + +状态：实现、局部回归、独立真实 API/控制台烟测与完整相关套件均完成；待人类审查。只提交 O3，不自动进入 O4 或原 FID 大计划更改 4–6；未创建 git 提交、未合并、未推送。 + +## 提交意图与批准范围 + +人类明确“审计通过，开始更改03”。本项只增加 Server/Client 同名本地查询 API 和 `/transfers` 控制台命令： + +```python +get_transfer_progress(*, include_persisted=True) -> dict +``` + +不增加远程查询协议、HTTP 端点、Web 字段、删除接口、僵尸判定、恢复触发或状态写回。O1/O2 的发送和接收生命周期保持。 + +## 改动内容 + +| 文件 | 关键变更 | +|---|---| +| `PyFlow/network_api/connect_tcp.py` | 新增共享 JSON-ready 观察格式化和磁盘读取包装；Server/Client 各暴露 `get_transfer_progress`; 两端本地 console 分别拦截 `/transfers`，参数错误仅打印用法 | +| `test/unit/network_api/test_transfer_progress_query.py` | 新增 Server/Client 参数化契约：作用域、运行时状态、内存/磁盘分离、relay 结构、错误状态、只读返回、副本隔离、关闭进度和 speed 开关 | +| `test/unit/network_api/test_TCP_client.py` | Client 实际交互循环 `/transfers` 输出与参数错误回归 | +| `test/integration/network_api/test_tcp_broadcast_alloc.py` | Server 实际 console `/transfers` 输出与参数错误回归 | +| `docs/source/File_Transfer/File_Transfer.rst` | 新增本地查询契约、字段与不变式 | +| `docs/source/Network_APIs/TCP_Server_APIs.rst` | 增加 Server API 和 console 命令说明 | +| `docs/source/Network_APIs/TCP_Client_APIs.rst` | 增加 Client console 命令说明 | +| `docs/source/changes/transfer-fid-routes.md` | 记录 O3 范围和限制 | +| 计划与本报告 | 记录授权、路径和证据 | + +## 返回契约与数据流 + +入口：本地 Python API 或本地 console `/transfers` → 复制现有内存快照与运行时状态 → 可选只读进度文件解析 → JSON-ready 独立结果 → 本地调用者/终端。 + +返回顶层字段： + +```text +{ + "progress_enabled": bool, + "speed_enabled": bool, + "memory": { + "uploading": [...], + "downloading": [...], + "relays": [...] + }, + "persisted": { + "path": str, + "status": "ok" | "missing" | "error" | "not_requested", + "snapshot": {...} | null, + "error": str | null + } +} +``` + +文件 row 保留 `fid`、上传 `receiver`、逻辑 `peer`、`name`、`path`、`transferred`、`total`、`speed`、`active`、`socket_registered`、`paused`。完整上传 `(receiver, fid)` 作为作用域；相同数字 fid 不折叠。下载的 receiver 和 paused 为 null。 + +- 内存 rows：`active` / `socket_registered` / `paused` 是查询时本地观察值。 +- 磁盘 rows：这些运行时字段均为 null，不伪装成 False。 +- Server relay：originator、targets、completed，且只列权威路由。 +- Client relay：既有恢复记录字段，不添加 Server 专用 completed 或 socket/lock 对象。 +- `include_persisted=False` 不读取磁盘；默认读取但不调用 `_load_transfer_progress`，不恢复到内存。 +- 缺失、空 JSON、损坏/旧格式、I/O 错误和跳过读取分别表达；错误不隐藏内存结果，也不修改文件。 +- 返回结构不包含 socket、lock、Event 或可修改的内部 list 引用；查询不读文件内容、不计算 hash、不连接、不发送、不持久化。 + +锁顺序：在 `_transfer_progress_lock` 内复制快照、活动 fid 与 socket keys；释放后单独读取暂停条件变量和磁盘；不在路由/暂停锁内做文件 I/O 或打印。结果不是跨状态源的事务快照。 + +## 改前/改后回归 + +新增局部命令： + +```bash +.venv/bin/python -m pytest test/unit/network_api/test_transfer_progress_query.py test/unit/network_api/test_TCP_client.py test/integration/network_api/test_tcp_broadcast_alloc.py -q +``` + +结果：**49 passed，0.25s，退出码 0**。 + +覆盖： + +- Server/Client 两种实例的相同契约； +- 同号不同 receiver、上传/下载同号的作用域隔离； +- active、socket_registered、paused 的本地状态； +- relay 权威字段和 Client 私有恢复字段； +- 当前内存与落后/磁盘独有 row 分离； +- missing、empty、malformed、unsupported、permission/read error、not_requested； +- 返回对象和嵌套 row 修改不会改变内存、磁盘或 route； +- progress/speed 关闭时已有记录仍可见，速度变为 null； +- Client/Server 实际交互循环输出完整 JSON，日志关闭仍可见，错误参数不发网络。 + +测试文件单独 `ruff --select F,E9` 已通过。现有 `test_tcp_broadcast_alloc.py` 的 `threading`/`time` 未使用告警是存量测试问题，本项不改无关导入；生产文件另有既有 console `response` F841。 + +## 独立真实运行时证据 + +一次性脚本： + +```bash +.venv/bin/python /tmp/pyflow-o3-smoke.py +``` + +退出码 0，输出 `O3_SMOKE_COMPLETE`，8.09s。脚本以真实 Server/Client、真实 TCP 文件端口和既有转发入口构造状态；不直接调用永久测试函数。明文、两个本地 console、离线实例及并发查询路径均实际运行。 + +### Server console 实例 + +观察到： + +- memory 中同一 fid 的 upload/download 独立列出；upload row 的 receiver、peer、active=true、socket_registered=true、paused=true；download row 的 receiver/paused 为 null，active/socket_registered=true。 +- Server relay 展示 originator、targets、completed；不出现 path 等 Server 未保存的副本。 +- persisted status 为 `ok`，只含实际已持久化的 relay；当前 memory 中的活动普通传输未被查询写回磁盘。 +- `/transfers --unknown` 和未闭合引号参数均输出 `Usage: /transfers`，没有发送网络命令。 +- 日志关闭时仍输出 JSON。 +- 查询前后内存 snapshot、进度文件 SHA-256、文件内容 SHA-256 均不变；原对象未加载磁盘 row。 + +### Client console 实例 + +观察到： + +- 同号 upload/download 独立列出；完整 receiver scope 保留。 +- retained forwarding upload 同时出现在 memory upload 和 client relay；磁盘 snapshot 独立保留该 row，运行时 active/socket/paused 为 null。 +- Client relay 不被伪装成 Server route；未输出 completed 或 send_lock。 +- `/transfers --unknown` 和未闭合引号同样仅打印本地 usage；日志关闭仍输出 JSON。 + +### 离线和并发 + +- 离线实例仅载入复制的进度文件：memory 为空，persisted relay/rows 可见；没有调用恢复入口或建立连接。 +- 查询返回对象被修改后，原内存 snapshot、进度文件和传输文件 hash 保持不变。 +- 两端实际传输暂停后循环查询 API 190 次，随后解除暂停；两条真实传输均返回 True，文件内容相同，owning 进度记录清理。查询期间没有 socket listener 收到 `/transfers`，没有状态回写。 + +Smoke 临时目录由 `TemporaryDirectory` 管理；报告完成后删除脚本。 + +## 编译、静态检查与完整相关套件 + +```bash +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py test/unit/network_api/test_transfer_progress_query.py test/unit/network_api/test_TCP_client.py test/integration/network_api/test_tcp_broadcast_alloc.py +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py test/unit/network_api/test_transfer_progress_query.py test/unit/network_api/test_TCP_client.py test/integration/network_api/test_tcp_broadcast_alloc.py +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py test/unit/transfer_web/test_web_transfers.py -q +``` + +编译已通过。新增查询测试文件单独 `ruff --select F,E9` 已通过；联合 F/E9 检查保留既有 `connect_tcp.py` console `response` F841 和既有 broadcast test 未使用导入，本项未修改无关存量问题。 + +完整相关套件结果：**264 passed，282.21s，退出码 0**。包含新增查询契约、O1/O2 传输回归、普通双向文件/目录、恢复、加密/公钥、转发、异步和现有 Web 消费者；定向测试属于该总数，不重复相加，也不挪用 O1/O2 历史数量。 + +一次性 smoke 脚本及临时目录已删除；只保留永久回归、文档和本报告。 + +## 明确未覆盖与风险 + +- O4 `delete_config` 未实施；不支持按名删除、僵尸自动判定或进度清理。 +- 查询不是一致性事务；内存、运行时开关与磁盘可能来自不同瞬间。 +- 100% 字节、active=false、无 socket、speed=null 均不被解释为僵尸或完成。 +- 不查询其他实例/进程，不保证共享路径独占，不读取源文件/接收文件内容。 +- 不改变 Web `/api/transfers`，不做远程查询，不解决 ACK 应用层收据窗口。 +- 未进行跨平台、负载、全仓测试或文档构建；完整相关套件只覆盖报告命令列出的范围。 + +## 人工闸门 + +O3 证据完成后交付人类审查。未获明确放行不进入 O4，不创建 git 提交，不合并，不推送。 diff --git a/docs/plans/ordinary-transfer-progress-inspection-o4-report.md b/docs/plans/ordinary-transfer-progress-inspection-o4-report.md new file mode 100644 index 0000000..4584a3b --- /dev/null +++ b/docs/plans/ordinary-transfer-progress-inspection-o4-report.md @@ -0,0 +1,118 @@ +# O4 原子报告：按名删除本地配置与主动放弃持久化进度 + +状态：实现、行为回归、独立真实程序与完整相关套件均完成，待人类审查。O1–O3 已获审计通过。本项是补充计划 O4，不是原 FID 大计划的原子更改 4；未创建 git 提交、未合并、未推送。 + +## 提交意图与批准范围 + +人类明确“审计完毕，开始O4 delete_config更改”。Server/Client 增加同名公开 API： + +```python +delete_config(config_name=None, *, is_list_config=False) -> bool | list[str] +``` + +仅列举或删除实例直接管理的进度文件与四种消息/事件日志，不接受任意路径；删除整个进度快照后，旧对象不得再次保存/加载或重启。不增加僵尸判定、逐 fid 清理、远程命令、HTTP 接口或自动删除策略。 + +## 改动与文件范围 + +| 文件 | 内容 | +|---|---| +| `PyFlow/network_api/connect_tcp.py` | 共享白名单解析/删除；两端公开方法；启动/关闭状态追踪；持久化/加载的主动放弃闸门 | +| `test/unit/network_api/test_delete_config.py` | 两端参数化行为回归：列表优先、白名单、文件错误、生命周期与发布/删除竞争 | +| `docs/source/File_Transfer/File_Transfer.rst` | API 返回、删除范围、调用顺序及整份快照放弃的风险 | +| 两端 `docs/source/Network_APIs/TCP_*_APIs.rst` | 公共接口入口与使用边界 | +| `docs/source/changes/transfer-fid-routes.md` | O4 变更记录，不改写前序报告的历史结论 | +| 两份计划、本报告、补充项总报告 | 授权、验收、范围和最终人工闸门 | + +原先拟改既有恢复集成测试；实施时复用其已有真实失败/恢复回归而未增加重复装配。新测试集中在删除 API 的行为边界，真实关闭/删除/新进程验证使用独立烟测。没有删除、跳过或弱化任何既有测试。 + +## 接口与数据流 + +### 列表模式 + +`is_list_config=True` 优先于名字参数，返回排序的 `list[str]`，没有目标则为 `[]`。仅检查白名单目标是否为已存在的普通文件，不改变运行状态、不创建文件、不调用关闭/刷盘。可在运行中使用；结果表示 API 允许操作，不保证随后具有删除权限。 + +### 按名删除 + +当前路径属性的 basename → 白名单准确匹配 → 生命周期与在途操作检查 → 必要的进度/记录锁 → 普通文件检查 → unlink → 返回 True，合法文件本来不存在返回 False。 + +白名单为 `file_trans_progress_file`、`messages_log_file`、`events_log_file`、`sent_messages_log_file`、`sent_events_log_file`。Server 默认进度名为 `server_file_trans_progress.json`，Client 为 `client_file_trans_progress.json`，四类日志默认名字相同。路径重定向后使用实际 basename；同名指向不同路径明确拒绝歧义。 + +无名、空名、未知名、路径、通配符、目录、符号链接抛 ValueError；删除名非字符串抛 TypeError。PermissionError 等文件系统异常直接返回给调用者，不报成功；不因文件内容损坏而禁止明确按名删除。 + +不触及文件传输内容、密钥/TOFU、协议表、端口锁、启动器/Web 配置、数据库。列表不是目录扫描；调用者提供的名称不会被拼接成文件路径。 + +### 关闭与防复活 + +- 仅未启动或明确完成 stop/close 的静止实例允许删除。启动中、关闭中、启动过但未明确关闭、仍有 active fid/socket/relay 时拒绝。 +- 生命周期包装器只在进度锁内更新短状态；不在该锁内执行整个阻塞连接/接受/关闭过程。既有 `_stopped` 及网络行为保留。 +- 进度删除与 persist/load 共用 `_transfer_progress_lock`，等待已经开始的快照 publication 完成后再删除。 +- 进度成功删除或合法目标已缺失时设置仅内存的 `_progress_discarded`。之后本对象 persist/load 分别提前返回、返回 0；旧对象启动入口在任何 socket 操作前明确抛 RuntimeError，要求新实例。 +- 不清空原内存进度/路由，不重置 FID 分配器，不新增 JSON 字段或磁盘 tombstone。只使旧状态不能再用于运行或恢复。 +- 日志删除锁住对应 record store，并拒绝仍有待刷记录的日志；不会建立永久日志禁写标志。重复关闭无新记录时不会重建日志。其他写入者及迟到控制 worker 应先结束。 + +## 定向回归与静态证据 + +```bash +.venv/bin/python -m pytest test/unit/network_api/test_delete_config.py test/unit/network_api/test_transfer_progress_query.py test/unit/network_api/test_TCP_client.py test/unit/network_api/test_TCP_server.py -q +``` + +结果：**101 passed，2.24s，退出码 0**。 + +覆盖两类实例的列表优先、运行中只读、逐个删除五种文件、默认参数与类型错误、未知/路径名称、目录/符号链接、重定向歧义、删除权限异常不损坏原文件或禁用默认保存、重复关闭、迟到 finish/load/persist、删除缺失进度、关闭未完成、活动 fid/socket/relay、未刷日志及发布/删除互斥。进度被其他写入者重新创建时，旧对象也不会读回或覆盖它。 + +```bash +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py test/unit/network_api/test_delete_config.py +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py test/unit/network_api/test_delete_config.py +ruff check --select F,E9 test/unit/network_api/test_delete_config.py +``` + +编译退出码 0。联合 Ruff 退出码 1，仅原控制台 `response` 未使用（F841，当前行 6959），不改无关存量代码、不宣称完整 lint 通过。新增测试单独 F/E9 输出 `All checks passed!`，退出码 0。 + +## 独立真实运行时证据 + +```bash +.venv/bin/python /tmp/pyflow-o4-smoke.py +``` + +结果：正常退出、`O4_SMOKE_COMPLETE`，**50.71s**。脚本使用独立临时目录，实际启动 Server/Client，双向发送消息、事件和文件；仅在最终 ACK 边界注入 False，产生真实的上传/接收保留快照，而非手工构造期望 JSON。 + +路径矩阵：明文/加密 × 上传/下载 × 默认关闭保留/主动删除，共 8 组双端场景。 + +| 路径 | 观察 | +|---|---| +| 运行中列表与删除 | 列表优先于传入的非法类型名字，文件字节和 running 保持；删除抛 RuntimeError | +| 意外控制断线 | Client 已 running=False 且 receive worker 结束，但未显式 close；删除仍被拒绝 | +| 默认关闭对照 | 结束全部 socket worker 后，两端各自快照在新进程中恢复 1 行，原 fid 存在 | +| 主动删除 | 两端各列出确切的进度文件 + 四类日志；删除进度不改变任何未选择日志；逐个删日志不改变其他文件 | +| 重复关闭/迟到调用 | 删除后 persist、load、重复 close/stop 不重建快照；再次删除返回 False；旧对象 start/connect 明确拒绝 | +| 新进程读取删除结果 | 删除后的两端快照各恢复 0 行，没有原 fid | +| 新实例 | 复用原配置位置新建对象后，真实文件成功发送；另一次 ACK 失败的新任务照常持久化，读回只含新 fid | +| 真实权限失败 | 目录 chmod 0500 后，实际 unlink 抛 PermissionError，原文件字节不变；未 mock 文件系统错误 | +| 内容边界 | 源文件与完整接收文件内容相同，清理前后保持 SHA-256；未删除传输内容 | + +新进程读回合计 32 次：16 次关闭后读取（含对照和删除前），8 次删除后读取，8 次新对象失败任务读取。4 组主动删除的双端场景均成功，此外 4 组默认保留场景保持原有恢复能力。加密场景完成真实握手并核对 `enc:` 文件连接标记；未声称本项进行了全路径抓包审计。 + +例：加密下载主动删除组的原 fid 为 9545167982；Server 与 Client 删除前均恢复 1 行、删除后均为 0，源/接收内容 SHA-256 保持 `acd6d25a759648fd2195f328b1d4151f15061d076eaf7c2540e2d5edaa38ab07`。随后新对象成功传输 fid 8974593928，并将新失败任务 fid 13269561224 正常保存读回。 + +## 完整相关回归 + +```bash +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py test/integration/network_api/test_message_event_store.py test/integration/network_api/test_send_message_store.py test/unit/transfer_web/test_web_transfers.py -q +``` + +结果：**332 passed，253.72s，退出码 0**。包括 O1/O2 真实 ACK 回归、O3 查询、两端关闭/连接竞态、普通文件/目录、转发恢复、加密公钥、异步、消息/事件存储及 Web 消费者。101 项定向回归与该总数有交集，不相加为唯一用例数量。 + +## 未覆盖、风险与人工闸门 + +- 整份快照删除会放弃正常中断任务及 relay 恢复关系，不只“僵尸”。用户自行核对两端 O3 输出后选择删除；对端不联动。 +- 共享路径不是实例私有文件。必须停止所有使用同一路径的实例/进程；没有跨进程所有权/隔离，不抵御调用者同时修改实例路径属性或外部文件替换。 +- 进度放弃不清空旧内存，仅封闭旧对象恢复/重启。需要继续运行必须新建对象；日志没有永久删除策略，新消息可以形成新日志。 +- API 不主动关闭/join 传输 worker，不改变网络 teardown 流程；正在关闭的 server 接受循环或活动传输未退出时仍会拒删。 +- 未改 ACK 应用层收据窗口、client_port 依赖、端点迁移、kind/tfid、现行恢复线路、ID 分配权、CI/依赖/覆盖率。 +- 没有新远程协议、HTTP 或控制台删除命令；未做全仓测试、全量 lint、文档构建、跨平台或负载验收。 + +规则沉淀为行为回归：列表不能删；名称不可逃逸白名单；正在运行/关闭的对象不能删；明确放弃后的旧对象不能重写、加载或启动。O4 现提交人类审查，补充项总报告不等于自动通过总验收或允许合并/推送。原 FID 计划更改 4–6 仍等待单独批准。 + +## 证据收尾 + +一次性 `/tmp/pyflow-o4-smoke.py` 已删除；其自建临时目录在 `TemporaryDirectory` 成功退出时清理。保留永久行为回归及本报告，不保留临时测试装配。补充 O1–O4 的总体路径对照和未覆盖风险见 `ordinary-transfer-progress-inspection-report.md`。 diff --git a/docs/plans/ordinary-transfer-progress-inspection-plan.md b/docs/plans/ordinary-transfer-progress-inspection-plan.md new file mode 100644 index 0000000..cc95ca9 --- /dev/null +++ b/docs/plans/ordinary-transfer-progress-inspection-plan.md @@ -0,0 +1,318 @@ +# 普通传输失败保留、进度查询与按名清理：需求卡和增量计划 + +状态：**O1–O3 已获人类审计通过；补充 O4 delete_config 已按批准完成实现、定向回归与独立真实运行验证，待人类审查及补充项总验收。本轮定向 101 passed、完整相关套件 332 passed。不进入原 FID 大计划的原子更改 4–6，未创建 git 提交、未合并、未推送。** + +本任务插入 `transfer-fid-and-resume-unify-plan.md` 的更改 3 与更改 4 之间。更改 3 的历史验收只证明其原有范围，不作为本任务修复或新增查询功能的验收证据。 + +## 1. 需求卡 + +### 目标 + +1. 修复 Server 普通发送收到错误符号后仍可能返回成功、删除上传记录的问题。 +2. 修复 Server 普通接收最终 ACK 明确写出失败后仍删除接收记录的问题。 +3. Server 和 Client 提供相同的公开、本地只读查询接口和控制台命令，查看当前实例全部已有文件进度及相关恢复记录，由用户自行判断是否属于疑似僵尸记录。 +4. 增加 O4：两端提供 `delete_config(config_name=None, *, is_list_config=False)`；列表模式返回可删除文件名 list，删除模式在实例停止后按白名单名字删除指定配置/状态文件，允许用户主动放弃整份持久化文件进度。 + +### 非目标 + +- 暂不解决“ACK 写入 TCP 成功，但对方应用没有收到”的不确定窗口；不增加收据、去重/重传协议、额外 ACK 或等待时限。 +- 不自动判断僵尸、不自动清理，不增加逐记录删除命令；仅 O4 允许显式按白名单删除整份配置/状态文件，不删除传输内容，不触发恢复、重发或连接操作。 +- 不推进原子更改 4–6；现行 kind/tfid、client 转发恢复记录、恢复命令及分配权保持。 +- 不改五项进度值、持久化 JSON 结构、文件字节协议、暂停持久化、端点迁移或加密机制。 +- 不增加远程查询协议、HTTP 端点或 Web 页面；现有 `/api/transfers` 的字段、权限与界面保持不变。 +- 不扫描其他进程、其他实例或整个 `.Flow` 目录；不新增全局注册表、历史完成表、时间戳或状态副本。 + +### 范围与安全边界 + +- 生产改动集中于 `PyFlow/network_api/connect_tcp.py`：Server 的普通发送/接收收尾、两个类的本地查询和控制台分支，以及 O4 的配置列举/按名删除与关闭后进度防复活边界。 +- 文件传输仍经过现有 `send_message`、`receive_message` 与加密入口；复用原进度、socket、暂停和路由状态，不新增发送旁路。 +- 查询可能包含完整本地路径、端点及目标关系，只返回给本进程调用者/本地控制台，不向对端发送，不注册到网络命令处理器，不写入消息/事件记录。 +- 返回结果是独立的 JSON-ready 观察快照，不暴露 socket、锁、Event 或可修改的内部记录引用。它不是跨连接事务快照。 +- O4 的删除只面向实例显式管理的配置/状态白名单，不接受任意路径，不触及密钥、协议表、用户数据库或传输内容；默认关闭不删除，用户明确选择删除才放弃恢复。共享路径的其他写入者必须先停止，不能声称实例属性代表独占文件。 + +### 风险等级与禁止事项 + +风险：中；O4 的显式删除属于不可逆的数据清理路径，须重点审查名称映射与关闭后写回。普通传输收尾影响失败保留与重连；接收端作用域、旧磁盘快照和运行时状态不能混淆。 + +禁止：删/跳过测试、弱化断言、改 CI/依赖/coverage、绕过统一收发入口、把字节 100% 当作完成、把无活动 socket 当作僵尸证明、静默忽略不可读进度文件、查询时加载/恢复/覆写状态、任意路径删除、运行中清理恢复文件、自行提交或进入下一原子更改。 + +## 2. 现状证据与修复决定 + +### 2.1 Server 普通发送 + +实施前，`TCP_Server_Base.file_transfer_mode` 的“等待开始符号”和“等待最终确认”两个循环中，错误符号分支调用 `close_socket()` 后只是 `break`；后者可继续执行 `file_sent = True`、成功日志和成功清理。O1 已将两处分支改为立即 False。Client 对应两个分支此前已经返回 False，本任务不重复改写已修逻辑。 + +决定：Server 两处错误符号分支立即返回 `False`。只有真实收到既有最终确认符号才能设置 `file_sent = True`。保留现有 `finally` 的 owning-entry 清理：已经登记且具备 peer 的失败上传保留，尚未登记的开始阶段失败不凭空创建记录，源文件不存在的原有处理保持。 + +数据流：普通文件/目录内文件发送入口 → 同一 `file_transfer_mode` → 开始符号 / 字节发送 / 最终确认 → 错误立即失败，成功 ACK 才成功收尾。 + +### 2.2 Server 普通接收 + +实施前，`TCP_Server_Base.file_transfer_mode_recv` 内部接收函数在文件写完后就设置 `finished_ok = True`，随后忽略最终 ACK 发送的布尔结果；发送异常也被局部捕获，导致 `finally` 删除接收进度。O2 已删除提前赋值，复用 Client 既有 `finished_ok = bool(self.send_message(...))` 模式。 + +决定：复用 Client 的现有模式，Server 只有最终 ACK 明确写出成功才将 `finished_ok` 置真。`False` 或异常时，利用既有 `_file_progress_finish(..., keep=True)` 保留并持久化可恢复接收行，即使字节数已等于总大小。完整文件不删除；记录继续指向原完整路径。 + +保持通知、事件和公钥 TOFU 注册先于 ACK 的顺序；公钥推送仍不创建普通文件进度。数据已接收的通知不升级为“对端已收到 ACK”的证据。 + +数据流：Client 文件/目录项上传 → Server 写入原目标文件 → 接收通知及必要的公钥注册 → 尝试原最终 ACK → 成功清理 / 明确失败保留 → 后续既有稳定 fid 恢复可在 EOF 重新确认。 + +### 2.3 不确定窗口的处理边界 + +最终 ACK 写出成功不等于对端应用确认收到。这一窗口暂不解决，可能出现一端已清理、另一端保留的状态。新增查询使用户能核对两端现有记录，但不声称消除不确定性或补齐分布式完成协议。 + +## 3. 新增公开查询契约 + +两个类分别提供同名方法，签名一致： + +```python +get_transfer_progress(self, *, include_persisted=True) -> dict +``` + +- 查询本实例；不要求连接已经建立，也不因查询启动网络连接。 +- 默认同时展示内存记录和本实例 `file_trans_progress_file` 的磁盘快照；`include_persisted=False` 不读取磁盘。 +- 内存与磁盘分开显示，不合并、不覆盖、不将磁盘记录恢复进内存。两者可能包含同一任务的不同时刻值，不能相加为任务数。 +- 复用 `_transfer_progress_snapshot` 的现有序列化来源和 `_read_transfer_progress` 的既有磁盘格式规则；新增展示字段不落盘。不调用 `_load_transfer_progress`、`_persist_transfer_progress` 或 `_request_resumes`。 + +### 3.1 返回结构 + +```text +{ + progress_enabled: bool, + speed_enabled: bool, + memory: { + uploading: [row, ...], + downloading: [row, ...], + relays: [record, ...] + }, + persisted: { + path: str, + status: "ok" | "missing" | "error" | "not_requested", + snapshot: {uploading: [...], downloading: [...], relays: [...]} | null, + error: str | null + } +} +``` + +内存和磁盘中的进度 row 采用相同字段： + +| 字段 | 语义 | +|---|---| +| `fid` | 原始整数传输 ID，不截断、不重新分配 | +| `receiver` | 上传 key 中的实际下一跳接收端；下载为 null | +| `peer` | 现有五项进度值中的逻辑对端，不与 receiver 混用 | +| `name` / `path` | 文件名和完整的已有本地路径 | +| `transferred` / `total` / `speed` | 原进度值；speed 未知或关闭时为 null | +| `active` | 该进度 key 是否位于本实例活动 fid 集合 | +| `socket_registered` | 该 key 是否仍在本实例活动传输 socket 表中；不是远端存活证明 | +| `paused` | 上传的本地暂停开关；下载不推断远端暂停状态，填 null | + +磁盘不持久化运行时活动/socket/暂停状态，因此磁盘 row 的 `active`、`socket_registered`、`paused` 均为 null，不伪装成 False。内存字段仅描述查询时观察到的本地状态，不增加完成 ACK 状态字段。 + +### 3.2 “全部已有记录”的边界 + +- 内存上传和下载全部列出，包括活动、暂停、中断、恢复读回及字节 100% 仍保留的记录，不只展示 active 项。 +- 上传按完整 `(receiver, fid)` 展示；不同 receiver 的同号记录不折叠。下载和上传也不按裸 fid 合并。 +- Server 的 `relays` 列出权威 `_fid_routes`：上游 fid、originator、完整目标与各下游 fid、completed；不把反向索引重复算作记录，不编造 Server 未保存的文件路径或逐文件字节进度。 +- O3 当时的 Client `relays` 元数据查询契约现由原 FID 计划更改 4 替代：Client `relays` 为空，转发进度只来自既有上传行，不新建替代表；Server 完成集合查询不变。 +- 进度开关关闭时仍返回所有实际已有记录，并明确 `progress_enabled=False`；本来未记录的传输无法补查。公钥推送不生成虚假进度。 +- 已成功删除的记录不是历史任务，不能查询回来。磁盘文件不存在与存在但为空分开表达；损坏、权限错误或不支持的旧格式显示 `error` 和文件路径，内存结果仍可查看，原文件不被改写。 +- 不新增最近更新时间、僵尸评分或“已确认完成”推断。`active=False`、socket 未登记、speed 为零/null、100% 字节均不足以独立证明是僵尸。 + +### 3.3 并发与开销 + +复用现有锁和既有快照顺序；只在需要的锁内复制对应状态，释放后格式化和输出，不在路由/暂停锁内读磁盘或打印。沿用 `_transfer_progress_lock` → Server 路由锁的快照顺序，不引入反向嵌套。更改 4 删除 Client 恢复元数据锁；瞬态 `_resume_sends` 使用进度锁。暂停状态单独观察。 + +返回的是可能随传输变化的观察快照，不承诺内存、磁盘、暂停开关在同一时刻原子一致。不遍历 socket 做探测、不读取文件内容或计算文件 hash，不添加周期扫描/后台线程。 + +## 4. 本地控制台命令 + +```text +/transfers +``` + +- Server 的 `console_input` 与 Client 的 `interactive_mode` 均在本地处理，调用同一个公开查询契约。 +- 输出完整、缩进的 JSON，包含内存与磁盘两部分，不截断记录；使用直接命令输出，不依赖 debug 或普通日志开关。 +- 不带过滤器、不增加删除命令。附带不支持的参数时打印用法，不回退成网络聊天消息。 +- Server 本地 `/help` 增加说明。Client 的既有 `/help` 仍发给 Server，本次不另建 Client 帮助系统；在 Client API 文档写明本地 `/transfers` 用法。 +- 命令依赖既有控制台循环在运行；实例未连接/控制台未启动时通过公开 Python 方法查询，不新增独立离线 CLI 或远程管理通道。 + +数据流：本地 API/控制台 → 既有内存快照 + 可选只读磁盘快照 → 带作用域和来源的 JSON → 用户核对。没有状态回写或网络出口。 + +## 5. 路径全集与验收映射 + +以下为本任务必须覆盖的路径；文件/目录项、普通/恢复、明文/加密按同一收发核心执行的组合验证，不把单个样例当作全集证明。 + +| 路径 | 必须修改点或保持项 | 验收证据 | +|---|---|---| +| Server→Client：开始符号阶段错误 | O1：立即 False；不发送后续文件元数据或生成虚假进度 | 真实 socket 返回错误符号；发送函数结果为 False；无成功事件 | +| Server→Client：文件发完后错误 | O1：立即 False；已有可恢复上传不删 | 文件/目录项、首次/恢复分别执行；结果 False，完整 scoped key 保留并可读回 | +| Client→Server、Server→Client 正常文件与目录项 | 保持两端成功 ACK 与清理语义；Client 已修分支不重复实现 | 两向真实传输，内容相同；正确 owning 记录清理，同号其他接收端不受影响 | +| Client→Server：最终 ACK 返回 False 或抛异常 | O2：以发送结果决定 finished_ok | 两种失败均保留完整目标文件、原 fid/peer/path/字节数和持久化接收行 | +| 已完整接收记录的恢复与再次失败 | O2：沿用现有 EOF 确认，不改内容和路径 | 原 fid、原完整路径恢复；EOF 不追加内容；再失败仍保留，成功后清理 | +| 数据未齐、断线、超时与源文件不存在 | 保持既有失败/保留边界，不扩展超时或重试 | 既有相关回归及局部真实失败场景；不提前报成功 | +| 上述双向普通文件及关键 ACK 失败路径的加密模式 | 继续经过既有安全收发入口 | 真实加密连接传输/失败保留/恢复；握手成功，不新增明文发送旁路 | +| `/crypto_pub_key` 接收与 ACK 失败 | O2：TOFU/ready 不因 ACK 失败跳过，仍不记普通进度 | 既有公钥 ACK 丢失回归及真实握手；普通进度中无公钥行 | +| 转发零/部分完成、原 ID 恢复 | 收尾改动不回退更改 3；O3 展示而不改变路由 | 现有逐目标 ACK、完成集合及稳定 ID 回归；查询后状态未变 | +| Server/Client 内存查询 | O3：上传/下载/各自 relays，含活动、暂停、中断、100% 保留记录 | 同号不同 receiver、跨方向同号均分别列出；活动标志正确，未知不编造 | +| Server/Client 磁盘查询与尚未加载记录 | O3：默认读取但不恢复；内存和磁盘分离 | 含磁盘独有行和落后快照；查询前后内存状态及磁盘文件字节不变 | +| 磁盘缺失/空文件快照/损坏/不支持格式/读失败/跳过读取 | O3：missing、ok 空快照、error、not_requested | 各状态显式可区分；内存可见；不覆盖/删除旧文件 | +| 未连接、进度开关关闭、速度关闭、空记录、并发完成/恢复 | O3：不因查询启动连接；不虚构未记录任务或强一致性 | 实例 API 可直接查询；速度未知为 null；反复查询不改变传输及记录生命周期 | +| 两端本地控制台 | O3：本地拦截 `/transfers`；参数错误也不发网 | 实际控制台输入、观察完整 JSON；普通日志关闭时仍可见；对端不收到该命令 | +| 现有 Web `/api/transfers` | 本轮不替换、不改字段或权限，不经 Web 暴露完整诊断数据 | 既有 Web 回归保持，接口继续原有轻量进度输出 | + +## 6. 原子拆分与文件列表 + +按下表顺序执行，每项完成局部验证和报告后停在人工审查闸门；不自行创建 git 提交，也不把一次批准视作可越过逐项审查。 + +| 项 | 单一意图 | 修改/新增文件 | 预计代码、测试及接口说明改动 | 局部验证 | +|---|---|---|---|---| +| O1 | Server 普通发送只按最终 ACK 成功 | 修改 `PyFlow/network_api/connect_tcp.py`、`test/integration/network_api/test_tcp_transfer_resume.py`、`docs/source/File_Transfer/File_Transfer.rst`、`docs/source/changes/transfer-fid-routes.md`；新增 `docs/plans/ordinary-transfer-progress-inspection-o1-report.md` | 40–120 行，另附报告 | 开始/最终阶段错误、正常成功、文件/目录项、恢复及加密的真实发送行为 | +| O2 | Server 普通接收 ACK 明确失败后保留 | 修改上述生产/恢复测试/传输说明/变更说明；新增 `docs/plans/ordinary-transfer-progress-inspection-o2-report.md` | 80–180 行，另附报告 | False/异常、原路径保留、进程读回、EOF 再确认、加密和公钥 ready | +| O3 | 两端公开本地完整只读观察入口 | 修改 `connect_tcp.py`、`test/unit/network_api/test_TCP_client.py`、`test/integration/network_api/test_tcp_broadcast_alloc.py`、`docs/source/File_Transfer/File_Transfer.rst`、`docs/source/Network_APIs/TCP_Server_APIs.rst`、`docs/source/Network_APIs/TCP_Client_APIs.rst`、`docs/source/changes/transfer-fid-routes.md`；新增 `test/unit/network_api/test_transfer_progress_query.py` 和 `docs/plans/ordinary-transfer-progress-inspection-o3-report.md` | 300–450 行，另附报告 | scoped identity 不合并、内存/磁盘分离、只读错误边界和两端实际命令输出；本轮新增回归已运行 | +| O4 | 两端按名列举/删除本地配置与关闭后放弃进度 | 修改 `connect_tcp.py`、两端 Network API 文档、文件传输说明和变更说明；新增 `test/unit/network_api/test_delete_config.py`、O4 原子报告与补充计划总报告；既有恢复集成测试复用且不改写 | API/生命周期为单一 cutover；回归覆盖调用边界和持久化竞争 | list 模式只读、按名白名单与错误边界、停止后清理、重复关闭/迟到写回不复活、新进程不恢复旧任务；详见第 9 节 | + +四个原子项共同更新本计划的状态；批准后产生本任务总报告 `docs/plans/ordinary-transfer-progress-inspection-report.md`。O4 是本补充计划的第四项，不等同于原 FID 大计划的原子更改 4。如实际规模超过约 500 行或出现新执行路径，先修订拆分，不边实现边扩张。 + +既有 `test_tcp_file_transfer.py`、`test_crypto_tcp.py`、Web 与转发测试先复用回归，不为相同路径新增重复测试；仅当必要的 consumer-visible 契约确实缺失时补充相应行为用例。永久测试关注误判成功、丢失可恢复记录、作用域混淆、只读边界与错误状态，不增加源码/文案/别名转调或格式快照测试。 + +## 7. 验收命令与运行时证据 + +以下是**批准后执行的命令和场景，不是本轮已执行结果**。 + +定向与完整相关回归： + +```bash +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py -q +.venv/bin/python -m pytest test/unit/network_api/test_transfer_progress_query.py test/unit/network_api/test_TCP_client.py test/integration/network_api/test_tcp_broadcast_alloc.py -q +.venv/bin/python -m pytest test/unit/network_api/test_delete_config.py test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_message_event_store.py test/integration/network_api/test_send_message_store.py -q +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py test/unit/transfer_web/test_web_transfers.py -q +``` + +- O1/O2 批准后先保留能击中旧行为的契约回归，修复后同路径通过；历史报告的成功数量不挪用为新证据。 +- 独立临时 smoke 驱动真实 Server/Client 与文件传输：只在指定的最终符号发送边界注入 False/异常，不 mock 整条传输。记录函数结果、保留的 scoped key、文件路径/大小/内容 hash、磁盘快照、EOF 恢复后的最终清理；加密场景同样真实运行。 +- O2 用新实例/新进程读回失败保留的记录，证明不是只存在于当前内存；不替换成手工构造的一份“期望快照”。 +- O3 的 smoke 通过两个真实控制台输入 `/transfers`，观察完整输出、磁盘读取状态及对端没有收到查询命令；另直接调用离线实例 API。用查询前后文件 hash 和状态比对证明只读。 +- 汇总观察值和实际执行命令到各原子报告；不以“测试通过”替代逐路径运行时证据。Smoke 临时文件在证据记录后删除。 +- 对实际改动 Python 文件执行编译和静态检查。既有 F841 等存量问题单独列出，不顺手修复，不将既有 lint 失败改报为通过。 + +## 8. 接口、兼容性及人工闸门 + +- 新公共契约为两端 `get_transfer_progress`、`delete_config` 和本地 `/transfers`；需人类批准后实施。既有公开方法签名不变；普通发送返回值纠正为其既定成功语义，O4 明确放弃进度后的旧对象不得重新启动/连接，必须新建实例。 +- 不触及 CI、依赖、覆盖率策略、五项持久化模型或远程协议。既有 Web 端点与 UI 刻意不改，避免把完整路径/历史快照扩散到其现有可访问范围。 +- 规则沉淀采用行为回归：错误符号不能成功清理、ACK 明确写出失败不能丢接收记录、查询不能恢复/写回或合并不同作用域身份、列表模式不能删除、显式清理后重复关闭/迟到写回不能复活进度文件。 +- 明确未覆盖：应用层 ACK 收据、自动去重/重传、逐 fid 人工删除、全局僵尸判定、跨实例/进程独占与共享文件隔离、变更端点迁移、历史完成任务以及未被进度开关记录过的任务。 +- 人类已审计通过 O3 并明确“开始O4 delete_config更改”，现仅执行补充 O4；实现、局部验证与原子/总体报告完成后交付人工审查。原 FID 大计划的更改 4–6 仍单独等待批准，不自行提交、合并或推送。 + +## 9. O4 补充：按名删除本地配置/状态文件 + +人类已确认此公共契约并明确开始 O4：Server 与 Client 均增加 `delete_config`；按传入的配置文件名删除；`is_list_config=True` 时只返回可删除文件名组成的 list，默认 False。用户可在关闭实例后主动删除持久化进度快照，放弃其中疑似僵尸任务的恢复记录。不增加自动清理或远程删除命令。 + +### 9.1 方法与返回值 + +```python +delete_config(self, config_name=None, *, is_list_config=False) -> bool | list[str] +``` + +- `is_list_config=True` 优先进入纯查询模式，即使同时传入名字也不删除；返回本实例白名单中当前存在、可作为删除目标的文件名，排序后以 `list[str]` 返回，没有则返回 `[]`。这里表示 API 允许删除，不保证调用者的文件系统权限；实际删除失败须报告。 +- 默认 `is_list_config=False`，每次明确传入一个文件名；成功删除返回 True，白名单内文件本来不存在返回 False。没有名字、空名字、未知名字或带路径的名字抛 ValueError；权限等实际文件系统错误向调用者抛出，不吞掉并声称成功。传入非字符串名字抛 TypeError。不提供默认删除全部、通配符、目录递归或批量删除。 +- 列表模式可在运行中调用，且不创建目录、文件、连接或关闭实例。删除模式只允许实例未启动或已显式 `stop()` / `close()` 完成；运行中、仅连接意外断开或仍有活动传输/中继时抛 RuntimeError，不隐式关闭或取消任务。 + +### 9.2 可删除范围与按名解析 + +范围采用 TCP 实例直接管理的可丢弃持久化状态白名单，不扫描 `.Flow` 中所有文件，也不把所有 `.json` 都当作配置。 + +| 实例路径属性 | 默认可传入的文件名 | 删除含义 | +|---|---|---| +| Server 的 `file_trans_progress_file` | `server_file_trans_progress.json` | 放弃本文件内全部上传、下载与 server relay 路由恢复记录 | +| Client 的 `file_trans_progress_file` | `client_file_trans_progress.json` | 放弃本文件内全部上传、下载与 client 转发恢复元数据 | +| 两端 `messages_log_file` | `messages_log.json` | 删除已落盘的接收消息记录 | +| 两端 `events_log_file` | `events_log.json` | 删除已落盘的接收事件记录 | +| 两端 `sent_messages_log_file` | `sent_messages_log.json` | 删除已落盘的发送消息记录 | +| 两端 `sent_events_log_file` | `sent_events_log.json` | 删除已落盘的发送事件记录 | + +名称取实例当前路径属性的 basename，实际删除使用该属性已配置的完整路径；支持既有实例文件路径重定向，不根据用户输入拼任意路径。Server 默认不列出/删除 Client 的进度文件，反之亦然。若两种属性重定向为相同 basename、却指向不同文件，则拒绝歧义，不猜测或一次删除两个文件。符号链接或目录不作为普通删除目标，删除时明确拒绝，不跟随链接删除其目标。 + +不纳入：协议必需的 `decode_command_table.json`、端口分配/锁文件、私钥/公钥/TOFU 注册表、启动器 `setup.json`、Web 配置/用户数据库/登录凭据、其他实例的文件以及传输的源文件或接收文件。扩大白名单须另行明确批准；本 API 不是任意文件删除工具。 + +当前默认路径由多实例共享，四种消息/事件日志还可能由 Server 与 Client 共用；对象归属不等于磁盘文件独占。删除前必须停止所有使用同一路径的实例/进程，否则其他写入者仍可能重建它。本次不建立跨进程文件所有权或隔离系统。 + +### 9.3 关闭、删除和防止进度复活 + +推荐顺序：O3 查询两端内存与磁盘记录 → 用户判断并备份仍需恢复的信息 → 停止涉及的实例并确认无活动传输/中继 → `delete_config` 删除指定文件 → 新建实例启动。列表/删除均为本地方法，不增加 `/delete_config`、HTTP 端点或在关闭时无条件清理的默认行为。 + +现有 `stop()` / `close()` 会调用 `_persist_transfer_progress()`；仅在关闭前 unlink，或者关闭后 unlink 但允许重复关闭/迟到 finally 继续写回，均不能达到清理目的。因此 O4 包含必要的生命周期约束： + +1. 没有请求删除时，关闭仍按原规则保存所有可恢复记录；不改默认持久化语义。 +2. 在 `_transfer_progress_lock` 内串行检查静止条件并删除进度文件；必须与正在执行的快照写回互斥，不能留下随后覆盖回来的临时快照。 +3. 成功删除、或明确删除的进度文件已不存在时，当前对象进入“此进度快照已主动放弃”的终结状态；其后 `_persist_transfer_progress` / `_load_transfer_progress` 不再写回或恢复这份已放弃快照。此状态仅存于当前对象，不写 tombstone 文件、不修改 JSON 格式，也不重置全局 FID 分配器。 +4. 同一对象重复 `stop()` / `close()` 和迟到收尾不得重建进度文件；主动放弃进度后，重新启动/连接该对象必须明确拒绝，使用新实例恢复正常工作，避免旧内存任务在新会话复活。不把原对象内存记录静默挪到新实例。 +5. 删除是文件级放弃，不逐 fid 识别僵尸，也不承诺只删僵尸。完整/部分接收文件和源文件始终保留；对端的进度快照不会被联动删除，需分别检查和操作。 +6. 消息/事件日志删除只清理指定落盘文件；不会改变消息协议、删除其他日志或建立永久禁写策略。关闭刷盘后执行；重复关闭没有新记录时不应生成被删日志，若迟到产生新消息记录则须先完成相关 worker 收尾再操作。 + +### 9.4 O4 路径与验收 + +| 路径 | 必须观察的证据 | +|---|---| +| Server / Client 列表；默认参数；同时传入名字和 True | 返回真实 list，仅含本实例白名单中存在的合法普通文件；列表查询前后文件字节、内存及运行状态不变，不能误删 | +| 关闭后分别按名删除进度和四类日志 | 指定文件消失、未指定文件字节不变；结果 True；新实例不再读回被删进度 | +| 合法文件缺失与重复删除 | 返回 False；不新建文件；对进度仍落实明确放弃后的防复活边界 | +| 无名字、非法类型、未知名字、绝对路径、`..`、通配符、目录/符号链接、重定向 basename 冲突 | 明确报错，范围外文件与链接目标不变 | +| 无删除权限等真实 I/O 错误 | 异常可见，不报成功，不留下已经删除的假象；不更改其他目标 | +| 运行中、意外掉线未关闭、关闭仍有活动传输/中继 | 拒绝删除，进度/路由不丢失；列表模式仍只读可用 | +| 与写回串行、重复关闭、迟到持久化/读取、同对象重启 | 删除返回后文件不复活；已放弃对象不能重启；新对象仍可正常保存新的任务 | +| 用户判断后放弃进度；磁盘格式损坏或不支持 | 不要求能够解析快照才允许按白名单删除；源文件/接收文件和其他配置保持不变 | +| 默认关闭未调用删除 | 可恢复进度照常写回并可由新实例读回，不弱化既有恢复 | + +O4 是独立原子项，接在 O3 后，不是原 FID 大计划中的“原子更改 4”。生产变更集中于 `connect_tcp.py`；新增 `test/unit/network_api/test_delete_config.py`，同步两端 API 文档、文件传输说明、变更说明与报告。既有恢复集成测试不重复加同路径装配，保留并完整回归；真实关闭/删除/新进程验证由独立烟测覆盖。生命周期与删除必须同次 cutover，不分离出会复活进度的 unlink-only 接口。 + +批准后的定向验收命令： + +```bash +.venv/bin/python -m pytest test/unit/network_api/test_delete_config.py test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_message_event_store.py test/integration/network_api/test_send_message_store.py -q +``` + +独立运行时 smoke 在临时目录用真实实例产生文件进度及日志，停止并完成 worker 收尾后列举和删除指定文件；观察重复关闭/写回不复活、新进程不恢复旧任务、其他配置与传输内容 hash 不变。用默认未删除的实例作保持原持久化行为的对照。报告只记录实际执行的结果，不把计划中的验收当成已通过。 + +### 9.5 已实现用法（默认文件名） + +```python +# 列表模式只读；运行中也可以查看。 +server_names = server.delete_config(is_list_config=True) +client_names = client.delete_config(is_list_config=True) + +# 由用户决定放弃整份进度；先关闭并确认活动传输/中继已退出。 +server.stop() +server.delete_config("server_file_trans_progress.json") + +client.close() +client.delete_config("client_file_trans_progress.json") +``` + +示例使用默认文件名；重定向路径时使用列表返回的实际 basename。未选择的日志不会被删除。新一次运行创建新的 Server/Client 实例，不复用已放弃进度的对象。 + +## 10. O1 已批准实施记录 + +- 授权：人类明确“审计完毕，开始01的更改”。本次只修复 Server 普通发送的开始等待与最终确认等待两处错误符号退出；不改接收端 ACK 生命周期，不实施查询或删除 API。 +- 改前真实回归：文件/目录项、offset=0/4096 四种组合均收到真实错误符号，却返回 True 并打印成功日志；原子报告记录命令与改后证据。 +- 实现：两处 `break` 改为 `return False`；既有 `finally` 的 owning-entry、失败持久化、活动 socket/暂停清理不变，公共方法签名、线路与最终 ACK 时限不变。 +- O1 验收报告：`docs/plans/ordinary-transfer-progress-inspection-o1-report.md`。当时新增回归 4 passed、完整相关套件 237 passed，独立真实程序验证明文/加密、文件/目录项、首次/恢复及 EOF 再确认。后续人类已审计通过 O1 并批准开始 O2；该历史数量不作为 O2 本轮证据。 + +## 11. O2 已批准实施记录 + +- 授权：人类明确“审计通过，开始02”。仅修复 Server 接收最终 ACK 明确失败后丢失接收行的问题，不更改协议、持久化结构、ACK 时限或 O1 发送端语义。 +- 决定：复用 Client 接收端现有模式，`finished_ok` 仅依据最终 ACK 的实际发送布尔结果设置；保持本地通知、事件更新和公钥注册在 ACK 前的顺序。 +- 本轮验收：新增回归 4 passed、完整相关套件 241 passed；独立真实程序验证 16 组明文/加密、文件/目录、首次/恢复、False/异常及重复 EOF 确认，并另起进程读回真实失败快照 32 次。进度关闭与连接前注入的公钥 ACK 失败均已运行。编译通过，F/E9 仅既有 F841；证据见 `docs/plans/ordinary-transfer-progress-inspection-o2-report.md`。一次性脚本已清理，O2 待人工审查，不进入 O3。 + +## 12. O3 实施与验收记录 + +- 授权：人类明确“审计通过，开始更改03”。仅增加 Server/Client 的同名本地 `get_transfer_progress` 与 `/transfers`；不增加远程协议、HTTP 端点或删除功能。 +- 复用两端 `_transfer_progress_snapshot` 和 `_read_transfer_progress`。返回的 relay 列表由既有快照生成，每条用整数 `fid` 标识快照主键；Server 保持 originator/targets/completed，Client 保持既有恢复元数据，不统一成虚假字段。 +- 运行时活动/socket 状态在进度锁内复制，暂停状态单独读取；磁盘读取不持有路由/暂停锁。只读格式化共用一个实现，不更改持久化来源或 Web 输出契约。 +- O3 当轮 49 个局部回归、独立真实 API/控制台烟测与完整相关套件 264 passed 已完成，证据见 `docs/plans/ordinary-transfer-progress-inspection-o3-report.md`；随后人类已审计通过并授权开始 O4。该历史数量不替代 O4 本轮验收。 + +## 13. O4 批准实施记录 + +- 授权：人类明确“审计完毕，开始O4 delete_config更改”。O1–O3 已通过人工审查，本项实施范围以第 9 节为准，不实施原 FID 大计划的原子更改 4。 +- 生命周期边界：区分未启动、启动过但尚未关闭、关闭正在执行和关闭完成；只允许静止实例删除。进度的删除与持久化/加载共用既有进度锁，明确放弃后旧对象不再保存/加载，启动入口在网络操作前报错。 +- O4 本轮结果：定向回归 101 passed，完整相关套件 332 passed；独立真实程序在明文/加密、上传/下载、默认保留/主动删除 8 组双端场景中验证关闭、按名清理、32 次新进程读回及新实例正常运行，实际权限失败可见且原文件不变。编译与新增测试 F/E9 通过；联合检查仅既有 F841。证据见 O4 原子报告，补充项路径全集与风险见总报告。O4 和补充项总验收待人类放行；临时烟测脚本已删除。 + diff --git a/docs/plans/ordinary-transfer-progress-inspection-report.md b/docs/plans/ordinary-transfer-progress-inspection-report.md new file mode 100644 index 0000000..0b1515a --- /dev/null +++ b/docs/plans/ordinary-transfer-progress-inspection-report.md @@ -0,0 +1,105 @@ +# 普通传输失败保留、进度查询与按名清理：总体报告 + +状态：O1–O3 已获人类审计通过；O4 已按批准完成实现与局部验收，待人类审查及补充项总验收。未创建 git 提交、未合并、未推送。本报告仅涵盖补充 O1–O4，不代表原 FID 大计划的原子更改 4–6 已实施。 + +## 目标与总体数据流 + +四项共同完成以下行为： + +1. Server 普通发送只有观察到原最终 ACK 才成功，错误符号不能误入成功清理。 +2. Server 接收只有最终 ACK 明确写出成功才清理；失败时即使完整文件已落盘也保留恢复行。 +3. Server/Client 提供一致的本地只读查询；内存进度、磁盘快照与运行时状态分开显示，不自动判定僵尸。 +4. 用户可在停止实例后按白名单名字删除文件级状态；放弃进度后旧对象不能使旧任务复活，新对象保持正常运行能力。 + +传输流：现有文件/目录/恢复入口 → 既有安全收发 → 本地文件与原五项进度行 → 原最终 ACK → 成功清理 / 明确失败保留 → 原 fid、原路径 EOF 再确认。 + +观察流：本地 API 或 `/transfers` → 既有内存快照 + 可选磁盘只读快照 → 带身份/来源的独立 JSON-ready 结果。不连接、不加载、不恢复、不写回。 + +清理流：用户核对并备份 → 停止所有共享文件的写入者，等待任务收尾 → `delete_config` 白名单解析及生命周期检查 → unlink 指定文件 → 仅在删除/确认缺失进度时永久封闭旧对象的 persist/load/start。源文件与接收文件不删除。 + +## 原子项汇总 + +| 项 | 实施内容 | 当前人工闸门 | 证据 | +|---|---|---|---| +| O1 | Server 两处错误符号分支立即 False；保留现有失败持久化 | 已审计通过 | `ordinary-transfer-progress-inspection-o1-report.md` | +| O2 | Server 接收最终 ACK 的布尔结果决定 finished_ok；失败保留原接收行 | 已审计通过 | `ordinary-transfer-progress-inspection-o2-report.md` | +| O3 | 两端 `get_transfer_progress` 与本地 `/transfers`，内存/磁盘及上传作用域不混淆 | 已审计通过 | `ordinary-transfer-progress-inspection-o3-report.md` | +| O4 | 两端 `delete_config`、只读列表、按名删除、关闭/写回互斥及进度放弃闸门 | 实现和局部证据完成，待审查 | `ordinary-transfer-progress-inspection-o4-report.md` | + +以上是原子实施切片，不是已经由本代理创建的 git commit。未跨越人工审查自动进入下一个原 FID 切片。 + +## 需求卡路径逐条对照 + +| 路径 | 结论 | 运行时或机械证据 | +|---|---|---| +| Server 普通发送开始符号错误 | 覆盖 O1 | 真实错误后 False，二进制发送尝试为 0；首次不造记录，恢复时不破坏原部分文件 | +| Server 发送全部字节后错误 | 覆盖 O1 | 文件/目录项、初次/4096 恢复均保留 scoped 上传及磁盘行；改前误返 True 的 4 例改后通过 | +| 两向正常文件/目录与原有恢复 | 覆盖相关回归 | 本轮完整套件 332 passed；独立程序新对象正常双向文件传输及新失败状态保存 | +| Server 最终 ACK False/异常 | 覆盖 O2 | 4 例改前 KeyError 丢行、改后保留；独立 16 组明文/加密、文件/目录、首次/恢复组合 | +| 文件完整后的重复 EOF 再确认 | 覆盖 O1/O2 | 失败再失败仍保留，成功才清理；原 fid、完整路径、内容不变 | +| 中途断线、超时、源文件不存在 | 保持既有边界 | 既有真实 socket 回归继续通过；本任务不扩展超时或源文件生命周期 | +| 加密收发及关键失败恢复 | 覆盖本任务运行场景 | O1/O2 独立程序真实握手/帧流验证；O4 明文/加密双向删除与新实例运行；非全局安全审计 | +| 公钥注册和 ACK 失败 | 覆盖 O2 及既有回归 | 在 connect 前注入两种 ACK 失败，实际命中且两端 framed，就绪前 TOFU 已保存，普通进度无公钥行 | +| 转发零/部分完成、稳定 ID 重放 | 保持并展示 O3 | 现有真实转发/恢复回归通过；O3 真实失败转发产生 Server route 与 Client metadata 并显示 | +| 两端全部内存进度 | 覆盖 O3 | 活动、暂停、中断、100% 保留行；同号不同 receiver 和收发同号不折叠 | +| 两端磁盘独有、落后快照 | 覆盖 O3 | 内存/磁盘分开返回，不加载磁盘；返回副本变更不影响原状态 | +| 缺失/空/损坏/不支持/读取失败/跳过读盘 | 覆盖 O3 | 状态区分为 missing/ok/error/not_requested；文件字节不变，内存仍可见 | +| 未连接、开关关闭、空记录和并发变化 | 覆盖 O3 | 离线查询不连接；实际保留记录仍展示，未知为 null；190 次并发观察不影响两条传输完成 | +| 本地两端 console | 覆盖 O3 | 实际 stdin/输出 JSON；日志关闭可见；额外参数和未闭合引号只打印用法，对端未收到查询命令 | +| 现有 Web `/api/transfers` | 刻意保持不变 | 既有 Web 回归通过，不扩散本地完整路径/恢复快照 | +| O4 列表优先与纯只读 | 覆盖 | 两端运行中可列举真实 5 类文件；名字参数不触发删除；回归检查返回 list、排序及文件/状态不变 | +| O4 按名删除进度与四类日志 | 覆盖 | 每次只消失指定文件；未选择的日志字节保持；32 次新进程读回对照默认保留/删除后/新对象任务 | +| O4 缺失与重复删除 | 覆盖 | 合法目标缺失返回 False；进度缺失也封闭旧对象，不能从内存重新持久化 | +| O4 非法类型、名称、路径、符号链接/目录、歧义 | 覆盖边界回归 | 明确报错，外部目标/链接目标不变；支持当前属性的合法 basename 重定向 | +| O4 权限等删除失败 | 覆盖 | 回归确认失败不禁用默认保存；独立真实 chmod 0500 导致 PermissionError，原字节不变 | +| O4 运行中、意外掉线、关闭中、活动传输/中继 | 覆盖 | 真实运行中/Client 意外掉线均拒删；关闭阻塞和 active fid/socket/relay/日志缓冲由行为回归拒删 | +| O4 发布/删除竞争、迟到收尾、重复关闭、旧对象启动 | 覆盖 | 进度锁内先完成 publication 再删；late finish/load/persist、stop/close 不复活；旧对象启动前报错 | +| O4 损坏文件按名清理、内容边界 | 覆盖 | 无须解析损坏进度即可删除；源文件/接收文件及未选择状态保持 | +| O4 默认关闭保留、新实例继续工作 | 覆盖 | 关闭后真实快照各恢复 1 行；删除后为 0；新对象真实传输成功，新失败任务又可恢复 1 行 | + +“覆盖”是上述范围和证据，不表示跨进程所有权、全局僵尸判定或 ACK 应用收据已解决。 + +## 最新整体回归与证据 + +O4 本轮定向命令： + +```bash +.venv/bin/python -m pytest test/unit/network_api/test_delete_config.py test/unit/network_api/test_transfer_progress_query.py test/unit/network_api/test_TCP_client.py test/unit/network_api/test_TCP_server.py -q +``` + +**101 passed，2.24s**。 + +本轮完整相关套件： + +```bash +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py test/integration/network_api/test_message_event_store.py test/integration/network_api/test_send_message_store.py test/unit/transfer_web/test_web_transfers.py -q +``` + +**332 passed，253.72s**。包括新增 O4 与前序相关回归；定向/历史命令有交集，不相加为唯一总数。独立 O4 程序在 50.71s 内完成 8 组双端真实路径、32 次新进程读回、4 组新实例传输及真实权限错误。各项 smoke 的原始观察与限制作于原子报告，历史通过数不替代本轮运行。 + +编译通过；新增 O4 测试 F/E9 检查通过；生产/测试联合 F/E9 仅既有 console `response` F841，不宣称全量 lint 通过。 + +## 架构、接口与副作用 + +- 公共 API 新增两端同名 `get_transfer_progress`、`delete_config`；新增的 `/transfers` 仅本地 console。 +- 进度仍是原五项值，Server route 与 Client metadata 保持既有格式；不新增权威恢复表或持久化“完成/僵尸”字段。 +- 删除通过本地 allowlist，旧对象的终结状态仅内存存在；未选择删除时默认保存保持。 +- 不改协议标志、字节流、ACK 时限、加密入口、Web 合同、CI/覆盖率策略或依赖版本。 +- 非本次迁移：kind/tfid、`/forward_resume`、`/forward_upload`、原 client 恢复表与 ID 分配权。原计划切换仍需分别批准。 + +## 未覆盖与需要接受的风险 + +1. ACK 写入 TCP 不等于对端应用确认；没有新增收据、自动重试或自动清除不确定状态的协议。 +2. 用户判断“疑似僵尸”后删除的是整份进度，包含其中正常中断任务；不可按 fid 部分删除，也不联动删除对端。 +3. 同一路径可能由多对象/进程共享。删除前必须停止所有写入者；本实现不提供隔离、锁定外部进程或阻止其重新创建文件。 +4. 内存状态在进度删除后仍可被观察，但旧对象已终结，不得连接或启动；应新建实例。日志没有永久禁写标志，迟到新消息可形成新日志。 +5. API 不替调用者主动停止/join 全部 worker；运行中或关闭/传输未静止时拒绝。临时网络 teardown 机制未在本次重构。 +6. 查询是观察快照，不是内存/磁盘/暂停开关的原子事务，也不能证明远端存活或完成。 +7. 固定 client_port、端点迁移、源文件外部改变、混合新旧版本、旧断点迁移及中继内容落盘均不在范围。 +8. 未做全仓测试、文档构建、全量 lint、跨平台或负载验收；报告不声称全局加密覆盖审计。 + +## 合并前闸门 + +- 规则沉淀：O1/O2 真实 ACK 行为回归、O3 只读/作用域回归、O4 白名单与进度防复活回归。 +- 文档收尾：文件传输说明、两端 Network API、变更记录、计划和各原子报告已同步。 +- O4 与本总体报告待人类审查；明确接受上述风险并完成最终闸门前不请求合并，不自行提交远端。 diff --git a/docs/plans/test-flake-fixes-report.md b/docs/plans/test-flake-fixes-report.md new file mode 100644 index 0000000..db074ab --- /dev/null +++ b/docs/plans/test-flake-fixes-report.md @@ -0,0 +1,74 @@ +# 测试端修复报告：Windows socketpair 写失败假设 + 加密用例的瞬时态采样 + +状态：已实施，待人工审查；未创建 git 提交、未合并、未推送。本报告记录的测试端修复不含产品改动；后续 ACK 交付修复见 [final-ack-delivery-report.md](final-ack-delivery-report.md)。 + +## 1. 提交意图 + +把三组只在特定平台/时序下失败的测试，改为断言**单调可观测的状态**，使其在任何调度下都给出确定结论，而不是靠轮询撞窗口。 + +## 2. 改动内容 + +### 2.1 `test/unit/network_api/test_fid_routes.py` + +`test_pending_resume_survives_failed_send_and_prunes_completed_or_missing_routes`：`peer.close()` 之后补 `broken.shutdown(socket.SHUT_WR)`。 + +原因：测试需要"发送一定失败"来验证 `_flush_pending_originator_resumes` 保留待发项。POSIX 上对已关闭对端的写入必 EPIPE，Windows 上 `socket.socketpair()` 是 TCP loopback 模拟，第一次 `sendall` 会进本地缓冲并**成功**，`WSAECONNRESET` 要到下一次写才出现。于是 `sent` 变成 1、待发项被按设计移除，`== 0` 失败。 + +`shutdown(SHUT_WR)` 让"本端已不可写"成为 socket 自身的属性。本机实测两种写法（Linux）： + +``` +write after peer close: OSError 32 [Errno 32] Broken pipe +write after shutdown(SHUT_WR): OSError 32 [Errno 32] Broken pipe +``` + +产品侧语义未变：`_flush_pending_originator_resumes` 的 docstring 就是"成功发送才移除"，返回 1 本身是自洽行为。 + +### 2.2 `test/integration/network_api/test_crypto_tcp.py` + +新增两个读取辅助： + +- `_server_streak(server, sock)`：读 `_crypto_state[addr]["decode_failures"]`。 +- `_server_framed(server, sock)`：读 `channel.secure`。 + +关键事实（探针实测）：`decode_failures` 跨重握手**累加**（0→1→2→3），因为 `_crypto_is_protocol_line` 让 `/crypto*` 协议行不触发 `_crypto_note_healthy` 清零。而 `channel.secure` / `_encrypted_sockets` 是瞬时态，50 ms 轮询可能整个错过。 + +三处改写： + +1. `_wait_server_reexchanged`：原来是 `wait_until(sock not in _encrypted_sockets)` 再 `wait_until(sock in _encrypted_sockets)`——两段都在采样瞬时态。改为先等 `decode_failures` 非零（单调），再等 `secure` 恢复（稳定态）。 +2. `test_decode_failure_burst_closes_connection`：注入前仍等 framed（注入在重握手窗口内会被当 stale 丢弃，这是协议设计），注入改为等 `streak >= 预期值 or not client.running`。原断言 `not server_framed()` 在重握手快速完成时永远采样不到。 +3. `test_stale_frame_coalesced_behind_a_plaintext_line_is_consumed`：把 `_crypto_try_server_flip` 的 no-op 替换改成**闸门**（`threading.Event`）。原写法有两个隐患：重握手在替换生效前就完成（注入的帧会被真正解密并二次计数）；以及替换期间若 flip 被调用则被永久吞掉，收尾等 framed 会挂 30 秒。闸门版本在 `finally` 里恢复并 `set()`，两种时序都收敛。 + +## 3. 数据流（以 burst 用例为例） + +```text +wait_until(server framed) # 帧只会在 framed 态被计数 +sendall(garbage frame) + -> server receive_message -> channel.feed -> _consume_frame -> _decode_rsa + -> _FrameError("malformed RSA frame") -> _crypto_channel_error -> _crypto_on_decode_failure + -> channel.disable() + state["decode_failures"] += 1 + -> /crypto_key_exchange 1 -> 对端 ack -> /crypto_ready -> /crypto_frames_start -> enable() +wait_until(streak >= n or not client.running) # 计数是单调量，不会被错过 +n == MAX_DECODE_FAILURES -> do_close -> 连接关闭 -> _wait_disconnected +``` + +## 4. 局部验证 + +- `test_crypto_tcp.py` 全量 **22 passed**，连跑 3 轮（83s / 93s / 92s）全绿。 +- 目标三个用例连跑 5 轮全绿；另 5 轮（闸门版本）全绿。 +- `test_fid_routes.py` + `test_crypto_channel.py` → **38 passed**。 +- ruff：改动前后同为 42 条（全为该文件基线既有问题），新增的两个 helper 已避免引入新的 D401。 + +## 5. 明确未覆盖 + +- **Windows 未实机验证**。`test_fid_routes` 的修复按 Windows 语义推理（`shutdown(SHUT_WR)` 后 send 必失败，WSAESHUTDOWN 10058），但本机是 Linux；需在 Windows CI 确认。加密三例同理——它们在 Windows 上没失败过，本轮改的是"防错过窗口"，不是针对 Windows 特有行为。 +- 未处理 `test_crypto_tcp.py` 里其余瞬时态断言（如 `test_second_connection_passes_tofu` 的 `not in server._encrypted_sockets`）。那条在 `client.close()` 之后，触发是同步的，不在本次失败清单内，故未动。 +- 若 `decode_failures` 在某次重握手中途被清零（目前只在"真实对端业务行"到达时发生），burst 用例会提前 30 秒超时而不是静默通过——这是有意的失败方向。 + +## 6. 风险与待确认 + +- 风险：低。改动只落在测试文件，未触碰产品行为。 +- 需要确认：`_server_streak` 直接读 `_crypto_state` 私有结构。若后续重构该字段名，这两个用例会 `AttributeError` 而不是悄悄失效——比现在的瞬时采样更容易发现，但仍是耦合内部实现的测试写法。是否接受这种耦合，请你定。 + +## 7. 与协议端待办的关系 + +本报告形成时，终态 ACK 交付缺陷尚未处理。随后需求方取消 ACK 发送时机重构，改为继续增量修复交付问题；实际根因、执行范围与验收见 [final-ack-delivery-requirement.md](final-ack-delivery-requirement.md) 和 [final-ack-delivery-report.md](final-ack-delivery-report.md)。本报告的“本轮未处理”不表示当前工作区仍未修复。 diff --git a/docs/plans/transfer-fid-and-resume-unify-atomic-1-report.md b/docs/plans/transfer-fid-and-resume-unify-atomic-1-report.md new file mode 100644 index 0000000..9fa57d4 --- /dev/null +++ b/docs/plans/transfer-fid-and-resume-unify-atomic-1-report.md @@ -0,0 +1,102 @@ +# 原子更改 1 报告：传输状态按接收端隔离 + +状态：实现与局部验收完成；待人类审查。不进入原子更改 2。未创建 git 提交、未合并、未推送。 + +## 提交意图 + +为现有传输进度、活动 socket、暂停与端口匹配补齐接收端作用域，防止不同接收端或收发方向的同数值 fid 相互覆盖和清理。复用既有五项进度值，不新增 client 恢复专用表。 + +## 改动内容与数据流 + +- `PyFlow/network_api/connect_tcp.py`：接收进度仍以本地整数 fid 为键；发送进度、出站活动和暂停使用 `(receiver, fid)`。普通发送、恢复、公钥推送和中继目标端口收集按控制 peer 与 fid 同时匹配端口。接收监听不再清除同号出站端口通知。分配器在已知接收端作用域中检查占用，保持现有生成布局与分配方。 +- Client 出站命名空间是配置的 server；转发上传进度值中的第五项仍是逻辑目标。Server 出站命名空间是接收 client 的控制端点。 +- Server 本地 `pause_trans(fid, peer)` / `start_trans(fid, peer)` 要求显式接收端；Client 可省略 peer 使用其 server，拒绝其他显式 peer。远程暂停线路不变。 +- 持久化 `upload` 键是紧凑 JSON 字符串 `[receiver, fid]`，`download` 仍为十进制字符串；五项值不变。遇到历史裸上传 fid 键，加载拒绝并记录原因；写回与关闭保留旧文件，不自动迁移或删除。归档旧文件后才能持久化新任务。 +- 两个 Web 后端按收发方向提取数值 fid，保留既有 `/api/transfers` 字段。前端 JS、模板不变。 +- 迁移文件/恢复/加密测试中的索引调用方；新增同号并发、跨方向清理、持久化、旧快照保护、Web 碰撞行和分配器占用行为回归。 +- TCP 两端 API、文件传输文档、模块相关 docstring、批准计划与 `docs/source/changes/transfer-receiver-scope.md` 同步更新。 + +入口 → 出口：端口响应从已有控制连接获得 receiver → 缓存与等待同时匹配 receiver/fid → 发送腿登记 scoped key → 暂停和断线清理仅定位该 key → 完成按原 entry/socket 所有权释放 → 同样的 scoped key 写入及读回进度文件 → Web 仅输出数值 fid 与五项进度内容。 + +## 改前复现 + +使用修改前捕获的函数定义执行内存/socketpair探针，观察到： + +1. 两个接收端的同号上传只剩 1 条进度，预期 2 条。 +2. 清理入站 peer X，误关闭原本属于出站 peer Y 的同号 socket。 +3. 两条同号端口响应中，等待 X 取走 Y 的端口。 + +对应永久回归保留在 `test_tcp_transfer_resume.py` 的同号真实 socket、跨方向 socketpair 和持久化场景中。 + +## 局部验证 + +### 相关传输与 Web 套件 + +```bash +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/unit/transfer_web/test_web_transfers.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_crypto_tcp.py test/unit/network_api/test_resume_matching.py -q +``` + +结果：108 passed，170.82s。包含真实文件/目录传输、转发、流控、普通/加密恢复、Web API 等现有及新增场景。 + +```bash +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_async_server.py -q +``` + +结果：109 passed，16.20s。 + +随后新增分配器作用域回归： + +```bash +.venv/bin/python -m pytest test/unit/network_api/test_fid_allocator.py -q +``` + +结果：2 passed，0.02s。另运行最终版本的旧快照保护用例，1 passed，0.01s；覆盖两端、加载优先/写入优先、空状态/新 scoped 状态写回及关闭的字节不变性。上述命令间有用例交集，不能直接相加为唯一用例总数。 + +### 独立真实运行证据 + +用一次性脚本启动实际 `TCP_Server_Base` 与两个 `TCP_Client_Base`，未 mock 发送/接收关键路径，未通过 pytest 驱动： + +- 两个接收端控制地址分别为 `127.0.0.1:54384`、`127.0.0.1:54394`；发送同一个数值 fid `708`，以相反接收端顺序领取端口 `37797`、`46447`。 +- 上传快照同时保留 `[["127.0.0.1",54384],708]` 和 `[["127.0.0.1",54394],708]` 两个 JSON 键；两条任务暂停时进度均为 0。 +- 将实际在途 server 交给真实 Flask `/api/transfers` 路由，登录后 HTTP 200；收到两行 fid 均为 708，文件名分别为 `source-0.bin`、`source-1.bin`，总大小 98,323 / 196,627 字节，未合并同号行。 +- 清理第一个 receiver：其 socket 关闭；第二条腿的暂停保持 True、socket 仍开启。释放第二条腿后，文件长度 196,627 字节，源/目标内容完全相等，scoped 发送状态清理完成。 +- 最终文件 SHA-256：`912e79f992326790f02f5b470f7fc473a7dc82fd366ec0ce5a1fd9d0005ee028`。 +- 最终脚本退出码 0。一次性脚本和自建临时目录已移除；永久可重跑场景为 `test_equal_server_upload_fids_are_isolated_by_receiver`。 +- 初次烟测脚本错误使用控制端口 0 导致超时；改为已分配端口后完成路径。随后脚本进程内清理目录与后台收尾写入发生竞争；改为进程退出后由父进程清理，最终退出码 0。两项均为烟测脚本修正，未为此改动协议实现。 + +### 静态检查 + +```bash +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py PyFlow/transfer_web/web_backend/server_backend.py PyFlow/transfer_web/web_front/client_backend.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_crypto_tcp.py test/unit/transfer_web/test_web_transfers.py test/unit/network_api/test_fid_allocator.py +``` + +退出码 1：仅 `connect_tcp.py` 既有 `response` 未使用（F841）。用本轮修改前捕获的源码运行同样规则，得到同一条诊断，证实为基线问题；未修改无关控制命令代码。未声称全量 lint 通过。 + +## 路径与验收结论 + +| 原子 1 路径 | 证据与结论 | +|---|---| +| 不同接收端同数值 fid | 真实两 client 并发传输与独立脚本通过；发送进度独立 | +| 跨方向同号清理 | 两端 socketpair 回归通过；收到的 EOF 仅属于被清理方向 | +| 定向暂停/继续 | 两 client 真 socket 回归与脚本通过；清理/释放一条不解除另一条暂停 | +| 普通/恢复端口匹配 | 按接收端匹配；现有普通双向与恢复套件通过 | +| 公钥及加密子连接 | `test_crypto_tcp.py` 通过；端口缓存测试验证控制 receiver | +| 转发目标端口和流控 | 既有转发文件、目录、多目标、流控及归属回归通过；不宣称已完成新版转发路由 | +| 持久化 | 同号两项读回且不覆盖 live 项；旧 upload 裸键快照按字节保留 | +| Web 消费者 | 两后端 Flask 回归通过；独立实际在途 server 的 Flask API 返回两条同号记录 | + +## 明确未覆盖与风险 + +- 原子更改 2–6 未开始。当前 tfid/kind、发送端分配、`/forward_resume`、转发恢复专用记录、原有最终目标完成语义均尚未切换，不报告整个需求完成。 +- 本次没有保证不同发送者入站同号都可共存；接收表仍按本端 fid 定位，最终由接收端分配的约束属于更改 6。 +- 没有解决 client_port 改变、旧版本断点迁移、暂停持久化、中继落盘或关闭进度后自动续传。 +- Web 只验证真实 Flask API 与数据适配，没有浏览器视觉验收；前端模板/JS 未改。 +- 未运行整个仓库测试、完整 lint、文档构建或 Windows/macOS 验证。 +- API 影响：发送进度字典的键、上传 JSON 索引，以及 server 本地暂停调用需要迁移。已有文件字节格式和线路命令不变。 +- 核心文件相对本轮修改前快照为 +170/-148 行（净 +22）；新增/迁移测试与文档另计，整体超出原表的粗略 250–400 行估算。仍是同一个 receiver-scoping 意图；未创建 git 提交，人类可据此审查规模，未以行数为理由省略路径。 + +## 红旗与规则沉淀 + +未删除/跳过测试、弱化断言、修改 CI/依赖/coverage 或添加发送旁路。没有生成第二份进度记录模型或兼容别名。永久行为回归与变更说明固定“出站数值 fid 不是完整身份”的约束。 + +请人类审查入口到出口、公开键契约和上述未覆盖项；放行之前不进入原子更改 2。 diff --git a/docs/plans/transfer-fid-and-resume-unify-atomic-2-report.md b/docs/plans/transfer-fid-and-resume-unify-atomic-2-report.md new file mode 100644 index 0000000..4443877 --- /dev/null +++ b/docs/plans/transfer-fid-and-resume-unify-atomic-2-report.md @@ -0,0 +1,108 @@ +# 原子更改 2 报告：Server 单份 FID 路由 + +状态：实现与局部验收完成，等待人类审查。未创建 git 提交、未合并、未推送；未进入原子更改 3。 + +## 提交意图 + +将 server 的多 ID 转发恢复记录收敛为一份权威路由与接收端作用域索引，保证目标协商失败后仍保留完整接受目标集合，并可从持久化数据重建路由。 + +## 改动内容与数据流 + +- `PyFlow/network_api/connect_tcp.py`：server 删除 `_forward_resume_records`，替换为 `_fid_routes[upstream_fid]` 与 `_fid_route_index[(receiver, downstream_fid)] -> upstream_fid`。前者唯一保存 `originator` 与完整 `{receiver: fid}` 目标映射，后者只保存索引，不复制路径、文件名、目录、速度或字节进度。 +- `_register_fid_route` 在主键或下游索引冲突时整体拒绝，不部分覆盖；`_lookup_fid_route` 区分本地上游与具体 receiver 的下游；`_drop_fid_route` 按对象所有权移除主记录和其索引，旧对象不能删除新登记状态。 +- 复用 `_forward_relays_lock`。`_forward_relays` 保持为在途条件变量与流控状态，不成为第二份持久化路由。删除路由后释放锁，再更新已有进度文件，避免与加载的进度锁/路由锁反向嵌套。 +- Server 的恢复分派、进度读回占用判断和分配器 live-table 引用迁移至新路由。收到目标恢复请求时，以实际控制 peer 和下游 fid 找到上游 fid，再发送原有 `/forward_resume`；未知 peer/fid 不回退至其他目标。 +- `relays` JSON 每个上游 fid 只存一行：`originator` 和 `targets: [[receiver, fid], ...]`。加载重建派生索引，恢复计数以权威路由为单位，不覆盖 live 主键或下游索引。 +- 旧 server relay 格式拒绝读取，并在写回/关闭时保留原文件，不建立兼容别名或新状态文件；client 的旧转发恢复记录及其 JSON 格式保持不变。 +- 中继前去重目标地址。可达性初筛后传入中继的目标构成接受集合；端口、连接、加密附着或开始握手失败不再从集合中消失。整组清理条件从 `delivered >= len(target_conns)` 改为 `delivered == len(targets)`。 + +数据流：接受目标集合 → 登记唯一上游路由与 scoped 下游索引 → 各目标协商、数据泵送 → 全部接受目标成功才清理整组；未全部成功则保留映射 → 断线持久化每组一行 → 新实例重建同一组索引 → 具体接收端的恢复请求转换为原始上游 fid。 + +## 文件清单 + +修改： + +- `PyFlow/network_api/connect_tcp.py` +- `test/integration/network_api/test_tcp_transfer_resume.py` +- `test/unit/network_api/test_fid_allocator.py` +- `docs/source/File_Transfer/File_Transfer.rst` +- `docs/source/Network_APIs/TCP_Server_APIs.rst` +- `docs/plans/transfer-fid-and-resume-unify-plan.md` + +新增： + +- `test/unit/network_api/test_fid_routes.py` +- `docs/source/changes/transfer-fid-routes.md` +- 本报告。 + +未修改 client 公共接口、Web 适配、CI、依赖、coverage 或文件字节格式。核心文件相对本轮开始捕获的已提交源码为 +95/-77 行（净 +18）；新增集成场景与单测使整体审查量高于原 200–350 行粗估，仍保持单一 server 路由意图，没有省略验收路径。 + +## 改前复现与改后独立烟测 + +使用一次性脚本启动真实 server、源 client、健康目标和故障目标；故障仅注入在目标监听协商边界，发送真实控制响应宣布拒绝连接的端口，不 mock 数据泵或成功结果。 + +改前：健康目标完整接收 131,109 字节，故障目标未建立数据连接，源端返回上传成功；server 的 `relays` 计数为 0。脚本因“接受目标协商失败却删除整组恢复路由”断言失败，退出码 1。 + +改后同一场景： + +- 健康目标完整接收 131,109 字节，目标文件与源文件完全一致。 +- 接受集合包括 `127.0.0.1:48548` 与拒绝监听的 `127.0.0.1:48560`；原始源端为 `127.0.0.1:48546`。 +- 上游 fid `6547520319`；两个下游 fid 分别为 `10842487615`、`15137454911`。 +- 保留 1 份权威路由、2 个下游索引；新实例中所有查找指向同一权威对象，未知 receiver 被拒绝。 +- 健康目标文件 SHA-256：`507205331a573dd31fb8f7959182403edb5232f7915297c7c8baec98d3e6eefd`。 +- 原传输进程退出码 0。随后另起独立 Python 进程，只从 `server-progress.json` 加载，恢复计数 1、接受目标数 2、索引数 2；共享权威对象与错误 receiver 拒绝检查均通过，退出码 0。 +- 独立进程读回验证的是持久化路由及索引重建，不是声称跨服务重启的自动文件续传已端到端完成。 +- 一次性烟测/读回脚本与自建临时目录已清理；永久真实 socket 场景保留在集成测试中。 + +重要边界：烟测中源端仍可能返回上传成功，这是现有最终 ACK 语义，属于原子更改 3。本次仅保证 server 不丢路由，不能据此声称源端记录生命周期已修复。 + +## 测试与检查 + +```bash +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py -q +``` + +结果：208 passed，179.72s。覆盖现有普通/转发文件、目录、加密、恢复与异步路径，以及新增路由和 socket 场景。套件输出包含一条 `Server error: [Errno 9] Bad file descriptor`，未造成测试失败；本次未将该输出当作已修复事项。 + +最终分配器断言加强后，单独执行： + +```bash +.venv/bin/python -m pytest test/unit/network_api/test_fid_routes.py test/unit/network_api/test_fid_allocator.py -q +``` + +结果：11 passed，0.03s。与上一命令有交集，不相加为唯一用例数。 + +新增永久行为验证： + +- 上游及多个下游查到同一权威路由。 +- 不同 receiver 的同数值下游 fid，包含跨路由场景，互不混淆。 +- 错误 peer/fid 拒绝，不部分覆盖主键/索引。 +- 非 owning 对象不能清理现有路由。 +- 单份 JSON 路由经新实例读回，索引重建且不覆盖 live owner。 +- 旧 server relay 快照在写回/关闭后按字节保留。 +- 分配器首个返回值同时避开已有上游与相应 receiver 的下游 fid。 +- 真 socket：一个目标拒绝连接、另一个完成后仍保留目标全集；未知恢复请求不转发，正确请求在源端离线时进入原有待发队列。 +- 真 socket：一个目标完成、另一个 writer 暂停时，路由与索引仍存在；释放后两份 payload 一致，路由与索引全部清理。 + +静态命令： + +```bash +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py test/unit/network_api/test_fid_routes.py test/unit/network_api/test_fid_allocator.py +``` + +退出码 1，仅既有 `response` 未使用（F841）。对本轮修改前捕获的已提交源码运行同样规则，得到同一诊断；不声称全量 lint 通过，也未修改无关控制命令代码。 + +## 明确未覆盖与风险 + +- 原子更改 3–6 未实施：源端最终完成通知与记录保留、逐目标完成状态、统一 resume、client 专用恢复表删除、握手合并、kind/tfid 删除、接收端分配权切换均仍待批准。 +- 当前若部分目标成功，路由保留整个接受映射；尚不单独清理成功目标或解决其重复恢复问题。 +- 当前 `/forward_resume`、`/forward_upload`、kind 与 tfid 继续生效。它们在更改 2 后存在不是遗漏。 +- 不改变可达性初筛的既有跳过行为：提交转发请求时已不可达或是 server 自身的目标仍被跳过；本次保证的是被接受后协商失败的目标不误算成功。 +- 不迁移旧 server relay 快照、不解决 client_port 改变、中继内容落盘或暂停持久化。旧快照需人工归档后再持久化新路由。 +- 无全仓测试、完整 lint、文档构建或跨平台验证；本次未改 UI。 + +## 审查与规则沉淀 + +未删除或跳过测试、弱化断言、改 CI/依赖/coverage，未增加直接发送旁路；server 旧表已切除，不保留兼容别名。测试和变更说明固定两条约束：下游身份包含 receiver；接受目标全集不能被连接成功子集替代。 + +请人类审查上述入口到出口、单份状态及未覆盖边界。未获下一项明确放行前，不进入原子更改 3。 diff --git a/docs/plans/transfer-fid-and-resume-unify-atomic-3-report.md b/docs/plans/transfer-fid-and-resume-unify-atomic-3-report.md new file mode 100644 index 0000000..8048618 --- /dev/null +++ b/docs/plans/transfer-fid-and-resume-unify-atomic-3-report.md @@ -0,0 +1,143 @@ +# 原子更改 3 报告：回传到达端原有最终 ACK + +状态：按最新人类要求完成第二结果通道的部分回退；其他已实施修复保留。单 ACK 实现、定向/完整回归及独立真实程序验收完成，待人类审查。未创建 git 提交、未合并、未推送；未进入原子更改 4–6。 + +## 最新决策与本次边界 + +人类要求：Server 直接等待到达端既有最终 ACK，再向源端回传同一种 ACK，保持协议简洁；其他问题的修改暂不回退。 + +因此本次删除独立结果控制行、额外结果等待与结果状态，恢复单一最终 ACK 的成功语义。没有整体回退更改 3，也没有撤销逐目标完成、稳定 ID 恢复、原路径衔接、并发控制、持久化或失败资源清理。 + +本报告以当前单 ACK 版本为准。此前双层确认版本的 231 passed 是历史证据，不作为本次切换的验证结果。 + +## 回退与保留清单 + +| 项目 | 当前处理 | +|---|---| +| 第二结果控制通道及其分派/处理函数 | 删除，不留别名或兼容入口 | +| `TRANSFER_RESULT_TIMEOUT`、`result`、`result_event` | 删除，不再有第二阶段等待 | +| 上传字节读完即发送的本跳成功 ACK | 删除；流结束不等于目标交付成功 | +| Server 逐目标 `completed` 与完整接受目标映射 | 保留；只有真实目标最终 ACK 才推进 | +| 稳定上游/下游 fid、`relay_fid=`、原始源端校验 | 保留；成功目标不重发 | +| 同 fid 发送锁及同目标重复合并 | 保留；完成后排队的旧 owning 记录不能重开任务 | +| 原路径、磁盘 offset、file/folder 匹配及 EOF 再确认 | 保留；不恢复已删除的新 ID/同路径桥接 | +| 完成集合 JSON、源上传行读回、快照锁 | 保留；五项进度值与 client JSON 字段不变 | +| 队列 receiver 作用域、失败缓冲释放、读取超时、资源收尾 | 保留 | + +## 最终实现与数据流 + +### Server：收到全部目标 ACK 才向源端回传 + +- `_forward_read_sign` 仍经 `receive_message` 有界读取，现在返回已识别的既有 marker 字节或 None，而不是只返回布尔值。开始握手继续按真假判断；最终确认调用保留到达端的实际 marker。不会把可能合并在同次读取中的 header ACK 一并作为完成信号转发。 +- 每个 writer 收到真实最终 ACK 后，调用原有 owning-route 检查的 `_complete_fid_target`。完整 `targets` 保持不变；`completed` 和未完成反向索引保持原修复。 +- `_forward_pump` 先等待本次 writer 收尾，再同时核对：本次上游输入完整、主表仍属于该 route、原接受目标全集均已完成、确实读到了本次目标 ACK。 +- 条件满足后，通过既有 `send_message` 在上传数据连接上回传该 ACK marker；否则发送既有错误符号。成功 ACK 写出后才调用 `_drop_fid_route`。写出返回 False 或抛异常都保留完成路由。 +- 这是转发同一种应用层 ACK，不是复制某一跳 TCP/密文帧；每跳继续经过既有发送与加密入口。 +- 中继最终清理仍释放 socket、端口和本次 owning 运行状态，但不再发送任何额外控制结果。 + +### 源端：只使用原有最终 ACK + +- `file_transfer_mode` 删除 forwarding record/result/event 的额外等待。既有最终数据 ACK 是成功的唯一完成标志。 +- 收到既有错误符号的两处 Client 发送分支改为立即返回 False，不能通过 `break` 落入 `file_sent=True`。这是聚合失败使用原有错误符号后所必需的修正；普通 Client 上传也共享这个分支。 +- 断线、错误或超时且源文件仍存在、进度可恢复时，既有 `_file_progress_finish(..., keep=True)` 保留上传行并持久化。没有新增另一份进度表。 +- `_forward_one` 仅在真实发送函数返回 True 后删除自己的 forwarding metadata，并写回已有进度文件。失败仍保留原 metadata 和完整目标列表。 +- 运行时 forwarding record 只保留 `send_lock` 与 `resuming` 等重放控制，不保存结果同步对象。 +- 最终 ACK 等待仍为原有 30 秒加文件大小额度，没有第二个 20 秒结果等待，也没有新收据或自动重试协议。 + +### 跨恢复闭环保持 + +目标恢复请求仍通过现行 `/resume_trans`、`/forward_resume` 驱动；源端用原 `relay_fid` 重放。先前成功目标留在完成集合中，不重发；本次仅恢复剩余目标。最后一个目标确认后，同一个原始最终 ACK 在本次上传连接回到源端，使原任务的上传行、metadata 和 route 清理。 + +原始源端、目标归属、child fid 和 route 对象所有权检查没有回退。Client 接收端最终 ACK 写出失败时仍保留原路径与进度，允许在 EOF 位置重试确认。 + +## 本次改动文件 + +- `PyFlow/network_api/connect_tcp.py`：第二通道删除、原始最终 ACK 回传和 owning metadata 清理；保留其他修复。 +- `test/integration/network_api/test_tcp_transfer_resume.py`：适配单 ACK 成功条件，替换已删除协议的用例，增加 held-ACK 与 ACK 写出失败行为验证。 +- `docs/source/File_Transfer/File_Transfer.rst` +- `docs/source/Network_APIs/TCP_Server_APIs.rst` +- `docs/source/Network_APIs/TCP_Client_APIs.rst` +- `docs/source/changes/transfer-fid-routes.md` +- `docs/plans/transfer-fid-and-resume-unify-plan.md` +- 本报告。 + +此前更改 3 的路由单测与加密转发最终状态断言保留。没有改分配权、kind/tfid、`/forward_resume`、`/forward_upload`、文件内容格式、Web 字段、CI、依赖或 coverage。 + +## 改前 / 改后真实程序证据 + +### 相同探针：上游 ACK 不得早于最后目标确认 + +一次性脚本直接启动实际 Server、源 Client 和两个目标 Client，传输 262,160 字节；仅在最后目标最终 ACK 发送边界阻塞，不 mock 数据泵。 + +| 探针观察 | 切换前双层确认版 | 当前单 ACK 版 | +|---|---|---| +| 最后目标已经写完文件但 ACK 被阻塞 | 是 | 是 | +| 已确认目标数 | 1 | 1 | +| 源端上传行 / metadata | 各 1 条 | 各 1 条 | +| 此时上游成功 ACK 数 | 1；违反最新要求 | 0 | +| 相同探针退出码 | 1，断言成功 ACK 过早 | 0 | + +当前版释放目标 ACK 后，只向源端回传一次 `/FILE_RECEIVE_CLIENT_DATA`，源端返回 True，上传行、metadata 和 Server route/index 均清空；两目标文件与源数据完全相同。 + +### 零 ACK、部分成功与原 ID 恢复 + +独立实际程序分别传输普通文件及目录内文件；只在目标最终 ACK 发送边界注入失败。每场景内容 131,090 字节，无 Server 文件内容落盘。 + +| 场景 | upstream / child fids | 首次失败状态 | 后续恢复 | 最终状态 | +|---|---|---|---|---| +| 零 ACK，普通文件 | 7288614307 / 11583581603、15878548899 | False；成功 ACK 数 0；源上传/metadata 各 1；未完成索引 2 | 原 ID，依次 False → True；共 4 条目标腿 | 仅最后一次回传 1 个既有 ACK，源/Server 状态清空 | +| 部分 ACK，目录内文件 | 4883320775 / 9178288071、13473255367 | False；成功 ACK 数 0；源上传/metadata 各 1；未完成索引 1 | 原 ID，仅重放失败目标，返回 True；共 3 条目标腿 | 同样只有 1 个最终 ACK，源/Server 状态清空 | + +两场景原目标映射与所有上下游 ID 保持不变；成功目标不重发。所有目标最终内容与源数据完全一致，SHA-256 为 `bac2bc23ee34db01070c104e661075be969be0aee6c3bb899c0db93963fcddb8`。脚本退出码 0。 + +另起独立 Python 进程加载首次失败时的快照：零 ACK 恢复 2 个未完成索引；部分 ACK 恢复 1 个，已完成目标不重新索引。两种情况源端均恢复 1 条五项上传行和 1 条 metadata；运行时只有发送锁/重放集合，没有结果同步对象。退出码 0。这验证状态读回，不宣称跨 Server 进程重启的自动续传。 + +## 测试与检查 + +### 定向回归 + +```bash +.venv/bin/python -m pytest test/unit/network_api/test_fid_routes.py test/unit/network_api/test_fid_allocator.py test/unit/network_api/test_resume_matching.py test/integration/network_api/test_tcp_transfer_resume.py -q +``` + +结果：**52 passed，68.98s，退出码 0**。 + +关键行为： + +- 原 file/folder × 零 ACK/部分 ACK 场景保留所有内容、状态、ID 和重复请求断言，并增加未完成时上游成功 ACK 数为 0、最终只有 1 个既有 ACK 的断言。 +- 删除已废弃第二结果协议的专属断言；原丢失结果用例替换为上游最终 ACK 写出失败，覆盖返回 False 和抛异常两种形式，仍检查源端状态、完成路由和新 Client 读回。 +- 新增实际目标 ACK 被阻塞时源端仍等待、上传进度完整但未清理的场景；释放后确认回传 marker 与既有目标 ACK 相同，最终状态清空。 +- 旧完成状态读回、非法快照保护、端点作用域、普通续传、公钥和加密相关用例没有删除或弱化。 + +### 完整相关套件 + +```bash +.venv/bin/python -m pytest test/unit/network_api test/integration/network_api/test_tcp_transfer_resume.py test/integration/network_api/test_tcp_file_transfer.py test/integration/network_api/test_crypto_tcp.py test/integration/test_forward_extension.py test/integration/test_forward_msg_attribution.py test/integration/network_api/test_tcp_async_server.py test/unit/transfer_web/test_web_transfers.py -q +``` + +结果：**233 passed，243.25s，退出码 0**。包含普通双向文件/目录、加密、公钥、原生/扩展转发、稳定 ID 续传、异步及 Web 消费者。与定向命令有用例交集，不相加为唯一用例总数。 + +### 静态检查 + +对当前修改的生产/测试 Python 文件运行 `python -m py_compile`，退出码 0。 + +```bash +ruff check --select F,E9 PyFlow/network_api/connect_tcp.py test/integration/network_api/test_tcp_transfer_resume.py +``` + +退出码 1，仅既有 `response` 未使用（F841），前两份原子报告已记录该存量问题。本次未修改无关控制台分支，不宣称全量 lint 通过。 + +## 明确未覆盖与风险 + +- 原子更改 4–6 未实施；现有 kind/tfid、client 转发恢复记录、恢复/上传命令和当前分配权保留。 +- 上游 ACK 成功写入 TCP 不等于源端应用层收据。丢失 ACK 或源端等待超时可导致保守保留状态；本次没有增加自动清除此状态的协议。ACK 写出明确失败时 Server 保留完成路由。 +- 原有最终 ACK 等待时限不扩大；慢目标或长时间手动暂停可能超过源端等待。目标确认读取超时保留，未声称所有阻塞写都有统一总时限。 +- 此前发现的 Server 普通发送错误符号 fallthrough、Server 接收端 ACK 写出失败仍清理接收行的存量缺口，本次没有扩展修复。Client 发送错误符号分支因转发失败回传所必需而修正。 +- 不解决固定 client_port、端点迁移、源文件外部删除/修改、混合新旧节点、旧断点迁移、中继内容落盘或暂停持久化。 +- 没有全仓测试、完整 lint、文档构建、跨平台/负载验证或完整加密覆盖审计。本次无 UI 改动。 + +## 规则沉淀与人工闸门 + +“到达端未确认不得向源端回成功 ACK”“失败保留源记录”“恢复复用原上下游 ID”“ACK 写出失败不丢完成路由”已落实为真实 socket 行为回归及同步协议说明。 + +完整相关套件与独立烟测均已完成；一次性脚本及本次自建临时目录已清理。请人类审查本次单 ACK 版本。未获下一项明确批准，不进入更改 4，不自行提交。 diff --git a/docs/plans/transfer-fid-and-resume-unify-atomic-4-report.md b/docs/plans/transfer-fid-and-resume-unify-atomic-4-report.md new file mode 100644 index 0000000..12587b8 --- /dev/null +++ b/docs/plans/transfer-fid-and-resume-unify-atomic-4-report.md @@ -0,0 +1,144 @@ +# 原子更改 4 报告：统一恢复入口与 Client 状态收敛 + +状态：已实施，待人类审查；未创建 git 提交、未合并、未推送；未进入原子更改 5/6。 + +## 1. 变更范围与数据流 + +本次只覆盖计划第 7 节。入口是普通重连、转发目标重连、目录项 EOF 再确认、离线源端待发和重复恢复请求；出口是原有 `/file`/`/forward_item` 数据连接、既有最终 ACK、稳定 FID 的进度清理或失败保留。 + +统一线路： + +```text +receiver -> /resume_trans -> connected peer +``` + +两个地址是 shell-quoted 二元组。`parse_resume_command` 只接受五个 token，返回 `(fid, offset, origin, target)`；负 fid/offset、非 tuple 地址、非字符串 host、非整数 port、旧三 token 和 `/forward_resume` 均拒绝。`format_resume_line` 是唯一构造入口。 + +Server 收到请求后先要求 `target ==` 实际控制连接地址；`origin == (server.host, server.port)` 走本地 scoped upload，否则必须同时匹配 `(requester, downstream_fid)` 路由索引和 route originator，再把请求转成稳定 upstream fid 给源端。未知、伪造来源、伪造目标和同数值跨 receiver FID 不降级到另一条路径。源端离线时，Server 以 `(originator, upstream_fid, target)` 合并待发项，只保留最新 offset；安全发送成功后删除，发送失败保留。 + +Client 只在 `origin == client_socket.getsockname()` 时处理 Server 推送的恢复请求。直接目标使用配置 Server 的 scoped upload 行；转发目标也只从该五项 upload 行取源路径，通过既有 `relay_fid=`/`/forward_item` 重放稳定 upstream FID。目标端保留原接收行和完整嵌套路径，因此源端不复制 targets、destination、folder metadata 或第二张恢复表。 + +同一 scoped upload 的 transient `_resume_sends` 只含 `lock`、`users`、`targets`：同目标重复加入，异目标串行，结束释放；不写入 JSON。普通恢复和转发恢复共用它。`Client._transfer_progress_snapshot()["relays"]` 固定为空；非空旧 Client relay 区域拒绝读取并保留原文件。Server `_fid_routes`/`relays` 权威路线不变。 + +FID 不再编码 plain/relay kind：删除 kind 常量、解析函数和 allocator kind 参数；保留低位为零，日志格式为 `e.`。新传输的既有分配方、`tfid` 命名、`/forward_upload` 和 `relay_fid=` 暂不改变，属于后续 5/6。 + +## 2. 修改文件 + +- `PyFlow/network_api/connect_tcp.py` + - 五 token resume parser/formatter 与所有发送、分派、Server route 翻译调用方； + - 删除 `/forward_resume`/kind/Client duplicate recovery state； + - Client upload-row-only recovery；Server offline pending queue；transient send serialization； + - module/class/docstring state contracts and legacy snapshot preservation. +- `test/unit/network_api/test_resume_matching.py` + - five-token format/parse round trip and malformed/old-wire rejection. +- `test/unit/network_api/test_fid_routes.py` + - endpoint forgery, scoped downstream lookup, offline latest-offset coalescing, failed-send retention and route pruning. +- `test/unit/network_api/test_transfer_progress_query.py` + - nonempty legacy Client relay snapshot rejection and byte-preserving write/shutdown behavior. +- `test/unit/network_api/test_delete_config.py` + - transient `_resume_sends` deletion guard (existing supplemental regression migrated). +- `test/integration/network_api/test_tcp_transfer_resume.py` + - unified direct/forwarded resume consumers; route identity; duplicate recovery; distinct-offset serialization; directory/full-path and EOF recovery. +- `test/integration/network_api/test_crypto_tcp.py` + - Client relay snapshot assertion migrated to empty relays. +- Active documentation: + - `docs/source/File_Transfer/File_Transfer.rst` + - `docs/source/Network_APIs/TCP_Server_APIs.rst` + - `docs/source/Network_APIs/TCP_Client_APIs.rst` + - `docs/source/changes/transfer-fid-routes.md` + - `docs/source/changes/transfer-receiver-scope.md` + - `docs/source/changes/transfer-resume-simplified.md` + - `docs/plans/ordinary-transfer-progress-inspection-plan.md` + - `docs/plans/transfer-fid-and-resume-unify-plan.md` + - `docs/plans/transfer-fid-and-resume-unify-requirements.md` marked superseded. +- Locale extraction catalogs: `docs/source/locale/{ja,ko,ru,zh_CN,zh_TW}/LC_MESSAGES/` entries for `File_Transfer/File_Transfer.po`, `Network_APIs/TCP_Client_APIs.po`, `Network_APIs/TCP_Server_APIs.po` and `api/PyFlow.network_api.connect_tcp.po` (20 catalogs). + +The catalogs were not fully regenerated with `sphinx-intl`. To stop removed APIs from remaining in localized output, 80 obsolete extracted-message blocks were removed from those 20 catalogs. `polib` parsed all 135 catalogs successfully (8,936 entries total); a workspace search found no `forward_resume`, `resume_transfers` or `parse_forward_resume_command` entries. This is cleanup, not translation of the updated descriptions; catalog regeneration and review remain out of scope. + +## 3. Verification evidence + +### Parser and static syntax + +```text +.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py \ + test/integration/network_api/test_tcp_transfer_resume.py +``` + +Result: exit code 0. + +Direct parser smoke: + +```text +format_resume_line(7, 123, ("sender host", 1), ("target host", 2)) +/resume_trans 7 123 '("sender host", 1)' '("target host", 2)' +parse_resume_command(line) +(7, 123, ('sender host', 1), ('target host', 2)) +``` + +### Focused unit regression + +```text +.venv/bin/python -m pytest \ + test/unit/network_api/test_resume_matching.py \ + test/unit/network_api/test_fid_routes.py \ + test/unit/network_api/test_transfer_progress_query.py \ + test/unit/network_api/test_delete_config.py -q +``` + +Result: **120 passed in 0.35s**. + +### Focused live socket recovery regression + +```text +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py \ + -k 'unified_resume or forwarded_file_resumes or forward_route_survives_a_refused_target_listener or download_resumes_over_the_encrypted_channel' -q +``` + +Result: **4 passed, 28 deselected in 14.53s**. + +### Real plaintext and encrypted path probe + +Throwaway `/tmp/pyflow-atomic4-smoke.py` started actual Server plus three Clients and exercised: + +| Path | Evidence | +|---|---| +| ordinary download/upload | partial offset `12345`; original path and SHA-256 matched | +| forwarded two-target replay | stable upstream `fid=500`; offsets `32123`/`76543`; duplicate target request coalesced; both payload hashes matched | +| offline origin | pending `(fid,target)` retained; latest offset `54321` replaced `4096`; reconnect delivered and cleaned route/upload | +| encrypted equivalents | same three paths over framed channel; all payload hashes matched | + +Probe output: **262,166 bytes**, same SHA-256 `2c189692b3fdcac0988b6563883b004a1165cd66f518b414c4c898e208ead97e` for all four paths in both plaintext and encrypted runs. + +An additional throwaway live-socket smoke (script removed after execution) used an encrypted Server and Client, a 131,089-byte folder item at `original/nested/folder-item.bin`, and a rejected final ACK after EOF. The receiver then sent the five-token `/resume_trans` at offset 131,089; the original FID resumed over an encrypted data connection, the exact bytes and nested destination were preserved, and both progress rows cleared. Result: passed; worker teardown completed cleanly. + +### Full affected suite + +The initial full run reached **338 passed, 1 failed** because the new duplicate-target assertion assumed callback ordering not promised by Server dispatch. The assertion now waits for the observable target reservation and active transfer. The focused rerun passed; the fresh full run after that correction and the pending-state helper rename passed: **339 passed in 299.79s**. + +### Documentation and static checks + +- `.venv/bin/python -m sphinx -b html docs/source /tmp/pyflow-atomic4-html-final`: succeeded for 27 sources with 2 warnings: configured `_static` path is absent, and `command_control_extension_tcp` has a docstring definition-list formatting warning at line 23. +- `polib` parsed all 135 `.po` catalogs (8,936 entries); no parse errors. Obsolete resume API terms are absent from the locale catalogs. +- `.venv/bin/python -m py_compile` on `connect_tcp.py` and the affected unit/integration files: exit code 0. +- `ruff check --select F,E9` reported `F841` at `connect_tcp.py:6937` (`response` assigned but unused in the custom-handler branch). That branch is outside the atomic-4 recovery changes and was left untouched. + +## 4. Path conclusions + +- Ordinary bidirectional recovery: existing real-socket integration tests and the plaintext/encrypted probe. +- Forwarded one-to-one/multi-target, partial completion and stable-FID replay: focused integration, the live plaintext/encrypted probe and the full affected suite. +- Directory/full nested path and EOF second-failure/success: parametrized real-socket folder ACK/recovery cases plus the executed encrypted folder EOF probe above. +- Offline origin/reconnect and latest-offset coalescing: live probe and route unit tests. +- Same-target duplicate/different-target serialization: focused concurrency regression and fresh full-suite run. +- Legacy Client relay snapshot preservation and O3/O4 state boundaries: focused unit regressions. +- Public-key, async and forwarding-extension consumers: unchanged in this cutover; included in the full affected suite, which passed. + +## 5. Explicit risks and non-coverage + +- Full `sphinx-intl` catalog regeneration and translation review were not performed. The obsolete strings named above were removed from 20 affected catalogs; current changed source descriptions are not fully translated. +- Ruff still reports the unrelated unused local at `connect_tcp.py:6937`; it was not changed as part of this recovery cutover. +- No client-port identity migration, handshake consolidation, receiver-owned allocation switch, dependency/CI/coverage change, automatic old-state migration, retry protocol, or relay disk persistence was added. +- No git commit, merge or push was performed. + +## 6. Human review gate + +This report stops at atomic change 4. Review the endpoint validation, route ownership, pending queue removal ordering, scoped upload lookup, transient lock lifecycle, old snapshot preservation, and the evidence limitations above. Atomic changes 5/6 remain gated and are not implemented. diff --git a/docs/plans/transfer-fid-and-resume-unify-atomic-5-report.md b/docs/plans/transfer-fid-and-resume-unify-atomic-5-report.md new file mode 100644 index 0000000..9b49ecb --- /dev/null +++ b/docs/plans/transfer-fid-and-resume-unify-atomic-5-report.md @@ -0,0 +1,119 @@ +# 原子更改 5 报告：一次性监听握手关联统一 + +状态：已实施并完成下列验证，待人类审查；未创建 git 提交、未合并、未推送；未进入原子更改 6。 + +## 1. 提交意图与范围 + +只落实主计划第 8 节：双向普通文件、目录内文件、公钥、每个中继目标 listener 和中继源上传，统一使用实际控制 socket 所属的一次性 request。删除旧端口缓存、专用中继上传响应、FIFO 和共享错误槽，不保留兼容入口。 + +不改变新传输 FID 的分配方、`tfid`/`relay_fid=`、稳定上下游 FID、五项进度行、Server route、文件字节格式、暂停/恢复、既有最终数据 ACK 或加密 framing。接收端分配权切换仍属于未放行的更改 6。 + +## 2. 数据流与公共线路 + +1. 发起方先生成全新 UUID hex，并登记 `(request_id, actual_control_socket, expected_fid)`，再发送命令。中继源上传尚无 upstream FID，只有该请求匹配后才能接收 Server 分配的 FID；没有改变分配权。 +2. `/file`、`/file_folder`、`/crypto_pub_key` 在末尾 FID 前携带 `request=`；`/forward_item` 在 kind 后携带该标记。目录创建的非传输命令不因此增加数据连接。 +3. 接收方建立真实 listener，经同一控制连接返回唯一响应： + + ```text + /server_file_transfer_port + ``` + +4. `_TransferHandshakeRegistry` 按实际 socket 对象、request 和预期 FID 匹配，将 port 与 stamp 作为一份结果交付唯一 waiter。错 socket/FID、重复、迟到和已取消结果直接丢弃，不缓存供后来请求消费。中继各目标独立登记，允许乱序响应。 +5. 成功消费移除登记；超时、异常、取消、控制连接退出及对象关闭退役相应登记并通知 waiter。上下文管理的 `_transfer_handshake` 负责退出时清理。客户端接收线程捕获其所属控制 socket，而不是迟到时读取已经更换的 `self.client_socket`。 +6. 删除生产 `/forward_upload` 命令、`_forward_upload_queue`、共享 `_forward_error` 槽以及旧 `file_server_port_list`/等待入口。中继源失败只用 `/forward_error ` 回到其实际控制 socket 所属请求。 +7. 匹配后的原有文件数据连接、加密 attach、ACK 与进度/route 成功清理或失败保留继续使用既有实现。公钥文件腿仍 plaintext；request 不写入进度、route JSON 或恢复身份。 + +这是不兼容的线路切换，不支持新旧节点混跑，不提供旧响应 fallback。 + +## 3. 修改范围 + +- `PyFlow/network_api/connect_tcp.py`：request 构造/解析、统一 port 响应、瞬态 registry、普通/目录/公钥与中继两侧消费者、实际控制 socket 分派和断开清理；删除旧协调状态与命令。 +- `test/unit/network_api/test_transfer_handshakes.py`：乱序、socket/FID/request 匹配、取消/超时/断开、定向错误、重复及旧/畸形响应拒绝。 +- `test/integration/helpers.py`：低层数据传输测试先通过真实控制连接请求其唯一 listener，不再读共享端口缓存。 +- 受影响真实调用方与回归：`test/integration/network_api/test_tcp_transfer_resume.py`、`test_tcp_file_transfer.py`、`test_crypto_tcp.py`、`test_tcp_async_server.py` 和 `test/integration/test_forward_msg_attribution.py`。保留最终 ACK、稳定 FID、嵌套目的路径与失败状态断言。 +- 活动文档：`docs/source/File_Transfer/File_Transfer.rst`、`docs/source/Network_APIs/TCP_Client_APIs.rst`、`TCP_Server_APIs.rst`、`docs/source/changes/transfer-fid-routes.md`、主计划及本报告。历史原子报告不回写。 +- 翻译清理按活动变更说明的清单，仅移除 19 个 catalog 中 54 个过期 gettext 块；未全局再生成、未提供新译文。涉及五种语言的 File_Transfer/TCP 两端 catalog，以及 ko/ru/zh_CN/zh_TW 的 connect_tcp API catalog。 + +## 4. 验证证据 + +### 4.1 受影响完整套件 + +```text +.venv/bin/python -m pytest test/unit/network_api \ + test/integration/network_api test/integration/test_forward_extension.py \ + test/integration/test_forward_msg_attribution.py \ + test/unit/transfer_web/test_web_transfers.py -q +``` + +结果：**413 passed in 308.30s**。范围含普通/目录、恢复/ACK、公钥/加密、async 控制通道、中继扩展和 Web transfer 消费者。 + +该轮之后补入旧会话回调隔离用例，单独执行： + +```text +.venv/bin/python -m pytest test/integration/network_api/test_tcp_transfer_resume.py \ + -k reconnected_handshake -q +``` + +结果：**2 passed, 35 deselected in 4.94s**。这两例未计入前述 413；不是一次运行得到的 415。 + +### 4.2 定向乱序、过期与重连回归 + +```text +.venv/bin/python -m pytest test/unit/network_api/test_transfer_handshakes.py \ + test/integration/network_api/test_tcp_transfer_resume.py \ + -k 'handshake or expired_listener' -q +``` + +结果：**12 passed, 32 deselected in 8.17s**，执行时间早于新增的两例重连测试。 + +- `test_concurrent_forward_upload_handshakes_match_out_of_order_replies`：真实 Server 与客户端故意阻塞第一上传的 port 响应；第二上传先完成，再释放第一上传。两个源文件内容分别正确，两个调用均成功，进度与路由清理。它直接约束旧 FIFO 串任务问题。 +- `test_expired_listener_reply_cannot_capture_a_new_attempt[upload/download]`：第一次 listener 请求超时后，第二次同 FID 请求先收到旧响应和错误 FID 响应，再收到自己的有效响应；实际文件只能经新端口完成。测试结束关闭放弃的旧 listener。 +- `test_reconnected_handshake_rejects_reply_from_old_control_socket[upload/download]`：断开真实连接并确认旧 request 退役，同一 Client 实例重新连接；把新 request 的响应经旧 socket 回调入口交付，观察新 waiter 的结果仍为空，再经真实新控制连接交付并完成 32,781 字节文件。直接覆盖地址复用不能代替 socket 身份的边界。 + +### 4.3 独立真实加密程序 + +临时 `/tmp/pyflow-atomic5-smoke.py` 由 `.venv/bin/python` 执行，而非调用 pytest 测试函数。实际启动一个加密 Server 和两个加密 Client，隔离 key/progress/log 路径；全部断言通过，结束后全部被跟踪 worker 均退出。脚本在验证完成后移除。 + +| 路径 | 实际观测 | +|---|---| +| 公钥/控制加密初始化 | 两个 Client 均进入 framed 通道 | +| 普通目录项双向传输 | 每方向 131,089 字节，`ordinary/nested/item.bin` 原路径及完整字节相等 | +| 普通测试 payload SHA-256 | `cec551561b4ceedbc5845b735317c04c73bf1b20627a76b960a5602de26b7e4c` | +| 同源并发中继 | 98,315 字节目录项与 81,927 字节普通文件；第一上传 port 阻塞期间第二文件先完成 | +| 中继目的路径 | 第一项仅在 `kept/nested/first.bin` 出现，不落到根目录 | +| 定向失败隔离 | 另发不存在目标的上传返回 False，不使两个有效上传失败 | +| port/stamp 配对 | 两个独立 request 分别带 `enc:` stamp；接收 attach 均使用加密 stamp，最终文件字节正确 | +| 数据通道原始输入 | 捕获 target SecureChannel.feed 的输入，不含完整测试文件明文 payload | +| 成功收尾 | source upload、Server route/relay 及三方 pending request 均为空 | + +这里的原始输入观测不是系统级 pcap、完整日志脱敏审计或所有安全路径的全局证明。 + +### 4.4 文档、翻译与静态检查 + +- `.venv/bin/python -m sphinx -b html docs/source /tmp/pyflow-atomic5-html`：27 个源文件构建成功，2 条警告：配置的 `_static` 不存在；`command_control_extension_tcp` docstring 第 23 行 definition list 缺空行。未改动这些非本次问题。 +- `polib` 读入全部 135 个 `.po`：8,882 entries，parse_errors 为空。没有全局 `sphinx-intl` 再生成或翻译评审。 +- 生产与活动文档范围检索未发现旧端口缓存、等待函数、FIFO 或共享错误槽消费者；`/forward_upload` 仅保留在拒绝旧响应的测试与说明其删除的文档中，不是兼容入口。 +- 最终 `ruff check --select F,E9` 覆盖 `connect_tcp.py`、helper、握手单测及受影响 integration 文件，结果仍为 **1 个 F841**：`connect_tcp.py:6975` 的 custom-handler 分支把返回值赋给未使用的 `response`。原子 4 报告已记录同一分支（当时行号 6937）；本次未改动该非握手问题。测试中迁移后遗留的两处无用变量已移除。不得将本项描述为 lint 全绿。 + +## 5. 路径逐项结论 + +| 需求路径 | 结论与证据 | +|---|---| +| 双向普通文件/目录内文件 | 完整 socket 套件通过；独立加密程序双向目录项内容/路径通过；multiple_file/multiple_file_folder 回归保留并执行 | +| 双向公钥及并发 key exchange | 加密套件含 `test_concurrent_clients_share_key_exchange`，完整套件通过；独立程序两端 key exchange 完成 | +| 中继各目标 listener | 一对多转发、目标失败后恢复/route 及并发上传回归通过，每个 request 独立匹配 | +| 中继源上传乱序 | 定向真实 socket 回归与独立加密程序均观测第二文件先完成 | +| 超时/取消/错误 FID/重复/定向错误 | registry 行为单测；双向过期 listener 回归；独立失败隔离通过 | +| 控制断开、重连及迟到回调 | registry 单测及新增双向真实重连用例通过；旧 socket 不能填充新 waiter | +| nonce 与端口配对 | 共同结果对象；加密/公钥回归和乱序加密程序实际内容完成 | +| async、forwarding API 与恢复消费者 | 完整受影响套件通过；helper 和 required request 参数调用方已迁移 | +| 进度/route/ACK/稳定 FID | 不新增持久化 request；既有失败保留与稳定恢复断言保留并通过相关套件 | + +## 6. 风险、未覆盖与人工闸门 + +- 新旧线路不兼容，必须同版本部署；未增加旧节点桥接或自动迁移。 +- 更改 6 未获授权，未切换 FID 分配方，未删除 `tfid`/`relay_fid=`。不处理 client_port 身份迁移、中继落盘、暂停持久化或新重试协议。 +- 本次没有 CI、依赖、coverage 阈值、加密算法或鉴权策略更改；保留既有断言，不以 mock 回声代替最终字节/路径/ACK 行为。 +- 遗留 Ruff F841、两条文档警告及新文本翻译缺口如上公开；不声明已消除。 +- 人工重点审查：实际 socket 捕获与断开清理、登记早于发送、消费/超时/异常退役、port/stamp 不拆分、各目标 request 独立、源上传错误不能唤醒另一请求、旧端口 cache 与命令无生产消费者。新增真实乱序/过期/重连回归和活动变更说明保留这些约束。 +- 本报告仅交付更改 5 的审查材料，不代表人类已审计通过，不授权继续更改 6；未创建提交、未合并、未推送。 diff --git a/docs/plans/transfer-fid-and-resume-unify-plan.md b/docs/plans/transfer-fid-and-resume-unify-plan.md new file mode 100644 index 0000000..33fb71c --- /dev/null +++ b/docs/plans/transfer-fid-and-resume-unify-plan.md @@ -0,0 +1,166 @@ +# 传输 FID 与恢复链路统一计划 + +状态：原子更改 1–4 及补充 O1–O4 的历史审查事实保留；原更改 6 的接收端分配 +方向由 `transfer-sender-fid-revision-plan.md` 取代。用户明确授权 R1 后继续 R2； +R1/R2 已实施，最终受影响套件 **434 passed in 278.89s**，明文/加密独立程序通过， +文档构建成功。证据及已知警告见 `transfer-sender-fid-r2-report.md`。 +当前待人工审查；未创建提交、未合并、未推送。 + +本文件替换先前“kind 位 + tfid 重放”的草案。后续实现不得沿用被替换的设计。 + +## 1. 需求卡 + +目标：普通文件与公钥仍由发送方分配新 FID；中继源端预先分配 upstream FID，Server 为每个目标分配 child FID；恢复复用既有上下游 FID 与逐跳路由。 + +现状：普通传输/公钥沿用发送方分配；R1 将中继 upstream FID 从 Server 分配改为源 Client 预分配，Server 仍独立分配各目标 child FID。详见当前修订方案。 + +非目标：不改文件字节流格式，不做中继内容落盘、暂停状态持久化、client_port 变化后的身份迁移；不改加密算法、鉴权体系、CI、依赖及覆盖率要求。关闭进度记录时保持不支持自动续传的既有约定。新旧节点不混跑，不承诺旧版断点自动迁移，不自动删除旧进度文件。 + +记录边界：复用 `file_upload_progress` 和 `file_recv_progress` 的五项值 `[bytes, total, speed, local_path, peer]`。不新增 client 恢复专用表，不复制已有路径和进度信息。接收记录的 peer 是原始发送者，发送记录的 peer 是逻辑接收者，不将二者混作 origin。 + +Server 将现有转发恢复记录收敛为一份 FID 路由关系：原始源端、上游身份、目标全集及下游身份、目标完成状态；不另建重复权威状态。已有 relays 持久化区域承载该关系。 + +恢复方向：`/resume_trans `；origin 是自己则发送本地文件，否则 server 核对原始请求者与路由并转换上下游 fid。最终接收端以稳定 fid、来源、文件名和总大小匹配旧接收记录，继续写其完整路径，不用新 ID/同路径桥接。 + +完成语义：Server 等完整接受目标集合的既有最终数据 ACK 后，才在上传数据连接上向源端回传同一种 ACK；任一目标未确认不回成功 ACK。删除第二结果通道和额外等待状态。源端仅凭最终数据 ACK 清理任务，失败保留进度；稳定 ID 恢复及逐目标状态保留。 + +风险：中高。改变中继 upstream FID 分配权并要求 `/forward_item` 带 `fid=`、显式 `resume=`；保留逐目标身份、五项进度值和文件内容格式。 + +禁止事项：不删/跳过测试，不弱化断言，不改 CI/依赖/coverage，不引入第二套 fid 生成器，不把握手 request 标记作为持久文件 ID，不绕过既有发送/接收/加密入口，不自行提交远端或进入下一原子更改。 + +## 2. 路径全集与原子拆分 + +| # | 原子意图 | 覆盖路径 | 预计代码/测试与对应说明改动 | 局部验证 | +|---|---|---|---|---| +| 1 | 现有索引补齐接收端作用域 | 普通双向发送/接收、转发上传/目标端口、公钥双向端口、暂停、断线清理、持久化、Web | 250–400 行 | 不同接收端同号、跨方向同号、并发端口匹配、重启读回和 Web 输出 | +| 2 | Server FID 路由表接管现有转发恢复状态 | 中继登记、上下游查找、目标全集、relays 持久化 | 200–350 行 | 一对多路由与重启读回、协商失败不误算完成 | +| 3 | 逐目标完成与源端记录生命周期（含已批准提前的稳定 ID 恢复依赖） | 数据泵逐目标 ACK、聚合原始最终 ACK 回传、发送保留与清理、沿用旧上下游 fid、接收端旧记录衔接 | 删除第二结果通道；审查重点为目标 ACK→原路由→同种 ACK 回传→源端清理 | ACK 不提前回传、零 ACK、部分成功、失败目标恢复与最后目标完成、上游 ACK 发送失败保留、完成状态重启读回 | +| 4 | 统一恢复入口 | 普通上传/下载、转发、目录内单文件、离线补发、二次断线、重复请求、不同 offset；复用更改 3 已落地的稳定 ID 衔接 | 300–450 行 | 旧 fid/原路径/内容不变；删除 forward_resume、client 专用恢复表及 kind | +| 5 | 统一并发握手关联 | 普通文件、目录内文件、公钥、中继 | 250–400 行 | 乱序与旧连接迟到响应不串任务；删除 forward_upload 专用通道 | +| 6 | 原接收端分配方案（已被修订取代） | 普通双向、公钥双向、中继每跳、恢复复用 | 由 `transfer-sender-fid-revision-plan.md` 的发送方分配 R1/R2 取代 | 用户已授权 R2，实施和验证进行中 | + + +每项包含对应测试、运行时证据、文档和报告，不留测试文档尾单。出现多意图或超出可审规模时先修订拆分。准备项不被宣称为整项任务完成；所有线路切换迁移其全部调用方，不留兼容别名或未使用接口。 + +## 3. 原子更改 1：已批准实施契约 + +### 3.1 索引与入口 + +- 接收进度维持本端 `fid -> 五项值`；发送进度改为 `(receiver_tuple, fid) -> 五项值`。 +- 入站运行状态仍以本地 fid 定位；出站 socket、活动标记与暂停开关使用 `(receiver_tuple, fid)`，两种键不可相互释放。 +- Client 出站 receiver 始终为配置 server，包括转发上传；转发值中的逻辑目标不改成 server。 +- Server 本地暂停/继续发送要求显式 receiver peer。Client 默认使用其 server。已有远程暂停线路格式不变。 +- 端口缓存增加该响应的 receiver peer；从控制连接的现有对端身份取得，不能只按 fid 领取或删除。 +- 普通发送、恢复发送、双向公钥推送、中继目标端口收集都使用相同的 peer 匹配约束。 +- 接收路径不再删除同数值 fid 的出站端口通知。 +- 分配器对恢复中的发送索引按 receiver 检查占用；不改变当前 kind 布局与分配权。 + +### 3.2 持久化与 Web + +- `download` 仍以十进制 fid 字符串为键。 +- `upload` 的键使用紧凑 JSON 编码的 `[receiver, fid]`；五项值及逻辑 peer 含义不变。 +- 不接受旧的裸 upload fid 键作为兼容别名。加载、写回及关闭均保留旧文件并记录拒绝原因；旧断点自动迁移不在本次范围。 +- Server/client Web 适配发送索引，API 继续输出数值 fid 和既有进度字段；同号两条任务均保留，不扩展 UI 功能。 + +### 3.3 变更文件 + +修改：`PyFlow/network_api/connect_tcp.py`；`PyFlow/transfer_web/web_backend/server_backend.py`；`PyFlow/transfer_web/web_front/client_backend.py`；`test/integration/network_api/test_tcp_file_transfer.py`；`test/integration/network_api/test_tcp_transfer_resume.py`；`test/integration/network_api/test_crypto_tcp.py`；`test/unit/transfer_web/test_web_transfers.py`；TCP 两端 API、文件传输说明及本计划。 + +新增：`test/unit/network_api/test_fid_allocator.py`（接收端作用域占用回归）；`docs/source/changes/transfer-receiver-scope.md`；`docs/plans/transfer-fid-and-resume-unify-atomic-1-report.md`。 + +不变：线路上的 fid 与文件字节格式、当前 kind 与 tfid、分配方、恢复命令；其后续切换均未授权。 + +### 3.4 验收与报告 + +- 先复现：同号上传进度覆盖、跨方向误关闭、端口误领取。 +- 用真实 socket 驱动同号并发发送，分别观察路径与进度；清理一个 peer 后另一个仍能接收完全一致的 payload。 +- 回归跨方向清理和同号持久化；每条值仍为五项，读回不得覆盖已存在的其他 receiver。 +- Web API 输出两条同号任务，字段和排序正确。 +- 运行现有文件、恢复、转发、公钥/加密相关套件；另做真实程序烟测，测试结果不能替代路径证据。 +- 原子报告列明命令、结果、路径覆盖、未覆盖、公共契约影响和风险。完成更改 1 后停在人工审查闸门，不推进更改 2。 + +## 4. 原子更改 2：已批准实施契约 + +- Server 用 `_fid_routes[upstream_fid]` 保存唯一权威路由：`originator` 与完整 `targets {receiver: fid}`。不保存源路径、目标路径、速度、文件名或目录元数据副本。 +- `_fid_route_index[(receiver, downstream_fid)]` 仅回指上游 fid；不复制记录。上游查找直接用本地 fid，下游查找必须带实际请求 peer。 +- 登记在主键或下游索引冲突时不做部分覆盖；读回不得覆盖当前 live 路由。删除按路由对象所有权清理主表与索引。 +- 复用 `_forward_relays_lock`。现有 `_forward_relays` 只保留在途条件变量、缓冲及流控状态，不是另一份持久化路由。 +- Server 的 `relays` JSON 每个上游 fid 只写一行，内容为 `originator` 和 `targets: [[receiver, fid], ...]`；加载时重建索引并以权威路由计数。旧 server relay 结构拒绝读取并保留原文件，不添加兼容别名。 +- Server 所有恢复分派、收发进度读回的路由占用判断和分配器占用检查迁移到新表。Client 专用转发恢复记录不在本次删除。 +- 可达性初筛后交给中继的目标构成本次接受目标全集；后续端口、连接或开始标志协商失败不能从此集合消失。只有交付数等于接受目标全集大小时，才允许删除整组路由。 +- 改动文件：`connect_tcp.py`、既有恢复集成测试与分配器单测、新增路由行为单测，以及文件传输说明、变更说明、计划和原子报告。 +- 验收：一对多上下游查找、同数值下游 fid 的接收端隔离、读回重建与冲突不覆盖、真实协商失败后仍保留完整路由、全目标成功后路由清理；现有转发/恢复/加密回归不退化。 +- 非目标：不发明新的完成通知、不修复源端最终 ACK 生命周期、不删除 kind/tfid、不改分配方或恢复/上传线路；更改 3–6 仍需分别批准。 +- 更改 2 的证据见 `docs/plans/transfer-fid-and-resume-unify-atomic-2-report.md`；server 路由已切换，client 旧恢复记录及现行线路仍保留。 + +## 5. 原子更改 3：已批准契约与依赖调整 +本节记录原子更改 3 在更改 4 切换前的已批准实现与验收，以下保留的旧线路/状态名称是历史边界；当前统一恢复契约以第 7 节为准。 + +改前真实复现：目标完整接收字节后丢失最终 ACK，零 ACK 和部分 ACK 两种情况下，源端 `_forward_one` 都返回 True，上传进度和 client 恢复记录均被删除；server 仍保留路由。这是更改 3 的改前问题证据，不是最终验收结论。 + +实施前 `_resume_forward` 会先 pop 旧进度和恢复记录，再调用 `_forward_one`；server 随后生成新上游 fid 和新目标 fid。原路由与新尝试没有关联。因此除了修复最终 ACK 的成功条件，还需保留已批准的稳定 ID 恢复关联，才能让后来成功的重放完成原路由。 + +人类已确认选择：将原子更改 4 中“恢复沿用旧上下游 fid、接收端按旧记录磁盘 offset 衔接”提前至 3，使逐目标完成能跨重放闭合。仍保留 `/forward_resume`、client 既有恢复记录、kind 和 tfid 命名；统一 resume 命令及旧表删除留在 4，握手合并和分配权切换留在 5/6。 + +最新人类决策：第二结果通道不再使用，Server 只回传到达端的既有最终 ACK。`_fid_routes[fid]` 保留 `originator`、完整 `targets` 和 `completed`，仅未完成目标保留反向索引。当前上游字节完整、原目标全集均确认后，才经既有数据连接与发送/加密入口回传同种 ACK；成功写出后按 owning route 清理。失败返回既有错误符号或关闭连接，源端保留可恢复状态。 + +删除新增结果命令、额外超时、result/event 字段和处理函数；不留兼容别名。源端上传函数仅按既有最终 ACK 判成功，错误符号必须返回 False，不能 fall through 成功清理。`_forward_one` 仅在成功后删除其 owning metadata 并持久化。恢复继续用可选 `relay_fid=` 复用原上下游 fid、不重发已完成目标，同 fid 串行及同目标重复合并保持。 + +接收端 ACK 发送失败时保留原完整路径与进度，即使磁盘字节已齐；稳定 fid、原始来源、文件名、总大小和磁盘 offset 衔接保留，允许 EOF 再确认。保留已有队列作用域、失败缓冲释放、读取超时及快照/读回修复。不增加结果收据/自动重试协议，不扩展普通发送的既有 ACK 等待时限，不自行创建 git 提交，不进入更改 4。 + +### 更改 3 验收与人工闸门 + +当前单 ACK 版本相关完整套件 233 passed，定向回归 52 passed；独立真实程序观察到最后目标 ACK 释放前上游成功 ACK 数为 0、释放后为 1，零/部分失败保留源状态并通过原 ID 恢复完成。ACK 写出失败和加密路径由相关回归覆盖，另起新进程验证完成集合及源上传/metadata 读回。新证据与风险见 `docs/plans/transfer-fid-and-resume-unify-atomic-3-report.md`；上一轮双层确认验证不替代本次验收。 + +更改 3 及补充 O1–O4 已完成审计，现获用户放行更改 4。更改 3 的历史报告保留其当时边界；更改 4 的具体迁移以下一节新增契约为准，握手合并及接收端分配权切换仍需更改 5/6 单独批准。 + +## 6. 更改 4 前的补充 O1–O4 + +人类要求先给方案、确认前不写代码：修复 Server 普通发送错误符号误判成功、Server 普通接收最终 ACK 明确写出失败仍清理记录，增加 Server/Client 本地只读的公开进度查询与控制台命令；另增补充项 O4：公开 `delete_config`，通过 `is_list_config=True` 返回可删除名字 list，默认 False 时按指定名字在停止后删除本地配置/状态文件，支持主动放弃持久化进度。 + +需求卡、查询/删除边界、O1→O2→O3→O4 拆分及验收证据见 [补充计划](ordinary-transfer-progress-inspection-plan.md)。人类已审计通过 O1–O3 并明确批准 O4；O4 不等于原 FID 大计划的原子更改 4。不增加 ACK 收据、自动重试、僵尸自动判定或逐记录删除命令，不自行进入原计划更改 4–6。 + +补充 O1–O4 已获人类审计通过；运行证据保留在各原子报告和 [补充总报告](ordinary-transfer-progress-inspection-report.md)。本次授权是原 FID 大计划的原子更改 4，不是补充 O4 的重复实现。 + +## 7. 原子更改 4：已批准实施契约与路径映射 + +- 唯一恢复线路为 `/resume_trans `，两个地址均为 shell-quoted tuple。删除三 token 旧线路、`/forward_resume` 常量/解析/分支，不保留别名；更改 4 当时保留的 `/forward_upload` 由已放行更改 5 删除，`relay_fid=` 保留至更改 6。 +- Server 接收请求时验证 target 等于实际控制请求端；origin 指向 Server 才走本地 `(receiver,fid)` 上传，否则必须准确匹配权威 route 的 originator 和请求端下游 fid，再转换为原上游 fid 转发。未知或伪造来源/目标不降级为另一条恢复路径。 +- Client 只有在 origin 为本次控制连接自身身份时发送本地上传；路径取自既有上传五项行。目标不是 Server 时通过原稳定上游 fid 重放单一目标，Server 再核对 route。源端不再复制 targets/kind/path/destination/目录元数据；目标通过原接收行恢复嵌套完整路径，普通 `/file` EOF 重放不要求重建目录命令。 +- 移除 Client `_forward_resume_records` 及专用读回/快照代码；Client `relays` 快照区域保持为空，非空旧区域明确拒绝并保留原文件，不静默迁移。Server 路由区域不变。O3 查询随之不再伪造 Client relay 行，O4 在途检查改用瞬态发送同步状态。 +- 仅保留在途锁与请求标记，不保存路径或持久化元数据。相同 fid 的不同目标串行，同目标重复请求合并，结束后释放瞬态项；失败保留上传，成功清理。普通恢复和转发恢复共用入口与此约束。 +- 删除 fid kind 常量/函数/参数；所有分配调用迁移为同一无 kind 分配器，低位固定零，日志格式为 `e.`。不改变谁分配新传输，也不删除现行 tfid 命名。 +- 离线源端请求保留于 Server，重连后经过同一格式化函数与安全发送入口；重复的同目标等待项合并，最新 offset 保留。发送未成功不得先删除待发请求，已完成/不存在路由不得补发陈旧请求。 + +修改范围：`connect_tcp.py`、分配器/解析/路由与既有真实恢复测试、补充查询/删除测试的受影响状态契约、TCP 两端与文件传输文档、当前变更说明、计划及原子更改 4 报告。历史报告不回写为当前格式；旧需求草案标明被本计划替代。逐项迁移现有消费者，不弱化 ACK/稳定 ID/原路径断言。 + +路径全集：普通双向文件/目录项 → 统一命令与本地上传行；转发一对一/多目标及部分完成 → scoped route 转换与单目标重放；源端离线/重连 → 待发队列；二次断线/EOF 再失败/成功 → 原 fid 与原文件；同目标重复/不同目标不同 offset → 瞬态合并与串行；Server/Client 读回与进度关闭 → 不新增恢复状态；O3 查询/O4 删除 → 新状态契约；加密/公钥/异步/转发扩展 → 无握手和发送旁路回退。 + +更改 4 验收要求：保留更改 3 与补充 O1/O2 真实失败证据的行为回归，增加五 token 格式/路由身份边界、无 kind 分派、上传行独立恢复、离线最新 offset 补发与二次断线证据；运行真实普通/转发/目录、并发/EOF 与加密烟测及相关完整套件，报告各路径结果。上述为当时验收契约；当前方向及授权见第 9 节。 + +## 8. 历史原子更改 5：已被 R2 取代的契约与实施结果 + +- 覆盖双向普通文件/目录内文件、双向公钥、每个中继目标 listener 和中继源上传。每次握手生成全新 `request=`，重试不复用。`/file`、`/file_folder`、`/crypto_pub_key` 将标记置于末尾 FID 之前；`/forward_item` 将标记置于 kind 之后。 +- request 不是 FID，不进入五项进度行、进度/route JSON 或恢复身份。它只绑定本次实际控制 socket 的一次端口请求；相同地址的新连接不能接收旧连接的结果。 +- 发送控制命令前登记实际 control socket、request id 和 expected FID；中继上传尚未分配时仅由匹配请求接收 Server 分配的 upstream FID，不改变现有分配权。每个目标独立登记，可乱序完成。 +- 唯一监听响应为 `/server_file_transfer_port `。实际 socket 身份、request id、预期 FID 必须共同匹配；端口与加密 stamp 一起交付给唯一 waiter。错误 socket/FID、重复和迟到响应直接丢弃，不缓存、不 fallback、不留兼容 alias。 +- 成功消费即移除登记；失败、超时、取消、控制连接断开和对象 teardown 均退役登记并唤醒 waiter。新任务不能接收旧结果；失败不改变既有进度/路由保留规则。 +- 删除 `/forward_upload` 生产命令、FIFO 与共享 `_forward_error` 槽。源上传使用上述共同响应；失败只用 `/forward_error ` 定向到同一实际控制 socket 的请求，不能唤醒另一个并发上传。 +- 不改 allocation authority、`tfid`/`relay_fid=`、稳定上下游 FID、五项进度行、Server route、暂停/恢复、文件字节、ACK 或加密 framing。公钥腿继续 plaintext，端口和 nonce 关联不改变密钥交换顺序。不得提前进入更改 6。 +- 迁移所有生产调用方、真实握手/恢复/加密/async/forwarding 测试和活动文档，不保留旧 cache、命令、参数别名或旁路。历史报告不回写。翻译目录只移除过期 gettext 块；新英文源文待翻译，不全局再生成。 +- 验收要求（非已运行证据）：真实 socket 并发普通/目录/公钥/中继请求；延迟第一响应并让后发请求先完成；取消/超时/断线后旧响应不能满足新请求；错误 socket/FID、重复响应不能串任务；端口与 nonce 必须成对。由整合负责人运行受影响普通/加密/async/forwarding/恢复套件与文档构建并另行记录结果。 + +更改 5 实施证据：受影响完整套件 413 passed；之后新增双向旧控制 socket 回调隔离用例，单独运行 2 passed，未计入前述 413。独立加密程序通过双向目录项、同源乱序中继、定向错误隔离、原嵌套路径与收尾清理。文档构建成功但保留两条警告；最终 Ruff 仍有一处历史 custom-handler F841。详细命令、路径结论和限制见原子更改 5 报告；当前待人类审查，不进入更改 6。 + +## 9. 发送方 FID 修订 R1/R2 + +原更改 6“接收端分配 FID”的契约由 `transfer-sender-fid-revision-plan.md` 取代，历史更改 1–5 报告保留其当时事实。当前 R1/R2 边界、全量路径与风险以修订方案为准。 + +R1：普通文件/目录项/公钥继续由各自发送方分配；中继源端在发送 `/forward_item` 前分配 upstream FID，并绑定现有上传行。Server 必须接收 `fid=` 并保留原值，只通过同一 `_FidAllocator` 为每个目标分配 child FID。新请求不带 `resume=`；恢复必须显式带 offset（包括 `resume=0`）并复用现有 route 与上下游 FID。 + +Server 在 route 登记前拒绝已被 upstream route、接收进度或同目标 outbound progress/socket/route/pause 占用的上游 FID，不覆盖既有所有者。R2 以实际控制 socket 和 FID 关联，并以 `/forward_error ` 返回错误；每次协商只有一个端口或错误终结响应。已发送且无终结结果的超时/取消 FID 在同会话保持未决，不能重试；其他任务不受影响。自然断线清理该会话等待、未决标记及所属 listener/数据腿，重连沿用原 FID；数据或 ACK 失败不断开健康控制连接。 + +R1 定向回归 13 passed；R1 阶段的 427 passed / 1 项旧 `request_id` 调用失败按用户要求由 R2 移除旧参数解决。最终 R1/R2 完整受影响套件 434 passed；独立明文/加密运行、自然重连与 EOF 证据及历史失败修复记录见 `transfer-sender-fid-r2-report.md`。当前只交付人工审查，不自动提交、合并或推送。 + + + diff --git a/docs/plans/transfer-fid-and-resume-unify-requirements.md b/docs/plans/transfer-fid-and-resume-unify-requirements.md new file mode 100644 index 0000000..663e46d --- /dev/null +++ b/docs/plans/transfer-fid-and-resume-unify-requirements.md @@ -0,0 +1,231 @@ +# 需求文档：转发与传输的统一 ID 分配 + 恢复通道合并 + +状态：**已被 [统一实施计划](transfer-fid-and-resume-unify-plan.md) 替代，仅保留为历史草案，不是实施依据。** optional-target、kind 及 Client 专用恢复表等旧方案不得沿用；当前授权范围见实施计划更改 4。 + +--- + +## 1. 背景与问题 + +原实现中，文件传输、文件夹传输、中继转发三条链路各自维护传输 ID（fid），存在以下问题： + +- **D1 收/发同号**：`file_client_id` 与 `_forward_fid` 都从 0 起，入方向 ID 来自对端、出方向 ID 来自本端，两者可能数值相同。`_active_transfer_fids`、`_active_transfer_sockets`、`_transfer_pause` 等共享注册表以 fid 为键，导致一方重连的清理误关另一方健康连接。实测：`_drop_stale_transfers(peer=X)` 误关发往 Y 的健康连接。 +- **D2 重启复用**：计数器每次启动从 0 起，重启后的新腿可能复用旧进度表中仍待恢复的 ID，覆盖其条目。 +- **D3 中继与普通共空间**：`_resume_request` 靠“先查中继表，查不到再走普通恢复”的顺序消歧，误判会路由错误。 +- **计数器递减**：腿结束时 `-= 1` 归还 ID，与分配点不配对，转发上传腿结束时会把别人的号减掉。 +- **转发恢复不幂等**：`_resume_forward` 在重放前删除记录，重放中二次断线后无法再次恢复。 +- **`/forward_resume` 与 `/resume_trans` 功能重叠**：两条命令承载同样的信息，增加代码路径。 + +## 2. 目标 + +1. 用单一分配器 `_FidAllocator` 替换两套计数器，ID 在实例间（随机 epoch）与重启间（新 epoch）不冲突，且同实例内单调不重号。 +2. 删除腿结束时的归还逻辑。 +3. 删除 tfid：转发时 server 不为目标预留 ID，目标 client 用自己的分配器分配 fid 并回报。 +4. 删除 kind 位：ID 低位固定，不参与任何判断。 +5. 废弃 `/forward_resume`，能力并入 `/resume_trans` 的可选目标字段。 +6. 转发识别改为 server 在 `/resume_trans` 时校验原始对端：不是自己即为转发消息。 +7. 转发重放沿用原 fid，使“成功才删”替代“先删后放”，达到幂等。 + +## 3. 非目标 + +- 不改 `/file`、`/file_folder` 既有字段语义（只允许新增可选字段）。 +- 不改进度表 JSON 结构、不改字节传输协议与加密帧格式。 +- 不做中继落盘（server 仍不触碰磁盘）。 +- 不做暂停状态跨重连持久化。 +- 不解决“上传/转发恢复依赖固定 `client_port`”（另开任务，仅补文档）。 +- 不做注册表、种子、僵尸扫描、区间压缩（已论证不需要）。 + +## 4. 已确认的决策 + +| # | 决策 | 依据 | +|---|---|---| +| 1 | epoch 为随机 31 位，每实例一次 | 时间戳同毫秒对撞 1977/2000；随机 2⁻³² 级碰撞 | +| 2 | counter 每进程从 0 起，无硬上界，置于高位 | counter 置低位时超过 2³³ 会溢出污染 epoch；高位无此问题 | +| 3 | 不做种子、不做注册表、不做僵尸扫描 | 注册表 20 万 ID 常驻 18.3 MiB，计数器恒定 0.3 KiB；种子多一条失败路径 | +| 4 | 唯一性由三表检查 + 随机 epoch 保证 | 三表检查确定性拦截同进程冲突 | +| 5 | 不做内存压缩 | 在飞 ID 规模由并发数决定（几十条） | +| 6 | 原 bit 31 的 dir/kind 位取消 | 实测 dir 位对入/出冲突无保护作用；kind 位判断后又被取消 | +| 7 | `/forward_resume` 废弃，并入 `/resume_trans` | 两者承载信息相同 | +| 8 | tfid 取消，目标 client 分配自身 fid 并回报 | 每条腿的 ID 只由分配该腿的一端产生，链路只剩一套来源 | +| 9 | 分配归属不变量：client→server 的 fid 由 server 分配；server→目标的 fid 由目标 client 分配 | 同上 | +| 10 | kind 位取消，低位固定为 0 | 转发识别不依赖 ID 位 | +| 11 | 转发识别：`/resume_trans` 到达 server 时若 `record["originator"] != 请求方` 则为转发消息 | 单次字段比较，无需位判断 | +| 12 | server 发 `/file` 时不带占位 ID，由目标在回报中给出 fid | 避免占位值被误用 | +| 13 | 重放时 server 向源端转发的 `/resume_trans` 保留目标原始地址 | 源端需要校验请求目标 | + +## 5. 设计 + +### 5.1 ID 布局 + +``` + 63 32 31 1 0 +┌──────────────────────────────┬───────────────────┬───┐ +│ counter（无上界） │ epoch (31b) │ 0 │ +└──────────────────────────────┴───────────────────┴───┘ +fid = (counter << 32) | (epoch << 1) +``` + +- 最低位恒为 0，保留为扩展位，不参与任何判断。 +- 线路上仍是十进制整数；`int()`、`shlex.split`、JSON 键 `str(fid)` 均无位宽要求（实测 20–61 位可往返）。 +- 日志格式 `format_fid(fid) = e.`。 + +### 5.2 分配器 + +```python +class _FidAllocator: + def __init__(self, live_tables): + self._epoch = int.from_bytes(os.urandom(4), "big") & 0x7FFFFFFF + self._counter = 0 + self._live_tables = live_tables # 引用，不复制 + self._lock = threading.Lock() + + def alloc(self): + with self._lock: + while True: + self._counter += 1 + fid = (self._counter << 32) | (self._epoch << 1) + if not any(fid in t for t in self._live_tables): + return fid +``` + +`live_tables` 为 `(file_upload_progress, file_recv_progress, _forward_resume_records)`。 + +**锁约束**：调用 `alloc()` 时不得持有 `_transfer_progress_lock` 或 `_forward_relays_lock`。 + +### 5.3 分配归属 + +| 腿 | 分配者 | 回报 | +|---|---|---| +| 发送 client → server（上传/普通） | server（入方向，对端 client 发来的 fid 由 server 登记） | 原有命令沿用 | +| server → 目标 client（投递） | 目标 client | `/server_file_transfer_port ` | +| 公钥推送 | 推送方 | 沿用 | + +说明：文件上传时 client 目前仍由自身分配器产生 fid 并发给 server（提交 1 的现状）。按决策 9，**client→server 的 fid 应由 server 分配**。这一点与提交 1 的现状不一致，需在提交 3 中一并调整，并在本文件 §8 标注。 + +### 5.4 `/resume_trans` 路由 + +``` +server 收到 /resume_trans （来自请求方 R）： + record = _forward_resume_records.get(fid) + if record is not None and record["originator"] != R: + # 转发消息：R 是目标，原始对端是 originator + 转发给 originator：/resume_trans + else: + resume_trans(R, fid, offset) # 直连恢复 +``` + +- 判据只有一次字段比较，不依赖 ID 位。 +- 源端收到 4-token 形式的 `/resume_trans ` 时，调用 `_resume_forward(target, fid, offset)`。 + +### 5.5 转发握手（新顺序） + +1. A → S：`/forward_item file report.pdf (C)`。 +2. S 分配源端 fid_S（上传腿，server 分配）。 +3. S 向 C 发 `/file report.pdf (A)`（不带 ID，决策 12）。 +4. C 分配 fid_C，回 `/server_file_transfer_port `。 +5. S 收到后登记 `_forward_resume_records[fid_S] = {originator: A, targets: [C], fid_targets: {C: fid_C}}`，并以 fid_C 作为该目标的投递键。 +6. S 向 A 发 `/forward_upload `；A 开始上传；S 扇出给 C（键 fid_C）。 +7. 完成后 S 删除 record；A 删除 upload 进度。 + +等待点由“等预先登记的端口”改为“等目标回报 fid+端口”，额外代价为一次往返。 + +### 5.6 重放（幂等） + +``` +C 残文件 → offset +C --/resume_trans --> S +S: record 存在且 originator≠C → 转发 +S --/resume_trans (C)--> A +A: _resume_forward(C, fid_S, offset) # 只读记录，不删 + resuming 已含 C → 只记日志 + 否则 _forward_one(..., fid=fid_S, resume_offset=offset) + --/forward_item file resume= ...--> S +S: 沿用 fid_S；向 C 投递，键仍为 fid_C（C 侧沿用原 fid） +成功：A 从记录移除 C；S 从 record 移除 C；空则删 +``` + +- `relay_fid`/`tfid` 参数取消：fid_S 由 S 侧记录沿用，fid_C 由 C 侧沿用。 +- C 的 `file_recv_progress` 键在恢复前后不变，幂等由“键稳定 + 成功才删”保证。 + +## 6. 变更文件列表 + +| 文件 | 动作 | 内容 | +|---|---|---| +| `PyFlow/network_api/connect_tcp.py` | 修改 | 分配器去 kind；删除 `FID_KIND_*`、`fid_kind`；`_resume_request` 改为 originator 校验；删除 `FORWARD_RESUME_LINE`、`parse_forward_resume_command`、`handle_server_command` 对应分支；`_forward_relay` 改为目标回报 fid；`_resume_forward` 去掉先删后放；`_forward_one` 接受既有 fid；模块 docstring 第 20–30 行同步 | +| `test/integration/network_api/test_tcp_transfer_resume.py` | 修改 | 删除硬编码奇偶 fid 约定；新增 D1/D2、C5 二次断线、C6 多目标用例；握手顺序测试 | +| `test/integration/network_api/test_tcp_file_transfer.py` | 修改 | 确认 fid 取值变化后的断言不依赖具体值 | +| `test/integration/test_forward_extension.py` | 修改 | 转发握手改为目标回报 fid | +| `test/unit/network_api/test_fid_allocator.py` | 新增 | 不变量：唯一、跨实例不同、跨重启不同、三表跳过、`format_fid` 可读 | +| `test/unit/network_api/test_resume_matching.py` | 修改 | `/resume_trans` 3/4 token 解析；originator 校验分支 | +| `docs/source/Network_APIs/TCP_Server_APIs.rst`、`TCP_Client_APIs.rst` | 修改 | `/resume_trans` 可选字段；fid 归属；`client_port` 与恢复关系 | +| `docs/source/File_Transfer/File_Transfer.rst` | 修改 | 转发握手与恢复链路；移除 `/forward_resume` | +| `docs/source/changes/transfer-fid-allocator.md` | 新增 | 变更说明：协议不兼容点、fid 取值域变化 | +| `docs/plans/transfer-fid-and-resume-unify-plan.md` | 删除 | 由本需求文档取代 | + +## 7. 公共契约变更 + +| 契约 | 前 | 后 | 兼容性 | +|---|---|---|---| +| `fid` 取值 | 小整数 | `(counter<<32)\|(epoch<<1)`，20–61 位十进制 | **不兼容**：外部假设小整数会失效 | +| `fid` 分配者 | 发送方各自分配 | 上传由 server 分配；投递由目标分配 | **不兼容** | +| `/server_file_transfer_port` | ` `，fid 为 server 预留 | 同格式，fid 为目标自分配值 | 格式兼容，语义变化 | +| `/resume_trans` | 3 token | 3 或 4 token（4 token 带目标地址） | 读兼容；旧实现收 4 token 丢行 | +| `/forward_resume` | 独立命令 | 删除 | **不兼容** | +| `/forward_item` | 无 ID 参数 | 无新增参数（重放沿用记录，不需 relay_fid/tfid） | 兼容 | +| `_forward_alloc_fid` | 公开方法 | 已删除（提交 1） | 外部调用方需改 | + +## 8. 已知偏差（需在提交 3 处理） + +提交 1 中，上传腿的 fid 仍由 client 的分配器产生并发给 server。按决策 9，上传腿的 fid 应由 server 分配。提交 3 需要： +- server 接收上传时为该腿分配 fid，回报给 client；或 +- 将上传握手改为 server 先给出 fid。 + +这是提交 1 的遗留问题，提交 3 合并时必须修正，否则归属不变量不成立。 + +## 9. 原子提交拆分 + +| # | 意图 | 主要路径 | 预估 | 局部验证 | +|---|---|---|---|---| +| 1（已完成） | 分配器 + 删两处归还 + 替换四处分配 | A1–A4, B1–B2 | ~130 行 | 已通过：10 + 53 + 22 条 | +| 2 | 删除 kind 位；ID 布局改为 `(counter<<32)\|(epoch<<1)`；`_resume_request` 改为 originator 校验 | D3, C3 路由 | ~60 行 | resume 套件全绿；originator 校验两分支单测 | +| 3 | 转发握手改为目标分配 fid（删 tfid 预留）；上传 fid 改由 server 分配（修正 §8） | A5, D1, §8 | ~120 行 | 转发集成测试全绿；握手顺序测试；上传归属测试 | +| 4 | `/forward_resume` 并入 `/resume_trans`（4-token 目标字段） | C3, C4 | ~70 行（净负） | 抓取 4-token 行；resume 套件全绿 | +| 5 | 重放沿用原 fid；删除 `_resume_forward` 先删后放 | A6, B3, B4, C5 | ~50 行 | 二次断线测试 | +| 6 | 多目标粒度测试；文档与变更说明；删除旧计划文件 | C6 | ~130 行 | 一成功一失败；文档构建 | + +提交 2 必须先合；提交 3 依赖提交 2；提交 4/5 依赖提交 3；提交 6 最后。 + +每个提交附报告：意图、改动内容、为什么、局部验证、明确未覆盖、风险。 + +## 10. 验收证据 + +1. D1 回归：入方向 fid 与出方向分配器 ID 不同；`_drop_stale_transfers(peer=X)` 不关闭发往 Y 的腿。 +2. D2 回归：新实例首批 ID 与读回的旧 ID 不同。 +3. 转发握手：目标回报 fid 前 server 不向源端开放上传；回报后才发 `/forward_upload`。 +4. C3/C5/C6 真实 socket 集成测试。 +5. 基线 64 条相关测试全绿（提交 1 后为 10 + 53 + 22 条，已验证）。 +6. 日志中一次转发恢复出现不带位判断的 4-token `/resume_trans`。 + +## 11. 风险 + +| 风险 | 影响 | 应对 | +|---|---|---| +| 协议握手顺序变化（新增目标回报往返） | 旧版本互不兼容 | `changes/` 标注；不提供兼容期 | +| ID 不含来源位，来源判断只能靠表或 originator 字段 | 判断逻辑必须覆盖两分支 | 单测分别覆盖 originator 为自己与非自己 | +| 目标分配 fid 后、回报前断线 | 该 fid 未登记，属于未开始的投递 | 单测确认无残留 | +| 上传归属修正（§8）影响提交 1 的测试 | 测试中预测 ID 的写法失效 | 提交 3 改为从回报中读取 fid | +| 三表检查锁顺序 | 死锁 | `alloc` 不得在两个 progress 锁内调用，逐点核查 | + +### 未覆盖（本次不做） + +- 中继落盘；暂停状态持久化；`is_enable_file_transfer_progress_bar=False` 时无法恢复；恢复对固定 `client_port` 的依赖（仅补文档）。 + +## 12. 对 CI / 测试 / 依赖 / 公共接口的影响 + +- CI：否。 +- 测试：新增 3 类，不删不改既有断言（测试中硬编码奇偶 fid 的写法会被删除，属于约定变更，需在提交 2 标注）。 +- 依赖：无新增。 +- 公共接口：**是**。 + +## 13. 待人类确认 + +确认本文件后，从提交 2 开始实施，每个提交单独报告并等待放行。 \ No newline at end of file diff --git a/docs/plans/transfer-sender-fid-r1-report.md b/docs/plans/transfer-sender-fid-r1-report.md new file mode 100644 index 0000000..d5cbb4c --- /dev/null +++ b/docs/plans/transfer-sender-fid-r1-report.md @@ -0,0 +1,44 @@ +# R1 报告：中继 upstream FID 由发送方预分配 + +状态：R1 主要更改已完成；用户明确要求继续 R2，不单独修补即将删除的测试 `request_id` 参数。本报告记录 R1 阶段实际证据，不声明 R1 已由人类审计通过，也不把 R2 后验证冒充 R1 独立验收。未创建提交、未合并、未推送。 + +## 意图、范围与数据流 + +对应 `transfer-sender-fid-revision-plan.md` 的 R1：所有新数据腿由该腿发送方通过唯一 `_FidAllocator` 分配。普通文件、目录项、公钥分配方不变；中继源 Client 在发命令前分配 upstream FID 并绑定既有五项上传行；Server 仅登记该 ID，再为每个目标分配独立 child FID。保留 route/targets/completed、进度 JSON、原路径、字节协议和最终数据 ACK。 + +执行路径：源端 `_forward_one` → upstream FID/上传行绑定 → `/forward_item ... fid=` → Server 核对占用并登记 route → 每目标 child FID → 原数据泵/目标 ACK → 原聚合 ACK → 成功才清理。失败保留上传及 route。 + +恢复必须显式携带 `resume=`，包括 `resume=0`；Server 核对实际源端、已有 route 和未完成目标，再复用原上下游 FID。新请求省略 `resume=`。旧 `relay_fid=` 线路不保留别名。 + +## 占用边界 + +Server 不覆盖已有同 upstream FID 的 route 或接收行，也不占用同目标下已有的上传、活动 FID/socket、暂停、relay 或反向 route 身份。冲突明确返回错误；不同 receiver 作用域不被误合并。登记遵循 progress → pause → route 锁顺序，分配器调用在这些锁之外。 + +## 修改范围 + +- `PyFlow/network_api/connect_tcp.py`：源分配、解析/恢复调用、逐目标分配与 route 占用拒绝。 +- `test/unit/network_api/test_fid_routes.py`：占用冲突和 receiver 作用域。 +- `test/integration/network_api/test_tcp_transfer_resume.py`：不同目标乱序、source-owned FID、零 offset 重放、两源同值冲突；现有 ACK/路径/进度断言保留。 +- TCP 两端 API、File_Transfer、活动变更说明与主/修订计划同步。历史 atomic-1–5/O 报告不回写。 + +R1 保留 UUID 关联直至 R2；当前最终线路与会话隔离以 R2 报告为准。 + +## R1 阶段验证 + +1. 新增冲突回归先运行：6 个 target-leg 所有者分支均失败，直接复现登记未拒绝已占用上游别名的问题。 +2. 修复并补齐接收/活动/暂停/route 边界后运行： + +```text +.venv/bin/python -m pytest test/unit/network_api/test_fid_routes.py \ + test/integration/network_api/test_tcp_transfer_resume.py \ + -k 'registration_rejects_upstream_fid_that_collides or registration_scopes_live_downstream_upload_conflicts or sender_owned_fids_to_distinct_targets_survive_out_of_order_replies or forward_resume_with_zero_offset_reuses_existing_route or duplicate_upstream_fid_is_rejected_without_replacing_existing_route' -q +``` + +实际结果：**13 passed, 55 deselected in 5.73s**。 + +3. R1 阶段合并受影响套件实际结果：**427 passed, 1 failed in 274.89s**。唯一失败为 `test_public_forward_api` 调用当时仍要求 `request_id` 的 `forward_target_command`，不是文件内容/恢复失败。用户明确要求不为旧参数修测试，而直接进入删除该参数的 R2；临时修补已撤回。 +4. R2 独立运行另验证明文/加密并发、逐目标 ACK 和 EOF 复用原 ID；它是最终整合实现证据，准确结果见 `transfer-sender-fid-r2-report.md`。 + +## 风险与闸门 + +不支持新旧线路混跑；不迁移旧进度，不处理 client_port 身份迁移，不新增自动重试或第二张路径表。R1 本身不能替代 R2 的未决 FID 隔离。R2 已获用户明确授权；最终报告完成后等待人工审查，不自动创建提交或推送。 diff --git a/docs/plans/transfer-sender-fid-r2-report.md b/docs/plans/transfer-sender-fid-r2-report.md new file mode 100644 index 0000000..a11298b --- /dev/null +++ b/docs/plans/transfer-sender-fid-r2-report.md @@ -0,0 +1,112 @@ +# R2 报告：实际控制 socket + FID 握手闭环，移除 UUID request + +状态：R2 已完成实现与本报告所列验证；最终受影响完整套件 **434 passed in 278.89s**，明文/加密独立真实程序通过，文档构建成功并保留两条既有警告。用户已明确授权在 R1 主要更改完成后继续 R2。当前停在人工审查闸门；未创建提交、未合并、未推送。不以此报告宣称人类已经审查通过。 + +## 1. 意图与范围 + +执行 `transfer-sender-fid-revision-plan.md` 的 R2，不恢复 FIFO、共享端口缓存、共享错误槽或额外尝试 token。保留 R1：每跳发送方分配新 FID，源端 upstream FID 预绑定现有五项上传行，Server 逐目标分配 child FID，`resume=0` 也明确走旧 route。保留进度/route JSON、原接收路径、逐目标完成状态、文件字节、既有最终数据 ACK 和加密入口。 + +R1 阶段的 427 passed / 1 项旧 `request_id` 调用失败没有通过给测试补旧参数来掩盖。用户要求 R2 删除该接口；当前 `forward_target_command`/`forward_item_to` 均不再有 UUID 参数。R1 阶段事实单独保留在 `transfer-sender-fid-r1-report.md`。 + +## 2. 当前线路与数据流 + +```text +/file <原字段> +/file_folder <原文件项字段> +/crypto_pub_key +/server_file_transfer_port +/forward_item fid= [resume=] <原文件与目标字段> +/forward_error +/resume_trans +``` + +目录创建不申请文件端口。旧五 token UUID 端口响应被拒绝，没有 fallback 或兼容别名。`uuid` 仍可用于与握手无关的公钥临时文件名，未因名称相同删除无关用途。 + +1. 发送前按实际控制 socket 对象及 FID 登记，使用 `_send_transfer_request` 在真正调用既有发送入口之前标记可能已发出；每个登记只允许一次发送。 +2. 正常端口或错误只交给该 socket/FID 的当前登记；端口和 stamp 是同一结果。无人等待、错误 socket/FID、已终结或未决 key 的响应被丢弃，不缓存到后续请求。 +3. `wait`/`wait_many` 与取消在同一 condition 锁内决定结果和未决归属。未发送退出可重登；已发送且无终结结果的超时/取消/发送不确定 key 保留于 `_unresolved`，同一会话不再发送该 FID 的新协商。已明确收到结果的目标不因另一目标超时被一律标为未决。 +4. 登记 record 是私有所有权句柄，不是新的线路或持久化 attempt ID；旧 context 的取消不能删除后来同 key 的新登记。普通与中继恢复沿用现有 per-upload 串行和同目标合并，不把所有文件全局串行。 +5. 中继生产方在发送端口或错误前标记该协商已终结；端口之后的文件数据/ACK 失败只走原数据腿，不再发送第二个控制错误。原腿结束前不启动同 FID 第二条写入。 + +## 3. 控制会话与资源归属 + +- 待 accept 的 listener 在对外公告前登记到所属控制 socket；accept 后的数据 socket、公钥/普通出站 socket，以及中继上下游 socket 也登记。中继下游 socket 同时受目标与源控制会话约束。 +- 控制会话自然断开、对象关闭或 Server 停止时，`cancel_socket` 只退役该 socket 的 waiters、未决 key 和所属 listener/数据 socket。关闭资源不要求另一条健康控制连接断开，未完成进度/route 保留。 +- 接收循环捕获实际控制 socket；旧 Server 会话注销不能删除同地址的新 client/crypto 状态，旧 Client receive 循环不能把新会话的 running 状态清掉。新 socket 可按原 FID 恢复，旧回调不能满足新等待。 +- 已消费端口后的数据/ACK 失败不等于未决端口，也不主动断开健康控制连接。暂停仍是原数据腿上的状态；控制会话关闭导致数据 socket 关闭后，普通发送和 relay 的暂停/背压等待能退出，不能挂住 worker。 + +## 4. 修改文件与不变边界 + +- 生产：`PyFlow/network_api/connect_tcp.py`，包括 registry、解析/构造、双向普通/目录/公钥/恢复发送、relay 两侧、公开 builder 与控制资源生命周期。 +- 测试：`test/unit/network_api/test_transfer_handshakes.py`、`test_fid_routes.py`；`test/integration/helpers.py`；`test/integration/network_api/test_tcp_transfer_resume.py`、`test_tcp_file_transfer.py`。crypto、async、forwarding attribution 与 Web 既有消费者纳入最终受影响回归；没有为它们保留旧参数别名。 +- 活动文档：File_Transfer、TCP 两端 API、transfer 三份活动变更说明、主计划及 sender-FID 修订方案。本报告和 R1 报告新增。历史 atomic-1–5/O 报告不回写；活动文档将被取代的旧 request/接收端分配说明明确标记为历史。 +- 无 CI、依赖、coverage、进度值结构、JSON schema、加密算法或鉴权策略变更。不修改与传输握手无关的 Web 联系人/FTP request ID。 +- Python LSP references 请求返回 `-32601 Unknown request: textDocument/references`，已记录工具问题；使用定向仓库检索迁移真实调用方，没有声称 LSP 引用枚举成功。 + +## 5. 实际验证 + +### 独立真实程序（不是调用 pytest 测试函数） + +`.venv/bin/python /tmp/pyflow-r2-smoke.py` 启动真实 Server、源 Client 与两个目标；每实例日志、进度、公私钥及 TOFU 路径隔离到临时目录。分别运行明文与加密，最终两种模式全部断言通过： + +| 路径 | 实际观察 | +|---|---| +| 双向普通/目录项 | source→Server 的 `nested/first.bin` 和 Server→Client 文件字节正确 | +| 两个不同目标并发 | 延迟第一 upstream 端口回复，第二先完成；60,017 / 70,029 字节各到正确目标 | +| FID 预绑定与线路 | 源控制命令发送前已有该 FID 的五项路径绑定；无 request 标记；所有 listener 回复四 token | +| 端口/stamp | 明文为 plain，加密为 enc:；文件通过对应数据通道完成 | +| 部分目标 ACK 丢失 | 源返回 False，原上传及 route 保留，另一目标完成集合保留 | +| EOF 恢复 | 请求 `resume=60017`，原 upstream/child FID 与原文件完成；成功后清理 | +| 未决超时 | 第一次 listener 回复被保留，等待超时；同 socket/FID 不能再登记或发请求 | +| 迟到端口/错误 | 注入原端口和 FID 错误响应，未解除未决 key | +| 健康控制连接 | 超时后另一 FID 完整传输成功；原控制 socket 仍存活，真实 `/time` 往返成功 | +| 自然断线与同 FID 新会话 | shutdown 原控制连接、释放旧 socket 后重连；原 FID 获新 listener，旧 socket 回调不能填充新等待，文件字节正确 | +| 资源收尾 | 明文 64 个、加密 116 个跟踪 worker 全部退出；旧 listener/资源随会话关闭 | + +首次烟测曾调用终结对象的 `close()` 后再 `connect()`，按既有关闭语义连接失败；脚本改为真实控制连接 shutdown → 接收循环退出 → 释放 socket → connect 后通过。未为烟测改变生产 close 语义。它与最终成功结果分开记录。 + +烟测为临时验证程序，未保留在仓库或其原临时脚本路径。 + +### 受影响完整套件 + +```text +.venv/bin/python -m pytest test/unit/network_api \ + test/integration/network_api test/integration/test_forward_extension.py \ + test/integration/test_forward_msg_attribution.py \ + test/unit/transfer_web/test_web_transfers.py -q +``` + +首轮 R2 整合：**431 passed, 3 failed in 293.30s**。三个迁移缺陷为：把合法四 token plain 回复遗留在拒绝列表、双目标测试漏建 event/lock、两个乱序测试从 stamp 槽而非 FID 槽取值。已修正测试驱动，保留原乱序、内容/路径/ACK 断言；没有通过生产特殊分支、跳过或弱化断言规避。新增原 ACK-loss 场景断言每次只有一个控制端口结果且控制连接保持健康。 + +次轮完整运行：**433 passed, 1 failed in 280.82s**。失败为 `test_unified_resume_serializes_distinct_offsets_without_local_fid_fallback`：测试先从两个不同控制连接发送请求，却未经同步就要求第一个调用对应的目标先进入发送。实际日志显示第二个目标的 offset 先到达源端，并非发生并行写入或 FID 串任务。已利用该测试原有的最终 ACK 屏障，在发第二请求前等待第一目标到达屏障；不修改生产调度，不删除串行、offset、旧 FID、原路径、最终字节及无本地 fallback 断言。 + +修正后的最终完整运行：**434 passed in 278.89s**。包含本报告的单元、真实 socket 集成、加密/公钥、async、转发扩展/归属和 Web 消费者；没有失败或跳过。前两轮失败及修复原因保留如上，不用最后一次成功覆盖历史事实。 + +### 文档与静态检查 + +- `.venv/bin/python -m sphinx -b html docs/source /tmp/pyflow-r2-html`：27 个源文件构建成功，保留两条既有警告：配置的 `_static` 不存在；`command_control_extension_tcp` docstring 第 23 行 definition list 缺空行。 +- `.venv/bin/python -m py_compile PyFlow/network_api/connect_tcp.py`：成功。 +- `.venv/bin/python -m ruff`：环境未安装 ruff 模块；改用已安装的 `ruff` 可执行文件。 +- `ruff check --select F,E9` 覆盖上述生产文件、helper 与相关单元/集成文件：修正测试迁移后只剩历史 custom-handler 的 `connect_tcp.py:7066 F841 response`。本轮未改该无关分支，不声明 lint 全绿。 + +## 6. 路径结论与剩余风险 + +| 修订计划路径 | 结论与实际证据 | +|---|---| +| 普通双向文件、目录项 | 通过；完整 file-transfer 套件及独立明文/加密双向程序，保持原嵌套路径和精确字节 | +| 双向公钥、并发 key exchange | 通过；完整 crypto 套件及独立加密 Server/三 Client 初始化与正常消息往返 | +| 首次一对一/多目标中继 | 通过；源 FID 预绑定、不同目标乱序、Server 仅分配 child 的真实回归与独立运行 | +| 无效目标、协商失败 | 通过；既有 forwarding/route 失败保留回归及两源同 FID 的定向拒绝，不覆盖另一个任务 | +| 普通/中继 resume | 通过；零 offset、部分 ACK、EOF 再确认、原 ID/路径/内容与完整目标集合 ACK 断言 | +| 同目标重复、不同 offset | 通过；重复请求合并及经 ACK 屏障确定性的 distinct-offset 串行回归 | +| 未决端口超时、取消、发送不确定 | 通过；registry sent/未发送取消分类、per-target wait_many 与双向真实超时隔离；独立程序证明另一 FID/控制命令仍工作 | +| 数据腿失败、EOF ACK 未完成 | 通过；失败保留进度/route；每次只有一个控制终结结果，原控制连接存活；独立程序原 ID EOF 恢复 | +| 控制连接自然断线 | 通过；双向旧 socket 回调拒绝、pending listener 退休、暂停数据 socket 关闭、新 socket 复用 FID；既有自动重连回归保留 | +| 源离线、目标重连 | 通过；完整 resume suite 的 pending 最新 offset、失败不提前删除与二次断线路径 | +| 占用冲突 | 通过；接收、活动 socket/FID、pause、relay/index 的 route 原子拒绝；跨源同值不覆盖，receiver 作用域不误合并 | +| 暂停/继续、O3/O4、读回 | 通过；现有 network_api 相关单元/集成回归，进度结构不增添持久化握手状态 | +| async、forwarding API、Web | 通过；完整 async、转发扩展/归属及 Web transfer 消费者套件，无旧 request 参数旁路 | + +取舍按已批准方案保持：同会话未决 FID 可能长期等待自然重连，未决集合内存随未决 FID 数量增长，不宣称 O(1)；不为未决 FID 新增主动重连、自动改号、attempt token 或新重试协议，既有 `is_auto_reconnect` 配置不变。不承诺区分错误实现对已完成协商重复发送、且恰落在下一次同 socket/FID 等待里的旧响应；单终结生产约束是去 UUID 的前提。不同源相同 upstream 数值冲突明确拒绝而不是同时成功，不引入 source-scoped route/JSON 迁移。 + +不支持新旧节点混跑，不自动迁移旧进度，不解决 client_port 变化的身份迁移，不做中继落盘或暂停持久化。不以本次烟测声称完成全路径抓包、全局日志脱敏或安全审计。最终完成后交付人工审查，不自动提交、合并或推送。 diff --git a/docs/plans/transfer-sender-fid-revision-plan.md b/docs/plans/transfer-sender-fid-revision-plan.md new file mode 100644 index 0000000..d47ef82 --- /dev/null +++ b/docs/plans/transfer-sender-fid-revision-plan.md @@ -0,0 +1,187 @@ +# 修订方案：发送方分配 FID，保留逐腿 ID，撤销 UUID request + +状态：用户已明确授权 R1 后继续 R2。R1/R2 已完成实现；最终受影响完整套件 **434 passed in 278.89s**，明文/加密独立真实程序通过，文档构建成功。历史 R1 的 427 passed / 1 项旧 `request_id` 调用失败按用户要求由 R2 删除旧参数解决，不给测试补旧参数。证据、失败修复记录及既有 lint/文档警告见 `transfer-sender-fid-r2-report.md`。当前等待人工审查；未创建提交、未合并、未推送。 + +## 1. 已确认方向与本次收敛 + +1. 普通双向文件、目录项和公钥已经由发送方分配新 FID。普通 resume 复用原上传行的旧 FID，不应重新分配。 +2. 修订前中继由 Server 分配上游 fid 及每个目标的 tfid；R1 已将源 Client 到 Server 的上游腿改为源 Client 预先分配。Server 发往目标的下游腿仍由 Server 分配。 +3. tfid 只是下游腿角色名。当前已经调用普通无 kind `_fid_alloc.alloc(receiver=target)`，不需要第二个分配器，也不需要删除逐腿映射。 +4. 发送方预分配 FID 能解决不同新任务的端口/错误关联，但不能单独区分稳定 FID 的多次尝试。不同尝试若使用同一实际控制 socket 和同一 FID，迟到的旧响应仍无法仅凭 FID 识别。 +5. 已有控制连接真正断线时，新连接自然是新会话，不需要额外主动换一次。文件数据子连接失败或关闭，不代表长期控制连接也已断开；两种情况必须分开。 +6. 端口申请已发出但终结响应未知的超时、取消或发送结果不确定，按 `(actual_control_socket, fid)` 留下未决标记：当前会话内不再为该 FID 发起新端口申请，迟到响应直接丢弃，不恢复旧等待。其他 FID、命令和健康控制连接不受该标记主动中断；待控制连接自然断开并重新建立后，才允许恢复原 FID。 +7. 端口响应已明确消费后的文件数据/最终 ACK 失败，不默认断控制连接，也不自动归入“端口未决”。原数据腿结束、原协商收尾后仍按既有串行/合并约束处理恢复，不能在旧腿仍拥有资源时再启动同 FID 写入。 + +**本次不再提出“所有失败统一换控制会话”。** 保守代价收窄为：没有终结响应的该 FID 等待自然重连，可能长期保留未完成进度；不自动重连、不改号、不用改名 attempt token 绕过此规则。完整去 UUID 与未决标记必须在同一个原子更改中落地，不能留下仅删 request 的不安全中间版本。 + +## 2. 需求卡 + +### 目标 + +- 所有新数据腿由该腿的发送方通过唯一 `_FidAllocator` 分配 FID。 +- 新中继上传在发控制命令前，源端已将 upstream FID 与文件绑定；不再等 Server 分配 upstream FID。 +- 保留每个目标独立的 downstream FID（现称 tfid）及 Server 逐腿 route/completed 状态。 +- 普通与中继恢复复用原 FID、原完整接收路径、磁盘 offset 和既有最终数据 ACK。 +- 删除 `request=`、相关参数/生成器和 UUID-keyed 等待登记，改用实际控制 socket 与 FID 关联；同时隔离未决 FID，不以文件失败主动断开健康控制连接作为前提。 + +### 非目标 + +- 不恢复旧递减计数器、专用 `_forward_alloc_fid`、FIFO、共享 `_forward_error` 槽或无主端口缓存。 +- 不迁移到接收端分配；原主计划更改 6 的该方向取消，不将其视为必要重构。 +- 不把每次 resume 变成新 FID，不添加改名后的 attempt token，不新增第二张路径/进度权威表。 +- 不改变五项进度值、现有 JSON key 结构、文件字节协议、加密算法、鉴权策略、CI、依赖和 coverage。 +- 不解决 client_port 变化后的身份迁移，不新增自动重试协议，不自动迁移或删除旧进度文件。 + +### 安全与数据边界 + +- FID 仍不是鉴权凭据。Server 继续核对实际请求端、原始 origin、目标全集和 route 所有权。 +- port 与 stamp 一次交付，不拆分消费；公钥腿仍 plaintext，普通加密腿仍走现有 framing/发送入口。 +- 外部源端提供 FID 后，Server 不得覆盖其他源的 route、接收行、活动腿或暂停状态。保留现有裸 upstream key 结构意味着数值占用冲突必须明确失败，不是无条件支持不同源同号并存；不得偷偷换号。 +- 本方案选择冲突拒绝以避免 JSON/索引迁移。若要求不同源同号都可同时成功，需要另行批准 source-scoped 接收/route key 及持久化契约；不夹带在本方案中。 + +### 风险与禁止事项 + +风险中高：改变中继上游分配权、控制线路和未决握手的重试条件。未决 FID 可能要等待自然重连；会话内未决标记的内存随未决 FID 数量增长，不宣称 O(1)，断开后释放。禁止删除/跳过行为测试、弱化内容/路径/ACK 断言、恢复 FIFO、仅靠概率宣称无冲突、无声接受旧响应、新旧节点混跑、直接 reset/clean 工作区、自动提交或推送。 + +## 3. 线路与接口契约 + +### 3.1 普通文件、目录项、公钥 + +保留命令原字段及末尾 FID，移除 request 标记： + +```text +/file <现有字段> +/file_folder <现有文件项字段> +/crypto_pub_key +/server_file_transfer_port +``` + +无文件内容的目录创建命令仍不申请数据端口。 + +### 3.2 中继源请求 + +```text +/forward_item fid= [resume=] <现有文件与目标字段> +/forward_error +``` + +- 新任务：源端 `_fid_alloc.alloc(receiver=server_peer)`；在发请求前将 FID 与源路径、总大小和既有五项上传行绑定。上传 key 仍为 `(server_peer, fid)`，五项值中的 peer 继续沿用逻辑接收者含义，不借此改进度结构。Server 只能登记这个 upstream FID，不能另行分配。 +- 新任务不带 `resume=`；恢复明确带 `resume=`，包括 `resume=0`。恢复要求旧 route 存在且属于源端，不因 offset 为零误建新 route。 +- Server 创建每个目标的 child FID 时继续调用 `_fid_alloc.alloc(receiver=target)`，保留 `targets` 和 `completed` 的逐目标身份。 +- `relay_fid=` 线路被必要的 `fid=` 加显式恢复语义替代，所有调用方迁移，不留旧线路别名。Python 内部参数统一表达 upstream FID，名称调整不得扩展为新类型体系。 +- 目标 listener 与源上传使用同一四 token 端口响应；不恢复 `/forward_upload`。 +- `forward_target_command`、`forward_item_to` 移除仅为 UUID 添加的 keyword-only `request_id`，其余已有字段保持。 + +### 3.3 统一恢复 + +```text +/resume_trans +``` + +格式不变。接收方使用旧接收记录和磁盘 offset；Server 按实际 requester、origin、target 和 route 转换 downstream/upstream FID；源端从既有上传行取路径。普通或转发恢复均不重新分配已存在的腿 ID。原路径、相同 fid、部分完成和 EOF 再确认约束全部保留。 + +## 4. 去 UUID 后的等待与会话不变量 + +- 瞬态等待 key 改成 `(actual_control_socket, fid)`，登记早于发送；每个 key 最多一个有效协商。复用现有同目标合并、同 upstream FID 的不同目标串行，不全局串行所有文件。 +- 每次生产协商只允许一个终结控制结果：端口响应或协商错误。端口返回后，数据传输失败走既有数据连接错误/ACK，不再补发同次协商的第二个控制结果。该单终结约束与去 UUID 同批验收。 +- 正常端口或错误响应只交付当前 key，端口与 stamp 成对消费。错误 socket、无人等待、已经有结果和未决 key 的响应均丢弃，不缓存为后续任务结果。接收循环捕获所属 socket 对象，不在旧回调中读取新的 `self.client_socket`。 +- 无歧义地在发送前退出：删除登记即可。命令可能已经发出、但本方尚未确定终结结果就超时或取消：该 key 标为未决；不能简单删除后在同一会话重登同 FID。等待返回与超时/取消必须在同一同步边界决定归属，避免把已到达的结果误分类。 +- 未决 key 不再触发新 `/file` 或 `/forward_item`，也不被迟到端口/错误自动解除；只保留必要的瞬态 FID/状态，不复制文件路径、不持久化。旧会话退出清理标记；新 socket 可以根据原进度/route 恢复。没有自动重连时，继续沿用用户手动重连的既有行为。 +- 多目标收集中每个 key 单独定性：已经明确取得结果的目标不因另一个目标超时被一律标为未决；源上传若已经收到明确协商错误，也不等同于它自身的端口响应未知。部分目标状态与完整 route/completed 保留。 +- 已消费端口结果后的数据/ACK 失败：关闭该数据腿、保留进度/route，不主动断健康控制连接；原腿未释放前，不能启动第二条同 FID 写入。暂停/继续仍作用于原数据连接，不当作失败。 +- 控制连接自然断开：取消其 waiters、清理未决标记及其所属 pending listener/数据腿，保留未完成进度/route，复用已有退出/重连生命周期。当前 pending accept listener 尚未进入 accept 后的数据 socket 登记（connect_tcp.py:5488–5520、10954–10986），需补齐运行时资源归属，不新增持久化状态。 +- 去 UUID 的关联以协议生产方单终结响应为前提；不承诺区分错误实现对已结束协商重复发送、且恰落在下一次同 socket/同 FID 等待中的旧响应。若还要求此保证，必须保留额外尝试身份或进一步禁止会话内复用该 FID，不能靠分配器证明。 + +## 5. 路径映射与验收要求 + +| 路径 | 必须修改或保持 | 验收证据 | +|---|---|---| +| 普通双向文件、目录项 | 发送端分配保持；去 request；socket+fid 等待 | 同连接延迟第一响应，第二文件先完成，精确字节与嵌套路径正确 | +| 双向公钥及并发 key exchange | 原发送端分配、明文 key 腿和安全切换顺序保持 | 多客户端真实 key exchange 完成，正常帧消息可往返 | +| 首次一对一/多目标中继 | 源先分配 upstream，Server 只分配 child | 源发命令前已有绑定；响应回同 FID；不同目标不同任务的文件不能互换 | +| 无效目标/协商失败 | 错误带 fid；不改其他请求状态 | 一项失败、另一项成功；失败行/已登记 route 保留 | +| 普通/中继 resume | 复用旧上下游 FID，不新建 route | 零 ACK、部分 ACK、EOF 再确认；原路径/内容正确，完整集合 ACK 前源不成功 | +| 同目标重复/不同 offset | 继续合并/串行，不恢复第二张元数据表 | 相同目标只一次有效发送；不同目标各用其 offset | +| 未决端口超时/取消/发送结果不确定 | 当前 socket 的该 FID 不再发新协商，迟到回复丢弃；其他 FID 不被主动中断 | 先超时后注入旧端口/错误，不唤醒新任务且不重发该 FID；另一 FID 完成、控制命令往返正常 | +| 数据腿失败/EOF ACK 未完成 | 已消费端口不等于端口未决，不默认关闭健康控制连接；前一腿结束后再恢复 | 数据 socket 释放、进度保留、控制连接仍收发；恢复保持原 ID/路径 | +| 控制连接自然断线 | 清理旧 waiters/标记/listener，沿用既有退出与重连，不额外主动换一次 | 同地址新 socket 可以恢复原 FID；旧回调不能满足新等待；自动重连开/关符合原配置 | +| 源离线、目标重连 | 保留 pending 最新 offset 与 route | 新会话补发后原 FID 完成；发送失败不提前删除 pending | +| 占用冲突 | 拒绝，不覆盖、不重分配 | 两源同数值 upstream、与已有接收/下游活动身份冲突均返回明确错误，旧任务状态不变 | +| 暂停/继续、O3 查询/O4 删除、读回 | 不新增持久化状态，不把暂停当失败 | 控制/查询/删除在途保护、重启读回回归通过 | +| async、forwarding API、Web 消费者 | 全部迁移，不留 request 调用旁路 | 相应既有集成/契约回归通过 | + +## 6. 原子拆分与文件范围 + +用户已明确授权按 **R1 → R2** 继续，R2 正在实施。每项附对应测试、文档、局部报告和真实运行证据；合并旧 A 的分配权/恢复接入；旧 B/C 的未决规则与去 UUID 必须合在 R2，避免无隔离的中间版本。测试、文档和报告不另拆为后续任务。 + +| 原子项 | 单一意图与估计 | 关键改动 | 局部验收 | +|---|---|---|---| +| R1：统一每跳发送方分配 | 预计 200–350 行；保留当前 request 关联直至 R2，不新增临时别名 | 源端先分配并绑定 upstream FID；Server 接收必填 `fid=`，只分配 downstream FID；用显式 `resume=` 区分旧 route 恢复；合并冲突检查、失败保留和稳定恢复调用方 | 两个不同目标并发乱序不互换文件；新上游 ID 不被 Server 改写；`resume=0` 不建新 route；部分 ACK/EOF 恢复仍用旧上下游 ID；占用冲突不覆盖 | +| R2：用 FID 完成握手闭环并去 UUID | 预计 350–500 行；握手关联与未决隔离不可拆开 | 所有普通/目录/公钥/中继改为 socket+FID；删除 request 全链参数；端口与错误直接携带 FID；未决 key 阻止同会话重试，正常数据失败不断控制连接；归并单终结响应、旧会话资源清理和全部消费者迁移 | 双向并发/乱序、nonce 配对；超时后该 FID 不重发/不吃迟到回复；其他任务/命令正常；自然重连后原 FID 恢复；数据失败与 ACK 丢失保留原进度且控制连接存活 | + +估计为待实施 diff 规模，不是已有结果。若单项实际超过 500 行或混入第二意图，先回到计划修订；不为凑两项跳过审查，也不把必需的测试/文档推迟到下一项。 + +### R1 文件范围 + +- `PyFlow/network_api/connect_tcp.py`：`_forward_one`、`_forward_item_handler`、`_forward_relay`、route/进度占用检查及恢复调用；保留现行 UUID 线路直至 R2。R1 中 `/forward_item` 保留 kind 后的 request 标记，同时新增必填 `fid=`；完成后不再存在“Server 为新上传分配 upstream ID”的分支。 +- `test/unit/network_api/test_fid_allocator.py`、`test_fid_routes.py`；`test/integration/network_api/test_tcp_transfer_resume.py`，以及受影响的转发/加密既有行为测试。 +- 同步中继分配权/恢复接口说明、主计划相应状态和 R1 原子报告；主计划原更改 6 的接收端分配方向改为被本修订取代，不回写历史报告。 + +### R2 文件范围 + +- `PyFlow/network_api/connect_tcp.py`：registry/解析构造/分派、普通双向/目录项/公钥/中继的全部生产者与消费者、control socket 生命周期、待接收资源所有权。保留正确的 socket 捕获、port/stamp 配对和数据 ACK 行为。 +- `test/unit/network_api/test_transfer_handshakes.py`、必要的 `test_fid_routes.py`/`test_resume_matching.py`；`test/integration/helpers.py`；`test/integration/network_api/test_tcp_transfer_resume.py`、`test_tcp_file_transfer.py`、`test_crypto_tcp.py`、`test_tcp_async_server.py`；`test/integration/test_forward_msg_attribution.py`。 +- 同步普通/公钥/中继全部新线路及未决 FID 等待自然重连的说明、R2 原子报告；受影响翻译条目局部清理，不全局再生成。 + +### 不再单独拆分或追加的事项 + +- tfid 当前已经是普通分配器产物，只保留角色名称/映射，不做第二轮分配器重写或全仓机械改名。 +- 不恢复 `/forward_resume`、`/forward_upload`、共享端口 cache/FIFO/错误槽；不增加第三张路径/恢复表，不改 progress/route JSON。 +- 普通发送分配权、五 token resume、逐目标完成与原最终 ACK、O3/O4、Web 输出契约均保留，只迁移确实受影响的消费者。 + +生产代码预计仅需 `PyFlow/network_api/connect_tcp.py`；不修改 Web 数据适配器或进度 JSON。测试具体范围： + +- `test/unit/network_api/test_fid_allocator.py`、`test_fid_routes.py`、`test_resume_matching.py`、`test_transfer_handshakes.py`。 +- `test/integration/helpers.py`。 +- `test/integration/network_api/test_tcp_transfer_resume.py`、`test_tcp_file_transfer.py`、`test_crypto_tcp.py`、`test_tcp_async_server.py`。 +- `test/integration/test_forward_msg_attribution.py`，以及真实消费者检索发现的其他受影响调用；实施前对公开 builder/dispatch 运行可用 LSP references，若服务不支持则记录并作全仓定位。 +- 文档：主计划、TCP 两端 API、File_Transfer、活动 transfer 变更说明、对应新原子报告；翻译仅同步受影响旧条目，不全局再生成。此前报告保留其当时事实，由修订计划标明替代关系，不伪造为新方案证据。 + +不通过直接 git reset 回到旧版本实现 R1/R2。当前 UUID 更改尚未提交；按修改意图收敛，保留正确的实际 socket 捕获、资源退役、端口/stamp 配对和行为回归。确认拆分不等于授权创建 git 提交、合并或推送。 + +## 7. 验证命令与已运行证据 + +### 方案前真实运行证据（不替代 R1/R2 验收） + +本轮在内存加载 HEAD `5002502` 的无 request 历史模块，临时目录隔离日志/进度，启动真实 Server 和 Client。两文件预先由源分配不同 FID，压住第一 port 响应，第二文件先完成，再释放第一响应。实际结果： + +- payload 60,018 / 60,035 字节，两文件完整内容各自正确；两个发送函数均成功。 +- 回应为四 token，无 UUID request；观察到端口 38079 / 46425。 +- 同数值 upstream FID 的第二 origin 被现行 route 登记拒绝，旧 route 不被覆盖。 +- 所有跟踪 worker 退出，未写入仓库代码；没有保留烟测脚本。 + +前轮证据继续有效，不重跑以掩盖事实：`7b84660` 同一 FID 第二握手领取迟到旧端口，两次握手期间分配器调用次数为 0；`5002502` 共享 FIFO 可使不同目标互收对方文件。这两项说明不能只靠分配器或恢复旧队列宣称安全。 + +当时尚未实施 R1/R2；前述预实施探索不是“源预分配中继、当前冲突拒绝、去 UUID 新 waiter、未决 FID 隔离及自然重连恢复”已完成的证明。 + +### 已按约定执行的整合验证 + +各原子项运行对应行为回归和真实 socket 烟测，最后执行： + +```text +.venv/bin/python -m pytest test/unit/network_api \ + test/integration/network_api test/integration/test_forward_extension.py \ + test/integration/test_forward_msg_attribution.py \ + test/unit/transfer_web/test_web_transfers.py -q +.venv/bin/python -m sphinx -b html docs/source /tmp/pyflow-sender-fid-html +``` + +最终受影响套件 434 passed；独立真实程序已覆盖明文/加密、不同目标并发、多目标部分 ACK/EOF、未决隔离、健康控制连接与自然重连原 FID；全部跟踪 worker 退出。Sphinx 构建成功，保留两条既有警告；Ruff 仅剩历史 custom-handler F841。路径结论、命令、修复记录和未覆盖边界详见 `transfer-sender-fid-r2-report.md`。 + +补充真实运行证据：在当前工作区版本中以既有错误符号替代文件最终 ACK，发送函数返回 False，数据 socket 已退出活动登记，上传行保留；原控制 socket 和运行状态保持，随后通过同一控制连接完成一次真实命令/响应。全部跟踪 worker 退出。该证据验证两类 socket 的生命周期不同，不是无 request 同 FID 重试已实现的证据。 + +## 8. R1 证据与 R2 执行状态 + +R1 覆盖源端预分配 upstream FID、显式 `resume=`（含零 offset）、Server 逐目标分配 child FID、占用冲突拒绝，以及对应测试与活动文档。定向回归 13 passed；合并受影响套件 427 passed / 1 项旧 `request_id` 调用失败。用户明确要求不单独修复该旧调用，因为 R2 删除此参数，并明确授权继续 R2。 + +R2 已实施并通过本报告关联的验证；去 UUID 与未决 FID 隔离同批完成。不因数据腿或 ACK 失败主动断开健康控制连接，未决 FID 等待自然会话断开后恢复，既有自动重连配置不变。R1/R2 当前交付人工审查，不将验证通过等同于人类审计通过，不创建提交、合并或推送。 diff --git a/docs/plans/transfer-sign-buffer-bounds-plan.md b/docs/plans/transfer-sign-buffer-bounds-plan.md new file mode 100644 index 0000000..b68e341 --- /dev/null +++ b/docs/plans/transfer-sign-buffer-bounds-plan.md @@ -0,0 +1,190 @@ +# 传输协议标记缓存有界化：消费后删除版方案 + +状态：需求方已明确“审计完成，开始更改”。B1/B2 已实施并完成定向测试与隔离运行烟测；最终验收结果与差异见 [transfer-sign-buffer-bounds-report.md](transfer-sign-buffer-bounds-report.md)。未创建 git 提交、未合并、未推送，人工最终放行与平台 CI 尚未发生。不使用 `threading.Event`。 + +## 1. 需求卡 + +### 目标 + +- 直传发送端的完整协议行进入有界待处理队列；消费线程记录已处理前缀，处理后删除，不保留完整行历史。 +- 未终止协议行同样有长度上限，不能通过一直不发送换行让 `partial` 累积增长。 +- 中继等待 start/final ACK 时不再无限累积 `buf`。 +- 保留此前修复的粘包、拆包及 ACK 后紧随 EOF 的行为；必须先取得判定结果再删除记录，不能因索引前移或连接关闭而跳过已入队 ACK。 +- 不使用 `threading.Event`，不增加线程或新的握手。 + +### 非目标与禁止项 + +- 不更改文件正文、文件元信息的二进制收发、文件分块大小、FID、进度模型或续传匹配。 +- 不更改 ACK 发送时机、既有等待时限、接收端关闭时机、加密握手或统一 send/receive 入口。 +- 不删除/跳过既有测试、不弱化断言、不增加 lint 忽略、不改 CI、依赖或覆盖率。 +- 不将“达到容量就清除”实现成无条件丢弃半条协议行或已识别状态。 +- 不把 `max_dict_size` 或 `file_transfer_chunk_len` 当成协议缓存上限：它们不是同一个契约。 +- 不声称整条连接或 Python 进程总占用小于 64 KiB。协议原始字节、待处理记录的线长配额受限；Python 字符串/容器、接收对象、加密层和文件缓冲仍有额外占用。 + +### 数据与安全边界 + +本方案只读取发送端入站的协议标记，不对正文执行分行或 strip。完整行最多暂存至下一次容量扫描，不新增原始内容日志。达到 64 KiB 时扫描并批量清理完整记录、保留 start/ACK/error 判定，然后继续解析后续字节；不能截断后把尾部误识别成新命令，也不能丢弃已识别 ACK。 + +风险等级：中。缓存回收影响 start/ACK/error 的可见性；误清理可能把成功传输误判为失败，错误识别成功则会破坏进度恢复。 + +## 2. 64 KiB 的准确含义 + +设内部常量 `TRANSFER_SIGN_BUFFER_LIMIT = 64 * 1024`，不是公共配置项。 + +- **批处理窗口上限**：每个扫描窗口的原始协议字节最多 65,536 字节。一次读取更大时分段处理，不先拼成更大的历史缓冲。 +- **直传单行上限**：一条协议行连同结尾的 `\n` 最多 65,536 字节。单条未终止行耗尽此上限时无法识别行边界，按既有失败关闭路径处理。 +- **直传待处理总量上限**：未扫描记录的原始线长之和，加上尚未终止的 `partial`，不超过 65,536 字节；在 strip/解码前按 bytes 计量。 +- **达到容量时**：处理完当前完整行后若总量达到 65,536 字节，立即扫描队列全部记录、保存识别结果、一次删除已扫描前缀并释放队列配额，再继续处理同一次读取的剩余数据。若追加一条完整行会越过上限，则先扫描/清空已有完整行，再追加该行。 +- **partial 占满可用配额时**：先扫描并清除已有完整行，保留 partial；如果 partial 本身达到 65,536 字节仍无换行，则它是一条超长协议行，走失败关闭。不会清掉半条行继续寻找伪造的后缀命令。 +- **及时检查**：每次读取后解析完整行，不等缓冲填满；容量扫描与调用线程正常消费走同一套锁内 marker 核对/前缀删除逻辑。 +- **无错误的正常聚合**：任意多条各自合法的短行即使单次读取累计超过 64 KiB，也分批扫描并继续；队列字节配额满本身不是传输错误。超限失败只针对单条未终止协议行，或无法安全分界的单条超长行。 + +64 KiB 是待扫描记录与 partial 的并存窗口上限，不是允许队列增长后失败的阈值。每到上限扫描完整记录、保存识别结果并批量清理；只有无法分割的超长单行不能通过清队列解决。 + +## 3. 直传设计：有界队列、消费索引与前缀删除 + +修改两端 `file_transfer_mode` 中的协议接收线程及 `_take_wire_lines`。 + +### 3.1 原始字节与临时行 + +1. 输入继续来自现有 `receive_message`，加密数据先经原有解密路径。 +2. 按窗口剩余空间分段扫描 `\n`，在分配/追加之前检查容量；完整行才 UTF-8 解码，沿用既有 strip/空行处理，不改文件内容路径。 +3. 完整非空行连同原始线长进入临时待处理队列；队列达到 64 KiB 时扫描所有记录、锁内保存识别结果后批量删除，不保留完整行历史。空行直接消费。 +4. 队列与 partial 的线长在锁内核算。若部分行占用剩余空间，先扫描/释放已有完整记录，再保留 partial。 +5. 合法完整短行达到批次上限时不报错、不丢行，扫描后继续当前读取。只有单条行自身超限(未终止尾部达到 64 KiB，或换行位置已超过单行上限)才走既有失败路径；不截断、不重同步到行内后缀。 +6. EOF 不当作换行，不把未终止 ACK 提交为成功；EOF/关闭不清空已排队记录。 + +实施落点：原 `_take_wire_lines` 的两处产品调用及边界测试统一迁移到私有 `_TransferSignBuffer`。`feed` 逐行解析，达到 64 KiB 时调用锁内扫描逻辑将完整 pending 前缀批量核对、锁存 marker 并删除；`consume` 在阶段检查时对未满批次执行相同操作。两者共用队列、线长配额和结果锁，没有保留旧辅助函数或适配入口，公开 API 不变。 + +### 3.2 一个生产者、一个消费者 + +- 接收线程是唯一生产者，负责解析与入队。 +- `file_transfer_mode` 调用线程负责阶段性消费；接收线程在达到容量时也调用同一锁内扫描/清理逻辑。 +- 批量扫描维护的 start/ACK/error 结果字段由两侧仅在同一把锁内读写；不依赖某个线程私有的共享状态。 +- 读取 marker 结果也通过同一锁快照；不使用 `threading.Event`，不增加线程或结果通道。 +- 接收线程发现本次输入含 error sign 后，在锁外走现有关闭路径；已扫描记录中的 error 状态仍保留，调用线程在 EOF 后消费快照。 +- 完整行队列达到上限时批量扫描并删除，不是错误。只有单条无法以 `\n` 安全分界的超长协议行沿用接收失败路径；发送端仍消费已入队记录再检查关闭，不改变 ACK/error 既有优先级。 + +### 3.3 索引与删除规则 + +1. 每次消费时，在锁内取得当前队列长度，`read_index` 从零开始核对这一批。 +2. 对当前行先完成判定：start/ACK/error 更新本次调用的固定结果；header 和未知完整行按既有语义忽略。 +3. 只有判定完成才增加 `processed_count`，并累加对应的已消费原始线长。 +4. 本批核对结束，执行一次 `del pending[:processed_count]`，同时扣减配额，再释放锁。结果先保存，原记录后删除。 +5. 下一批索引重新从零开始，不跨删除操作保留旧下标。新追加的记录只会在锁释放后入队，不会被上一批的前缀删除误删。 + +这是“处理过的记录不进入下一批”的消费删除，与处理一条就删除一条具有相同的回收语义，但避免每次删列表头都移动后续全部元素。不能在 `enumerate(pending)` 中 `del pending[index]` 后继续递增：列表左移会跳过下一项。 + +### 3.4 处理完成的含义与阶段衔接 + +- 每条记录先判定并更新固定结果，再删除；“已经遍历过”不等于处理完成。 +- 同一批出现 header 和 ACK，两条都核对并删除，ACK 结果保留。容量触发的 producer scan 与发送线程 consume 共用同一锁和扫描逻辑；每轮等待仍先快照 `file_running` 再消费，然后依据锁内 marker 快照处理关闭。 +- 队列达到 64 KiB 时立即扫描/清理；在启动等待、正文分块发送前及最终 ACK 等待处也可主动消费部分批次。阶段切换不重置判定结果。 + +核对并发时需要区分：生产者单纯 append 不会移动旧记录的下标；下标前移来自删除自身。锁保护的是“核对、发布结果、删除、更新配额”这一组操作，不能把 CPython 某个单独 list 操作的原子性当成整组操作的保证。 + +## 4. 中继设计：扫描后仅保留跨边界匹配所需后缀 + +修改 `TCP_Server_Base._forward_read_sign`；其调用点是目标连接的 start 等待和 writer 的最终 ACK 等待。 + +- 输入仍经 `receive_message`，返回仍为 `target bytes | None`。 +- 沿用当前字节标记查找语义；本次不顺手改成“必须整行匹配”，也不额外要求 marker 自带换行。 +- 扫描窗口最多 64 KiB。一次读更大时分段扫描；同一次读取内仍保留现有先判 error、再判目标的顺序。 +- 未发现目标或 error 时，清除已扫描历史，只保留最后 `max(len(target), len(error)) - 1` 个字节。这已足够识别跨两次读取/窗口的 marker。 +- deadline、超时设置/恢复、EOF/异常返回 `None` 均保持不变。窗口清理不重置 deadline。 +- 中继不是按行解析器，因此这里限制的是等待标记时保留的历史窗口，不额外引入超长行拒绝规则。错误标记和目标标记的跨边界匹配都必须回归。 + +目标 writer 只有收到真实最终 ACK 才完成对应路由；全目标完成门槛、上游 ACK 换行发送以及 ACK 写成功才删除路由的规则均不变。 + +## 5. 数据流与路径全集 + +直传：协议入站 bytes → 有界分行/配额检查 → 待处理队列 → 调用线程核对并保存固定结果 → 按已消费索引删除前缀/释放配额 → 原有等待与进度处理。 + +中继：目标入站 bytes → 有界窗口查找 error/target → 匹配则返回 → 否则只留短后缀 → 原有路由完成/失败处理。 + +| 路径 | 修改点 | 必须保留的行为与验收 | +|---|---|---| +| 客户端上传普通文件 | 客户端 `file_transfer_mode` 接收线程 | 初传、拆包/粘包 ACK 返回成功，正文相同、上传进度清理 | +| 服务端下发普通文件 | 服务端 `file_transfer_mode` 接收线程 | 同上，覆盖相反方向 | +| 文件夹项 | 两端相同发送腿 | 相对目录及正文不变，失败仍保留进度 | +| 双向续传与 EOF 重放 | 两端相同发送腿 | offset=0、部分 offset、offset=total；header/ACK 同批仍成功 | +| 双向公钥推送 | 两端同一发送函数 | 无进度的 plaintext 公钥腿仍完成，原有 TOFU/握手不变 | +| 直传 error/EOF/单条超长行 | 两端协议接收线程与消费点 | 容量批次扫描后保留错误状态；完整 ACK 不丢；只有无边界超长行失败，不清理恢复记录 | +| 中继目标 start | `_forward_read_sign` 的 start 调用 | 目标/error 跨读取和跨窗口仍可识别，失败沿用跳过/清理路径 | +| 中继目标最终 ACK | writer 的 `_forward_read_sign` 调用 | 真实 ACK 才完成目标，全目标聚合和上游路由删除门槛不变 | +| 中继错误/超时/断开 | 同上两处调用 | 返回 `None`，deadline 不被持续数据延长，未确认目标不标完成 | +| plaintext/encrypted | 两端协议读取和中继入口 | 不绕过现有接收/解密；正常发送结果与落盘内容相同 | + +## 6. 文件与接口影响 + +计划修改： + +- `PyFlow/network_api/connect_tcp.py`：内部容量常量、分行/原始线长核算、两端协议待处理队列与索引消费、中继滚动窗口。 +- `test/unit/network_api/test_wire_lines.py`：长度/批次/尾部边界契约；追加后与删除后的配额一致性、索引不跳项；保留原有拆包行为断言。 +- `test/integration/network_api/test_tcp_transfer_resume.py`：复用现有真实 socket/worker 生命周期辅助，覆盖延后消费的 ACK、消费与追加交错、正文发送/暂停时的积压、EOF 后消费及中继跨窗口行为。 +- `docs/source/File_Transfer/File_Transfer.rst`：明确 64 KiB 是协议缓存/直传行长界限，与正文分块无关；记录错误方向和中继清理规则。 +- 全量验证发现并局部修正 `test/integration/network_api/test_tcp_file_transfer.py` 的多文件目标目录用例：原先仅等待文件存在，实际读到空文件。改为复用已有 `wait_for_text` 等待准确内容，保留两条原始 bytes 相等断言；无产品代码或测试策略变化。 + +任务载体：本方案；实施后的证据写入 `docs/plans/transfer-sign-buffer-bounds-report.md`，不把本方案写成已验收。 + +公共方法签名、协议 marker 字面值、FID、数据模型、配置文件均不变。新增的是“直传拒绝超过上限的协议行”这一异常输入契约，必须在审查中单独确认；不更改中继的现有匹配契约。 + +## 7. 原子变更拆分（实施前定稿） + +### B1：直传协议缓存有界化 + +- 单一意图：两端直传按索引消费后删除，原始尾部/完整行/待处理积压均有硬上限。 +- 文件：`connect_tcp.py`、`test_wire_lines.py`、`test_tcp_transfer_resume.py`、对应运行文档段落。 +- 预估：约 300–500 行，包括队列/锁/配额与行为回归；若超出单意图审查范围，先修订方案，不边实现边追加范围。 +- 局部验证：边界契约、双向粘包初传/EOF、并发追加时前缀删除不跳项、不误删；真实持续短行和暂停期间积压。附返回值、进度、worker 退出及缓存峰值；人工审查后再进入 B2。 + +### B2：中继等待 marker 的历史缓存有界化 + +- 单一意图：既有字节 marker 匹配所需的窗口/后缀有界。 +- 文件：`connect_tcp.py`、协议标记与中继行为测试、对应运行文档段落。 +- 预估：约 150–300 行。 +- 局部验证：start/ACK/error 跨所有 marker 切分点及窗口边界；真实转发最终确认、ACK 丢失/发送失败和全目标门槛。随后运行受影响集与全量回归，附最终报告；不自行创建提交或推送。 + +## 8. 验收证据要求（实施结果见报告） + +### 边界与状态契约 + +- 小于 64 KiB 的正常 start/ACK 立即解析，不为凑窗口而等待。 +- 大量完整短行累计达到或超过多个 64 KiB 批次时均被扫描并批量清理，后续有效 ACK 仍成功，不因聚合容量报错。 +- ACK 被容量触发的 producer scan 清理后，调用线程仍读取到已锁存结果；ACK 与 header 同读、ACK 后 EOF、调用线程晚于接收线程关闭均不丢确认。 +- 65,535 字节内容加换行的单行边界可接受；单条未终止行达到 65,536 字节被拒绝。单次读取包含多个合法短行且总量超过 64 KiB 时持续分批处理。 +- 单条超长行内部藏有 ACK 时不得把后缀重同步成有效 ACK；未终止 ACK 不提前成功。 +- 中继 target 与 error 跨窗口仍识别；持续噪声期间缓存不随累计输入增长、总等待时间不重置。 + +### 建议命令 + +```sh +uv run --python 3.10 --with pytest --with flask pytest \ + test/unit/network_api/test_wire_lines.py \ + test/integration/network_api/test_tcp_transfer_resume.py \ + test/integration/network_api/test_tcp_file_transfer.py \ + test/integration/network_api/test_crypto_tcp.py -q + +uv run --python 3.10 --with pytest --with flask pytest -q +``` + +另执行隔离 loopback 进程烟测：双向普通文件/文件夹项、双向续传/EOF、公钥推送、明文/加密转发。注入仅有界的长行和多批短行，记录实际峰值、ACK 结果、内容逐字节相同及失败进度；不尝试真实 OOM，不留下临时脚本或数据。仅通过单测不能代替这些运行证据。 + +## 9. 明确未覆盖与审查闸门 + +- 本次方案不是连接级总内存、CPU 限流或所有协议入口的安全审计；只解决指定两个无界缓存路径。 +- 不限制正文长度，不改变消息 API 对 bytes 不自动补换行的现有契约。 +- Python 对象和解密层总占用没有被宣称为 64 KiB；验收分别观察未消费原始线长配额、尾部字节与固定判定结果。 +- Windows/macOS 未因本计划获得运行证据；本地通过不得写成平台 CI 已通过。 +- 用户已批准开始更改，当前 B1/B2 作为未提交工作区交付；本文件的预估和原型不充当最终验收，人工最终审查与平台运行证据仍以报告中的明确边界为准。 + +## 10. 本轮独立验证（不是产品修改后的验收） + +在 Python 3.10.20 运行的独立原型记录了算法风险，均不是最终产品验收： + +- 对 `[HEADER, ACK, ERROR]` 使用 `enumerate` 并原位 `del[index]`，实际只访问 HEADER、ERROR，ACK 留在列表中，确认索引前移会漏处理。 +- 核对 `[HEADER, ACK]` 后前缀删除：消费两项，队列清空，局部 `ack_seen` 保留为 True。 +- 控制生产者与消费者交错：消费者持锁核对并删除一项 header，生产者随后追加 ACK；下一批正确消费 ACK，没有跳项或误删。原型中的 Event 仅用于安排验证时序，不属于产品设计。 +- 早期原型曾在 64 KiB 聚合容量满时拒绝下一条短行；需求方澄清后此规则已被实现层的“扫描全部完整行、保存结果、批量删除并继续”取代，不能作为当前验收结论。 +- 中继滚动窗口原型对 start 的 36 个切分点、ACK 的 24 个切分点，以及 error 跨界/同次读取目标后出现 error 的情况均验证通过；窗口 65,536 字节，后缀不超过 36/24 字节。最终产品证据见实施报告。 + diff --git a/docs/plans/transfer-sign-buffer-bounds-report.md b/docs/plans/transfer-sign-buffer-bounds-report.md new file mode 100644 index 0000000..94ba71d --- /dev/null +++ b/docs/plans/transfer-sign-buffer-bounds-report.md @@ -0,0 +1,129 @@ +# 传输协议缓存有界化：实施与验收报告 + +状态：按最终“达到 64 KiB 即扫描、批量清理”语义完成实现与验收；阈值清理版受影响集 **118 passed in 328.93s**，Python 3.10 完整回归 **685 passed in 528.50s**。完整短行累计至上限不会报错；只有单条未终止行自身达到 64 KiB 才失败。未创建 git 提交、未合并、未推送，待人工最终审查。 + +## 1. 需求与实施边界 + +用户先要求消费后删除与 64 KiB 上限，随后明确容量到达时应主动扫描全部完整协议行并批量清理，不应因短行聚合达到容量失败。最终实现遵循此语义：达到阈值锁内扫描、保存 start/ACK/error 状态、删除记录并继续解析；单条无法分界的超长行才走失败关闭。不使用产品 `threading.Event`，不改 ACK 时机、超时或正文路径。 + +## 2. B1：直传消费后删除 + +### 实现 + +`PyFlow/network_api/connect_tcp.py` 中，原 `_take_wire_lines` 与两端完整 sign 历史统一迁移到私有 `_TransferSignBuffer`；没有保留旧辅助函数或兼容入口。 + +- `feed` 按 TCP byte stream 分段、逐行解析，完整行才 UTF-8 解码/strip；单次读取大于 64 KiB 时分窗口，不先建立任意大的历史拼接串。 +- 原始完整线长（含换行）在解码/strip 前入账；空行直接消费。`pending_bytes + len(partial)` 不超过 65,536。 +- 完整行入队使窗口达到 64 KiB 时，接收线程持锁核对所有完整行，锁存 start/ACK/error，再一次删除整段并释放配额，然后继续当前输入。添加一行会越界时先扫描已有完整行再添加。 +- 消费线程的 `consume()` 在相同锁内扫描未满批次并返回 marker 状态快照。producer 阈值扫描与调用线程阶段扫描都在判定后批量删除。 +- 任意数量的合法短行按多个窗口扫描；**累计队列达到上限不会报错**。单条未终止行达到 64 KiB 时仍拒绝，因为不能安全清空半行后恢复解析。 +- 正文阻塞期间 producer 可自扫描；固定 marker 识别结果跨阶段保留。接收/消费对标记和队列的操作使用同一局部锁，不增加 Event。 +- header/未知行没有永久历史。socket/file I/O、暂停等待和关闭不在队列锁内执行。 + +### ACK/EOF 顺序 + +每轮等待先快照 `file_running`，再消费队列，最后使用该快照检查关闭状态。如果只按“消费空队列 → 读最新关闭状态”的顺序，接收线程可能刚好在两者之间提交 ACK 并关闭，导致误报失败。当前顺序保证：读到关闭快照时，之前提交的记录有机会被消费；读到运行快照时，稍后的关闭留到下一轮判断。 + +已完整 ACK 与随后 error/关闭的原有判定优先级未重新定义；未终止 ACK 仍不能成功。 + +## 3. B2：中继滚动窗口 + +`TCP_Server_Base._forward_read_sign` 保留现有字节子串 marker 匹配及 `bytes | None` 返回契约。 + +- 每个扫描窗口最多 65,536 字节；单次读取较大时分段扫描。 +- 未匹配时只保留 `max(len(target), len(error)) - 1` 字节的后缀。默认 start 保留最多 36 字节，最终 ACK 保留最多 24 字节。 +- 同一次读取必须扫描完再接受目标 marker；该读取后半段出现 error 仍按原有顺序拒绝，不能因内部窗口切分而让前半段 ACK 提前成功。 +- deadline 不因扫描/清理而重新开始，socket timeout 在出口恢复。无法在有界窗口内表示的超长目标 marker 直接返回 `None`，避免非前进循环；既有固定 marker 均远小于上限。 +- 目标只有真实最终 ACK 才标记完成；全目标门槛、上游换行 ACK、写成功才删除路由等规则不变。 + +## 4. 文件与公共契约 + +| 文件 | 本轮变更 | +|---|---| +| `PyFlow/network_api/connect_tcp.py` | 有界解析/待处理配额、按索引消费删除、两端等待接入、中继滚动窗口 | +| `test/unit/network_api/test_wire_lines.py` | 将原有分行边界契约迁移到新消费者；增加最大行、积压、配额释放、并发交接及真实 socket 中继跨边界/error 优先级覆盖 | +| `test/integration/network_api/test_tcp_transfer_resume.py` | 双向真实接收线程测试：达到 64 KiB 的完整短行批量清理后真实传输成功；单条无换行长行拒绝并保留恢复记录 | +| `test/integration/network_api/test_tcp_file_transfer.py` | 全量发现的就绪竞态止血：两处 exists 等待替换为既有 `wait_for_text`，两条原始 bytes 相等断言保留 | +| `docs/source/File_Transfer/File_Transfer.rst` | 消费删除、wire-byte 配额、中继后缀及异常输入边界 | +| `docs/plans/transfer-sign-buffer-bounds-plan.md` | 记录批准、实际落点、ACK/EOF 快照与就绪用例修正 | +| `docs/plans/final-ack-delivery-report.md` | 保留历史数字并链接本次有界化，避免把上一阶段结果冒称当前验收 | +| 本报告 | 两项变更意图、路径证据、测试失败与修正经过、未覆盖限制 | + +公开 API、FID、进度行、依赖、CI、覆盖率、加密握手与文件正文不变。唯一拒绝边界是直传协议单行无法在 64 KiB 内分界；完整短行累计容量不是错误。不删除或弱化既有断言。 + +## 5. 机械验证 + +### 修复前证据 + +本轮开始时在真实旧辅助函数上输入 65,536 个无换行字节：输出 `before: unterminated bytes retained = 65536`，容量不变量断言失败。说明不是仅按代码推断旧缓存无界。 + +### 定向回归 + +- 阈值清理语义下，`test_wire_lines.py`、resume、file transfer、crypto 四文件受影响集：**118 passed in 328.93s**。 +- 结果包含两个方向真实 socket 在正文暂停时注入超过 64 KiB 的完整短行；producer 批量扫描后发送继续、正文相同、真实 ACK 成功、进度移除。两个方向注入单条无换行长行则失败并保留进度。 +- Ruff 新增/重写 parser 单测 0 条、transfer integration 新代码无诊断；两个既有 10 秒等待处诊断及其他存量问题未因本次改动产生。 + +### 首轮全量失败与就绪修正 + +命令： + +```sh +uv run --python 3.10 --with pytest --with flask pytest -q +``` + +首轮结果：**681 passed, 1 failed in 501.18s**。失败为 `test_client_uploads_multiple_files_to_destination`：只等 `one.bin`/`two.bin` 存在，就检查内容；实际 `one.bin` 为 `b''` 而非 `b'one'`。接收端在正文写入之前就会创建文件，因此 exists 不是完成条件，测试相邻用例已经使用 `wait_for_text`。 + +局部修正仅把两个 exists 等待换成等待准确文本内容，保留 `read_bytes() == b"one"` 和 `read_bytes() == b"two"`。独立进程真实调用 `/multiple_file` 后，两个内容等待和两个原始 bytes 断言均通过。没有为此改动产品传输行为、删用例、加 sleep 或放宽断言。 + +历史记录：旧语义版本在就绪等待修正后为 **682 passed in 518.92s**；它发生在需求方澄清“容量到达应批量清理”之前，不属于当前最终实现验收。 + +最终阈值清理语义的完整回归： + +```sh +uv run --python 3.10 --with pytest --with flask pytest -q +``` + +结果：**685 passed in 528.50s**，无失败、无跳过。 + +## 6. 独立运行证据 + +以下均为隔离进程中真实 socket 传输，日志、进度、目标目录及密钥重定向到临时目录；不是仅靠单测通过推断路径覆盖。 + +| 路径 | 本轮实际结果 | +|---|---| +| 明文/加密 × 上传/下载 × 普通初传 | 全部返回 `True`；68,876 字节含 `222\n\n111`、空白、NUL、全部 0–255 字节的正文逐字节相同，双侧进度清理 | +| 明文/加密 × 上传/下载 × 文件夹项 | `bundle/nested` 内容逐字节相同，返回 `True`，进度清理 | +| 明文/加密 × 上传/下载 × 部分续传 | offset=64、total=68,876；目标完整、返回 `True`，进度清理 | +| 明文/加密 × 上传/下载 × EOF 重放 | offset=total=68,876；正文无需重发，确认成功，进度清理 | +| 明文/加密 × 文件/文件夹项中继 | 每项 90,009 字节；`_forward_one=True`，目标内容相同，源上传记录及服务端路由清理 | +| 两个加密客户端的双向公钥推送 | 共四条 plaintext 公钥腿真实调用均返回 `True`，原有握手完成 | +| 单条无换行长行 | 未终止内容达到 65,536 字节，发送腿返回 `False`，上传恢复记录保留 | +| 完整短行达到总量阈值 | 双向真实 socket 注入 65,538 字节完整短行：每个 65,536 字节批次 scan/delete，余下数据继续处理；随后文件逐字节相同、ACK 成功、上传进度清理 | + +parser 单测还以恰好 65,536 字节记录验证 marker 被锁存且队列/计数同步清空；同一大读取跨多个窗口的完整短行不失败。单条未终止行测试验证仅其自身耗尽线长上限时关闭。该配额按 wire bytes 计，不代表 RSS。 + +### 实际中继函数窗口观测 + +通过真实 socketpair 输入、只观测局部缓冲尺寸的 tracer 运行产品 `_forward_read_sign`： + +| 场景 | 输入字节 | 窗口峰值 | 保留后缀峰值 | 返回 | +|---|---:|---:|---:|---| +| start 跨窗口 | 196,627 | 65,536 | 36 | 匹配 | +| ACK 跨窗口 | 196,621 | 65,536 | 24 | 匹配 | +| 同一次读取 ACK 后继续出现 error | 262,161 | 65,536 | 24 | `None` | + +连续噪声输入下，0.08 秒 deadline 实际约 0.0811 秒返回 `None`，原 socket timeout 恢复成功，未被持续输入延长。 + +### 强制 ACK/EOF 消费竞态 + +独立实际文件上传在最终等待中人为安排：消费线程先检查空队列，随后接收线程收到 ACK 并进入关闭，再允许消费线程继续。本轮 **110,000 字节**正文相同，发送返回 `True`，上传记录移除，验证运行状态快照不会把“旧空队列 + 新关闭”误判为失败。用于安排顺序的 Event 仅在临时烟测进程中使用，产品没有新增 Event。 + +## 7. 风险、未覆盖与交付闸门 + +- Windows/macOS 与 free-threaded Python 未在本轮实际运行；没有声称平台 CI 或所有解释器已验收。 +- 64 KiB 是原始协议行/未消费配额或中继扫描窗口限制，不是整条连接、Python 堆、socket 缓冲、加密帧或文件正文总内存限制。 +- CPU/连接数限流、其他协议入口和全局安全属性不在本轮范围。中继仍沿用既有字节子串匹配，没有冒称已切换为严格整行认证。 +- TCP 写成功但 ACK 实际未到达、断网、原有最终等待超时等仍按既有失败/续传规则处理；未增加重试、ACK-of-ACK、drain、shutdown 握手或延迟关闭。 +- 完整短行达到一个或多个 64 KiB 批次时会自动 scan/clear 后继续，不会因聚合积满退出。只有单条无法分界的超长行拒绝并保留恢复记录；大于 64 KiB 的正文成功传输仍有覆盖。 +- 临时烟测均为进程内脚本字符串和 `TemporaryDirectory`，未在仓库留下脚本或测试数据。永久规则沉淀为边界/消费删除/中继错误顺序测试与运行文档。 +- 阈值清理语义的全量验证已通过：**685 passed in 528.50s**。人工最终审查、跨平台证据和平台 CI 仍是剩余交付闸门；未提交、未合并或推送。 diff --git a/docs/plans/unified-port-socket-allocation-plan.md b/docs/plans/unified-port-socket-allocation-plan.md new file mode 100644 index 0000000..e771a46 --- /dev/null +++ b/docs/plans/unified-port-socket-allocation-plan.md @@ -0,0 +1,170 @@ +# 统一端口分配与 socket 创建接口计划 + +状态：需求方向已确认：新增内部统一 socket 分配接口，保留现有 `palloc()` / `pfree()` 的整数与释放契约。当前仅完成调查，未为本计划修改产品代码；工作区已有前一轮 Windows 10013 修复及其测试/文档改动，本计划将重构该实现，避免再叠加旁路。 + +## 1. 需求卡 + +### 目标 + +在不破坏公共 `palloc()` / `pfree()` API 的前提下，统一“逻辑端口候选 → 真实 socket bind/connect → 实际端口 → 释放逻辑预留”的内部路径。 + +新增内部 socket 分配层，供以下路径复用： + +- 文件传输发送端：server/client `file_transfer_mode()` 的 bind/connect。 +- 文件传输接收端：server/client `file_transfer_mode_recv()` 的 listener bind 与实际端口公告。 +- forward relay 的 upstream listener bind 与实际端口公告。 +- 两类 `create_temporary_server()` listener，以及客户端 `create_temporary_client()` 的 palloc 绑定路径。 +- 当前已有的 Windows `EADDRINUSE`/10048/10013 回退统一收口，不保留 `_bind_local_port_with_fallback()` 与发送端重复的两套规则。 + +主控制连接的固定 `client_port` 不经过 `palloc()`：客户端必须先建立控制连接，服务端的 `/client_alloc_port_range` 协商之后才有客户端分配范围。因此控制连接继续使用同一内部 bind/connect 规则，但保留其专用重连循环，不伪造 palloc 预留。 + +### 非目标 + +- 不把 `palloc()` 改成返回 socket、tuple 或 lease；不改变 `palloc()` 自动模式返回 `0`、手动模式返回整数端口的现有契约。 +- 不让 `pfree()` 负责关闭 socket；它继续只释放逻辑分配记录。socket 生命周期仍由调用路径和 transfer registry 管理。 +- 不在 `palloc()` 内提前 bind 探测：探测与真正使用之间存在 TOCTOU，不能解决 Windows 保留端口，也会改变公共分配 API 的语义。 +- 不改变服务端主 listener、无固定本地端口的 relay target connect、加密、FID、协议 marker、文件内容、重试次数或 CI/依赖。 +- 不以临时兼容别名保留第二套分配入口；迁移完成后删除重复的端口回退 helper/逻辑。 + +### 受影响范围 + +- `PyFlow/network_api/connect_tcp.py`：内部 socket/port lease helper；server/client 的 temporary、file transfer、relay、控制连接调用点；现有 `palloc()`/`pfree()` 实现保留。 +- `test/unit/network_api/test_TCP_server.py`、`test/integration/network_api/test_tcp_broadcast_alloc.py`：保留并扩展 palloc/pfree 整数语义、手动范围、自动模式和释放行为契约。 +- `test/integration/network_api/test_tcp_file_transfer.py`、`test/integration/network_api/test_tcp_transfer_resume.py`：真实 loopback socket 的 bind/connect 回退、listener 实际端口公告、relay 清理。 +- 可能新增 `test/unit/network_api/test_socket_port_allocation.py`：只测试内部 helper 的端口 lease 状态机，不替代真实 TCP 集成证据。 +- `docs/source/Port_Allocation/Port_Allocation.rst`、`docs/source/Network_APIs/TCP_Client_APIs.rst`、`docs/source/Network_APIs/TCP_Server_APIs.rst`：区分“逻辑端口分配 API”和“真实 socket 分配层”，记录实际端口与回退语义。 +- `docs/plans/unified-port-socket-allocation-report.md`：实施与验收报告；更新前一轮 10013 计划中的实现状态/链接，避免两份报告描述不同实现。 + +### 安全与数据边界 + +socket 分配层只处理本地地址、端口、socket 生命周期和本地日志；不读取、记录或改变 payload、密钥、文件内容、FID 或远端鉴权数据。所有 socket 仍必须经过现有 `_transfer_handshakes`、加密 attach、统一 send/receive 入口。 + +风险等级：中。风险集中在 socket 替换时的 registry 释放、palloc 记录释放、listener 端口公告和并发重试。 + +## 2. 内部接口与数据流 + +### 2.1 保留的公共分配契约 + +`palloc()` / `pfree()` 保持不变： + +- 自动模式：`palloc() -> 0`，由 socket bind(`0`) 让内核选择实际端口；`pfree(0)` 无操作。 +- 手动模式：`palloc() -> int`，只记录候选端口；`pfree(int)` 删除逻辑记录并调整现有游标。 +- palloc 不保证端口已被 OS bind；真实可用性由下一层 socket bind/connect 确认。 + +### 2.2 新的内部 socket 分配层 + +增加一个模块级私有入口（名称以实现时为准，计划契约称为 `_open_port_socket`）： + +```text +_open_port_socket( + bind_host, + requested_port, + *, + connect_to=None, + configure=None, + register=None, + release=None, +) -> (socket, actual_port, used_fallback) +``` + +约束： + +1. `requested_port` 是调用方已经从 `palloc()` 得到的候选整数，或显式固定端口；该层不改变 palloc 返回类型。 +2. helper 创建新 socket并重复应用 `SO_REUSEADDR`、timeout 等配置；若提供 `connect_to`，在 bind 后 connect。 +3. bind 或 connect 报本地端口冲突（`EADDRINUSE`、WinError 10048/10013、Windows-only `errno.EACCES`）且候选端口非零时： + - 释放/关闭失败 socket及其 registry 记录； + - 创建全新 socket； + - 用 bind port `0` 重试； + - 返回 `used_fallback=True` 与 `getsockname()` 的实际端口。 +4. 非端口冲突错误原样交给现有 timeout/refused/重试分支；未绑定/端口 `0` 不触发“固定端口回退”。 +5. `register`/`release` 回调只用于 transfer registry 生命周期；helper 不绕过 `_TransferHandshakeRegistry`。 +6. 调用方负责在完整 socket 生命周期结束后调用现有 `pfree(requested_port)`；helper 不让 `pfree` 关闭 socket，也不留下“OS 实际端口”到逻辑分配表。 + +### 2.3 数据流 + +```text +palloc() -> requested candidate + -> _open_port_socket: configure -> bind(candidate) -> connect(optional) + success -> actual_port -> listener announcement / transfer handshake + local conflict -> close + registry release -> bind/connect from port 0 + -> actual_port -> same announcement / handshake + -> socket close -> existing pfree(requested candidate) +``` + +控制连接：`client_port` 是显式固定值或 `None/0`，不产生 palloc lease；仍由 `TCP_Client_Base.connect()` 的 wait/reconnect 循环调用同一 socket 规则。 + +## 3. 全部执行路径与修改点 + +| 路径 | 当前来源 | 统一入口后的行为 | 必须验证 | +|---|---|---|---| +| server/client `palloc` 自动模式 | `palloc() == 0` | bind(0)，返回真实 ephemeral port；pfree(0) 不变 | 实际 `getsockname()[1] > 0` | +| server/client 手动模式候选可用 | `palloc() -> concrete` | bind/connect 候选，传输/监听结束后 pfree 原候选 | 候选与实际端口一致；记录释放 | +| sender bind 冲突 | 两端 `file_transfer_mode()` | 关闭失败 socket，registry release，bind/connect port 0；正文和 ACK 不变 | bind 10013/10048，逐字节文件相同 | +| sender connect 阶段本地冲突 | 两端 `file_transfer_mode()` | 同上；后续重试不再反复使用失败固定端口 | connect 10013/10048，实际源端口变化 | +| receiver listener bind 冲突 | 两端 `file_transfer_mode_recv()` | 新 socket bind(0)，公告实际 listener 端口；pfree 原候选 | 请求方收到真实端口并完成文件传输 | +| relay upstream listener bind 冲突 | server `_forward_relay()` | bind(0)，公告实际端口，finally pfree 原候选 | 真实 relay resume 完成，原候选释放 | +| temporary server listener | 两个类 `create_temporary_server()` | palloc 候选经统一 bind；失败回退并返回实际端口 | stop 后 socket 和逻辑记录都释放 | +| temporary client palloc 分支 | client `create_temporary_client()` | palloc 候选经统一 bind/connect；返回实际 socket | side-channel 收发真实数据 | +| 控制连接固定端口 | `TCP_Client_Base.connect()` | 不调用 palloc；使用同一冲突识别和专用 wait/reconnect | fixed bind/connect 10013；None/0 不回退 | +| relay target 无固定源端口 | `_forward_relay()` target connect | 不改变；无 palloc lease | 原有转发/失败清理回归 | +| 主 server listener | `start_TCP_Server()` | 不改变 | 既有启动/停止测试 | +| 非端口冲突、远端拒绝、timeout | 所有路径 | 继续现有异常、等待和最大重试语义 | 原错误路径断言/真实失败 smoke | + +## 4. 原子实施拆分 + +### A：内部接口与契约 + +- 文件：`connect_tcp.py`、allocator 单测、Port Allocation 文档。 +- 意图：新增统一 socket 打开/替换/实际端口返回入口；不迁移业务调用点之外的行为。 +- 预估：150–250 行。 +- 验证：helper 的 bind 成功、端口 0、10013/10048 回退、非端口错误、registry callback 顺序；现有 `palloc/pfree` 契约测试全部保留。 + +### B：临时/监听路径迁移 + +- 文件：`connect_tcp.py`、`test_tcp_file_transfer.py`、必要的 temporary API 测试、Network API 文档。 +- 意图：迁移 temporary server/client、两端 file receive listener、relay upstream listener；删除旧 listener 专用 helper。 +- 预估：150–300 行。 +- 验证：真实 loopback listener 公告、OS 实际端口、manual allocation 记录释放、stop/error 清理；覆盖 server/client 两方向。 + +### C：发送/控制路径收口与最终验收 + +- 文件：`connect_tcp.py`、`test_tcp_file_transfer.py`、`test_tcp_transfer_resume.py`、报告及文档收尾。 +- 意图：迁移两端 file sender 的 bind/connect 回退；控制连接复用冲突判定但保留专用重试生命周期；删除重复分支。 +- 预估：150–300 行。 +- 验证：固定源端口 bind/connect 10013/10048、控制连接固定/None/0、真实文件/续传/relay、失败 socket registry 无残留。 + +每个原子阶段单独局部验证；不创建提交、不合并、不推送，等待人工审查后进入下一阶段。 + +## 5. 验收命令与运行时证据 + +```sh +uv run pytest \ + test/unit/network_api/test_TCP_server.py \ + test/unit/network_api/test_TCP_client.py \ + test/integration/network_api/test_tcp_broadcast_alloc.py \ + test/integration/network_api/test_tcp_file_transfer.py \ + test/integration/network_api/test_tcp_transfer_resume.py -q +``` + +必须提供的运行时证据： + +1. 自动模式真实 bind：`palloc() == 0`，socket 实际端口非零，`pfree(0)` 不改变行为。 +2. 手动模式真实 bind：候选端口被 OS 允许时，实际端口等于候选；socket 关闭后原记录从 allocator 中消失。 +3. 手动候选被占用/注入 10013：实际端口不是候选，listener 公告实际端口或 sender 完成连接；原候选仍由 pfree 释放。 +4. connect 阶段本地冲突：失败 socket 已从 registry 释放；替换 socket重新注册；最终 transfer/relay ACK 成功。 +5. 两个并发 transfer：候选分配不重复，清理后所有逻辑记录归零；不以“测试不抛异常”作为证据。 +6. Windows CI：Linux 只能注入 `winerror=10013` 验证分支；必须明确记录真实 Windows WSAEACCES/Hyper-V/WinNAT 未由本机证明。 + +## 6. 风险与明确未覆盖 + +- 最大风险是 socket 替换时先 release registry、后注册新 socket的时序；必须保持 cancel/close 竞态安全。 +- `palloc/pfree` 当前是范围记账，不是 OS 端口锁；本计划不宣称它能消除跨进程 TOCTOU。 +- 不迁移主 server listener 和无固定源端口的 relay target connect；这些路径的端口语义不同。 +- 不改翻译 `.po` 文件；源 `.rst` 是本次接口契约来源，翻译生成流程不在本任务范围。 +- 若人工要求把 `palloc()` 本身改为返回 socket/lease，需要另开破坏性公共 API 迁移计划，本计划不覆盖。 + +## 7. 规则与文档收尾 + +- 更新 `Port_Allocation.rst`：palloc 是逻辑候选分配；真实可用性由内部 socket 接口确认。 +- 更新 client/server API 文档：`0`、固定端口、实际 `getsockname()` 端口和回退边界。 +- 报告逐路径列出覆盖/未覆盖、实际命令和平台限制。 diff --git a/docs/plans/winerror-10013-local-port-fallback-plan.md b/docs/plans/winerror-10013-local-port-fallback-plan.md new file mode 100644 index 0000000..e03ddb0 --- /dev/null +++ b/docs/plans/winerror-10013-local-port-fallback-plan.md @@ -0,0 +1,159 @@ +# Windows 本地端口 WSAEACCES 回退修复计划 + +状态：需求卡与扩展计划已批准并实施。定向 smoke 22 passed；最终受影响套件 97 passed。未在 Windows runner 上验证真实 WSAEACCES；未创建提交、未合并、未推送。 + +## 1. 需求卡 + +### 目标 + +当固定非零本地端口因 Windows `WSAEACCES`（`winerror == 10013`，或 Windows `errno.EACCES`）不可用时，退回 OS 分配端口，并保持连接/传输成功。 + +适用路径： + +- `TCP_Client_Base.connect()` 控制连接：固定非零 `client_port` 的 `bind()` 或 `connect()`。 +- `TCP_Server_Base.file_transfer_mode()` 与 `TCP_Client_Base.file_transfer_mode()` 传输发送 socket：固定非零 `client_port` 的 `bind()` 或 `connect()`。 +- `TCP_Server_Base.file_transfer_mode_recv()` 与 `TCP_Client_Base.file_transfer_mode_recv()` 传输接收 listener：固定非零 `server_file_port` 的 `bind()`。回退端口必须通过 `getsockname()` 取得并公告给发送方。 +- `TCP_Server_Base._forward_relay()` 的上游 `up_sock` listener：固定非零 palloc 端口 `bind()`。回退后的实际端口公告给 forwarder；端口分配器必须释放原申请端口，而不是 OS 端口。 + +既有 `errno.EADDRINUSE` 与 `winerror == 10048` 回退保持不变。`None`/`0` 表示未固定端口，不把 WSAEACCES 当成固定端口冲突。发送 socket 的固定端口冲突只触发一次切换到 `port=0`，之后沿用原有传输重试；不增加最大重试次数。 + +### 非目标与禁止项 + +- 不修改通用 `create_temporary_client()` / `create_temporary_server()` side-channel API、服务端主 listener、无固定本地端口的 forwarding target `connect()`、端口分配算法、传输协议/FID/恢复身份、公共函数签名、CI、依赖或覆盖率策略。 +- 不将未固定端口上的 10013 当成固定端口冲突；不对其他 socket 错误做新降级；不新增重试次数。 +- 不删除或弱化现有测试断言。 + +### 数据与风险边界 + +本次只改变本地端口失败时的选择。控制协议、远端 listener 地址、文件内容、加密、FID 和 ACK 不变。文件接收与转发 listener 回退后必须公告 `getsockname()` 的真实端口；`_forward_relay` 还必须释放原 palloc 端口，避免手动分配表泄漏。风险：中，涉及控制、直传和转发的 socket 生命周期。 + +## 2. 数据流与路径全集 + +```text +控制连接：固定非零 client_port -> 新 socket -> bind -> connect + 本地端口冲突 -> close -> 不再绑定固定地址 -> connect 一次 + 控制端口为 None/0、文件传输 palloc 返回 0，或其它错误 -> 原路径 + +直传发送端（server/client）：palloc client_port -> track_socket -> bind -> connect + 本地端口冲突 -> release_socket + close -> 后续重试用 port=0 -> OS 源端口 + +直传接收端（server/client）：palloc server_file_port -> listener bind + 本地端口冲突 -> 同 socket bind(host, 0) -> getsockname -> 公告真实 listener 端口 + +转发发送 listener：palloc -> track up_sock -> bind + 本地端口冲突 -> bind(host, 0) -> 公告 getsockname 端口 -> finally 释放原 palloc 端口 +``` + +| 路径 | 代码落点 | 必须保持/新增行为 | +|---|---|---| +| 控制连接固定端口 bind/connect 冲突 | `TCP_Client_Base.connect()` | 10013 与既有 10048/EADDRINUSE 一样改用 OS 端口并连通 | +| 控制连接本地端口为 None/0 | `TCP_Client_Base.connect()` | 不进入固定端口回退；原错误/等待语义不变 | +| 服务端/客户端发送文件：固定源端口 bind/connect | 两端 `file_transfer_mode()` | 释放已登记失败 socket，再以 port 0 继续原重试；传输内容和 ACK 不变 | +| 服务端/客户端接收文件：固定 listener bind | 两端 `file_transfer_mode_recv()` | listener 回退到 port 0；公告真实端口，发送端仍完成传输 | +| 转发 relay 的 upstream listener bind | `TCP_Server_Base._forward_relay()` | listener 公告真实端口；`pfree()` 释放原申请端口 | +| 自动端口 0、其它错误 | 上述路径 | 不进入固定端口专用降级；沿用原处理/重试 | +| 无固定源端口的 forwarding target connect | `_forward_relay()` | 不变；connect 不绑定固定本地端口 | +| 通用 temporary client/server API、主 listener | 两个类的 API | 本次不修改；不声称覆盖 | + +## 3. 文件与契约影响 + +计划修改： + +- `PyFlow/network_api/connect_tcp.py`：识别 `EADDRINUSE`、Windows 10048/10013 与 Windows-only `errno.EACCES`；新增固定非零端口判定及共用 listener bind 回退；更新主连接、两端发送 socket、两端接收 listener、转发 upstream listener 的错误/资源处理；forward relay 记录并释放原 palloc 端口。 +- `test/integration/helpers.py`：加入一次性注入 `OSError(errno.EACCES)` + `winerror=10013` 的 socket 测试工具。 +- `test/unit/network_api/test_TCP_client.py`：覆盖 Windows-only `errno.EACCES` 识别边界。 +- `test/integration/network_api/test_tcp_transfer_resume.py`：控制连接 10013 bind/connect 回退、无固定端口不回退；在真实转发续传中覆盖 relay listener 回退及原 palloc 端口释放；保留现有真实“配置端口被占用”用例。 +- `test/integration/network_api/test_tcp_file_transfer.py`：真实 loopback 覆盖服务端/客户端发送 socket 的 bind/connect 回退；覆盖两端接收 listener bind 回退并断言实际端口公告与文件字节。 +- `docs/source/Network_APIs/TCP_Client_APIs.rst`、`docs/source/Network_APIs/TCP_Server_APIs.rst`：记录固定本地传输端口不可用时回退到 OS 端口；更新控制连接说明。 +- 本计划及 `docs/plans/winerror-10013-local-port-fallback-report.md`：记录需求、路径证据与平台限制。 + +不新增/修改公共 API。CI、依赖、数据模型均不变。 + +## 4. 原子变更与局部验收 + +一个原子意图：统一固定本地端口冲突识别及控制/文件传输出站 socket 回退。预计总 diff 不超过约 400 行；若实现需要修改列出的范围之外的公共 API、主 listener 或通用 side-channel，先退回本计划，不边写边扩围。 + +局部验证： + +1. **错误识别**：`winerror=10013` 在当前 Linux 测试进程也被识别；单独 `errno.EACCES` 只在 `os.name == "nt"` 时匹配；EADDRINUSE/10048 保持匹配。 +2. **控制连接**：真实 loopback server/client；注入 10013 到固定端口 bind 与 connect，均断言 `connect()` 成功且实际本地端口不同于配置值。无固定端口时注入 10013，断言不执行固定端口回退且保持失败。 +3. **直传发送**：真实 loopback 两方向；对 server/client sender 的固定源端口 bind/connect 注入 10013，断言成功传输、文件逐字节相等、失败 socket 从 handshake registry 释放。 +4. **直传接收 listener**：真实 loopback 两方向；对固定 listener bind 注入 10013，断言公告端口不同于固定端口且客户端连通并完成文件传输。 +5. **转发 listener**：真实转发续传中对 relay upstream listener 注入 10013；断言目标收到完整文件、回退端口可连接、原 palloc 端口在 finally 中释放。 +6. **原回归与套件**：运行 `test_client_connects_when_its_configured_port_is_taken`、相关上传/下载/续传用例及三个受影响测试文件。 +7. 测试必须通过真实 TCP listener 观察连接与文件内容；不 mock 掉关键连接/传输路径。 + +建议命令： + +```sh +uv run pytest test/unit/network_api/test_TCP_client.py \ + test/integration/network_api/test_tcp_transfer_resume.py \ + test/integration/network_api/test_tcp_file_transfer.py -q +``` + +当前环境为 Linux：合成 `winerror=10013` 可验证产品分支；真实 Windows WSAEACCES/Hyper-V/WinNAT 与 Windows CI 尚不能由本机证明，报告必须标明，不得声称已通过。 + +## 5. 人工审查点 + +- 全部覆盖路径为：控制连接、两端直传发送、两端直传接收 listener、转发 upstream listener；主 listener 与通用 side-channel 明确不在范围。 +- 10013/`errno.EACCES` 只在固定非零本地端口时触发；端口 0 和未绑定连接不做固定端口降级。 +- 直传发送失败 socket 必须先从 `_transfer_handshakes` 释放再关闭；重建 socket 后方可重试。 +- 监听回退需公告真实 `getsockname()` 端口；转发 listener 失败回退后必须 `pfree()` 原 palloc 请求值。 +- OS 分配端口后协议远端地址、加密协商、数据与 ACK 保持原样。 +- 不提交、不合并、不推送；最终工作区与 Windows CI 证据交由人类审查。 + +## 6. 实施结果与验证 + +### 变更结果 + +- `connect_tcp.py`：控制连接、server/client 文件发送 socket 均只在固定非零源端口冲突时改用 OS 端口；两端文件接收 listener 与 `_forward_relay` listener 在固定 bind 失败时 bind 到端口 0，并公告实际端口。转发 relay 在清理时释放原始 palloc 端口。 +- 错误识别：保留 `errno.EADDRINUSE` 与 WinError 10048；新增 WinError 10013；仅 Windows 上的 `errno.EACCES` 被视为端口冲突。端口 `None` 和 `0` 不符合固定端口回退条件。 +- 更新客户端/服务端 TCP API 文档，说明固定 transfer 端口是 best-effort 及接收 listener 端口公告行为。 +- 新增 loopback 集成回归：控制连接 bind/connect 10013；控制连接未绑定与端口 0 不回退；两端文件 sender bind/connect 10013；两端接收 listener bind 10013；真实 relay resume listener bind 10013 与原 palloc 端口释放。传输测试比较收件文件实际 bytes。 + +### 验证证据 + +1. 定向 smoke： + +```text +uv run pytest test/unit/network_api/test_TCP_client.py \ + test/integration/network_api/test_tcp_transfer_resume.py::test_client_falls_back_when_fixed_port_returns_wsaeacces \ + test/integration/network_api/test_tcp_transfer_resume.py::test_client_does_not_fallback_on_wsaeacces_without_fixed_port \ + test/integration/network_api/test_tcp_transfer_resume.py::test_client_connects_when_its_configured_port_is_taken \ + test/integration/network_api/test_tcp_transfer_resume.py::test_forwarded_file_resumes_through_the_server \ + test/integration/network_api/test_tcp_file_transfer.py::test_sender_falls_back_when_fixed_source_port_returns_wsaeacces \ + test/integration/network_api/test_tcp_file_transfer.py::test_receiver_listener_falls_back_when_fixed_port_returns_wsaeacces -q +22 passed in 14.34s +``` + +2. 最终受影响套件： + +```text +uv run pytest test/unit/network_api/test_TCP_client.py \ + test/integration/network_api/test_tcp_transfer_resume.py \ + test/integration/network_api/test_tcp_file_transfer.py -q +97 passed in 190.14s +``` + +3. 新增 `None`/`0` 端口边界与 errno 单测：3 passed in 0.35s。 + +### 明确未覆盖 + +- Linux loopback 用例注入了带 `winerror=10013` 的 `OSError`，并真实连接本机 TCP listener/收发文件；这验证应用分支与 socket 生命周期，不证明 Windows 内核的 Hyper-V/WinNAT 真实 WSAEACCES 行为。 +- Windows runner / 实际 Hyper-V/WinNAT 端口排除范围未测试；提交或合并前仍需 Windows CI 运行时证据。 +- 通用 `create_temporary_client()` / `create_temporary_server()` 与服务端主 listener 未修改，未声称覆盖。 +- `uv run ruff check` 的一次尝试对 5 个文件给出来自 56 个文件的诊断，输出范围与参数不符，不能作为 lint 结果；已向 `xd://report_issue` 报告该工具行为。未据此声称 lint 通过。 + +### 总验收对照 + +| 路径 | 结果 | 证据 | +|---|---|---| +| 控制连接固定端口 bind 10013 | 覆盖 | 定向集成用例成功连接并检查实际源端口变化 | +| 控制连接固定端口 connect 10013 | 覆盖 | 定向集成用例成功连接并由 server 接收端确认客户端 | +| 控制连接 None/0 | 覆盖 | 两种配置分别注入 10013；无第二次 connect；不绑定或只绑定 `0` | +| errno.EACCES 平台边界 | 覆盖 | Linux 原生判断为 false；模拟 Windows 时为 true；None/0 不触发 fallback | +| 服务端/客户端 sender bind/connect | 覆盖 | 四种组合真实传输文件并逐字节比较 | +| 服务端/客户端 receive listener bind | 覆盖 | 两个方向验证 fallback 端口公告与真实文件内容 | +| forward relay listener bind 与 palloc cleanup | 覆盖 | 真实续传完成；观察到 bind `0` 和原申请端口释放 | +| 真实 Windows WSAEACCES / Hyper-V/WinNAT | 未覆盖 | 当前验收环境无法提供 Windows runner 证据 | +| 通用 side-channel 与主 server listener | 未覆盖（非目标） | 本次未修改 | diff --git a/docs/source/File_Transfer/File_Transfer.rst b/docs/source/File_Transfer/File_Transfer.rst index c47dccf..750d2c5 100644 --- a/docs/source/File_Transfer/File_Transfer.rst +++ b/docs/source/File_Transfer/File_Transfer.rst @@ -69,13 +69,73 @@ attributes used by the transfer loop are: occurs, causing the transfer socket to be closed. +Transfer signs are newline-terminated protocol lines, including the relay's +upstream final ACK. A socket read is not a message boundary: both senders +consume complete signs and buffer an unterminated tail for the next read. +Thus a header sign and final ACK sharing a read, or a sign split across reads, +have the same meaning as separately received lines. EOF does not discard an +already parsed ACK, and an unterminated sign is not a completion ACK. This +applies to plaintext and decrypted transfer bytes without changing ACK timing, +timeouts or the receiver's close behavior. + +Each direct sender caps the combined **unscanned wire-byte quota** (queued +line lengths before stripping/decoding, plus the partial tail) at 64 KiB. +Every complete line is checked on receipt. When the quota reaches the limit, +all complete queued lines are scanned, recognized start/ACK/error results are +latched under the queue lock, and the scanned prefix is deleted as one batch; +parsing then continues. Reaching the aggregate quota is not itself a transfer +error, even when one read contains many batches of short lines. If adding one +complete line would cross the limit, the current complete batch is scanned +first. A single line, including its terminating newline, must fit within 64 +KiB; an unterminated line that reaches the limit is rejected because its +boundary cannot yet be recovered. Unknown and header lines leave no permanent +history. The sender scans pending records before deciding that a closed data +connection failed. File payload bytes do not use this parser and are not +limited to 64 KiB. Python object overhead, socket receive data and encrypted- +frame buffers are separate from this wire-byte quota. + +Relay target start/ACK waits retain their existing byte-marker matching and +deadline. They scan at most 64 KiB per window, then retain only the suffix +needed to match a marker split across reads: 36 bytes for the default start +marker, 24 for the default final ACK. An error marker still takes precedence +over a target marker in the same read; scanning windows does not restart the +timeout. The upstream final ACK remains a newline-terminated protocol line. + The remaining keys of the table cover the other three -direction/phase combinations. Protocol control lines that are -not part of this table (``/server_file_transfer_port``, -``/client_alloc_port_range``, ``/pause_trans``, -``/start_trans``, ``/forward_item``, ``/forward_upload``, -``/forward_error``, ``/crypto_*``) are hardcoded in -``connect_tcp.py``. +direction/phase combinations. Protocol control lines not +listed here (``/server_file_transfer_port ``, +``/client_alloc_port_range``, ``/pause_trans``, ``/start_trans``, +``/forward_item``, FID-scoped ``/forward_error`` and ``/crypto_*``) +are hardcoded in ``connect_tcp.py``. + +Temporary Listener Correlation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All ordinary file/folder, public-key, relay-target and relay-upload listeners +use one handshake. The sender registers the actual control socket and sender-owned +FID before sending the command. ``/file`` and ``/file_folder`` retain their +fields and trailing FID; public-key requests use ``/crypto_pub_key ``. +The listener replies with exactly four tokens: +``/server_file_transfer_port ``. + +Dispatch matches the actual control socket and FID, not an address alone. Each +relay target has an independent registration. The relay source allocates its +upstream FID and sends ``/forward_item fid= [resume=] +``; the Server preserves it and allocates a separate child FID per target. +Every retry includes ``resume=``, including ``resume=0``, and reuses the +route and child IDs. An upstream collision with an existing route, receive +record or target-scoped live transfer/pause identity is rejected without +replacing existing state. Forward failures use ``/forward_error ``. + +Each negotiation has one terminal control reply, either a port and stamp or an +error. Late, duplicate and mismatched replies are discarded, not cached. If a +sent request times out or is cancelled without a terminal reply, its FID remains +unresolved and cannot be retried on that control session; other FIDs and commands +continue. Natural disconnect clears that session's waiters, unresolved marks, +pending listeners and data sockets; reconnect can resume using the same stable +FID. Data or ACK failure does not disconnect a healthy control connection. There +is no forwarding-upload FIFO or shared error slot. Data framing, public-key +plaintext bytes, ACKs and five-item progress rows are unchanged. ### Metadata Exchange @@ -160,23 +220,15 @@ Its logic is as follows: 1. Allocate a unique client ID using the lock ``file_client_id_lock``. -2. Extend the original command string with the client - ID (e.g., - ``/file mydoc.txt`` becomes ``/file mydoc.txt 42``) - and send it to the server - over the main control connection using +2. Register the actual server control socket and expected FID, then send the + command with its trailing ID (for example, ``/file mydoc.txt 42``) using ``send_message()``. 3. Obtain a local port for the outgoing file transfer socket by calling ``palloc()``. -4. Wait (up to 20 seconds) for the server to reply with - a port number. The - server's reply (``/server_file_transfer_port - ``) is - received by the main connection and stored in - ``file_server_port_list``. - When a matching entry is found, the port is - extracted. +4. Wait (up to 20 seconds) for + ``/server_file_transfer_port ``. Only the matching + control socket and FID can deliver the port and stamp; there is no port cache. 5. Call the low-level send function ``file_transfer_mode(filename, server_host, server_port, client_port)``. @@ -184,11 +236,10 @@ Its logic is as follows: For folder transfers, the worker function ``folder_file_transfer_client_recv_client_start(message)`` walks the -directory tree on the console thread. It first sends a -``/file_folder `` command for the top-level folder itself -to create the -corresponding directory on the server, then sends the same command for -every subdirectory it walks into. For each file, it starts a daemon +directory tree on the console thread. Only directories that hold a file, +directly or below them, are announced to the receiver with a +``/file_folder `` command; a directory with no file in its +subtree is never created on the far side. For each file, it starts a daemon thread that holds ``file_semaphore`` around the actual transfer, with a command of the form @@ -203,8 +254,8 @@ around its own transfer, so the concurrent transfers are capped by ### Server-Side Handler (Receiving) -When the server receives an extended command like ``/file -mydoc.txt 42`` on the +When the server receives an extended command like +``/file mydoc.txt 42`` on the main control connection, ``handle_command`` detects it and calls: @@ -224,7 +275,7 @@ The server then: 4. This low-level receive function sets up a listening socket on the allocated port, sends the port number back to the client (as - ``/server_file_transfer_port ``), + ``/server_file_transfer_port ``), accepts the client's connection, and performs the protocol steps described in @@ -271,16 +322,11 @@ performs the following steps: ``(127.0.0.1,54321)``) using ``ast.literal_eval()``. Obtain the client's socket from ``self.clients``. -2. Allocate a unique client ID and extend the command - with it, then send the - extended command to the client over the main - connection. +2. Allocate a unique FID, register it against the actual client control socket, + then send the extended command with the trailing FID. 3. Allocate a local port with ``palloc()``. -4. Wait for the client to reply with its own port - number (via - ``/server_file_transfer_port``), which is stored in - ``file_server_port_list`` - by the main connection's message handler. +4. Wait for ``/server_file_transfer_port ``. The message + handler matches the actual control socket and FID without caching unmatched replies. 5. Call ``file_transfer_mode(filename, client_host, client_port, local_port)`` to send the file. @@ -294,7 +340,7 @@ This starts a receiver thread analogous to the server's receiver for client-initiated transfers: it allocates a port, replies with -``/server_file_transfer_port``, and calls +``/server_file_transfer_port ``, and calls ``file_transfer_mode_recv`` to receive and save the file. @@ -324,7 +370,11 @@ or creates the corresponding directory under the base receive directory (``received_files/`` for the server, the same directory -for the client). The +for the client). A command with no client id carries no data: the directory +is created on the control line itself and no transfer leg is opened. The +optional destination directory (``/file_folder ``) +is resolved by the same ``_receive_dir_for`` used by the file legs, so a +created directory and the files landing under it always agree. The relative path uses forward slashes to be platform-independent. @@ -644,10 +694,13 @@ Both classes record every running file transfer: - ``file_recv_progress`` — the files being received (downloads); - ``file_upload_progress`` — the files being sent (uploads). -The keys are the transfers' fids: the transfer id the sender stamps on the -transfer command (``/file ``), which also identifies the relay leg -of a forwarded file. The protocol-internal public-key push is not tracked. -Each value is a five-item list: +Downloads use integer fid keys. Uploads use ``(receiver, fid)`` keys, where +receiver is the actual next-hop control endpoint, not the logical target of a +forward. Client uploads always use ``((host, port), fid)``; server uploads use +the receiving client's control address. Equal numeric fids from different +receivers do not share progress, pauses, sockets or port announcements. +The protocol-internal public-key push is not tracked. Each value remains a +five-item list: - ``[0]``: bytes transferred so far; - ``[1]``: total size of the file, in bytes; @@ -660,12 +713,29 @@ Each value is a five-item list: a download's entry, the receiver in an upload's. A forwarded transfer stores the original peer on both sides, never the relaying server. -An entry appears when a transfer starts and is removed when it ends, so the -stores hold exactly the transfers that are in flight at that moment (iterate -over a copy while transfers are running). With -``is_enable_file_transfer_progress_bar`` set to ``False`` no entry is ever -recorded — and, since the stores are the state a reconnect resumes from, no -transfer can be resumed either. +An entry appears when a transfer starts. Successful direct transfers remove +their entry; interrupted transfers and forwarded uploads awaiting final target +completion retain it (iterate over a copy while transfers are running). A +forwarded upload can therefore show all bytes sent while still pending. With +``is_enable_file_transfer_progress_bar`` set to ``False`` no progress entry is +recorded, and automatic partial-file recovery remains unsupported. + +Direct senders return ``True`` only after the receiver's final data ACK. +An error sign at startup returns ``False`` before sending file metadata; +an error sign after the body also returns ``False``, even when every byte +has been sent. A tracked upload with a peer retains and persists its +``(receiver, fid)`` row after failure. Startup rejection creates no new row +and leaves an existing interrupted row intact. The server and client share +this rule for files, folder items and resumed legs. ACK wait limits and the +uncertainty of an ACK lost after a successful TCP write are unchanged. + +Both receivers also retain and persist a tracked receive row with a peer +when writing the final ACK returns ``False`` or raises, even if the complete +file is already on disk. The original fid, full path, source and total remain +available for EOF confirmation; a failed confirmation keeps the row again, +and a successful ACK write removes it. Local receive notifications, events +and public-key TOFU registration still precede that ACK. Key pushes remain +untracked, and disabling progress recording does not create recovery rows. ``is_enable_real_time_network_speed`` (``True`` by default) measures the protocol's current traffic — the control layer and every temporary @@ -679,19 +749,100 @@ The web tool (``transfer_web``) reads these stores: both backends serve frontend renders the live speed readout in the topbar and one progress bar per running transfer. +Local progress inspection +------------------------- + +Both ``TCP_Server_Base`` and ``TCP_Client_Base`` expose the same read-only +method: + +.. code-block:: python + + get_transfer_progress(*, include_persisted=True) -> dict + +The result contains ``progress_enabled`` and ``speed_enabled`` flags, a +``memory`` section, and a separate ``persisted`` section. Memory rows include +the integer ``fid``, the complete upload ``receiver`` scope when applicable, +the logical ``peer``, path/name, byte counts, speed, local ``active`` and +``socket_registered`` observations, and upload ``paused`` state. Uploads with +the same numeric fid but different receivers remain separate. Server relay +rows show ``originator``, all accepted ``targets`` and ``completed`` targets. +Clients expose forwarded uploads through their existing upload rows; their +``relays`` section is empty, with no second recovery-metadata table. + +By default, ``persisted`` reads the instance's existing progress JSON without +loading it into memory or changing it. Its ``status`` is ``ok``, ``missing``, +``error`` or ``not_requested``; disk rows set runtime-only fields to ``None``. +``include_persisted=False`` skips file I/O. The method never starts a +connection, resumes a transfer, writes state, reads file contents or infers a +zombie/completion status. A local interactive console can print the same +snapshot with ``/transfers``; unsupported arguments print usage locally. + +The server and client console commands are local-only. The existing Web +``/api/transfers`` response remains the smaller live-progress contract and is +not replaced by this diagnostic snapshot. + +Explicit local state deletion +------------------------------ + +Both classes expose ``delete_config(config_name=None, *, is_list_config=False)``. +List mode takes precedence over the name and returns a sorted ``list[str]`` +of existing regular files in this instance's allowlist, without changing state. +It is allowed while running. With the default ``False``, pass one basename: +deletion returns ``True`` or ``False`` when the allowed file is already absent. + +The allowlist contains the instance's ``file_trans_progress_file`` and its four +message/event logs. Default progress names are +``server_file_trans_progress.json`` and ``client_file_trans_progress.json``; +log names are ``messages_log.json``, ``events_log.json``, +``sent_messages_log.json`` and ``sent_events_log.json``. Redirected path +attributes retain their configured locations; use their current basenames. +Paths, globs, ambiguous names, directories and symlinks are refused. Keys, +TOFU registries, protocol tables, port locks, launcher/Web settings and file +contents are outside the allowlist. Invalid names raise ``ValueError``; +non-string deletion names raise ``TypeError``; filesystem errors propagate. + +Deletion requires an unstarted instance or a successful explicit shutdown, +with no startup/shutdown operation, active transfer or relay remaining. +An unexpected disconnect is not shutdown. Pending log buffers must be flushed +before deleting their file. Otherwise ``RuntimeError`` is raised; the API does +not stop workers or flush files implicitly. + +.. code-block:: python + + names = server.delete_config(is_list_config=True) + server.stop() # wait for startup/control/transfer workers to finish too + server.delete_config("server_file_trans_progress.json") + client.close() + client.delete_config("client_file_trans_progress.json") + +Deleting progress abandons the entire upload/download/relay snapshot, including +any legitimate interrupted tasks. Missing progress is also an explicit discard. +The old object cannot reload, republish or restart; use a new instance for the +next run. Source and received files remain untouched. Logs have no permanent +disable flag. Default shutdown without deletion still persists normally. +Stop every writer sharing the selected path first: no cross-process ownership +or isolation is provided. This is a local Python API, not a console/HTTP command. + + .. _transfer-pause-and-reconnect: Pausing, Reconnecting and Resuming Transfers -------------------------------------------- -An outbound transfer can be held and released at any time with the -``pause_trans(fid)`` and ``start_trans(fid)`` methods (both classes), or from -the peer with the ``/pause_trans `` and ``/start_trans `` lines: -``pause_trans_remote(peer, fid)``/``start_trans_remote(peer, fid)`` send them. +An outbound transfer can be held and released with the server methods +``pause_trans(fid, peer)`` / ``start_trans(fid, peer)``; the receiving peer is +required, including when arming a transfer that has not started. The client +methods default to its connected server and reject a different explicit peer. +Remote ``/pause_trans `` / ``/start_trans `` lines use their requesting +control peer as the scope; ``pause_trans_remote(peer, fid)`` / +``start_trans_remote(peer, fid)`` send them. Their wire format is unchanged. The leg stops before its next chunk and stays open; a leg that is paused keeps its socket, so nothing is torn down. Every outbound transfer honours its own fid — a plain upload as much as the upload leg of a forwarded file — so the flow control the server uses on a relay works on any transfer. +A paused leg that belongs to a session swept at reconnect is released by +that sweep: its pause switch is cleared, its socket is closed, and the leg +ends, so a resume request queued behind its per-fid send lock can run. The temporary connections of a transfer are resumed after a disconnect: @@ -700,22 +851,98 @@ The temporary connections of a transfer are resumed after a disconnect: ``.Flow/client_file_trans_progress.json`` (the ``file_trans_progress_file`` attribute) — every upload, every download and, on the server, the relay records a disconnected forward is re-driven from; +- upload entries use compact JSON strings containing ``[receiver, fid]`` as + keys, while download keys remain decimal fid strings. Values remain five + items. A snapshot with unsupported bare upload-id keys is logged and left + untouched, including during shutdown; archive it before enabling persistence + of new transfers. Automatic migration is not provided; - an interrupted transfer keeps its progress entry and its partial file instead of dropping them, so the file on disk is exactly the state to continue from; -- after a reconnect the receiver drives the resume (`resume_transfers`): it - opens a fresh transfer listener, announces it and sends - ``/resume_trans `` with the byte offset its partial file - holds. The sender re-establishes the leg, reads its source from that offset - and only the remainder of the file travels. A forwarded transfer is asked - for through the server, which routes ``/forward_resume`` to the originating - client so it re-forwards just the missing part. - -The server resumes a peer's transfers when it reconnects, and a client with -``is_auto_reconnect=True`` reconnects on its own and resumes every transfer it -can reach; `resume_transfers` is public, so an application can also drive the -resume itself. Legs the previous session left behind are closed before the -resume runs, so two legs never write the same transfer. +- after a reconnect each receiver sends ``/resume_trans + `` for its interrupted downloads (``_request_resumes``). + Both addresses are shell-quoted tuple literals. ``origin`` is the original + sender, ``target`` the requesting receiver, and ``offset`` the bytes on disk. + The server requires target to equal the actual control requester. A server + origin selects its scoped local upload; otherwise the downstream fid and + origin must match its authoritative route before translation to the upstream + fid and forwarding of the same command to the source. A client sends only + when origin matches its current control-connection identity. + Each side writes the concrete endpoint of that control connection + (``getsockname()`` locally, ``getpeername()`` for the peer) into + ``origin``/``target``; the configured ``(host, port)`` is accepted as an alias + for the same connected peer. A wildcard bind (``0.0.0.0``) or a host alias + (``localhost`` vs ``127.0.0.1``) therefore does not change the identity. A + direct row persisted before this rule, which recorded the configured server + address, is migrated in memory when read; the progress file format is + unchanged. Addresses behind NAT and a changed ``client_port`` remain + unsupported identities: the former is not comparable from the two sides, the + latter cannot be matched to the stored peer after reconnecting. + Local uploads answer with ``/file ``. Forward replay sends + ``/forward_item fid= resume=`` with its fields. + New transfers omit ``resume=``; retries include ``resume=``, even at + zero. The source owns the upstream FID and the Server allocates one child FID + per target. Every listener replies through the same four-token + ``/server_file_transfer_port `` line; the server validates + the sender and unresolved target and reuses the upstream and child FIDs. Stale + or mismatched responses cannot satisfy another request. The receiver matches + fid, original sender, basename + and total size, then writes its retained full path, including nested folder + items. A size mismatch preserves the partial file. No new-ID/path alias or old + resume command is used. + +On the server, each upstream fid has one authoritative route containing its +original sender, full accepted ``receiver -> downstream fid`` mapping, and +``completed`` set. Only a target's final data ACK adds it to that set. The +derived ``(receiver, downstream fid) -> upstream fid`` index contains unresolved +targets only. No file paths, progress values or directory metadata are copied; +live relay buffers and pause conditions remain separate runtime state. + +The server's ``relays`` snapshot writes one row per upstream fid, containing +``originator``, ``targets: [[receiver, fid], ...]`` and +``completed: [receiver, ...]``. Loading preserves completion and rebuilds only +unresolved indexes without replacing live owners. A canonical route snapshot +without ``completed`` means no target has acknowledged yet. Unsupported older +server relay schemas or invalid completion sets are rejected and preserved on +write and shutdown; archive them before persisting new routes. Client snapshots +keep ``relays`` empty; nonempty older client relay metadata is rejected and its +file preserved, including on shutdown. Five-item upload/download values remain +unchanged. The source recovers a forwarded file from its upload path alone; +directory/destination metadata is not duplicated because the target retains +its full receive path. Per-upload locks and in-flight target sets are transient +and released after use, not persisted. + +Targets accepted after reachability filtering remain in the mapping even if +their listener or start handshake fails. A successful target loses its reverse +index immediately, but the accepted mapping remains until the current input +stream is fully read and every accepted target has acknowledged. The server +then forwards the existing final data-ACK marker upstream using the normal +per-hop send path (encrypted when enabled). It drops the route only after that +ACK write succeeds; an unsuccessful write leaves the completed route available +for recovery. The source's original final-ACK wait is 30 seconds plus 10 seconds +per 100 MiB. A long pause or slow downstream delivery can outlast that wait; +timeout or a lost ACK conservatively retains the source upload progress, +even if the server completed delivery. No separate receipt or retry +protocol is used. If a target cannot send its final ACK, its receive entry +survives even when the disk file is complete; an EOF replay with the same fid +can confirm it without retransmitting the body. Retries for different targets +of one upstream fid are serialized at the source, concurrent duplicate requests +for the same target are coalesced, and completed targets are not replayed. +Ordinary resumes share the same in-flight coalescing. Offline-source requests +are kept on the server per upstream fid/target, retaining the latest offset. +Reconnect sends through the same secure control path; a failed send keeps the +request, and completed or missing routes are never replayed. +Fids no longer encode a plain/relay kind: the reserved low bit is always zero, +and diagnostics use ``e.``. Ordinary senders own their FIDs; +the relay source owns the upstream FID and the Server allocates one child FID +per target. Resumes reuse the existing upstream and child FIDs. +Public-key pushes remain untracked and TOFU verification still precedes their +ACK. + +The server requests a peer's interrupted downloads when that peer reconnects, +and a client with ``is_auto_reconnect=True`` does the same after every connect, +so no separate call is needed. Legs the previous session left behind are closed +before the resume runs, so two legs never write the same transfer. .. _related-apis: @@ -802,10 +1029,10 @@ file to disk. control_socket=None, crypto=None, fid=None, peer=None, resume_offset=0) Low-level send function that connects to the receiver and transmits the file. -``fid`` keys the leg in ``file_upload_progress`` and is what -``/pause_trans`` / ``/start_trans`` addresses; ``peer`` is the transfer's -original receiver and ``resume_offset`` continues an interrupted file from -the byte the receiver already holds. +``(receiver, fid)`` keys the upload, with receiver taken from ``peer`` or, +for an untracked logical peer, the control endpoint (transfer listener when +no control socket exists). Pause/start use this same receiver scope; +``resume_offset`` continues from the byte the receiver already holds. ### Client-Side File Transfer APIs @@ -895,7 +1122,7 @@ reference. - ``file_folder_transfer_server_recv_server_start_thread`` (folder receiving) - ``pause_trans`` / ``start_trans`` (hold and release an outbound transfer) - ``pause_trans_remote`` / ``start_trans_remote`` (ask a client to do it) -- ``resume_transfers`` / ``resume_trans`` (continue interrupted transfers) +- ``resume_trans`` (continue one interrupted transfer, sender side) (The low-level helpers ``file_transfer_server_recv_server_start``, @@ -915,7 +1142,7 @@ public but are documented for completeness.) - ``file_folder_transfer_client_recv_server_start_thread`` - ``pause_trans`` / ``start_trans`` (hold and release an outbound transfer) - ``pause_trans_remote`` / ``start_trans_remote`` (ask the server to do it) -- ``resume_transfers`` / ``resume_trans`` (continue interrupted transfers) +- ``resume_trans`` (continue one interrupted transfer, sender side) (The low-level helpers are documented but not part of the public API.) diff --git a/docs/source/MAP.md b/docs/source/MAP.md index e2b45ed..40b5f09 100644 --- a/docs/source/MAP.md +++ b/docs/source/MAP.md @@ -17,7 +17,7 @@ | `docs/readme_translations/` | Machine-translated `README.md` (`README..md`) plus its translation memory | ✅ present | | `docs/source/conf.py` | Sphinx configuration | ✅ present | | `docs/source/templates/` | Change-note, design-note and how-to templates | ✅ present | -| `docs/source/changes/` | Auto-generated per-PR change notes | ❌ not created yet | +| `docs/source/changes/` | Per-change notes (named by feature until a PR number exists) | ✅ present | | `docs/source/design/` | Design notes (ADR) archive | ❌ not created yet | | `CHANGELOG` | Destination of `templates/changelog-entry.md` | ❌ not created yet | | `docs/source/MAP.md` | This file: the documentation map | ✅ this file | @@ -60,7 +60,7 @@ keeps `readme_cache.json` as the memory that makes a rerun request only changed ## Known gaps -- `docs/source/changes/`, `docs/source/design/` and the `CHANGELOG` do not exist yet (their templates are ready). +- `docs/source/design/` and the `CHANGELOG` do not exist yet (their templates are ready). `docs/source/changes/` exists and holds per-change notes; the first is `connect-tcp-send-listeners-and-control-records.md` and is not yet tied to a PR number. - API pages under `docs/source/api/` are generated by `sphinx-apidoc` (`-T`, so it writes no `modules.rst`); rerun it after adding or removing a *module*. The entry point is the hand-written `docs/source/api/index.rst` ("API Reference"), linked from the main toctree; page options live in `autodoc_default_options` in `docs/source/conf.py`. - Docstring backlog: 106 open ruff `D` findings (59 of them undocumented public API); public-API coverage measured by interrogate is 66.1%, gated by `fail-under = 65.0` in `pyproject.toml`. - No Doxygen configuration (no `Doxyfile`); `PyFlow/crypto_api/include/` uses plain block comments. diff --git a/docs/source/Network_APIs/TCP_Client_APIs.rst b/docs/source/Network_APIs/TCP_Client_APIs.rst index 1fffa7e..19d65be 100644 --- a/docs/source/Network_APIs/TCP_Client_APIs.rst +++ b/docs/source/Network_APIs/TCP_Client_APIs.rst @@ -75,13 +75,23 @@ The parameters of the ``__init__`` method are as follows: running file transfer is tracked in ``file_recv_progress`` (downloads) and ``file_upload_progress`` (uploads). Each entry is a ``[transferred bytes, total bytes, current speed, local file path, peer]`` - list keyed by the transfer's fid — ``peer`` is the ``(ip, port)`` of the - transfer's original sender (downloads) or receiver (uploads), never the - relay of a forwarded transfer. The entry is removed once the transfer ends; - an interrupted one keeps its entry (and its partial file) so the reconnect - logic can resume it. + list. Downloads use integer fid keys; uploads use ``((host, port), fid)`` + keys, scoped to the connected server even for forwarded uploads. + The value's ``peer`` remains the original sender (downloads) or logical + receiver (uploads), not the relay. For a direct download it is the connected + server's concrete endpoint (``getpeername()``); rows persisted before this + rule recorded the configured address and are normalized in memory on the next + resume. The configured address remains accepted as an alias; NAT and a changed + client port are not supported. Successful direct transfers remove their + entries; interrupted ones keep their partial files for reconnect. Forwarded + uploads retain their existing five-item upload row until the final data ACK + is observed; bytes sent alone are not completion. No client recovery table + duplicates paths, targets or directory metadata; ``relays`` stays empty. ``file_trans_progress_file`` names the file under the protocol's ``.Flow`` directory these stores are written to when the connection drops. + Upload JSON keys encode ``[receiver, fid]`` as compact JSON strings. + Older bare upload-id or nonempty client-relay snapshots are rejected and + preserved, not migrated. - ``is_enable_real_time_network_speed``: A flag indicating whether the protocol's real-time network speed is measured. With ``True`` (the default) every send and receive — the control layer and every temporary connection, @@ -95,10 +105,11 @@ Every parameter has default values: - ``host``: Default is ``None`` (must be provided before connecting) - ``client_host``: Default is ``'127.0.0.1'`` - ``port``: Default is ``65432`` -- ``client_port``: Default is ``None`` (OS assigns ephemeral port). A - configured port that cannot be bound — a session that just ended can hold - it in the operating system's ``TIME_WAIT`` — is reported and the connection - is made from an OS-assigned port instead. +- ``client_port``: Default is ``None`` (the OS assigns an ephemeral port). A + configured nonzero port that conflicts during ``bind`` or ``connect`` — for + example, while an earlier session remains in ``TIME_WAIT``, including Windows + ``WSAEACCES`` (10013) — is logged and the control connection uses an + OS-assigned port instead. ``None`` and ``0`` let the OS choose. - ``timeout``: Default is ``None`` - ``port_add_step``: Default is ``1`` - ``max_thread_num``: Default is ``10`` @@ -272,16 +283,34 @@ commands sent by the server, such as: - ``/client_alloc_port_range``: configures the client's manual port allocation range from the line the server sends it on connect. -- ``/server_file_transfer_port``: receives the file transfer - port assigned by the server for an ongoing file operation. -- ``/file`` and ``/file_folder``: handle file transfer requests - initiated by the server (server-to-client transfers). +- ``/server_file_transfer_port ``: four-token listener reply, + dispatched only to the registered actual control socket and FID. Unmatched or + late replies are discarded without cache or fallback. Each negotiation has one + terminal port or error reply. A sent request timed out or cancelled without a + terminal reply leaves that FID unresolved and blocked on this session; other + FIDs and commands continue. Natural disconnect clears session-owned waiters, + unresolved marks, pending listeners and data sockets; reconnect can resume the + same stable FID. Data or ACK failure does not disconnect healthy control. +- ``/file`` and ``/file_folder``: server-initiated requests retain their fields + and trailing sender FID; public-key requests use ``/crypto_pub_key ``. - ``/crypto_mode``, ``/crypto_key_exchange``, ``/crypto_key_exchange_ack``, ``/crypto_reject``, ``/crypto_pub_key``, ``/crypto_ready``: the encrypted-channel handshake messages; a ``/crypto_mode`` mismatch closes the connection. -- ``/forward_upload``, ``/pause_trans``, ``/start_trans``, - ``/forward_error``: forwarding and transfer-control messages. +- ``/pause_trans``, ``/start_trans`` and ``/forward_error ``: + transfer control and FID-scoped forwarding failures. Forward uploads use + the common ``/server_file_transfer_port`` reply; no ``/forward_upload`` line. +- ``/resume_trans ``: the sole resume command, + with shell-quoted tuple endpoints. Only an origin matching this control + connection's own identity may resume a local upload. The existing upload row + supplies the source path; a non-server target is replayed through the server + with its stable upstream fid. The receiver's retained full path handles folder + items without copying directory or destination metadata at the source. + +For a new relay upload, the source client allocates the upstream FID before it +sends ``/forward_item ... fid=``. The server returns that same upstream FID +and allocates a separate child FID for each target. A resume includes +``resume=``, including ``resume=0``, and reuses the existing route. If you have registered custom commands using the command extension API, `handle_server_command` will also check @@ -313,7 +342,8 @@ and newline framing are handled by the caller. def send_message( self: Self, client_socket: Any, - message: Any) -> True|False: + message: Any, + is_send_raw: bool = False) -> True|False: ... `send_message` sends data from the client to the server. @@ -321,13 +351,18 @@ It verifies the client is still running and the socket is valid, then: - accepts both ``str`` and ``bytes`` message payloads - trims string payloads and appends a newline if missing -- when the socket is in encrypted mode (the public-key exchange has - completed), the plaintext is signed with the ``_VALID`` marker, - RSA-OAEP encrypted with the server's public key, base64 encoded and - newline-terminated instead of being sent in cleartext -- encodes string payloads as UTF-8 -- sends the complete message with ``client_socket.sendall(data)`` +- when the channel is encrypted, writes the payload as frames through the + secure channel (see the framing description in the ``connect_tcp`` module + docstring) instead of as plaintext +- with ``is_send_raw=True`` writes a plaintext line even on an encrypted + channel; this is reserved for the crypto handshake and transfer + coordination lines +- sends the complete payload with ``client_socket.sendall(data)`` - returns ``True`` on success, otherwise logs the error and returns ``False`` +- after a successful write, reports the payload to the send listeners; a + write to the control socket is also recorded in the sent stores, while a + transfer socket is only reported. ``is_send_raw`` lines are neither + reported nor recorded, and ``bytes`` payloads are reported but not recorded The client also provides an interactive input loop (`interactive_mode`) that reads user input from the console and sends messages @@ -335,8 +370,8 @@ to the server. .. _tcp-client-event-listener-api: -TCP Client inbound event listener API -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TCP Client event listener API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ External code can observe traffic without replacing the protocol loop: @@ -345,27 +380,90 @@ protocol loop: def add_message_listener(self, listener) def remove_message_listener(self, listener) + def add_send_message_listener(self, listener) + def remove_send_message_listener(self, listener) def add_file_listener(self, listener) def remove_file_listener(self, listener) -``add_message_listener`` registers ``listener(sender_id, message)`` -for every inbound line, commands included, so a listener observes all -inbound traffic; a message another client forwarded here through the -``/send_msg_from`` envelope is reported as its payload. ``sender_id`` is -the author's ``"ip:port"`` — the forwarding client for such a forwarded -message, and it is ``None`` for a direct push from the server. -``add_file_listener`` -registers ``listener(client_id, full_path, name, size, command)`` and -fires after a file sent by the server has been fully written to disk. -Both listeners run on a protocol thread, so they must not block, and an -exception inside a listener is logged and swallowed. The ``remove_*`` -counterparts unregister a listener and ignore unknown ones. - -Received messages and events are also stored per sender socket in -``messages_dict`` / ``events_dict`` (each entry a ``[content, -timestamp]`` pair). When a store reaches ``max_dict_size`` (64 KiB) it -is flushed to ``messages_log.json`` / ``events_log.json`` under -``network_api/.Flow`` and cleared; ``close`` flushes both stores. +``add_message_listener`` registers ``listener(sender_id, message)`` for +every complete inbound control line. Lines are newline-delimited, stripped, +and empty lines are skipped; commands (including the crypto handshake lines) +are included, so the listener observes all inbound traffic while line +dispatch continues unchanged. A message another client forwarded here +through the ``/send_msg_from`` envelope is reported as its payload, not the +envelope. Raw chunks read from a non-control socket (file transfer, +forwarding, temporary connections) are reported to the same listeners as +``bytes``: one callback per read or decrypted chunk, with no line or frame +alignment. ``sender_id`` is the peer endpoint of the socket that produced +the event: the forwarding/origin client's ``"ip:port"`` for a forwarded +message, ``None`` for a direct push from the server, and that temporary TCP +connection's ``"ip:port"`` for a side-channel chunk (not the control server +address). Control lines are ``str``; side-channel chunks are ``bytes``. + +Threading: control-line callbacks run on the client receive thread; raw +side-channel chunks and file callbacks run on the thread that reads or +transfers that socket; send callbacks run on the thread that calls +`send_message`. Callbacks may run concurrently for different sockets, and +send callbacks for concurrent sends on the same socket may also overlap; +they must be thread-safe. Per-socket inbound order is preserved, +cross-socket order is not. + +``add_send_message_listener`` registers ``listener(target_id, message)`` +and calls it synchronously on the sending thread after every payload that +`send_message` writes successfully, for ``str`` and ``bytes`` alike. +``str`` payloads are reported stripped of leading/trailing whitespace; +``bytes`` are reported unchanged. ``target_id`` is the peer endpoint of the +socket that performed the write: the server's ``"ip:port"`` for the control +socket, or the temporary TCP connection's ``"ip:port"`` for a file/forward +side channel. Lines written with ``is_send_raw=True`` and internal protocol +frames such as the ECDH public-key publication are never reported. + +When several threads call `send_message` on the same socket, the socket +writes remain serialized, but the send-listener callbacks are not guaranteed +to run in the same order as the wire writes. This affects observation order +only: it does not change socket write order, receive order, or file data +integrity. Callers that need a strict observation order must serialize their +own `send_message` calls. + +``add_file_listener`` registers +``listener(full_path, name, size, command)`` and fires once after a file +sent by the server (a direct send, or a forwarded item) has been fully +written to disk. ``command`` is the wire command that triggered the +transfer, so protocol pushes such as ``/crypto_pub_key`` can be recognised. +The callback runs on the transfer thread and fires before the completion +sign is sent; it does not wait for the server's ACK. Per-chunk progress is +reported to the message listeners as ``bytes``, not by this callback. For +``/crypto_pub_key`` the callback runs before the server key is registered; a +listener must not move or delete the file. + +Listeners must not block, and an exception inside one is logged and +swallowed. The ``remove_*`` counterparts unregister a listener and ignore +unknown ones; add/remove take effect for subsequent notifications, while a +notification already in progress may still run. ``close()`` stops new +delivery, clears the listener lists, shuts down the socket and joins the +receive thread (unless called from that thread), so no control-line callback +starts after it returns. A side-channel/file callback already past the +stopped check on a transfer thread, or a send callback already running on +another thread, may still finish. With ``is_auto_reconnect`` a reconnect +through ``_release_client_socket()`` keeps the listeners for the next +session; only the terminal ``close()`` clears them. Listener lists are per +instance and are not replayed; after ``close()`` create a new instance to +register listeners again. + +Control traffic is also kept per peer socket in four stores, each value a +list of ``[content, timestamp]`` pairs: + +- ``messages_dict``: inbound plain-text lines (lines not starting with ``/``) +- ``events_dict``: inbound ``/`` commands, including the ``/send_msg_from`` + envelope of a forwarded message +- ``sent_messages_dict`` / ``sent_events_dict``: the same split for payloads + written to the control socket + +Only the control socket is recorded. Transfer chunks, transfer sign tokens +and ``bytes`` payloads are never recorded. When a store reaches +``max_dict_size`` (64 KiB) it is flushed to ``messages_log.json``, +``events_log.json``, ``sent_messages_log.json`` or ``sent_events_log.json`` +under ``network_api/.Flow`` and cleared; ``close`` flushes all four stores. .. _tcp-client-encrypted-channel-api: @@ -470,14 +568,14 @@ Encrypted transfer connections File and folder transfers, forwarded items and the forward relay run on temporary TCP connections. The listening side stamps each leg with its mode -and a fresh nonce in the port advertisement (``/server_file_transfer_port``, -``/forward_upload``); with ``is_enable_encrypto`` enabled both sides then -frame that connection exactly like the control channel, with the same -session key material, so file contents, names and sizes no longer cross the -wire in the clear. The public-key pushes themselves stay plaintext by -construction: they carry the keys the session is built on. A leg whose -stamp promises encryption is aborted when the session key material is not -ready rather than silently downgraded. +and a fresh nonce in the common port advertisement +(``/server_file_transfer_port ``). With +``is_enable_encrypto`` enabled, both sides then frame that connection exactly +like the control channel, with the same session key material, so file contents, +names and sizes no longer cross the wire in the clear. Public-key pushes stay +plaintext because they carry the keys the session is built on. A leg whose +stamp promises encryption is aborted when key material is not ready rather than +silently downgraded. Decode verification and key rotation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -657,6 +755,9 @@ Supported console commands (user-typed) include: - ``/multiple_file ...``: sends multiple files to the server, one thread per file, limited to ``max_thread_num`` at a time. - ``/file_folder ``: sends an entire folder to the server, recursively. - ``/multiple_file_folder ...``: sends multiple folders to the server. +- ``/transfers``: prints this client's local memory rows and separate persisted + transfer snapshot as JSON. It is handled locally, never sent to the server, + and accepts no arguments. - ``/forward_file``, ``/forward_folder``, ``/forward_send_msg``: the forwarding extension's console commands. - Any other non-empty line is sent as-is to the server — the same path chat @@ -741,10 +842,22 @@ And the low-level send mode function is: self, filename, server_address, server_port, client_port, control_socket=None, crypto=None, fid=None, peer=None, resume_offset=0) -``fid`` keys the leg in ``file_upload_progress`` and is what -``/pause_trans`` / ``/start_trans`` (or `pause_trans` / `start_trans`) -addresses; ``peer`` is the transfer's original receiver and ``resume_offset`` -continues an interrupted file from the byte the receiver already holds. +``((host, port), fid)`` keys the leg in ``file_upload_progress``. The +``/pause_trans`` / ``/start_trans`` lines and local methods address that server +scope; an explicit local ``peer`` must be the server, not the logical target. +The value's ``peer`` is the original receiver; ``resume_offset`` continues +the interrupted file from the byte the receiver already holds. + +For a forwarded upload, the server forwards the existing final data-ACK marker +after the full input stream and all accepted target ACKs complete. The source +keeps its existing upload row unless its original final-ACK wait observes that +marker. The wait is 30 seconds plus 10 seconds per 100 MiB; a long pause or +slow delivery can outlast it, and a lost ACK conservatively retains state. No +separate receipt or retry protocol is used. Direct transfer and public-key +confirmation remain data-ACK based. The server retains the upstream/child IDs +and full target mapping. Transient source locks serialize different target +requests on one upload and coalesce concurrent duplicates; they hold no file +metadata and are released after the last in-flight request. We recommend that for client-to-server file transfers, you use the console commands (``/file``, ``/file_folder``, @@ -761,6 +874,13 @@ a command over the main connection, then a temporary port is negotiated, and the actual file data is transferred over a separate TCP connection.* +For built-in file transfers, a configured nonzero sender source port is +best-effort: a local bind/connect conflict retries from an OS-assigned port. +If a receiver's configured nonzero listener port conflicts during ``bind``, +the listener uses port ``0`` and advertises the actual bound port. Port ``0`` +already delegates selection to the OS. These transfer fallbacks do not change +the generic temporary client/server APIs. + Port allocation API ------------------- @@ -837,8 +957,9 @@ is as follows: 2.1. The inbound event listener APIs: - `add_message_listener` / `remove_message_listener` + - `add_send_message_listener` / `remove_send_message_listener` - `add_file_listener` / `remove_file_listener` - - `messages_dict` / `events_dict` stores + - `messages_dict` / `events_dict` / `sent_messages_dict` / `sent_events_dict` stores 3. The client encrypted channel APIs (see :ref:`tcp-client-encrypted-channel-api`): - `_crypto_exchange_thread` @@ -864,18 +985,23 @@ is as follows: - `file_transfer_client_recv_server_start_thread` - `file_folder_transfer_client_recv_server_start_thread` - `file_transfer_mode` + - `get_transfer_progress` + - `delete_config`: local allowlisted state-file deletion after shutdown; + ``is_list_config=True`` returns names without deleting. Discarding progress + requires a new instance for future connections; file contents are kept. 5.1. The transfer pause and reconnect APIs (see :ref:`transfer-pause-and-reconnect`): - `pause_trans` / `start_trans` - `pause_trans_remote` / `start_trans_remote` - - `resume_transfers` / `resume_trans` + - `resume_trans` 6. The client console commands (interactive mode): - `/quit` - `/file` - `/multiple_file` - `/file_folder` + - `/transfers` - `/multiple_file_folder` See Also diff --git a/docs/source/Network_APIs/TCP_Server_APIs.rst b/docs/source/Network_APIs/TCP_Server_APIs.rst index 31c76a3..6b8e837 100644 --- a/docs/source/Network_APIs/TCP_Server_APIs.rst +++ b/docs/source/Network_APIs/TCP_Server_APIs.rst @@ -80,13 +80,24 @@ The parameters of the ``__init__`` method are as follows: running file transfer is tracked in ``file_recv_progress`` (downloads) and ``file_upload_progress`` (uploads). Each entry is a ``[transferred bytes, total bytes, current speed, local file path, peer]`` - list keyed by the transfer's fid — ``peer`` is the ``(ip, port)`` of the - transfer's original sender (downloads) or receiver (uploads), never the - relay of a forwarded transfer. The entry is removed once the transfer ends; + list. Downloads use integer fid keys; uploads use ``(receiver, fid)`` keys, + where receiver is the actual control-channel peer's ``(ip, port)``. + The value's ``peer`` remains the original sender (downloads) or logical + receiver (uploads), not a forwarding relay. It is the concrete endpoint of + the control connection each leg used; resume identity compares that endpoint + and accepts the configured address as an alias, so a wildcard bind or a host + alias does not change it. The entry is removed once the transfer ends; an interrupted one keeps its entry (and its partial file) so the reconnect logic can resume it. ``file_trans_progress_file`` names the file under the protocol's ``.Flow`` directory these stores are written to when a connection drops. + Upload JSON keys encode ``[receiver, fid]`` as compact JSON strings. + Older bare upload-id snapshots are rejected and preserved, not migrated. + Forward routes are persisted once per upstream fid in ``relays``, with an + original sender, the full receiver/fid target mapping and ``completed`` peers. + Only unresolved targets have receiver-scoped reverse indexes after load; + loading never replaces live owners. Unsupported older server relay schemas + and invalid completion sets are rejected and preserved, not auto-migrated. - ``is_enable_real_time_network_speed``: A flag indicating whether the protocol's real-time network speed is measured. With ``True`` (the default) every send and receive — the control layer and every temporary connection, @@ -327,7 +338,8 @@ caller. def send_message( self: Self, client_socket: Any, - message: Any) -> True|False: + message: Any, + is_send_raw: bool = False) -> True|False: ... `send_message` sends data back to a specific connected client. @@ -335,17 +347,22 @@ It verifies the server is running and the socket is valid, then: - accepts both ``str`` and ``bytes`` message payloads - trims string payloads and appends a newline if missing -- when the socket is in encrypted mode (the public-key exchange has - completed), the plaintext is signed with the ``_VALID`` marker, - RSA-OAEP encrypted with the peer's public key, base64 encoded and - newline-terminated instead of being sent in cleartext -- encodes string payloads as UTF-8 +- when the channel is encrypted, writes the payload as frames through the + secure channel (see the framing description in the ``connect_tcp`` module + docstring) instead of as plaintext +- with ``is_send_raw=True`` writes a plaintext line even on an encrypted + channel; this is reserved for the crypto handshake and transfer + coordination lines - sends the message with ``client_socket.sendall(data)`` when the server serves clients with one thread each, or through a non-blocking ``send()`` loop when ``is_asynic_clients_io`` is ``True`` - returns ``True`` on success; it returns ``False`` for an unsupported payload type, and an invalid/closed connection raises ``RuntimeError`` (other socket failures are logged and re-raised) +- after a successful write, reports the payload to the send listeners; a + write to a control socket is also recorded in the sent stores, while a + transfer socket is only reported. ``is_send_raw`` lines are neither + reported nor recorded, and ``bytes`` payloads are reported but not recorded And there are also some other functions which are used to send message in bulk to the clients, such @@ -456,14 +473,14 @@ Encrypted transfer connections File and folder transfers, forwarded items and the forward relay run on temporary TCP connections. The listening side stamps each leg with its mode -and a fresh nonce in the port advertisement (``/server_file_transfer_port``, -``/forward_upload``); with ``is_enable_encrypto`` enabled both sides then -frame that connection exactly like the control channel, with the same -session key material, so file contents, names and sizes no longer cross the -wire in the clear. The public-key pushes themselves stay plaintext by -construction: they carry the keys the session is built on. A leg whose -stamp promises encryption is aborted when the session key material is not -ready rather than silently downgraded. +and a fresh nonce in the common port advertisement +(``/server_file_transfer_port ``). With +``is_enable_encrypto`` enabled, both sides then frame that connection exactly +like the control channel, with the same session key material, so file contents, +names and sizes no longer cross the wire in the clear. Public-key pushes stay +plaintext because they carry the keys the session is built on. A leg whose +stamp promises encryption is aborted when key material is not ready rather than +silently downgraded. Decode verification and key rotation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -524,8 +541,8 @@ TCP clients. .. _tcp-server-event-listener-api: -TCP Server inbound event listener API -------------------------------------- +TCP Server event listener API +----------------------------- External code can observe traffic without replacing the protocol loop: @@ -534,27 +551,88 @@ protocol loop: def add_message_listener(self, listener) def remove_message_listener(self, listener) + def add_send_message_listener(self, listener) + def remove_send_message_listener(self, listener) def add_file_listener(self, listener) def remove_file_listener(self, listener) -``add_message_listener`` registers ``listener(client_id, message)`` -for every inbound line, commands included (``/help``, ``/file ...``, -the crypto handshake lines, ...), so a listener observes all inbound -traffic; line dispatch is unaffected, a command still goes to the -command handlers. ``add_file_listener`` -registers ``listener(client_id, full_path, name, size, command)`` and -fires after a file uploaded by a client has been fully written to the -transfer directory (``command`` is the wire command that triggered it, -so protocol pushes such as ``/crypto_pub_key`` can be recognised). -Both listeners run on a protocol thread, so they must not block, and an -exception inside a listener is logged and swallowed. The ``remove_*`` -counterparts unregister a listener and ignore unknown ones. - -In addition, received messages and events are stored per sender socket -in ``messages_dict`` / ``events_dict`` (each entry a ``[content, -timestamp]`` pair). When a store reaches ``max_dict_size`` (64 KiB) it -is flushed to ``messages_log.json`` / ``events_log.json`` under -``network_api/.Flow`` and cleared; ``stop`` flushes both stores. +``add_message_listener`` registers ``listener(client_id, message)`` for +every complete inbound control line. Lines are newline-delimited, stripped, +and empty lines are skipped; commands (``/help``, ``/file ...``) and the +crypto handshake lines are included, so the listener observes all inbound +traffic while line dispatch continues unchanged. Raw chunks read from a +non-control socket (file/folder transfers, forwards, temporary connections) +are reported to the same listeners as ``bytes``: one callback per read or +decrypted chunk, with no line or frame alignment, containing transfer +headers, file data and ``sign`` tokens alike. ``client_id`` is the peer +endpoint of the socket that produced the event; for a side-channel event +this is that temporary TCP connection's ``"ip:port"``, not the control +connection's client id. Control lines are ``str``; side-channel chunks are +``bytes``. + +Threading: on the synchronous server path, control-line callbacks run on +the per-client receive thread; on the asynchronous path +(``is_asynic_clients_io=True``) they run on the worker thread used by +``asyncio.to_thread``. Raw side-channel chunks and file callbacks run on the +thread that reads or transfers that socket, never on the event loop. Send +callbacks run on the thread that calls `send_message`. Listeners can +therefore run concurrently for different clients/sockets, and send callbacks +for concurrent sends on the same socket may also overlap; they must be +thread-safe. Per-socket inbound order is preserved, cross-socket order is +not. + +``add_send_message_listener`` registers ``listener(target_id, message)`` +and calls it synchronously on the sending thread after every payload that +`send_message` writes successfully, for ``str`` and ``bytes`` alike. +``str`` payloads are reported stripped of leading/trailing whitespace; +``bytes`` are reported unchanged. ``target_id`` is the peer endpoint of the +socket that performed the write: the client's ``"ip:port"`` for a control +socket, or the temporary TCP connection's ``"ip:port"`` for a file/forward +side channel. Lines written with ``is_send_raw=True`` and internal protocol +frames such as the ECDH public-key publication are never reported. + +When several threads call `send_message` on the same socket, the socket +writes remain serialized, but the send-listener callbacks are not guaranteed +to run in the same order as the wire writes. This affects observation order +only: it does not change socket write order, receive order, or file data +integrity. Callers that need a strict observation order must serialize their +own `send_message` calls. + +``add_file_listener`` registers +``listener(client_id, full_path, name, size, command)`` and fires once after +a file uploaded by a client (a direct send, or a forwarded item staged on +the server) has been fully written to its destination. ``client_id`` is the +uploader's control-connection ``"ip:port"``; ``command`` is the wire command +that triggered the transfer, so protocol pushes such as +``/crypto_pub_key`` can be recognised. The callback runs on the transfer +thread and fires before the completion sign is sent; it does not wait for +the peer's ACK. Per-chunk progress is reported to the message listeners as +``bytes``, not by this callback. For ``/crypto_pub_key`` the callback runs +before the key is registered; a listener must not move or delete the file. + +Listeners must not block, and an exception inside one is logged and +swallowed. The ``remove_*`` counterparts unregister a listener and ignore +unknown ones; add/remove take effect for subsequent notifications, while a +notification already in progress may still run. ``stop()`` stops new +delivery, clears the listener lists and suppresses the remaining callbacks +of an in-flight dispatch; a callback already past that check on another +thread may still finish. Listener lists are per instance and are not +replayed; after ``stop()`` create a new instance to register listeners +again. + +Control traffic is also kept per peer socket in four stores, each value a +list of ``[content, timestamp]`` pairs: + +- ``messages_dict``: inbound plain-text lines (lines not starting with ``/``) +- ``events_dict``: inbound ``/`` commands +- ``sent_messages_dict`` / ``sent_events_dict``: the same split for payloads + written to a control socket + +Only control sockets are recorded. Transfer chunks, transfer sign tokens and +``bytes`` payloads are never recorded. When a store reaches ``max_dict_size`` +(64 KiB) it is flushed to ``messages_log.json``, ``events_log.json``, +``sent_messages_log.json`` or ``sent_events_log.json`` under +``network_api/.Flow`` and cleared; ``stop`` flushes all four stores. .. _tcp-server-command-api: @@ -588,21 +666,45 @@ Built-in client commands include: down, so a client that stays after sending ``/quit`` is disconnected by the server. - ``/file [destination_file_path]``: starts a file transfer from - the client to the server. The trailing token of the wire line is the - sender's own transfer id, appended by the sending client. + the client to the server. The wire command ends with the sender's transfer ID. - ``/file_folder [destination_file_path]``: starts a folder transfer from the client to the server (same trailing id). - ``/crypto_mode <0|1>``, ``/crypto_pub_key``, ``/crypto_pub_key_request``: internal handshake messages of the encrypted channel. - ``/forward_item``, ``/forward_send_msg``, ``/pause_trans``, ``/start_trans``: internal forwarding/transfer-control messages. ``/pause_trans `` and - ``/start_trans `` hold and release the outbound transfer with that fid - (`pause_trans`/`start_trans` are the same switch as a public API). -- ``/resume_trans `` and ``/forward_resume ``: - internal transfer-reconnect messages: the receiver asks the sender (or, for - a forwarded transfer, the originator) to continue the file from the bytes - already on disk. See `resume_transfers`. -- ``/server_file_transfer_port ``: internal protocol message used to coordinate file transfer ports. + ``/start_trans `` hold and release the outbound transfer to the requesting + peer. Local calls require ``pause_trans(fid, peer)`` / ``start_trans(fid, peer)``; + the receiver scope is required even when arming a future transfer. + Forwarded requests carry a source-allocated ``fid=``. A new + upload omits ``resume=``; replay includes ``resume=``, including zero, + and must match an existing route before the Server reuses its upstream and + child FIDs. The Server rejects a colliding upstream FID without replacing an + existing route, receive record or target-scoped live transfer state. +- ``/resume_trans ``: the only internal + transfer-reconnect command. Addresses are shell-quoted tuple literals; target + must match the actual requesting peer. A server origin selects its scoped + local upload; otherwise the server validates the downstream route and original + sender, translates to the stable upstream fid, and forwards the same command. + Unknown or mismatched identities never fall back to another route + (see :ref:`transfer-pause-and-reconnect`). No fid kind selects this branch. +- Forwarded transfers return the existing final data-ACK marker upstream only + after the current input stream is complete and every accepted target has + acknowledged. The normal send path reframes the marker for the upstream hop; + the route is dropped only after the ACK write succeeds. A failed write keeps + the completed route for recovery. This does not change ordinary sender or + server receive-ACK behavior. +- ``/server_file_transfer_port ``: the four-token listener + reply for ordinary, public-key and relay legs. Replies match the actual control + socket and FID (the source-provided upstream FID for relay upload). Each + negotiation produces one terminal port or error reply; unmatched, duplicate or + late replies are discarded without a cache. Forward failures use + ``/forward_error ``. There is no forwarding-upload FIFO. + If a sent request times out or is cancelled without a terminal result, that + FID is blocked for the session while unrelated FIDs and commands continue. + Natural disconnect clears its waiters, unresolved marks, pending listeners and + data sockets; reconnect can resume the same stable FID. Data or ACK failure + does not disconnect a healthy control connection. In `handle_command`, the server will first check if the command matches any built-in commands. @@ -789,6 +891,8 @@ Supported console commands include: - ``/file_folder <(ip, port)> [destination_file_path]``: sends a folder from the server to a specific client. - ``/multiple_file_multiple_client ... <(ip, port)1> <(ip, port)2> ...``: sends multiple files to multiple clients. Tokens are classified as targets when they are wrapped in parentheses and as files otherwise, so files and targets may be interleaved. - ``/diff_multiple_file_diff_multiple_client ... <(ip, port)1> <(ip, port)2> ...``: sends the files that precede a group of targets to that group. A new group starts whenever a file token follows one or more target tuples; an optional trailing destination path (a token that is not a tuple) is accepted and passed on. +- ``/transfers``: prints the local memory rows and the separate persisted + transfer snapshot as JSON; it does not send a network command or change state. - ``/help``: prints a help summary of console commands. The forward commands (``/forward_send_msg``, ``/forward_file``, @@ -888,17 +992,25 @@ and server-initiated transfers. They handle the coordination of transfer ports, client connections, and the actual file data transfer operations.* +For built-in file transfers, a nonzero port returned by ``palloc`` is +best-effort. If a sender's local ``bind`` or ``connect`` reports a port +conflict, it retries with an OS-assigned source port. If a receive or relay +listener's ``bind`` conflicts, it binds port ``0`` and advertises the actual +port returned by ``getsockname()``. An allocated listener port remains reserved +until cleanup releases the original ``palloc`` value. + Client-to-server transfer flow: -1. The client sends ``/file`` or ``/file_folder`` to - request a transfer. +1. The client registers the sender FID on its actual control socket and sends + ``/file`` or ``/file_folder`` with that trailing FID. Public-key requests use + ``/crypto_pub_key ``; relay targets register independently. 2. ``handle_command`` starts a dedicated file-server thread: ``file_transfer_server_recv_server_start_thread`` for ``/file`` and ``file_folder_transfer_server_recv_server_start_thread`` for ``/file_folder``. 3. The server allocates an ephemeral transfer port with ``palloc`` and sends - ``/server_file_transfer_port `` + ``/server_file_transfer_port `` back to the client. 4. The client connects to that transfer port and sends file metadata, including length-prefixed filename @@ -1030,8 +1142,9 @@ are as follows: 2.1. The inbound event listener APIs (see :ref:`tcp-server-event-listener-api`): - `add_message_listener` / `remove_message_listener` + - `add_send_message_listener` / `remove_send_message_listener` - `add_file_listener` / `remove_file_listener` - - `messages_dict` / `events_dict` stores + - `messages_dict` / `events_dict` / `sent_messages_dict` / `sent_events_dict` stores 3. The server encrypted channel APIs (see :ref:`tcp-server-encrypted-channel-api`): - `_crypto_on_client_hello` @@ -1054,13 +1167,17 @@ are as follows: - `multiple_file_multiple_client_transfer_server_recv_client_start` - `diff_multiple_file_diff_multiple_client_transfer_server_recv_client_start` - `file_transfer_mode` / `file_transfer_mode_recv`: one transfer leg + - `get_transfer_progress` + - `delete_config`: local allowlisted state-file deletion after shutdown; + ``is_list_config=True`` returns names without deleting. See the file + transfer documentation for whole-snapshot discard and lifetime rules. - `forward_item_to` / `forward_message_to`: single forward destinations 5.1. The transfer pause and reconnect APIs (see :ref:`transfer-pause-and-reconnect`): - `pause_trans` / `start_trans` - `pause_trans_remote` / `start_trans_remote` - - `resume_transfers` / `resume_trans` + - `resume_trans` 6. The server console commands: - `/stop` @@ -1071,6 +1188,7 @@ are as follows: - `/file_folder` - `/multiple_file_multiple_client` - `/diff_multiple_file_diff_multiple_client` + - `/transfers` - `/help` See Also diff --git a/docs/source/changes/connect-tcp-send-listeners-and-control-records.md b/docs/source/changes/connect-tcp-send-listeners-and-control-records.md new file mode 100644 index 0000000..ac5d430 --- /dev/null +++ b/docs/source/changes/connect-tcp-send-listeners-and-control-records.md @@ -0,0 +1,67 @@ +# Change note: connect_tcp send listeners and control-only records + +> Per-change note for the `connect_tcp` send-side listeners, the outbound +> record stores, the control-socket split, and the `is_send_raw` merge. No +> PR number is assigned yet; rename the file when the PR is opened. + +## Core logic of this change +- Problem: observers could see inbound traffic only on the control socket. + Outbound traffic had no listener, and transfer-layer bytes (file, folder, + forward) were invisible, which blocks streaming on top of the transport. +- Approach: every write goes through one `send_message` path and every + side-channel read through `receive_message`; the listener and record calls + sit at those two points only. Records are kept for control sockets only, so + large transfer payloads never enter a store. +- `_send_raw` is removed. Plaintext protocol lines are written by + `send_message(..., is_send_raw=True)`. + +## Modules and interactions +- `PyFlow/network_api/connect_tcp.py`: `TCP_Server_Base` and `TCP_Client_Base` + each gain `_control_sockets`, `_send_message_listeners`, four record stores, + `_write_message` / `_after_send` (send side), `_notify_transfer_chunk` + (receive side for non-control sockets), and `_append_record`. +- Call direction: `send_message` -> `_write_message` -> `_after_send` -> + `_record_sent` and `_notify_message_sent`. `receive_message` and + `_async_receive` -> `_notify_transfer_chunk`. Control lines still go + `_process_client_line` / `receive_messages` -> `_notify_message_received` -> + `_record_message` / `_record_event`. +- Tests: `test/integration/network_api/test_send_message_store.py`, + `test_side_channel_records.py`; `test_tcp_transfer_resume.py` waits for + server registration and the transfer port. +- Docs: `Network_APIs/TCP_Server_APIs.rst`, `TCP_Client_APIs.rst`. + +## Implicit assumptions and easy-to-forget details +- A socket is a control socket only while it is in `_control_sockets`. It is + added on accept (server `_register_client`) and after a successful connect + (client `connect`). It is removed in server `_unregister_client`, client + `close`, and client `_release_client_socket`. A new control path that skips + these points will either lose records or leak transfer chunks into a store. +- Transfer sign tokens start with `/` (for example `/FER`). They are not + recorded because their socket is not a control socket. The `/` rule alone + would be wrong here. +- A write with `is_send_raw=True` is neither reported nor recorded. Any new + caller that sends a protocol line through it must accept that it is hidden + from listeners. +- Listeners run on the thread that does the I/O. On the asynchronous server + path, raw chunk listeners run on the event loop thread. +- `messages_dict` holds plain text: any inbound control line not starting with + `/`. The console `/send_msg` strips its own command name and sends only the + payload, so a chat line arrives here as plain text. A raw `/send_msg` line + sent through the public `send_message` API would be an event instead. +- Flushing: the four stores share `_flush_dict_locked`. The sender-key cleanup + keeps a socket key while any of the four stores buffers that socket. + +## Changing this in six months +- Duplication: the record, listener and notify code is written twice (server + and client). This was a deliberate deferral (decision a). Extracting a + shared mixin is the next refactor and should be its own change. +- Encrypted path (P7) is not covered by a runtime test for control-only + records or chunk notification. Add one before relying on it. +- Log files now include `sent_messages_log.json` and `sent_events_log.json` + under `network_api/.Flow`. Anything reading the logs must accept four files. +- Translations: the `.po` catalogues under `docs/source/locale/` still describe + the previous `send_message` signature and store list. They are not updated + here. +- Constraint: `send_message(..., is_send_raw=True)` must remain the only way to + send plaintext on an encrypted channel. Bypassing it breaks the handshake's + framing order. diff --git a/docs/source/changes/transfer-fid-routes.md b/docs/source/changes/transfer-fid-routes.md new file mode 100644 index 0000000..779dd77 --- /dev/null +++ b/docs/source/changes/transfer-fid-routes.md @@ -0,0 +1,320 @@ +# Change note: server FID routes, sender ownership and socket/FID recovery + +> Atomic changes 1–4 and supplemental O1–O4 passed human review; their historical +> reports remain unchanged. Sender-ownership revision R1 and socket/FID revision +> R2 are implemented and verified, awaiting human review. The final affected suite +> passed 434 tests; independent plaintext/encrypted runs and the documentation +> build passed. Known lint/doc warnings and earlier failed runs are disclosed in +> `docs/plans/transfer-sender-fid-r2-report.md`. No commit, merge or push. + +## Revision R1: sender-owned relay upstream FID + +- A relay source allocates and binds a fresh upstream FID before `/forward_item`. + It sends the required `fid=` field; the Server preserves that ID + in the route and listener response. The Server continues to allocate a distinct + child FID for each target using the existing allocator and receiver scope. +- A new upload omits `resume=`. Every replay includes `resume=`, including + `resume=0`; the Server accepts it only when the existing route belongs to the + actual originator and contains every requested unresolved target. Upstream and + child FIDs, progress rows, paths and final data ACK behavior remain stable. +- The Server rejects an upstream FID already used by a route, receive row, or a + target-scoped active upload, socket, route index, relay or pause state. The + FID-scoped `/forward_error` reports rejection; the existing owner's route, + indexes and progress remain unchanged. Distinct receiver scopes remain distinct. +- Ordinary file, folder and public-key senders retain their allocation. R2 removes + the historical UUID handshake without changing progress JSON or adding a route table. + +R1 behavior evidence and remaining review items are in +`docs/plans/transfer-sender-fid-r1-report.md`. + +## Revision R2: actual socket and sender FID correlation + +- `/file` and `/file_folder` retain their fields and trailing sender FID; + `/crypto_pub_key ` and `/forward_item fid= [resume=] ` + contain no request marker. The terminal reply is exactly + `/server_file_transfer_port ` or `/forward_error `. +- Waiters match actual control socket plus FID. A sent request without a terminal + result on timeout or cancellation leaves its FID unresolved for that session; + late replies cannot unblock it, but unrelated FIDs and commands continue. + Natural disconnect cleans that session's waiters, unresolved marks, listeners + and data legs; reconnect can resume with the same FID. Data or ACK failure + does not disconnect healthy control. Each negotiation has one terminal reply. + +## Historical atomic change 5: superseded listener-correlation protocol + +- Ordinary file/folder pushes, public-key legs, relay target listeners and relay + source uploads use a one-shot `request=` marker. The marker belongs to + one control request only: it is not a FID, is not persisted, and is not reused + for a retry. `/file`, `/file_folder` and `/crypto_pub_key` put the marker + before the trailing FID; `/forward_item` puts it immediately after its kind. + The common reply is `/server_file_transfer_port `. +- The requester registers before sending and dispatches only when the actual + control socket identity, request ID and expected FID match. Relay upload + accepts its server-assigned upstream FID without changing allocation authority. + Relay targets register independently and may reply in any order. Success + consumes registration; failure, timeout, cancellation and teardown retire it + and wake its waiter. Late, duplicate or mismatched replies are discarded, + never cached, with no fallback or alias. Forward setup errors carry their + request ID and cannot wake another upload. +- `/forward_upload`, its FIFO queue and the shared `_forward_error` slot are + removed. Stable upstream/child FIDs, progress rows, route state, transfer + payload, ACK behavior and crypto framing are unchanged. Public-key transfer + bytes remain plaintext; the temporary server-assigned port is associated by + request ID without changing the key-exchange order. +- Keep the real-socket concurrent-forward regression that deliberately blocks + the first port response while the second request completes. It catches + request-agnostic FIFO consumption; also retain timeout, cancellation, wrong + peer/FID, duplicate response and public-key/async regressions. +- Locale cleanup removes only 54 obsolete extracted gettext blocks from 19 + catalogs: the File_Transfer and TCP_Client_APIs/TCP_Server_APIs catalogs for + `ja`, `ko`, `ru`, `zh_CN` and `zh_TW`, plus the connect_tcp API catalogs for + `ko`, `ru`, `zh_CN` and `zh_TW`. Removed blocks teach the legacy port cache, + dedicated forwarding-upload command or outdated port response. No global + regeneration or replacement translations are supplied; new source text uses + English fallback pending translation. + +## Atomic change 4: one recovery command and one source of file metadata + +- The sole resume wire is `/resume_trans `, with + shell-quoted tuple endpoints. Server checks target against the actual requester, + then either selects its own scoped upload or validates the downstream route + and origin before translating to the upstream fid. Client accepts only its + own current connection identity as origin. No old command alias remains. +- Client recovery reads only existing five-item upload/download rows. Forwarded + folder items replay as files into the target's retained full path; no copied + targets/path/destination/folder metadata table remains. Client `relays` is empty; + nonempty legacy client relay snapshots are rejected and preserved on writes + and shutdown. Server routes/completed sets retain their existing authority. +- Transient per-upload locks and target sets serialize distinct targets and + coalesce duplicates for both ordinary and forwarded resumes; they are released + after use and never persisted. Failed sends retain source progress. Server + offline requests coalesce to the latest offset per upstream fid/target; failed + delivery keeps them queued, while completed/missing routes cannot be replayed. +- Fids carry no plain/relay kind; allocation calls have no kind argument, the + reserved low bit is zero, and diagnostic formatting is `e.`. + Allocation authority, `tfid` naming and `relay_fid=` remain unchanged until + separately approved step 6. Atomic 5 removes the old upload handshake. + Final data-ACK aggregation and stable upstream/downstream IDs are unchanged. + +## Atomic change 3: completion across stable-ID replay + +- Forwarding completion uses the original final data ACK only. The server + preserves the full accepted `targets` mapping and tracks `completed`; only an + actual target final ACK adds that receiver to the set and releases its reverse + index. Setup failure, interrupted input, stream exhaustion and error signs + cannot count as target completion. +- After the current input stream is read in full and every accepted target has + acknowledged, the server forwards the existing final ACK marker upstream + through the normal per-hop send path. It drops the route only after the ACK + write succeeds; a failed write leaves the completed route available for + recovery. This is not byte-for-byte packet forwarding and adds no message. +- The source keeps its forwarding record and upload row unless its original + final-ACK wait observes that marker. That wait remains 30 seconds plus 10 + seconds per 100 MiB. A long pause or slow downstream delivery can outlast it; + timeout or a lost ACK conservatively retains source state even if the receiver + completed. There is no receipt or automatic retry protocol. +- Replay adds optional `relay_fid=` alongside `resume=` in `/forward_item`. The + server validates the original sender and unresolved target, then reuses the + upstream and child IDs without replacing the accepted target mapping. Source + retries for one upstream ID share a send lock; concurrent duplicates for the + same target coalesce. Completed targets are not sent again, and queued replay + cannot start after its record has been removed. +- A receiver that cannot write its final ACK retains the receive row even at + EOF. A file or folder-item command matches fid, original source and basename; + the incoming size must match before reopening the original full path. +- A receiver takes its offset from disk through the unified command; stable-ID + replay uses the existing upload row. No new-ID or same-path bridge is used. +- ``relays`` JSON adds ``completed: [receiver, ...]``. Reload reconstructs only + unresolved indexes; canonical snapshots lacking the optional field mean no + completed targets. Invalid completed sets and unsupported older schemas are + preserved, not overwritten. The current Client snapshot is empty and has no + replay metadata; runtime locks and target sets are not persisted. Snapshot + creation is serialized by the progress lock before route state is read. +- The pump keys queues by ``(receiver, child-fid)``, accounts for failed-writer + buffers, wakes backpressure waiters on reader failure, and bounds the final + target-sign read. The sender does not treat an error sign as success. Existing + direct sends and public-key pushes retain their data-ACK success contract; + TOFU still precedes the key ACK. This change does not claim to fix the + Server's ordinary-sender or receive-ACK gap. No CI, dependencies, coverage + policy, file bytes, allocation authority, or ``/forward_upload`` cutover is + included. + +Keep regressions for zero ACK, partial ACK, EOF confirmation, file/folder replay, +duplicate requests, lost final ACK, completed-state reload and malformed-state +file preservation. Runtime evidence and explicit exclusions are recorded in +``docs/plans/transfer-fid-and-resume-unify-atomic-3-report.md``. + +## Atomic change 2 history + +### Core logic of this change + +- Problem: the server indexed the same forwarding record by an upstream fid and + every target fid. Its snapshot duplicated that record; loading reconstructed + independent copies. Lookup also lacked the receiver scope of a target fid. +- `_fid_routes[upstream_fid]` is the only authoritative route. It stores the + original sender and a complete `{receiver: downstream_fid}` mapping. +- `_fid_route_index[(receiver, downstream_fid)]` points to the upstream fid, + not a copied route. Scoped lookup and local upstream lookup return the same + record. The route contains no file path, name, directory, size or byte-progress + copies; those belong to the existing endpoint progress records. +- `_forward_relays` remains transient buffer/condition/pause state only. Client + forwarding recovery now uses its ordinary upload row; no client relay record + exists. + +### Modules and interactions + +- `PyFlow/network_api/connect_tcp.py` adds server registration, lookup and + identity-safe removal of route/index owners under `_forward_relays_lock`. + Registration is all-or-nothing when a primary or downstream key is occupied. +- Ordinary progress restoration skips only route-owned identities in the + appropriate local or downstream namespace. The allocator consults both + route tables alongside existing progress stores. +- Unified `/resume_trans` handling validates the requesting peer, origin and + target, then routes either to the local scoped upload or the authoritative + upstream route. It no longer emits a second resume command. +- The `relays` JSON region now contains one row per upstream fid, with + `originator` and `targets: [[receiver, fid], ...]`. Load rebuilds the derived + index, counts authoritative routes, and does not overwrite live owners. +- Unsupported old server relay schemas are logged and preserved on load, + persistence and shutdown. Client relay snapshots are empty; nonempty legacy + client metadata is rejected and preserved. + +### Implicit assumptions and easy-to-forget details + +- Targets passed to the relay after reachability filtering form the accepted + set. Duplicate addresses denote one target. Port, connection, crypto attach + or start-sign failure must not shrink that set in the recovery route. +- Whole-route deletion compares acknowledged deliveries with all accepted + targets, not merely the subset of target connections that opened. +- The route does not yet track individual completion transitions. If a subset + succeeds, the entire accepted mapping remains. Per-target completion and + final-source notification belong to atomic change 3. +- In particular, the source's existing data ACK can still report upload success + despite an incomplete target set. Retaining the server route does not fix + source-record lifetime; do not claim final end-to-end recovery is complete. +- A route must be removed only by its owning object. An old leg finishing after + another registration cannot remove the new primary record or its indexes. +- Snapshot loading acquires the progress lock and then route lock; allocation + must remain outside these locks. Removal releases the route lock before + persisting, avoiding a reverse lock nesting. + +### Changing this in six months + +- Keep the real-socket regression where one target advertises a bound but + non-listening port and another receives the complete file. It catches the + misleading `delivered == len(target_conns)` cleanup condition. +- Preserve the full accepted mapping across restart and ensure all reverse + lookups reach one authoritative object after loading. +- Do not reintroduce raw target-fid aliases: downstream identity always includes + its receiver. `tfid` remains until step 6; `/forward_upload`, kind bits and + `/forward_resume` are not current contracts. + + +## Follow-up O1: direct Server error signs never complete an upload + +- Human approval covers O1 only, before the separately gated O2–O4 follow-ups. +- `TCP_Server_Base.file_transfer_mode` now returns `False` immediately on an + error sign in both the startup wait and the final-data-ACK wait, rather than + falling through after a `break`. Only the existing final ACK permits success. +- Existing owning-entry cleanup persists a resumable failed upload under its + complete `(receiver, fid)` key. Startup failure creates no progress row and + does not discard a row retained from an earlier attempt. No protocol marker, + public signature, JSON shape, retry or timeout is added. +- A real-socket regression covers file/folder items at offsets zero and 4096, + receiver rejection after the body, durable retention and successful original-ID + EOF confirmation. Independent runtime checks also exercise startup rejection, + successful transfers and framed/encrypted failure and recovery. +- Server receive-ACK retention (O2), progress inspection (O3) and `delete_config` + (O4) are not implemented here. Evidence and limitations are recorded in + `docs/plans/ordinary-transfer-progress-inspection-o1-report.md`. + +## Follow-up O2: failed Server receive ACKs retain recovery state + +- O1 passed human review; O2 was separately approved. Server receive completion + no longer sets `finished_ok` immediately after writing the file. It records + `bool(send_message(...))` for the existing final ACK, matching the Client + receive path. A false result or exception keeps and persists the original + tracked receive row with a peer, even when all bytes are already on disk. +- Notifications, event updates and public-key TOFU registration stay before + the ACK. Key pushes remain untracked. No ACK receipt, retry protocol, timeout, + new field, public signature or persistence format is introduced. +- The regression uses real file/folder uploads, both supported ACK failure + forms, another failure at EOF and a successful EOF confirmation. It checks + the original fid/path/content and durable removal only after success. +- Independent runtime checks cover encrypted and resumed uploads, fresh-process + reload of failure snapshots, disabled progress tracking and deterministic + key-ACK failures before handshake completion. Evidence is recorded in + `docs/plans/ordinary-transfer-progress-inspection-o2-report.md`. +- Progress inspection (O3), `delete_config` (O4) and the original FID plan's + later cutovers remain outside this change; O2 awaits human review. + +## Follow-up O3: local transfer inspection + +- O1 and O2 were separately reviewed before O3. Both TCP classes now expose + `get_transfer_progress(*, include_persisted=True)`. It returns detached, + JSON-ready memory rows and an optional separate read of the existing progress + file; it never loads, persists, resumes or writes transfer state. +- Upload identity keeps the complete `(receiver, fid)` scope. Server and client + relay rows retain their existing authority and shape. Runtime flags are + observed only from local sets; disk rows use `null` for active/socket/pause + state. Missing, empty, malformed and unreadable files are distinct outcomes. +- `/transfers` is intercepted by both local consoles, prints JSON even when + normal logging is disabled, and rejects arguments without sending them over + the network. Existing Web `/api/transfers` is unchanged. +- Real API/console evidence, mutation checks and limitations are recorded in + `docs/plans/ordinary-transfer-progress-inspection-o3-report.md`. O4's + `delete_config` remains unimplemented and O3 awaits human review. + + +## Follow-up O4: explicit named local state deletion + +- O1–O3 passed human review before O4 was approved. Both classes now expose + `delete_config(config_name=None, *, is_list_config=False)`. List mode returns + sorted existing allowlisted regular-file names without side effects; deletion + accepts only one current basename of progress or the four message/event logs. +- Running, not-explicitly-closed, starting/stopping and active-transfer/relay + instances refuse deletion. Progress deletion shares the persist/load lock. + A successful or already-absent progress deletion permanently disables old-object + reload/publication/restart, preventing repeated shutdown or late finalization + from resurrecting the abandoned snapshot. New instances behave normally. +- Log deletion checks its buffer under the existing record lock and does not + permanently disable logging. Other files, keys, locks, transfer contents, + remote protocols, JSON formats, CI and dependencies remain unchanged. +- API-boundary and write/delete race regressions live in `test_delete_config.py`; + existing real transfer/ACK recovery regressions are reused unchanged. Runtime + evidence and limitations are in the O4 report; the supplemental O1–O4 overview + is `docs/plans/ordinary-transfer-progress-inspection-report.md`. + + +## Follow-up: resume identity uses the connected endpoint + +- `/resume_trans` keeps its five tokens and both fields remain `(ip, port)` + tuples. What changed is the value source: the server writes the concrete + local endpoint of the accepted control connection (`getsockname()`) as + `target`, and a client records the connected server endpoint + (`getpeername()`) as the peer it later sends as `origin`. +- Both sides accept either the concrete endpoint or the configured + `(host, port)` as "this peer", so a wildcard bind (`0.0.0.0`) or a host alias + (`localhost` vs `127.0.0.1`) no longer breaks a reconnect. A legacy direct + row that recorded the configured server address is migrated in memory on + read; the progress JSON shape is unchanged. +- Forwarded resume keeps its existing actual-address route scoping; no wire + command, FID allocation, relay record or public signature changes. +- NAT endpoints and a changed `client_port` remain unsupported identities. + +## Follow-up: reconnect sweep releases paused legs + +- `_drop_stale_transfers` now clears the pause switch of every upload leg it + sweeps (after closing the socket, outside the progress lock) and wakes the + pause condition. `_wait_transfer_paused` also takes the leg's own transfer + socket and exits once that socket is closed, because the sweep removes the + socket from `_active_transfer_sockets` before closing it. +- A leg paused for relay backpressure or by `pause_trans` therefore ends when + its session is swept instead of waiting forever while holding the per-fid + send lock. A resume request already queued for that fid runs after the lock + is released; no manual `start_trans` is needed. +- The send loop stops on the first failed write, so it does not keep reading + the rest of the file after the peer is gone. A final ACK already in flight is + still consumed by the existing ACK wait. Wire commands, progress rows, + public signatures and the pause/start semantics of a live session are + unchanged. diff --git a/docs/source/changes/transfer-receiver-scope.md b/docs/source/changes/transfer-receiver-scope.md new file mode 100644 index 0000000..23b2fc1 --- /dev/null +++ b/docs/source/changes/transfer-receiver-scope.md @@ -0,0 +1,37 @@ +# Change note: receiver-scoped transfer state (atomic change 1) + +> Historical scope of atomic change 1 in `docs/plans/transfer-fid-and-resume-unify-plan.md`. +> Later atomic changes supersede its old resume, kind and port-cache descriptions. +> The sender-FID revision supersedes receiver-owned allocation; R2 in progress +> replaces atomic 5's request-token correlation with actual socket and FID. + +## Core logic of this change + +- Problem: bare fid keys allowed two receivers' uploads to overwrite progress, a receive cleanup to close an equal-id send socket, and port waits to consume another receiver's announcement. +- Upload progress now uses `(receiver, fid)` keys. Download progress retains local integer fid keys. Values remain `[bytes, total, speed, local_path, logical_peer]`. +- Outbound activity, sockets and pause switches use the upload identity; inbound activity and sockets use the local integer key. Cleanup and ownership checks never cross those namespaces. +- The receiver is the actual next-hop control endpoint. Client uploads, including forwarded uploads, use the configured server. A forwarded upload's fifth progress item still names its original logical target. +- Atomic 1 originally scoped cached port announcements by peer and fid. Atomic 5 replaced the cache with request-token registration; R2 supersedes that historical contract with actual-control-socket/FID matching and no cache or fallback. + +## Modules and interactions + +- `PyFlow/network_api/connect_tcp.py`: scoped sender keys, receiver-specific pause/start, scoped port consumption, scoped upload snapshot keys and loading, allocator occupancy checks against scoped uploads. +- Server local calls now require `pause_trans(fid, peer)` / `start_trans(fid, peer)`. Client calls default to their configured server and reject another explicit receiver. Remote pause/start wire lines do not change. +- Both Web backends explicitly extract numeric fid from upload keys. `/api/transfers` fields and the frontend remain unchanged; equal numeric fids produce separate rows. +- Regression coverage: real equal-id sends to two clients, opposite-direction socket cleanup, scoped snapshot reload without overwriting a live receiver, historical snapshot preservation, Web API collision rows and allocator occupancy. + +## Implicit assumptions and easy-to-forget details + +- Never infer the receiver namespace from a forwarded upload's logical `peer` slot. Use its next-hop server. +- Opening an incoming listener must not retire another outgoing request with the same numeric fid. Request lifecycle belongs to its actual control socket and one-shot token. +- Upload JSON keys are compact JSON strings encoding `[receiver, fid]`, for example `[["127.0.0.1",6000],9]`. Download JSON keys remain decimal strings. +- Old bare upload-id snapshots cannot select a receiver unambiguously under this contract. Loading rejects them; persistence and shutdown preserve the original file rather than silently replacing it. Archive the file before persisting new tasks. No automatic migration or compatibility alias is supplied. +- The existing progress-recording flag still disables resumable progress when False. This does not disable in-flight scoped pauses or port matching. +- At this step ordinary/relay allocation, kind bits, tfid fields and `/forward_resume` remained in service; the atomic-4 cutover supersedes kind and resume-command handling only. + +## Changing this in six months + +- Keep the real-socket same-number regression: random epochs reduce collisions but do not provide identity isolation. +- Any new outbound progress, activity, pause or port lookup must carry its actual receiver namespace; numeric fid alone is not a complete outgoing identity. +- The server/client helper copies remain duplicated. This change intentionally does not introduce a mixin or a replacement record model. +- New protocol stages must migrate all callers without reintroducing a raw-fid fallback. Forward-route redesign and final-target completion semantics remain atomic changes 2 and 3. diff --git a/docs/source/changes/transfer-resume-simplified.md b/docs/source/changes/transfer-resume-simplified.md new file mode 100644 index 0000000..b912d4b --- /dev/null +++ b/docs/source/changes/transfer-resume-simplified.md @@ -0,0 +1,64 @@ +# Change note: transfer resume is driven by the reconnect, not by a public call + +> Historical note; details below record the pre-atomic-4 implementation. The +> atomic-4 section in `transfer-fid-routes.md` supersedes the old wire and +> Client-state descriptions; reconnect remains the driver. R2 in progress +> supersedes atomic 5's request-token correlation with actual socket and FID, +> unresolved same-session isolation and cleanup on natural disconnect. + +## Core logic of this change +- Problem: resuming an interrupted transfer had two drivers (`resume_transfers` + and a `/resume_trans` receive leg), a sender-side `_run_resume_send_leg` + wrapper, and a separate progress-table lookup. A stale port announcement could + also be picked up on resume. +- Approach: on every connect each side calls `_request_resumes`. The receiver + sets the offset from its partial file and sends `/resume_trans `. + The sender answers with `/file ` on the control channel and sends + the remainder through `file_transfer_mode(resume_offset=...)`. The receiver + matches that `/file` line to its progress entry by fid, sender and file name, + and appends only when the size equals the recorded total. +- Old port announcements are dropped (`_forget_transfer_port`) before the resume + `/file` is sent, so the sender waits for the new listener, not a dead one. + +## Modules and interactions +- `PyFlow/network_api/connect_tcp.py`: + - `_resumable_entry`: module-level matcher for a received `/file` line. + - `_request_resumes`: one per side (`TCP_Server_Base`, `TCP_Client_Base`). + The server calls it from `_resume_after_connect`; the client from the + connect thread. + - `resume_trans`: one per side, the only sender-side resume entry. + - `file_transfer_mode_recv`: refuses a continued leg whose size differs from + the recorded total (`resume rejected: ...`); keeps the partial file and + the progress entry. +- Tests: `test/unit/network_api/test_resume_matching.py` (matching rules); + `test/integration/network_api/test_tcp_transfer_resume.py` (upload and + download resume, encrypted resume, total mismatch refused). +- Docs: `File_Transfer/File_Transfer.rst`, `Network_APIs/TCP_Server_APIs.rst`, + `Network_APIs/TCP_Client_APIs.rst`. + +## Implicit assumptions and easy-to-forget details +- A `/file` line is a new transfer unless `_resumable_entry` matches. A new + transfer truncates the target path, so any change to the matcher must keep the + three-way check (fid, sender, basename). +- The progress file stores peers as JSON lists. Matching compares `tuple(...)` + on both sides; a comparison without `tuple()` silently never matches after a + reload. +- A continued leg is refused before `full_path` is assigned, so the error path + never removes the partial file. +- `_forget_transfer_port` must run before the `/file` is sent, not after: the + announcement can arrive before the sender starts waiting. + +## Changing this in six months +- `resume_trans` is kept as the single sender entry and is still callable as a + public API for an on-demand resume. Do not reintroduce a receiver-side leg. +- Gap: the auto-reconnect of a client downloading from the server is covered by + an integration test that calls `_request_resumes` directly, not by a real + disconnect followed by `connect()`. Add an end-to-end disconnect test before + relying on the automatic path. +- Gap: the encrypted path is covered for download resume only. +- At the time this note was written, its extracted `.po` entries still described + `resume_transfers`; the atomic-4 cleanup later removed 80 obsolete entries + from 20 catalogs. Current descriptions were not fully regenerated or translated. +- Duplication: the server and client copies of `file_transfer_mode_recv`, + `resume_trans` and `_request_resumes` are still written twice. Merging them is + a separate refactor. diff --git a/docs/source/locale/ja/LC_MESSAGES/File_Transfer/File_Transfer.po b/docs/source/locale/ja/LC_MESSAGES/File_Transfer/File_Transfer.po index 25fc5cd..43cdec5 100644 --- a/docs/source/locale/ja/LC_MESSAGES/File_Transfer/File_Transfer.po +++ b/docs/source/locale/ja/LC_MESSAGES/File_Transfer/File_Transfer.po @@ -148,21 +148,6 @@ msgid "" "closed." msgstr "``error_sign`` - エラーが発生したときにどちらかの側から送信され、転送ソケットが閉じられます。" -#: ../../source/File_Transfer/File_Transfer.rst:72 -#: 41f6d3d6e0e848bc8f6aecb995223622 -msgid "" -"The remaining keys of the table cover the other three direction/phase " -"combinations. Protocol control lines that are not part of this table " -"(``/server_file_transfer_port``, ``/client_alloc_port_range``, " -"``/pause_trans``, ``/start_trans``, ``/forward_item``, " -"``/forward_upload``, ``/forward_error``, ``/crypto_*``) are hardcoded in " -"``connect_tcp.py``." -msgstr "" -"表の残りのキーは、他の3つの方向/位相の組み合わせをカバーしています。このテーブルの一部ではないプロトコル制御ライン（''/ " -"server_file_transfer_port ''、''/ client_alloc_port_range ''、''/ " -"pause_trans ''、''/ start_trans ''、''/ forward_item ''、''/ forward_upload " -"''、''/ forward_error ''、''/ crypto _*''）は、'' connect_tcp.py " -"''にハードコードされています。" #: ../../source/File_Transfer/File_Transfer.rst:80 #: 45a36aa9239d4caba8752e4dc4fd551b @@ -373,17 +358,6 @@ msgid "" "``palloc()``." msgstr "``palloc()`` を呼び出して、送信ファイル転送ソケットのローカルポートを取得します。" -#: ../../source/File_Transfer/File_Transfer.rst:172 -#: 37226a4f2e1e4a878ba325fca05df9bf -msgid "" -"Wait (up to 20 seconds) for the server to reply with a port number. The " -"server's reply (``/server_file_transfer_port ``) is " -"received by the main connection and stored in ``file_server_port_list``. " -"When a matching entry is found, the port is extracted." -msgstr "" -"サーバーがポート番号を応答するまで (最大 20 秒) 待ちます。サーバーの応答 (``/server_file_transfer_port " -" ``) はメイン接続によって受信され、``file_server_port_list`` " -"に保存されます。一致するエントリが見つかると、ポートが抽出されます。" #: ../../source/File_Transfer/File_Transfer.rst:180 #: e9cd68239fe3498597f07ccfa774ae22 @@ -478,21 +452,6 @@ msgstr "" "「file_transfer_mode_recv(host, port, client_socket, client_id, " "new_save_path, file_name, command)」を呼び出します。" -#: ../../source/File_Transfer/File_Transfer.rst:224 -#: 2f6675b7698c41938542b09cbfb482e8 -msgid "" -"This low-level receive function sets up a listening socket on the " -"allocated port, sends the port number back to the client (as " -"``/server_file_transfer_port ``), accepts the client's " -"connection, and performs the protocol steps described in :ref:`file-" -"transfer-protocol`. The received file is saved under ``received_files/`` " -"(or a subdirectory if a folder transfer was requested)." -msgstr "" -"この低レベルの受信関数は、割り当てられたポート上で待機ソケットを設定し、ポート番号をクライアントに送り返し " -"(``/server_file_transfer_port `` " -"として)、クライアントの接続を受け入れ、:ref:`file-transfer-protocol` " -"で説明されているプロトコル手順を実行します。受信したファイルは ``received_files/'' " -"(フォルダ転送が要求された場合はサブディレクトリ) に保存されます。" #: ../../source/File_Transfer/File_Transfer.rst:234 #: 9fe861572b8746aeaa8266d8d30f963f @@ -582,16 +541,6 @@ msgstr "一意のクライアント ID を割り当て、それを使用して msgid "Allocate a local port with ``palloc()``." msgstr "``palloc()`` でローカルポートを割り当てます。" -#: ../../source/File_Transfer/File_Transfer.rst:279 -#: 723a05b6c3974b65a539af723f1e0872 -msgid "" -"Wait for the client to reply with its own port number (via " -"``/server_file_transfer_port``), which is stored in " -"``file_server_port_list`` by the main connection's message handler." -msgstr "" -"クライアントが独自のポート番号 (`/server_file_transfer_port` 経由) " -"で応答するのを待ちます。このポート番号は、メイン接続のメッセージ ハンドラーによって `file_server_port_list` " -"に保存されます。" #: ../../source/File_Transfer/File_Transfer.rst:284 #: 878572ca3d2948a9acbb6741f00ddd62 @@ -1340,28 +1289,6 @@ msgid "" "continue from;" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:706 -#: c7017b370d4147128e54a94458a7b536 -msgid "" -"after a reconnect the receiver drives the resume (`resume_transfers`): it" -" opens a fresh transfer listener, announces it and sends ``/resume_trans " -" `` with the byte offset its partial file holds. The sender " -"re-establishes the leg, reads its source from that offset and only the " -"remainder of the file travels. A forwarded transfer is asked for through " -"the server, which routes ``/forward_resume`` to the originating client so" -" it re-forwards just the missing part." -msgstr "" - -#: ../../source/File_Transfer/File_Transfer.rst:714 -#: ee858a76e93f44d5934cb2d89f6709a5 -msgid "" -"The server resumes a peer's transfers when it reconnects, and a client " -"with ``is_auto_reconnect=True`` reconnects on its own and resumes every " -"transfer it can reach; `resume_transfers` is public, so an application " -"can also drive the resume itself. Legs the previous session left behind " -"are closed before the resume runs, so two legs never write the same " -"transfer." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:723 #: fe9377db9cf04b9da0dbe4d07c730adf @@ -1590,12 +1517,6 @@ msgstr "" msgid "``pause_trans_remote`` / ``start_trans_remote`` (ask a client to do it)" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:898 -#: ../../source/File_Transfer/File_Transfer.rst:918 -#: 9c62709c2c06456dbfddee1f1ce6871e de496a0be32c490585ae0cbafc931a71 -msgid "``resume_transfers`` / ``resume_trans`` (continue interrupted transfers)" -msgstr "" - #: ../../source/File_Transfer/File_Transfer.rst:900 #: 11476e0eaa3b4a4286612812e4e2c004 msgid "" diff --git a/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po b/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po index 6c2034a..a3c45c2 100644 --- a/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po +++ b/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po @@ -651,14 +651,6 @@ msgid "" "allocation range from the line the server sends it on connect." msgstr "``/client_alloc_port_range``: サーバーのブロードキャストに基づいてクライアントの手動ポート割り当て範囲を設定します。" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:275 -#: f84404f7844d4779af2e6030e795b5d0 -msgid "" -"``/server_file_transfer_port``: receives the file transfer port assigned " -"by the server for an ongoing file operation." -msgstr "" -"``/server_file_transfer_port``: " -"進行中のファイル操作のためにサーバーによって割り当てられたファイル転送ポートを受け取ります。" #: ../../source/Network_APIs/TCP_Client_APIs.rst:277 #: 4eb4fe22d5db4b56b622dfd29459c3b7 @@ -681,14 +673,6 @@ msgstr "" " ''、''/ crypto_reject ''、''/ crypto_pub_key ''、''/ crypto_ready " "''：暗号化されたチャネルハンドシェイクメッセージ。''/ crypto_mode ''ミスマッチは接続を閉じます。" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:283 -#: 31a62335177b49a19f708aee2aa5b0b1 -msgid "" -"``/forward_upload``, ``/pause_trans``, ``/start_trans``, " -"``/forward_error``: forwarding and transfer-control messages." -msgstr "" -"''/ forward_upload ''、''/ pause_trans ''、''/ start_trans ''、''/ " -"forward_error '':転送および転送制御メッセージ。" #: ../../source/Network_APIs/TCP_Client_APIs.rst:286 #: b4c3c6ea61f344b8a3d3baf6b659d9c7 @@ -1028,21 +1012,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:471 -#: a47f9db0c3674a12ab6d3ab5d056a8c4 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:483 #: 6740137a28b84e5dbb3f85e8d58d32b5 @@ -1882,11 +1851,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:872 -#: 0f3b14e3c193426191323ee76959278e -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Client_APIs.rst:874 #: 5bf1072901734d3d8d0b119e5f50627c msgid "The client console commands (interactive mode):" diff --git a/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po b/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po index 689f9e6..e3c8c97 100644 --- a/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po +++ b/docs/source/locale/ja/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po @@ -924,21 +924,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:457 -#: 8ca69d91005b459a8a56f21902eaa690 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Server_APIs.rst:469 #: b32b4fa4788d4f9d88926958c81ad73f @@ -1162,23 +1147,7 @@ msgid "" " switch as a public API)." msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:601 -#: 7f3801d57ac34670bbe16736683db30e -msgid "" -"``/resume_trans `` and ``/forward_resume " -"``: internal transfer-reconnect messages: the receiver asks the " -"sender (or, for a forwarded transfer, the originator) to continue the " -"file from the bytes already on disk. See `resume_transfers`." -msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:605 -#: abd7d3032e15468e92f96a0b5af1f926 -msgid "" -"``/server_file_transfer_port ``: internal protocol " -"message used to coordinate file transfer ports." -msgstr "" -"``/server_file_transfer_port ``: " -"ファイル転送ポートを調整するために使用される内部プロトコル メッセージ。" #: ../../source/Network_APIs/TCP_Server_APIs.rst:607 #: 1e9e32071bb1425b8617a419a7ef87f6 @@ -1639,14 +1608,6 @@ msgstr "" "``handle_command`` は ``file_transfer_server_recv_server_start_thread`` " "を使用して専用のファイルサーバースレッドを開始します。" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:899 -#: e5ef46040255421fa92c6de33d976671 -msgid "" -"The server allocates an ephemeral transfer port with ``palloc`` and sends" -" ``/server_file_transfer_port `` back to the client." -msgstr "" -"サーバーは ``palloc`` で一時転送ポートを割り当て、``/server_file_transfer_port " -"`` をクライアントに送り返します。" #: ../../source/Network_APIs/TCP_Server_APIs.rst:903 #: 6eaa70d4d985489e8f0e683baa6b52a1 @@ -2078,11 +2039,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:1063 -#: 187f0ce4a65e41adb7ee99b021de4d2b -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Server_APIs.rst:1065 #: 944bb0f890764ad7a62584cc73782927 msgid "The server console commands:" diff --git a/docs/source/locale/ja/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po b/docs/source/locale/ja/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po index ef95d60..8df8dc0 100644 --- a/docs/source/locale/ja/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po +++ b/docs/source/locale/ja/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po @@ -64,28 +64,6 @@ msgid "" "on the same socket and share its command namespace;" msgstr "ファイル/フォルダ転送、メッセージ転送、およびポート割り当ては、同じソケット上に重ねられ、そのコマンド名前空間を共有します。" -#: 3488995e2a8349a58393d9f2d1155b99 PyFlow.network_api.connect_tcp:18 of -msgid "" -"an interrupted transfer is resumed after a disconnect: both sides persist" -" their transfer stores under the protocol's ``.Flow`` directory and the " -"receiver asks the sender to continue the file from the bytes already on " -"disk (``resume_transfers``; the wire lines are ``/resume_trans`` and " -"``/forward_resume``)." -msgstr "" - -#: 4cc5743bdfe54148ad38f54f10e688c9 PyFlow.network_api.connect_tcp:24 of -#, fuzzy -msgid "" -"The forwarding extensions use the module-level parsers " -"`parse_forwarded_message`, `parse_forward_items_and_addrs`, " -"`parse_forward_originator` and `forward_skip_message`; the reconnect " -"logic uses `parse_resume_command`, `parse_forward_resume_command` and " -"`_resume_offset_for`." -msgstr "" -"転送拡張機能は、モジュールレベルのパーサー`parse_forwarded_message '、` " -"parse_forward_items_and_addrs '、`parse_forward_originator '、および` " -"forward_skip_message `を使用します。" - #: 49796529cb7f49ee8131b257212b5420 PyFlow.network_api.connect_tcp:30 of msgid "" "Concepts live in ``docs/source/Network_APIs/TCP_Server_APIs.rst`` and " @@ -95,10 +73,6 @@ msgstr "" "コンセプトは'' docs/source/NETWORK_APIs/TCP_Server_APIs.rst ''および'' " "TCP_Client_APIs.rst ''にあります。引数、戻り値、例外の契約は以下のdocstringにあります。" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_resume_command:1 of -msgid "Split a ``/resume_trans `` line." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 14dcda8b059b480982e3fb03399e665b 99035a24f6e94207b84545f8c71a451e @@ -106,36 +80,14 @@ msgstr "" msgid "Returns" msgstr "返品" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_resume_command:3 of -#, fuzzy -msgid "``(fid, offset)``, or None when the line is malformed." -msgstr "送信者の''\" ip: port \"''、または行が整形式のエンベロープでない場合はNone。" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 19eb6fca79324b0791f8ec7488a5246c msgid "Return type" msgstr "戻り値" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:1 of -msgid "Split a ``/forward_resume `` line." -msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:3 of -msgid "``(target, fid, offset)``, or None when the line is malformed." -msgstr "" -#: 04a0882a3a5e4099a114728b4d1c79c0 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:5 of -msgid "``(target, fid, offset)``, or None when the line is" -msgstr "" - -#: 4ead8d8f93644adeb812150e760f7792 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:6 of -msgid "malformed." -msgstr "" #: 39274682c4bd46489ed6fe535b50ede5 #: PyFlow.network_api.connect_tcp.parse_forwarded_message:1 of @@ -443,14 +395,6 @@ msgid "" "never the relay." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Server_Base:96 -#: f9528e6c490d4807b949f6730c6ece8e of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when a connection drops and read back on the next reconnect " -"(``resume_transfers``)." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base:110 #: PyFlow.network_api.connect_tcp.TCP_Server_Base:104 #: ed04a3bf285b4f57bb6a5de0c4f7284f of @@ -1669,51 +1613,12 @@ msgid "" "byte." msgstr "" -#: 1634dbe84a95421a92a660f7c6fdd10e -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:1 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:1 of -msgid "Continue the interrupted transfers of a peer that is back." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:3 -#: eb46142fa4e44147ad3ad8c680d5bee0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and, for " -"every download this instance was receiving from ``peer``, opens a fresh " -"transfer listener and asks the sender to continue the file from the bytes" -" already on disk. Restored uploads need no action here: the peer drives " -"their resume from its own side." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:9 -#: b7d54977ca494a8b9968365dffd9bd3d of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer that is " -"connected right now." -msgstr "" - -#: 15c0118df8da4454a7fea8d529989237 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:13 of -msgid "Number of resumed receive legs started." -msgstr "" - #: 52cb873f015542b6a94506cebfe735a1 737ed182911443bc8cfff3c4d094b9a9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:1 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:1 of msgid "Continue one interrupted transfer this instance was sending." msgstr "" -#: 50f292ac853841c9b082eb3c3ff4d6f4 6c968a5b77b342ed891883336cfcc255 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:3 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:3 of -msgid "" -"Runs when a receiver asks to continue a transfer (``/resume_trans``, see " -"`resume_transfers` on the receiving side): the leg is re-established " -"against the receiver's fresh listener and the source file is read from " -"the byte offset the receiver already holds. Also the public API for " -"resuming one transfer on demand." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:9 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:9 #: b541e8446e374bc8b38cd7912a1fa33c of @@ -1860,14 +1765,6 @@ msgid "" "resumes the interrupted transfers)." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base:102 -#: c194bbee7cf14d268c330e06f94ae973 of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when the connection drops and read back on the next reconnect " -"(`resume_transfers`)." -msgstr "" - #: 40f2a9e1b730422db3dd58bc2a6046e3 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:1 of msgid "Create the client and, unless extended, connect and start reading." @@ -1934,16 +1831,6 @@ msgid "" "client's forward path does not read it today. Defaults to 2048." msgstr "サーバークラスとのパリティのために保持されるMiB単位のバッファ上限。クライアントのフォワードパスは今日それを読み取りません。デフォルトは2048です。" -#: 14c4889553fb42f7b0393a4a7ab46b4a -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:72 of -msgid "" -"Reconnect (and resume the interrupted transfers, see `resume_transfers`) " -"when the connection drops while `start_TCP_client` runs. The transfers " -"that died with the connection are persisted to the protocol's ``.Flow`` " -"directory and read back on the next connect. Defaults to False: a dropped" -" connection ends the client." -msgstr "" - #: 0f06ff3b6db940da96ec09bee3dba526 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:80 of msgid "If ``is_wait_server`` is True and ``timeout`` is not None." @@ -2337,24 +2224,6 @@ msgstr "ソケットから最大'' msg_length ''バイトを読み取ります msgid "Dispatch one command line pushed by the server." msgstr "サーバーによってプッシュされた1つのコマンドラインをディスパッチします。" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.handle_server_command:3 -#: a7e781fde939475cb081204adac16ded of -#, fuzzy -msgid "" -"Handles the protocol's own lines: ``/crypto_mode`` (a mismatch closes the" -" connection), ``/client_alloc_port_range``, the ``/crypto_*`` exchange " -"lines, and the transfer lines ``/file``, ``/file_folder``, " -"``/server_file_transfer_port``, ``/forward_upload``, ``/pause_trans``, " -"``/start_trans``, ``/resume_trans``, ``/forward_resume`` and " -"``/forward_error``. Any other name goes to the handlers registered for " -"the \"server\" side via `register_command`; an unknown command is only " -"reported on the console." -msgstr "" -"プロトコル自身の行を処理します： ''/ crypto_mode ''（ミスマッチは接続を閉じます）、''/ " -"client_alloc_port_range ''、''/ crypto _*''交換行、および転送行''/ file ''、''/ " -"file_folder ''、''/ forward_upload ''、''/ pause_trans ''、''/ start_trans " -"''、''/ forward_error " -"''。他の名前は、`register_command`を介して\"server\"側に登録されたハンドラに送られます。不明なコマンドはコンソールでのみ報告されます。" #: 91942d88960c4793b5eab8c3654c1400 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_messages:1 of @@ -2430,25 +2299,6 @@ msgid "" "there instead of its first byte." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:3 -#: a3c59554b8de41b280df7536b307bed0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and drives " -"every download that can be continued: a file the server was sending is" -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:6 -#: fae378e888be4821a14ed2483cfde1e0 of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer the " -"client can reach." -msgstr "" - -#: 48061103ed9d4bcaae8a8a837b363557 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:10 of -msgid "Number of resumed transfers started or requested." -msgstr "" - #: 6dbae803a56c442588b28af664b7a0e9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_file_console:1 of msgid "" @@ -2505,15 +2355,6 @@ msgstr "" "''がTrueの場合は'interactive_mode`を入力します。そうでない場合は、接続がアップしている間はプロセスをアクティブにします。接続を確立できない場合、ステータス1でプロセスを終了します。Ctrl" " - Cと接続の終了は両方とも「閉じる」を実行します。" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.start_TCP_client:9 -#: eec663f58484446ebce460e6402d7b5f of -msgid "" -"With ``is_auto_reconnect`` True a dropped connection does not end the " -"client: it reconnects (waiting for the server when ``is_wait_server`` is " -"set) and every session carries the interrupted transfers of the previous " -"one (`resume_transfers`). A close or ``/quit`` still stops the client." -msgstr "" - #~ msgid "" #~ "Concepts live in " #~ "``docs/Network_APIs/TCP_Server_APIs.rst`` and " diff --git a/docs/source/locale/ko/LC_MESSAGES/File_Transfer/File_Transfer.po b/docs/source/locale/ko/LC_MESSAGES/File_Transfer/File_Transfer.po index 0d09bfe..be01847 100644 --- a/docs/source/locale/ko/LC_MESSAGES/File_Transfer/File_Transfer.po +++ b/docs/source/locale/ko/LC_MESSAGES/File_Transfer/File_Transfer.po @@ -140,16 +140,6 @@ msgid "" "closed." msgstr "``error_sign`` - 오류가 발생하면 양쪽에서 전송되어 전송 소켓이 닫힙니다." -#: ../../source/File_Transfer/File_Transfer.rst:72 -#: 41f6d3d6e0e848bc8f6aecb995223622 -msgid "" -"The remaining keys of the table cover the other three direction/phase " -"combinations. Protocol control lines that are not part of this table " -"(``/server_file_transfer_port``, ``/client_alloc_port_range``, " -"``/pause_trans``, ``/start_trans``, ``/forward_item``, " -"``/forward_upload``, ``/forward_error``, ``/crypto_*``) are hardcoded in " -"``connect_tcp.py``." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:80 #: 45a36aa9239d4caba8752e4dc4fd551b @@ -359,17 +349,6 @@ msgid "" "``palloc()``." msgstr "``palloc()``을 호출하여 나가는 파일 전송 소켓에 대한 로컬 포트를 얻습니다." -#: ../../source/File_Transfer/File_Transfer.rst:172 -#: 37226a4f2e1e4a878ba325fca05df9bf -msgid "" -"Wait (up to 20 seconds) for the server to reply with a port number. The " -"server's reply (``/server_file_transfer_port ``) is " -"received by the main connection and stored in ``file_server_port_list``. " -"When a matching entry is found, the port is extracted." -msgstr "" -"서버가 포트 번호로 응답할 때까지 기다립니다(최대 20초). 서버의 응답(``/server_file_transfer_port " -" ``)은 기본 연결에서 수신되어 ``file_server_port_list``에 저장됩니다. " -"일치하는 항목이 발견되면 포트가 추출됩니다." #: ../../source/File_Transfer/File_Transfer.rst:180 #: e9cd68239fe3498597f07ccfa774ae22 @@ -463,20 +442,6 @@ msgstr "" "``file_transfer_mode_recv(host, port, client_socket, client_id, " "new_save_path, file_name, command)``를 호출합니다." -#: ../../source/File_Transfer/File_Transfer.rst:224 -#: 2f6675b7698c41938542b09cbfb482e8 -msgid "" -"This low-level receive function sets up a listening socket on the " -"allocated port, sends the port number back to the client (as " -"``/server_file_transfer_port ``), accepts the client's " -"connection, and performs the protocol steps described in :ref:`file-" -"transfer-protocol`. The received file is saved under ``received_files/`` " -"(or a subdirectory if a folder transfer was requested)." -msgstr "" -"이 저수준 수신 함수는 할당된 포트에 청취 소켓을 설정하고, 포트 번호를 클라이언트에 다시 " -"보내고(``/server_file_transfer_port ``로) 클라이언트의 연결을 수락하고 " -":ref:`file-transfer-protocol`에 설명된 프로토콜 단계를 수행합니다. 수신된 파일은 " -"``received_files/``(또는 폴더 전송이 요청된 경우 하위 디렉터리)에 저장됩니다." #: ../../source/File_Transfer/File_Transfer.rst:234 #: 9fe861572b8746aeaa8266d8d30f963f @@ -569,15 +534,6 @@ msgstr "고유한 클라이언트 ID를 할당하고 이를 사용하여 명령 msgid "Allocate a local port with ``palloc()``." msgstr "``palloc()``을 사용하여 로컬 포트를 할당합니다." -#: ../../source/File_Transfer/File_Transfer.rst:279 -#: 723a05b6c3974b65a539af723f1e0872 -msgid "" -"Wait for the client to reply with its own port number (via " -"``/server_file_transfer_port``), which is stored in " -"``file_server_port_list`` by the main connection's message handler." -msgstr "" -"클라이언트가 기본 연결의 메시지 핸들러에 의해 ``file_server_port_list``에 저장된 자체 포트 " -"번호(``/server_file_transfer_port``를 통해)로 응답할 때까지 기다립니다." #: ../../source/File_Transfer/File_Transfer.rst:284 #: 878572ca3d2948a9acbb6741f00ddd62 @@ -1326,28 +1282,6 @@ msgid "" "continue from;" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:706 -#: c7017b370d4147128e54a94458a7b536 -msgid "" -"after a reconnect the receiver drives the resume (`resume_transfers`): it" -" opens a fresh transfer listener, announces it and sends ``/resume_trans " -" `` with the byte offset its partial file holds. The sender " -"re-establishes the leg, reads its source from that offset and only the " -"remainder of the file travels. A forwarded transfer is asked for through " -"the server, which routes ``/forward_resume`` to the originating client so" -" it re-forwards just the missing part." -msgstr "" - -#: ../../source/File_Transfer/File_Transfer.rst:714 -#: ee858a76e93f44d5934cb2d89f6709a5 -msgid "" -"The server resumes a peer's transfers when it reconnects, and a client " -"with ``is_auto_reconnect=True`` reconnects on its own and resumes every " -"transfer it can reach; `resume_transfers` is public, so an application " -"can also drive the resume itself. Legs the previous session left behind " -"are closed before the resume runs, so two legs never write the same " -"transfer." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:723 #: fe9377db9cf04b9da0dbe4d07c730adf @@ -1566,12 +1500,6 @@ msgstr "" msgid "``pause_trans_remote`` / ``start_trans_remote`` (ask a client to do it)" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:898 -#: ../../source/File_Transfer/File_Transfer.rst:918 -#: 9c62709c2c06456dbfddee1f1ce6871e de496a0be32c490585ae0cbafc931a71 -msgid "``resume_transfers`` / ``resume_trans`` (continue interrupted transfers)" -msgstr "" - #: ../../source/File_Transfer/File_Transfer.rst:900 #: 11476e0eaa3b4a4286612812e4e2c004 msgid "" diff --git a/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po b/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po index c7c7608..aac0b7c 100644 --- a/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po +++ b/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po @@ -632,12 +632,6 @@ msgid "" "allocation range from the line the server sends it on connect." msgstr "``/client_alloc_port_range``: 서버 브로드캐스트를 기반으로 클라이언트 수동 포트 할당 범위를 구성합니다." -#: ../../source/Network_APIs/TCP_Client_APIs.rst:275 -#: f84404f7844d4779af2e6030e795b5d0 -msgid "" -"``/server_file_transfer_port``: receives the file transfer port assigned " -"by the server for an ongoing file operation." -msgstr "``/server_file_transfer_port``: 진행 중인 파일 작업을 위해 서버에서 할당한 파일 전송 포트를 받습니다." #: ../../source/Network_APIs/TCP_Client_APIs.rst:277 #: 4eb4fe22d5db4b56b622dfd29459c3b7 @@ -655,12 +649,6 @@ msgid "" "the connection." msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:283 -#: 31a62335177b49a19f708aee2aa5b0b1 -msgid "" -"``/forward_upload``, ``/pause_trans``, ``/start_trans``, " -"``/forward_error``: forwarding and transfer-control messages." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:286 #: b4c3c6ea61f344b8a3d3baf6b659d9c7 @@ -982,21 +970,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:471 -#: a47f9db0c3674a12ab6d3ab5d056a8c4 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:483 #: 6740137a28b84e5dbb3f85e8d58d32b5 @@ -1817,11 +1790,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:872 -#: 0f3b14e3c193426191323ee76959278e -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Client_APIs.rst:874 #: 5bf1072901734d3d8d0b119e5f50627c msgid "The client console commands (interactive mode):" diff --git a/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po b/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po index 9c9323b..8d92638 100644 --- a/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po +++ b/docs/source/locale/ko/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po @@ -912,21 +912,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:457 -#: 8ca69d91005b459a8a56f21902eaa690 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Server_APIs.rst:469 #: b32b4fa4788d4f9d88926958c81ad73f @@ -1146,23 +1131,7 @@ msgid "" " switch as a public API)." msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:601 -#: 7f3801d57ac34670bbe16736683db30e -msgid "" -"``/resume_trans `` and ``/forward_resume " -"``: internal transfer-reconnect messages: the receiver asks the " -"sender (or, for a forwarded transfer, the originator) to continue the " -"file from the bytes already on disk. See `resume_transfers`." -msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:605 -#: abd7d3032e15468e92f96a0b5af1f926 -msgid "" -"``/server_file_transfer_port ``: internal protocol " -"message used to coordinate file transfer ports." -msgstr "" -"``/server_file_transfer_port ``: 파일 전송 포트를 조정하는 데 사용되는 " -"내부 프로토콜 메시지입니다." #: ../../source/Network_APIs/TCP_Server_APIs.rst:607 #: 1e9e32071bb1425b8617a419a7ef87f6 @@ -1616,14 +1585,6 @@ msgstr "" "``handle_command``는 ``file_transfer_server_recv_server_start_thread``를 " "사용하여 전용 파일 서버 스레드를 시작합니다." -#: ../../source/Network_APIs/TCP_Server_APIs.rst:899 -#: e5ef46040255421fa92c6de33d976671 -msgid "" -"The server allocates an ephemeral transfer port with ``palloc`` and sends" -" ``/server_file_transfer_port `` back to the client." -msgstr "" -"서버는 ``palloc``으로 임시 전송 포트를 할당하고 ``/server_file_transfer_port " -"``를 클라이언트로 다시 보냅니다." #: ../../source/Network_APIs/TCP_Server_APIs.rst:903 #: 6eaa70d4d985489e8f0e683baa6b52a1 @@ -2054,11 +2015,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:1063 -#: 187f0ce4a65e41adb7ee99b021de4d2b -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Server_APIs.rst:1065 #: 944bb0f890764ad7a62584cc73782927 msgid "The server console commands:" diff --git a/docs/source/locale/ko/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po b/docs/source/locale/ko/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po index c2cec42..c69c3d3 100644 --- a/docs/source/locale/ko/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po +++ b/docs/source/locale/ko/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po @@ -61,24 +61,6 @@ msgid "" "on the same socket and share its command namespace;" msgstr "" -#: 3488995e2a8349a58393d9f2d1155b99 PyFlow.network_api.connect_tcp:18 of -msgid "" -"an interrupted transfer is resumed after a disconnect: both sides persist" -" their transfer stores under the protocol's ``.Flow`` directory and the " -"receiver asks the sender to continue the file from the bytes already on " -"disk (``resume_transfers``; the wire lines are ``/resume_trans`` and " -"``/forward_resume``)." -msgstr "" - -#: 4cc5743bdfe54148ad38f54f10e688c9 PyFlow.network_api.connect_tcp:24 of -msgid "" -"The forwarding extensions use the module-level parsers " -"`parse_forwarded_message`, `parse_forward_items_and_addrs`, " -"`parse_forward_originator` and `forward_skip_message`; the reconnect " -"logic uses `parse_resume_command`, `parse_forward_resume_command` and " -"`_resume_offset_for`." -msgstr "" - #: 49796529cb7f49ee8131b257212b5420 PyFlow.network_api.connect_tcp:30 of msgid "" "Concepts live in ``docs/source/Network_APIs/TCP_Server_APIs.rst`` and " @@ -86,10 +68,6 @@ msgid "" " the docstrings below." msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_resume_command:1 of -msgid "Split a ``/resume_trans `` line." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 14dcda8b059b480982e3fb03399e665b 99035a24f6e94207b84545f8c71a451e @@ -97,35 +75,14 @@ msgstr "" msgid "Returns" msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_resume_command:3 of -msgid "``(fid, offset)``, or None when the line is malformed." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 19eb6fca79324b0791f8ec7488a5246c msgid "Return type" msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:1 of -msgid "Split a ``/forward_resume `` line." -msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:3 of -msgid "``(target, fid, offset)``, or None when the line is malformed." -msgstr "" -#: 04a0882a3a5e4099a114728b4d1c79c0 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:5 of -msgid "``(target, fid, offset)``, or None when the line is" -msgstr "" - -#: 4ead8d8f93644adeb812150e760f7792 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:6 of -msgid "malformed." -msgstr "" #: 39274682c4bd46489ed6fe535b50ede5 #: PyFlow.network_api.connect_tcp.parse_forwarded_message:1 of @@ -416,14 +373,6 @@ msgid "" "never the relay." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Server_Base:96 -#: f9528e6c490d4807b949f6730c6ece8e of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when a connection drops and read back on the next reconnect " -"(``resume_transfers``)." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base:110 #: PyFlow.network_api.connect_tcp.TCP_Server_Base:104 #: ed04a3bf285b4f57bb6a5de0c4f7284f of @@ -1599,51 +1548,12 @@ msgid "" "byte." msgstr "" -#: 1634dbe84a95421a92a660f7c6fdd10e -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:1 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:1 of -msgid "Continue the interrupted transfers of a peer that is back." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:3 -#: eb46142fa4e44147ad3ad8c680d5bee0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and, for " -"every download this instance was receiving from ``peer``, opens a fresh " -"transfer listener and asks the sender to continue the file from the bytes" -" already on disk. Restored uploads need no action here: the peer drives " -"their resume from its own side." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:9 -#: b7d54977ca494a8b9968365dffd9bd3d of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer that is " -"connected right now." -msgstr "" - -#: 15c0118df8da4454a7fea8d529989237 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:13 of -msgid "Number of resumed receive legs started." -msgstr "" - #: 52cb873f015542b6a94506cebfe735a1 737ed182911443bc8cfff3c4d094b9a9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:1 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:1 of msgid "Continue one interrupted transfer this instance was sending." msgstr "" -#: 50f292ac853841c9b082eb3c3ff4d6f4 6c968a5b77b342ed891883336cfcc255 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:3 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:3 of -msgid "" -"Runs when a receiver asks to continue a transfer (``/resume_trans``, see " -"`resume_transfers` on the receiving side): the leg is re-established " -"against the receiver's fresh listener and the source file is read from " -"the byte offset the receiver already holds. Also the public API for " -"resuming one transfer on demand." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:9 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:9 #: b541e8446e374bc8b38cd7912a1fa33c of @@ -1772,14 +1682,6 @@ msgid "" "resumes the interrupted transfers)." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base:102 -#: c194bbee7cf14d268c330e06f94ae973 of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when the connection drops and read back on the next reconnect " -"(`resume_transfers`)." -msgstr "" - #: 40f2a9e1b730422db3dd58bc2a6046e3 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:1 of msgid "Create the client and, unless extended, connect and start reading." @@ -1846,16 +1748,6 @@ msgid "" "client's forward path does not read it today. Defaults to 2048." msgstr "" -#: 14c4889553fb42f7b0393a4a7ab46b4a -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:72 of -msgid "" -"Reconnect (and resume the interrupted transfers, see `resume_transfers`) " -"when the connection drops while `start_TCP_client` runs. The transfers " -"that died with the connection are persisted to the protocol's ``.Flow`` " -"directory and read back on the next connect. Defaults to False: a dropped" -" connection ends the client." -msgstr "" - #: 0f06ff3b6db940da96ec09bee3dba526 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:80 of msgid "If ``is_wait_server`` is True and ``timeout`` is not None." @@ -2224,18 +2116,6 @@ msgstr "" msgid "Dispatch one command line pushed by the server." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.handle_server_command:3 -#: a7e781fde939475cb081204adac16ded of -msgid "" -"Handles the protocol's own lines: ``/crypto_mode`` (a mismatch closes the" -" connection), ``/client_alloc_port_range``, the ``/crypto_*`` exchange " -"lines, and the transfer lines ``/file``, ``/file_folder``, " -"``/server_file_transfer_port``, ``/forward_upload``, ``/pause_trans``, " -"``/start_trans``, ``/resume_trans``, ``/forward_resume`` and " -"``/forward_error``. Any other name goes to the handlers registered for " -"the \"server\" side via `register_command`; an unknown command is only " -"reported on the console." -msgstr "" #: 91942d88960c4793b5eab8c3654c1400 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_messages:1 of @@ -2303,25 +2183,6 @@ msgid "" "there instead of its first byte." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:3 -#: a3c59554b8de41b280df7536b307bed0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and drives " -"every download that can be continued: a file the server was sending is" -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:6 -#: fae378e888be4821a14ed2483cfde1e0 of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer the " -"client can reach." -msgstr "" - -#: 48061103ed9d4bcaae8a8a837b363557 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:10 of -msgid "Number of resumed transfers started or requested." -msgstr "" - #: 6dbae803a56c442588b28af664b7a0e9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_file_console:1 of msgid "" @@ -2373,15 +2234,6 @@ msgid "" "while still connecting returns quietly, because that stop was asked for." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.start_TCP_client:9 -#: eec663f58484446ebce460e6402d7b5f of -msgid "" -"With ``is_auto_reconnect`` True a dropped connection does not end the " -"client: it reconnects (waiting for the server when ``is_wait_server`` is " -"set) and every session carries the interrupted transfers of the previous " -"one (`resume_transfers`). A close or ``/quit`` still stops the client." -msgstr "" - #~ msgid "" #~ "Concepts live in " #~ "``docs/Network_APIs/TCP_Server_APIs.rst`` and " @@ -2570,19 +2422,6 @@ msgstr "" #~ " and releases the port range." #~ msgstr "" -#~ msgid "" -#~ "Handles the protocol's own lines: " -#~ "``/crypto_mode`` (a mismatch closes the " -#~ "connection), ``/client_alloc_port_range``, the " -#~ "``/crypto_*`` exchange lines, and the " -#~ "transfer lines ``/file``, ``/file_folder``, " -#~ "``/forward_upload``, ``/pause_trans``, ``/start_trans``," -#~ " ``/forward_error``. Any other name goes" -#~ " to the handlers registered for the" -#~ " \"server\" side via `register_command`; an" -#~ " unknown command is only reported on" -#~ " the console." -#~ msgstr "" #~ msgid "" #~ "Stops the receive loop, releases the " diff --git a/docs/source/locale/ru/LC_MESSAGES/File_Transfer/File_Transfer.po b/docs/source/locale/ru/LC_MESSAGES/File_Transfer/File_Transfer.po index b36e10c..3cd9064 100644 --- a/docs/source/locale/ru/LC_MESSAGES/File_Transfer/File_Transfer.po +++ b/docs/source/locale/ru/LC_MESSAGES/File_Transfer/File_Transfer.po @@ -155,16 +155,6 @@ msgstr "" "``error_sign`` — отправляется любой стороной при возникновении ошибки, " "приводящей к закрытию сокета передачи." -#: ../../source/File_Transfer/File_Transfer.rst:72 -#: 41f6d3d6e0e848bc8f6aecb995223622 -msgid "" -"The remaining keys of the table cover the other three direction/phase " -"combinations. Protocol control lines that are not part of this table " -"(``/server_file_transfer_port``, ``/client_alloc_port_range``, " -"``/pause_trans``, ``/start_trans``, ``/forward_item``, " -"``/forward_upload``, ``/forward_error``, ``/crypto_*``) are hardcoded in " -"``connect_tcp.py``." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:80 #: 45a36aa9239d4caba8752e4dc4fd551b @@ -394,18 +384,6 @@ msgstr "" "Получите локальный порт для исходящего сокета передачи файлов, вызвав " "``palloc()``." -#: ../../source/File_Transfer/File_Transfer.rst:172 -#: 37226a4f2e1e4a878ba325fca05df9bf -msgid "" -"Wait (up to 20 seconds) for the server to reply with a port number. The " -"server's reply (``/server_file_transfer_port ``) is " -"received by the main connection and stored in ``file_server_port_list``. " -"When a matching entry is found, the port is extracted." -msgstr "" -"Подождите (до 20 секунд), пока сервер ответит номером порта. Ответ " -"сервера (``/server_file_transfer_port ``) принимается " -"основным соединением и сохраняется в ``file_server_port_list``. Когда " -"соответствующая запись найдена, порт извлекается." #: ../../source/File_Transfer/File_Transfer.rst:180 #: e9cd68239fe3498597f07ccfa774ae22 @@ -503,22 +481,6 @@ msgstr "" "Вызывает ``file_transfer_mode_recv(хост, порт, client_socket, client_id, " "new_save_path, file_name, команда)``." -#: ../../source/File_Transfer/File_Transfer.rst:224 -#: 2f6675b7698c41938542b09cbfb482e8 -msgid "" -"This low-level receive function sets up a listening socket on the " -"allocated port, sends the port number back to the client (as " -"``/server_file_transfer_port ``), accepts the client's " -"connection, and performs the protocol steps described in :ref:`file-" -"transfer-protocol`. The received file is saved under ``received_files/`` " -"(or a subdirectory if a folder transfer was requested)." -msgstr "" -"Эта низкоуровневая функция приема устанавливает сокет прослушивания на " -"выделенном порту, отправляет номер порта обратно клиенту (как " -"``/server_file_transfer_port ``), принимает соединение " -"клиента и выполняет шаги протокола, описанные в :ref:`file-transfer-" -"protocol`. Полученный файл сохраняется в папке ``received_files/`` (или в" -" подкаталоге, если была запрошена передача папки)." #: ../../source/File_Transfer/File_Transfer.rst:234 #: 9fe861572b8746aeaa8266d8d30f963f @@ -618,16 +580,6 @@ msgstr "" msgid "Allocate a local port with ``palloc()``." msgstr "Выделите локальный порт с помощью palloc()." -#: ../../source/File_Transfer/File_Transfer.rst:279 -#: 723a05b6c3974b65a539af723f1e0872 -msgid "" -"Wait for the client to reply with its own port number (via " -"``/server_file_transfer_port``), which is stored in " -"``file_server_port_list`` by the main connection's message handler." -msgstr "" -"Подождите, пока клиент ответит своим собственным номером порта (через " -"``/server_file_transfer_port``), который хранится в " -"``file_server_port_list`` обработчиком сообщений основного соединения." #: ../../source/File_Transfer/File_Transfer.rst:284 #: 878572ca3d2948a9acbb6741f00ddd62 @@ -1438,28 +1390,6 @@ msgid "" "continue from;" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:706 -#: c7017b370d4147128e54a94458a7b536 -msgid "" -"after a reconnect the receiver drives the resume (`resume_transfers`): it" -" opens a fresh transfer listener, announces it and sends ``/resume_trans " -" `` with the byte offset its partial file holds. The sender " -"re-establishes the leg, reads its source from that offset and only the " -"remainder of the file travels. A forwarded transfer is asked for through " -"the server, which routes ``/forward_resume`` to the originating client so" -" it re-forwards just the missing part." -msgstr "" - -#: ../../source/File_Transfer/File_Transfer.rst:714 -#: ee858a76e93f44d5934cb2d89f6709a5 -msgid "" -"The server resumes a peer's transfers when it reconnects, and a client " -"with ``is_auto_reconnect=True`` reconnects on its own and resumes every " -"transfer it can reach; `resume_transfers` is public, so an application " -"can also drive the resume itself. Legs the previous session left behind " -"are closed before the resume runs, so two legs never write the same " -"transfer." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:723 #: fe9377db9cf04b9da0dbe4d07c730adf @@ -1689,12 +1619,6 @@ msgstr "" msgid "``pause_trans_remote`` / ``start_trans_remote`` (ask a client to do it)" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:898 -#: ../../source/File_Transfer/File_Transfer.rst:918 -#: 9c62709c2c06456dbfddee1f1ce6871e de496a0be32c490585ae0cbafc931a71 -msgid "``resume_transfers`` / ``resume_trans`` (continue interrupted transfers)" -msgstr "" - #: ../../source/File_Transfer/File_Transfer.rst:900 #: 11476e0eaa3b4a4286612812e4e2c004 msgid "" diff --git a/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po b/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po index bad9800..c28247d 100644 --- a/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po +++ b/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po @@ -690,14 +690,6 @@ msgstr "" "``/client_alloc_port_range``: настраивает диапазон ручного выделения " "портов клиентам на основе широковещательной рассылки сервера." -#: ../../source/Network_APIs/TCP_Client_APIs.rst:275 -#: f84404f7844d4779af2e6030e795b5d0 -msgid "" -"``/server_file_transfer_port``: receives the file transfer port assigned " -"by the server for an ongoing file operation." -msgstr "" -"``/server_file_transfer_port``: получает порт передачи файлов, " -"назначенный сервером для текущей файловой операции." #: ../../source/Network_APIs/TCP_Client_APIs.rst:277 #: 4eb4fe22d5db4b56b622dfd29459c3b7 @@ -717,12 +709,6 @@ msgid "" "the connection." msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:283 -#: 31a62335177b49a19f708aee2aa5b0b1 -msgid "" -"``/forward_upload``, ``/pause_trans``, ``/start_trans``, " -"``/forward_error``: forwarding and transfer-control messages." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:286 #: b4c3c6ea61f344b8a3d3baf6b659d9c7 @@ -1079,21 +1065,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:471 -#: a47f9db0c3674a12ab6d3ab5d056a8c4 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:483 #: 6740137a28b84e5dbb3f85e8d58d32b5 @@ -1975,11 +1946,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:872 -#: 0f3b14e3c193426191323ee76959278e -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Client_APIs.rst:874 #: 5bf1072901734d3d8d0b119e5f50627c msgid "The client console commands (interactive mode):" diff --git a/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po b/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po index 0604428..f3e486f 100644 --- a/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po +++ b/docs/source/locale/ru/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po @@ -999,21 +999,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:457 -#: 8ca69d91005b459a8a56f21902eaa690 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Server_APIs.rst:469 #: b32b4fa4788d4f9d88926958c81ad73f @@ -1254,23 +1239,7 @@ msgid "" " switch as a public API)." msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:601 -#: 7f3801d57ac34670bbe16736683db30e -msgid "" -"``/resume_trans `` and ``/forward_resume " -"``: internal transfer-reconnect messages: the receiver asks the " -"sender (or, for a forwarded transfer, the originator) to continue the " -"file from the bytes already on disk. See `resume_transfers`." -msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:605 -#: abd7d3032e15468e92f96a0b5af1f926 -msgid "" -"``/server_file_transfer_port ``: internal protocol " -"message used to coordinate file transfer ports." -msgstr "" -"``/server_file_transfer_port <порт> ``: сообщение внутреннего " -"протокола, используемое для координации портов передачи файлов." #: ../../source/Network_APIs/TCP_Server_APIs.rst:607 #: 1e9e32071bb1425b8617a419a7ef87f6 @@ -1779,14 +1748,6 @@ msgstr "" "``handle_command`` запускает выделенный поток файлового сервера, " "используя ``file_transfer_server_recv_server_start_thread``." -#: ../../source/Network_APIs/TCP_Server_APIs.rst:899 -#: e5ef46040255421fa92c6de33d976671 -msgid "" -"The server allocates an ephemeral transfer port with ``palloc`` and sends" -" ``/server_file_transfer_port `` back to the client." -msgstr "" -"Сервер выделяет временный порт передачи с помощью palloc и отправляет " -"``/server_file_transfer_port `` обратно клиенту." #: ../../source/Network_APIs/TCP_Server_APIs.rst:903 #: 6eaa70d4d985489e8f0e683baa6b52a1 @@ -2253,11 +2214,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:1063 -#: 187f0ce4a65e41adb7ee99b021de4d2b -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Server_APIs.rst:1065 #: 944bb0f890764ad7a62584cc73782927 msgid "The server console commands:" diff --git a/docs/source/locale/ru/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po b/docs/source/locale/ru/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po index 30ceecb..90d3357 100644 --- a/docs/source/locale/ru/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po +++ b/docs/source/locale/ru/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po @@ -62,24 +62,6 @@ msgid "" "on the same socket and share its command namespace;" msgstr "" -#: 3488995e2a8349a58393d9f2d1155b99 PyFlow.network_api.connect_tcp:18 of -msgid "" -"an interrupted transfer is resumed after a disconnect: both sides persist" -" their transfer stores under the protocol's ``.Flow`` directory and the " -"receiver asks the sender to continue the file from the bytes already on " -"disk (``resume_transfers``; the wire lines are ``/resume_trans`` and " -"``/forward_resume``)." -msgstr "" - -#: 4cc5743bdfe54148ad38f54f10e688c9 PyFlow.network_api.connect_tcp:24 of -msgid "" -"The forwarding extensions use the module-level parsers " -"`parse_forwarded_message`, `parse_forward_items_and_addrs`, " -"`parse_forward_originator` and `forward_skip_message`; the reconnect " -"logic uses `parse_resume_command`, `parse_forward_resume_command` and " -"`_resume_offset_for`." -msgstr "" - #: 49796529cb7f49ee8131b257212b5420 PyFlow.network_api.connect_tcp:30 of msgid "" "Concepts live in ``docs/source/Network_APIs/TCP_Server_APIs.rst`` and " @@ -87,10 +69,6 @@ msgid "" " the docstrings below." msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_resume_command:1 of -msgid "Split a ``/resume_trans `` line." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 14dcda8b059b480982e3fb03399e665b 99035a24f6e94207b84545f8c71a451e @@ -98,35 +76,14 @@ msgstr "" msgid "Returns" msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_resume_command:3 of -msgid "``(fid, offset)``, or None when the line is malformed." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 19eb6fca79324b0791f8ec7488a5246c msgid "Return type" msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:1 of -msgid "Split a ``/forward_resume `` line." -msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:3 of -msgid "``(target, fid, offset)``, or None when the line is malformed." -msgstr "" -#: 04a0882a3a5e4099a114728b4d1c79c0 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:5 of -msgid "``(target, fid, offset)``, or None when the line is" -msgstr "" - -#: 4ead8d8f93644adeb812150e760f7792 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:6 of -msgid "malformed." -msgstr "" #: 39274682c4bd46489ed6fe535b50ede5 #: PyFlow.network_api.connect_tcp.parse_forwarded_message:1 of @@ -417,14 +374,6 @@ msgid "" "never the relay." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Server_Base:96 -#: f9528e6c490d4807b949f6730c6ece8e of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when a connection drops and read back on the next reconnect " -"(``resume_transfers``)." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base:110 #: PyFlow.network_api.connect_tcp.TCP_Server_Base:104 #: ed04a3bf285b4f57bb6a5de0c4f7284f of @@ -1600,51 +1549,12 @@ msgid "" "byte." msgstr "" -#: 1634dbe84a95421a92a660f7c6fdd10e -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:1 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:1 of -msgid "Continue the interrupted transfers of a peer that is back." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:3 -#: eb46142fa4e44147ad3ad8c680d5bee0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and, for " -"every download this instance was receiving from ``peer``, opens a fresh " -"transfer listener and asks the sender to continue the file from the bytes" -" already on disk. Restored uploads need no action here: the peer drives " -"their resume from its own side." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:9 -#: b7d54977ca494a8b9968365dffd9bd3d of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer that is " -"connected right now." -msgstr "" - -#: 15c0118df8da4454a7fea8d529989237 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:13 of -msgid "Number of resumed receive legs started." -msgstr "" - #: 52cb873f015542b6a94506cebfe735a1 737ed182911443bc8cfff3c4d094b9a9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:1 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:1 of msgid "Continue one interrupted transfer this instance was sending." msgstr "" -#: 50f292ac853841c9b082eb3c3ff4d6f4 6c968a5b77b342ed891883336cfcc255 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:3 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:3 of -msgid "" -"Runs when a receiver asks to continue a transfer (``/resume_trans``, see " -"`resume_transfers` on the receiving side): the leg is re-established " -"against the receiver's fresh listener and the source file is read from " -"the byte offset the receiver already holds. Also the public API for " -"resuming one transfer on demand." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:9 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:9 #: b541e8446e374bc8b38cd7912a1fa33c of @@ -1773,14 +1683,6 @@ msgid "" "resumes the interrupted transfers)." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base:102 -#: c194bbee7cf14d268c330e06f94ae973 of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when the connection drops and read back on the next reconnect " -"(`resume_transfers`)." -msgstr "" - #: 40f2a9e1b730422db3dd58bc2a6046e3 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:1 of msgid "Create the client and, unless extended, connect and start reading." @@ -1847,16 +1749,6 @@ msgid "" "client's forward path does not read it today. Defaults to 2048." msgstr "" -#: 14c4889553fb42f7b0393a4a7ab46b4a -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:72 of -msgid "" -"Reconnect (and resume the interrupted transfers, see `resume_transfers`) " -"when the connection drops while `start_TCP_client` runs. The transfers " -"that died with the connection are persisted to the protocol's ``.Flow`` " -"directory and read back on the next connect. Defaults to False: a dropped" -" connection ends the client." -msgstr "" - #: 0f06ff3b6db940da96ec09bee3dba526 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:80 of msgid "If ``is_wait_server`` is True and ``timeout`` is not None." @@ -2225,18 +2117,6 @@ msgstr "" msgid "Dispatch one command line pushed by the server." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.handle_server_command:3 -#: a7e781fde939475cb081204adac16ded of -msgid "" -"Handles the protocol's own lines: ``/crypto_mode`` (a mismatch closes the" -" connection), ``/client_alloc_port_range``, the ``/crypto_*`` exchange " -"lines, and the transfer lines ``/file``, ``/file_folder``, " -"``/server_file_transfer_port``, ``/forward_upload``, ``/pause_trans``, " -"``/start_trans``, ``/resume_trans``, ``/forward_resume`` and " -"``/forward_error``. Any other name goes to the handlers registered for " -"the \"server\" side via `register_command`; an unknown command is only " -"reported on the console." -msgstr "" #: 91942d88960c4793b5eab8c3654c1400 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_messages:1 of @@ -2304,25 +2184,6 @@ msgid "" "there instead of its first byte." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:3 -#: a3c59554b8de41b280df7536b307bed0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and drives " -"every download that can be continued: a file the server was sending is" -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:6 -#: fae378e888be4821a14ed2483cfde1e0 of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer the " -"client can reach." -msgstr "" - -#: 48061103ed9d4bcaae8a8a837b363557 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:10 of -msgid "Number of resumed transfers started or requested." -msgstr "" - #: 6dbae803a56c442588b28af664b7a0e9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_file_console:1 of msgid "" @@ -2374,15 +2235,6 @@ msgid "" "while still connecting returns quietly, because that stop was asked for." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.start_TCP_client:9 -#: eec663f58484446ebce460e6402d7b5f of -msgid "" -"With ``is_auto_reconnect`` True a dropped connection does not end the " -"client: it reconnects (waiting for the server when ``is_wait_server`` is " -"set) and every session carries the interrupted transfers of the previous " -"one (`resume_transfers`). A close or ``/quit`` still stops the client." -msgstr "" - #~ msgid "" #~ "Concepts live in " #~ "``docs/Network_APIs/TCP_Server_APIs.rst`` and " @@ -2571,19 +2423,6 @@ msgstr "" #~ " and releases the port range." #~ msgstr "" -#~ msgid "" -#~ "Handles the protocol's own lines: " -#~ "``/crypto_mode`` (a mismatch closes the " -#~ "connection), ``/client_alloc_port_range``, the " -#~ "``/crypto_*`` exchange lines, and the " -#~ "transfer lines ``/file``, ``/file_folder``, " -#~ "``/forward_upload``, ``/pause_trans``, ``/start_trans``," -#~ " ``/forward_error``. Any other name goes" -#~ " to the handlers registered for the" -#~ " \"server\" side via `register_command`; an" -#~ " unknown command is only reported on" -#~ " the console." -#~ msgstr "" #~ msgid "" #~ "Stops the receive loop, releases the " diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/File_Transfer/File_Transfer.po b/docs/source/locale/zh_CN/LC_MESSAGES/File_Transfer/File_Transfer.po index 8bc8f16..c0a13c1 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/File_Transfer/File_Transfer.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/File_Transfer/File_Transfer.po @@ -130,16 +130,6 @@ msgid "" "closed." msgstr "``error_sign`` - 当发生错误时由任何一方发送，导致传输套接字关闭。" -#: ../../source/File_Transfer/File_Transfer.rst:72 -#: 41f6d3d6e0e848bc8f6aecb995223622 -msgid "" -"The remaining keys of the table cover the other three direction/phase " -"combinations. Protocol control lines that are not part of this table " -"(``/server_file_transfer_port``, ``/client_alloc_port_range``, " -"``/pause_trans``, ``/start_trans``, ``/forward_item``, " -"``/forward_upload``, ``/forward_error``, ``/crypto_*``) are hardcoded in " -"``connect_tcp.py``." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:80 #: 45a36aa9239d4caba8752e4dc4fd551b @@ -343,16 +333,6 @@ msgid "" "``palloc()``." msgstr "通过调用“palloc()”获取传出文件传输套接字的本地端口。" -#: ../../source/File_Transfer/File_Transfer.rst:172 -#: 37226a4f2e1e4a878ba325fca05df9bf -msgid "" -"Wait (up to 20 seconds) for the server to reply with a port number. The " -"server's reply (``/server_file_transfer_port ``) is " -"received by the main connection and stored in ``file_server_port_list``. " -"When a matching entry is found, the port is extracted." -msgstr "" -"等待（最多 20 秒）服务器回复端口号。服务器的回复（``/server_file_transfer_port " -"``）由主连接接收并存储在``file_server_port_list``中。当找到匹配的条目时，将提取端口。" #: ../../source/File_Transfer/File_Transfer.rst:180 #: e9cd68239fe3498597f07ccfa774ae22 @@ -436,18 +416,6 @@ msgid "" "new_save_path, file_name, command)``." msgstr "调用“file_transfer_mode_recv(主机、端口、client_socket、client_id、new_save_path、file_name、command)”。" -#: ../../source/File_Transfer/File_Transfer.rst:224 -#: 2f6675b7698c41938542b09cbfb482e8 -msgid "" -"This low-level receive function sets up a listening socket on the " -"allocated port, sends the port number back to the client (as " -"``/server_file_transfer_port ``), accepts the client's " -"connection, and performs the protocol steps described in :ref:`file-" -"transfer-protocol`. The received file is saved under ``received_files/`` " -"(or a subdirectory if a folder transfer was requested)." -msgstr "" -"这个低级接收函数在分配的端口上设置一个监听套接字，将端口号发送回客户端（如“/server_file_transfer_port " -"”），接受客户端的连接，并执行文件传输协议中描述的协议步骤。接收到的文件保存在“received_files/”下（如果请求文件夹传输，则保存在子目录下）。" #: ../../source/File_Transfer/File_Transfer.rst:234 #: 9fe861572b8746aeaa8266d8d30f963f @@ -538,13 +506,6 @@ msgstr "分配唯一的客户端 ID 并用它扩展命令，然后通过主连 msgid "Allocate a local port with ``palloc()``." msgstr "使用 palloc() 分配本地端口。" -#: ../../source/File_Transfer/File_Transfer.rst:279 -#: 723a05b6c3974b65a539af723f1e0872 -msgid "" -"Wait for the client to reply with its own port number (via " -"``/server_file_transfer_port``), which is stored in " -"``file_server_port_list`` by the main connection's message handler." -msgstr "等待客户端回复其自己的端口号（通过“/server_file_transfer_port”），该端口号由主连接的消息处理程序存储在“file_server_port_list”中。" #: ../../source/File_Transfer/File_Transfer.rst:284 #: 878572ca3d2948a9acbb6741f00ddd62 @@ -1235,28 +1196,6 @@ msgid "" "continue from;" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:706 -#: c7017b370d4147128e54a94458a7b536 -msgid "" -"after a reconnect the receiver drives the resume (`resume_transfers`): it" -" opens a fresh transfer listener, announces it and sends ``/resume_trans " -" `` with the byte offset its partial file holds. The sender " -"re-establishes the leg, reads its source from that offset and only the " -"remainder of the file travels. A forwarded transfer is asked for through " -"the server, which routes ``/forward_resume`` to the originating client so" -" it re-forwards just the missing part." -msgstr "" - -#: ../../source/File_Transfer/File_Transfer.rst:714 -#: ee858a76e93f44d5934cb2d89f6709a5 -msgid "" -"The server resumes a peer's transfers when it reconnects, and a client " -"with ``is_auto_reconnect=True`` reconnects on its own and resumes every " -"transfer it can reach; `resume_transfers` is public, so an application " -"can also drive the resume itself. Legs the previous session left behind " -"are closed before the resume runs, so two legs never write the same " -"transfer." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:723 #: fe9377db9cf04b9da0dbe4d07c730adf @@ -1472,12 +1411,6 @@ msgstr "" msgid "``pause_trans_remote`` / ``start_trans_remote`` (ask a client to do it)" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:898 -#: ../../source/File_Transfer/File_Transfer.rst:918 -#: 9c62709c2c06456dbfddee1f1ce6871e de496a0be32c490585ae0cbafc931a71 -msgid "``resume_transfers`` / ``resume_trans`` (continue interrupted transfers)" -msgstr "" - #: ../../source/File_Transfer/File_Transfer.rst:900 #: 11476e0eaa3b4a4286612812e4e2c004 msgid "" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po b/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po index 2f3c73f..611b93d 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po @@ -600,12 +600,6 @@ msgid "" "allocation range from the line the server sends it on connect." msgstr "``/client_alloc_port_range``：根据服务器广播配置客户端手动端口分配范围。" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:275 -#: f84404f7844d4779af2e6030e795b5d0 -msgid "" -"``/server_file_transfer_port``: receives the file transfer port assigned " -"by the server for an ongoing file operation." -msgstr "``/server_file_transfer_port``：接收服务器为正在进行的文件操作分配的文件传输端口。" #: ../../source/Network_APIs/TCP_Client_APIs.rst:277 #: 4eb4fe22d5db4b56b622dfd29459c3b7 @@ -623,12 +617,6 @@ msgid "" "the connection." msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:283 -#: 31a62335177b49a19f708aee2aa5b0b1 -msgid "" -"``/forward_upload``, ``/pause_trans``, ``/start_trans``, " -"``/forward_error``: forwarding and transfer-control messages." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:286 #: b4c3c6ea61f344b8a3d3baf6b659d9c7 @@ -935,21 +923,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:471 -#: a47f9db0c3674a12ab6d3ab5d056a8c4 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:483 #: 6740137a28b84e5dbb3f85e8d58d32b5 @@ -1734,11 +1707,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:872 -#: 0f3b14e3c193426191323ee76959278e -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Client_APIs.rst:874 #: 5bf1072901734d3d8d0b119e5f50627c msgid "The client console commands (interactive mode):" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po b/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po index 12d9a82..3ea2e9d 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po @@ -870,21 +870,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:457 -#: 8ca69d91005b459a8a56f21902eaa690 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Server_APIs.rst:469 #: b32b4fa4788d4f9d88926958c81ad73f @@ -1092,21 +1077,7 @@ msgid "" " switch as a public API)." msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:601 -#: 7f3801d57ac34670bbe16736683db30e -msgid "" -"``/resume_trans `` and ``/forward_resume " -"``: internal transfer-reconnect messages: the receiver asks the " -"sender (or, for a forwarded transfer, the originator) to continue the " -"file from the bytes already on disk. See `resume_transfers`." -msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:605 -#: abd7d3032e15468e92f96a0b5af1f926 -msgid "" -"``/server_file_transfer_port ``: internal protocol " -"message used to coordinate file transfer ports." -msgstr "``/server_file_transfer_port ``：用于协调文件传输端口的内部协议消息。" #: ../../source/Network_APIs/TCP_Server_APIs.rst:607 #: 1e9e32071bb1425b8617a419a7ef87f6 @@ -1529,14 +1500,6 @@ msgid "" "``/file_folder``." msgstr "“handle_command”使用“file_transfer_server_recv_server_start_thread”启动专用文件服务器线程。" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:899 -#: e5ef46040255421fa92c6de33d976671 -msgid "" -"The server allocates an ephemeral transfer port with ``palloc`` and sends" -" ``/server_file_transfer_port `` back to the client." -msgstr "" -"服务器使用“palloc”分配临时传输端口并将“/server_file_transfer_port " -"”发送回客户端。" #: ../../source/Network_APIs/TCP_Server_APIs.rst:903 #: 6eaa70d4d985489e8f0e683baa6b52a1 @@ -1945,11 +1908,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:1063 -#: 187f0ce4a65e41adb7ee99b021de4d2b -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Server_APIs.rst:1065 #: 944bb0f890764ad7a62584cc73782927 msgid "The server console commands:" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po b/docs/source/locale/zh_CN/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po index 2c77325..5cf1dc8 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po @@ -62,24 +62,6 @@ msgid "" "on the same socket and share its command namespace;" msgstr "文件/文件夹传输、消息转发和端口分配分层在同一套接字上，并共享其命令命名空间。" -#: 3488995e2a8349a58393d9f2d1155b99 PyFlow.network_api.connect_tcp:18 of -msgid "" -"an interrupted transfer is resumed after a disconnect: both sides persist" -" their transfer stores under the protocol's ``.Flow`` directory and the " -"receiver asks the sender to continue the file from the bytes already on " -"disk (``resume_transfers``; the wire lines are ``/resume_trans`` and " -"``/forward_resume``)." -msgstr "" - -#: 4cc5743bdfe54148ad38f54f10e688c9 PyFlow.network_api.connect_tcp:24 of -msgid "" -"The forwarding extensions use the module-level parsers " -"`parse_forwarded_message`, `parse_forward_items_and_addrs`, " -"`parse_forward_originator` and `forward_skip_message`; the reconnect " -"logic uses `parse_resume_command`, `parse_forward_resume_command` and " -"`_resume_offset_for`." -msgstr "" - #: 49796529cb7f49ee8131b257212b5420 PyFlow.network_api.connect_tcp:30 of msgid "" "Concepts live in ``docs/source/Network_APIs/TCP_Server_APIs.rst`` and " @@ -87,10 +69,6 @@ msgid "" " the docstrings below." msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_resume_command:1 of -msgid "Split a ``/resume_trans `` line." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 14dcda8b059b480982e3fb03399e665b 99035a24f6e94207b84545f8c71a451e @@ -98,35 +76,14 @@ msgstr "" msgid "Returns" msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_resume_command:3 of -msgid "``(fid, offset)``, or None when the line is malformed." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 19eb6fca79324b0791f8ec7488a5246c msgid "Return type" msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:1 of -msgid "Split a ``/forward_resume `` line." -msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:3 of -msgid "``(target, fid, offset)``, or None when the line is malformed." -msgstr "" -#: 04a0882a3a5e4099a114728b4d1c79c0 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:5 of -msgid "``(target, fid, offset)``, or None when the line is" -msgstr "" - -#: 4ead8d8f93644adeb812150e760f7792 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:6 of -msgid "malformed." -msgstr "" #: 39274682c4bd46489ed6fe535b50ede5 #: PyFlow.network_api.connect_tcp.parse_forwarded_message:1 of @@ -417,14 +374,6 @@ msgid "" "never the relay." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Server_Base:96 -#: f9528e6c490d4807b949f6730c6ece8e of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when a connection drops and read back on the next reconnect " -"(``resume_transfers``)." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base:110 #: PyFlow.network_api.connect_tcp.TCP_Server_Base:104 #: ed04a3bf285b4f57bb6a5de0c4f7284f of @@ -1600,51 +1549,12 @@ msgid "" "byte." msgstr "" -#: 1634dbe84a95421a92a660f7c6fdd10e -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:1 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:1 of -msgid "Continue the interrupted transfers of a peer that is back." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:3 -#: eb46142fa4e44147ad3ad8c680d5bee0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and, for " -"every download this instance was receiving from ``peer``, opens a fresh " -"transfer listener and asks the sender to continue the file from the bytes" -" already on disk. Restored uploads need no action here: the peer drives " -"their resume from its own side." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:9 -#: b7d54977ca494a8b9968365dffd9bd3d of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer that is " -"connected right now." -msgstr "" - -#: 15c0118df8da4454a7fea8d529989237 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:13 of -msgid "Number of resumed receive legs started." -msgstr "" - #: 52cb873f015542b6a94506cebfe735a1 737ed182911443bc8cfff3c4d094b9a9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:1 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:1 of msgid "Continue one interrupted transfer this instance was sending." msgstr "" -#: 50f292ac853841c9b082eb3c3ff4d6f4 6c968a5b77b342ed891883336cfcc255 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:3 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:3 of -msgid "" -"Runs when a receiver asks to continue a transfer (``/resume_trans``, see " -"`resume_transfers` on the receiving side): the leg is re-established " -"against the receiver's fresh listener and the source file is read from " -"the byte offset the receiver already holds. Also the public API for " -"resuming one transfer on demand." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:9 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:9 #: b541e8446e374bc8b38cd7912a1fa33c of @@ -1773,14 +1683,6 @@ msgid "" "resumes the interrupted transfers)." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base:102 -#: c194bbee7cf14d268c330e06f94ae973 of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when the connection drops and read back on the next reconnect " -"(`resume_transfers`)." -msgstr "" - #: 40f2a9e1b730422db3dd58bc2a6046e3 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:1 of msgid "Create the client and, unless extended, connect and start reading." @@ -1847,16 +1749,6 @@ msgid "" "client's forward path does not read it today. Defaults to 2048." msgstr "" -#: 14c4889553fb42f7b0393a4a7ab46b4a -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:72 of -msgid "" -"Reconnect (and resume the interrupted transfers, see `resume_transfers`) " -"when the connection drops while `start_TCP_client` runs. The transfers " -"that died with the connection are persisted to the protocol's ``.Flow`` " -"directory and read back on the next connect. Defaults to False: a dropped" -" connection ends the client." -msgstr "" - #: 0f06ff3b6db940da96ec09bee3dba526 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:80 of msgid "If ``is_wait_server`` is True and ``timeout`` is not None." @@ -2225,18 +2117,6 @@ msgstr "" msgid "Dispatch one command line pushed by the server." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.handle_server_command:3 -#: a7e781fde939475cb081204adac16ded of -msgid "" -"Handles the protocol's own lines: ``/crypto_mode`` (a mismatch closes the" -" connection), ``/client_alloc_port_range``, the ``/crypto_*`` exchange " -"lines, and the transfer lines ``/file``, ``/file_folder``, " -"``/server_file_transfer_port``, ``/forward_upload``, ``/pause_trans``, " -"``/start_trans``, ``/resume_trans``, ``/forward_resume`` and " -"``/forward_error``. Any other name goes to the handlers registered for " -"the \"server\" side via `register_command`; an unknown command is only " -"reported on the console." -msgstr "" #: 91942d88960c4793b5eab8c3654c1400 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_messages:1 of @@ -2304,25 +2184,6 @@ msgid "" "there instead of its first byte." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:3 -#: a3c59554b8de41b280df7536b307bed0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and drives " -"every download that can be continued: a file the server was sending is" -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:6 -#: fae378e888be4821a14ed2483cfde1e0 of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer the " -"client can reach." -msgstr "" - -#: 48061103ed9d4bcaae8a8a837b363557 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:10 of -msgid "Number of resumed transfers started or requested." -msgstr "" - #: 6dbae803a56c442588b28af664b7a0e9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_file_console:1 of msgid "" @@ -2374,15 +2235,6 @@ msgid "" "while still connecting returns quietly, because that stop was asked for." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.start_TCP_client:9 -#: eec663f58484446ebce460e6402d7b5f of -msgid "" -"With ``is_auto_reconnect`` True a dropped connection does not end the " -"client: it reconnects (waiting for the server when ``is_wait_server`` is " -"set) and every session carries the interrupted transfers of the previous " -"one (`resume_transfers`). A close or ``/quit`` still stops the client." -msgstr "" - #~ msgid "" #~ "Concepts live in " #~ "``docs/Network_APIs/TCP_Server_APIs.rst`` and " @@ -2564,19 +2416,6 @@ msgstr "" #~ " and releases the port range." #~ msgstr "" -#~ msgid "" -#~ "Handles the protocol's own lines: " -#~ "``/crypto_mode`` (a mismatch closes the " -#~ "connection), ``/client_alloc_port_range``, the " -#~ "``/crypto_*`` exchange lines, and the " -#~ "transfer lines ``/file``, ``/file_folder``, " -#~ "``/forward_upload``, ``/pause_trans``, ``/start_trans``," -#~ " ``/forward_error``. Any other name goes" -#~ " to the handlers registered for the" -#~ " \"server\" side via `register_command`; an" -#~ " unknown command is only reported on" -#~ " the console." -#~ msgstr "" #~ msgid "" #~ "Stops the receive loop, releases the " diff --git a/docs/source/locale/zh_TW/LC_MESSAGES/File_Transfer/File_Transfer.po b/docs/source/locale/zh_TW/LC_MESSAGES/File_Transfer/File_Transfer.po index 317210b..e311a47 100644 --- a/docs/source/locale/zh_TW/LC_MESSAGES/File_Transfer/File_Transfer.po +++ b/docs/source/locale/zh_TW/LC_MESSAGES/File_Transfer/File_Transfer.po @@ -130,16 +130,6 @@ msgid "" "closed." msgstr "``error_sign`` - 當發生錯誤時由任何一方發送，導致傳輸套接字關閉。" -#: ../../source/File_Transfer/File_Transfer.rst:72 -#: 41f6d3d6e0e848bc8f6aecb995223622 -msgid "" -"The remaining keys of the table cover the other three direction/phase " -"combinations. Protocol control lines that are not part of this table " -"(``/server_file_transfer_port``, ``/client_alloc_port_range``, " -"``/pause_trans``, ``/start_trans``, ``/forward_item``, " -"``/forward_upload``, ``/forward_error``, ``/crypto_*``) are hardcoded in " -"``connect_tcp.py``." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:80 #: 45a36aa9239d4caba8752e4dc4fd551b @@ -341,16 +331,6 @@ msgid "" "``palloc()``." msgstr "透過呼叫“palloc()”來取得傳出檔案傳輸套接字的本機連接埠。" -#: ../../source/File_Transfer/File_Transfer.rst:172 -#: 37226a4f2e1e4a878ba325fca05df9bf -msgid "" -"Wait (up to 20 seconds) for the server to reply with a port number. The " -"server's reply (``/server_file_transfer_port ``) is " -"received by the main connection and stored in ``file_server_port_list``. " -"When a matching entry is found, the port is extracted." -msgstr "" -"等待（最多 20 秒）伺服器回覆連接埠號碼。伺服器的回應（``/server_file_transfer_port " -"``）由主連線接收並儲存在``file_server_port_list``中。當找到符合的條目時，將提取連接埠。" #: ../../source/File_Transfer/File_Transfer.rst:180 #: e9cd68239fe3498597f07ccfa774ae22 @@ -434,19 +414,6 @@ msgid "" "new_save_path, file_name, command)``." msgstr "呼叫「file_transfer_mode_recv(主機、連接埠、client_socket、client_id、new_save_path、file_name、command)」。" -#: ../../source/File_Transfer/File_Transfer.rst:224 -#: 2f6675b7698c41938542b09cbfb482e8 -msgid "" -"This low-level receive function sets up a listening socket on the " -"allocated port, sends the port number back to the client (as " -"``/server_file_transfer_port ``), accepts the client's " -"connection, and performs the protocol steps described in :ref:`file-" -"transfer-protocol`. The received file is saved under ``received_files/`` " -"(or a subdirectory if a folder transfer was requested)." -msgstr "" -"這個低階接收函數在指派的連接埠上設定一個監聽套接字，將連接埠號碼傳回客戶端（如「/server_file_transfer_port " -" " -"」），接受客戶端的連接，並執行檔案傳輸協定中所述的協定步驟。接收到的檔案保存在「received_files/」下（如果請求資料夾傳輸，則儲存在子目錄下）。" #: ../../source/File_Transfer/File_Transfer.rst:234 #: 9fe861572b8746aeaa8266d8d30f963f @@ -537,13 +504,6 @@ msgstr "分配唯一的客戶端 ID 並用它擴展命令，然後透過主連 msgid "Allocate a local port with ``palloc()``." msgstr "使用 palloc() 分配本地連接埠。" -#: ../../source/File_Transfer/File_Transfer.rst:279 -#: 723a05b6c3974b65a539af723f1e0872 -msgid "" -"Wait for the client to reply with its own port number (via " -"``/server_file_transfer_port``), which is stored in " -"``file_server_port_list`` by the main connection's message handler." -msgstr "等待用戶端回覆自己的連接埠號碼（透過「/server_file_transfer_port」），該連接埠號碼由主連線的訊息處理程序儲存在「file_server_port_list」中。" #: ../../source/File_Transfer/File_Transfer.rst:284 #: 878572ca3d2948a9acbb6741f00ddd62 @@ -1234,28 +1194,6 @@ msgid "" "continue from;" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:706 -#: c7017b370d4147128e54a94458a7b536 -msgid "" -"after a reconnect the receiver drives the resume (`resume_transfers`): it" -" opens a fresh transfer listener, announces it and sends ``/resume_trans " -" `` with the byte offset its partial file holds. The sender " -"re-establishes the leg, reads its source from that offset and only the " -"remainder of the file travels. A forwarded transfer is asked for through " -"the server, which routes ``/forward_resume`` to the originating client so" -" it re-forwards just the missing part." -msgstr "" - -#: ../../source/File_Transfer/File_Transfer.rst:714 -#: ee858a76e93f44d5934cb2d89f6709a5 -msgid "" -"The server resumes a peer's transfers when it reconnects, and a client " -"with ``is_auto_reconnect=True`` reconnects on its own and resumes every " -"transfer it can reach; `resume_transfers` is public, so an application " -"can also drive the resume itself. Legs the previous session left behind " -"are closed before the resume runs, so two legs never write the same " -"transfer." -msgstr "" #: ../../source/File_Transfer/File_Transfer.rst:723 #: fe9377db9cf04b9da0dbe4d07c730adf @@ -1471,12 +1409,6 @@ msgstr "" msgid "``pause_trans_remote`` / ``start_trans_remote`` (ask a client to do it)" msgstr "" -#: ../../source/File_Transfer/File_Transfer.rst:898 -#: ../../source/File_Transfer/File_Transfer.rst:918 -#: 9c62709c2c06456dbfddee1f1ce6871e de496a0be32c490585ae0cbafc931a71 -msgid "``resume_transfers`` / ``resume_trans`` (continue interrupted transfers)" -msgstr "" - #: ../../source/File_Transfer/File_Transfer.rst:900 #: 11476e0eaa3b4a4286612812e4e2c004 msgid "" diff --git a/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po b/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po index b92d211..ae083f5 100644 --- a/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po +++ b/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Client_APIs.po @@ -600,12 +600,6 @@ msgid "" "allocation range from the line the server sends it on connect." msgstr "``/client_alloc_port_range``：根據伺服器廣播設定客戶端手動連接埠分配範圍。" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:275 -#: f84404f7844d4779af2e6030e795b5d0 -msgid "" -"``/server_file_transfer_port``: receives the file transfer port assigned " -"by the server for an ongoing file operation." -msgstr "``/server_file_transfer_port``：接收伺服器為正在進行的檔案操作分配的檔案傳輸連接埠。" #: ../../source/Network_APIs/TCP_Client_APIs.rst:277 #: 4eb4fe22d5db4b56b622dfd29459c3b7 @@ -623,12 +617,6 @@ msgid "" "the connection." msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:283 -#: 31a62335177b49a19f708aee2aa5b0b1 -msgid "" -"``/forward_upload``, ``/pause_trans``, ``/start_trans``, " -"``/forward_error``: forwarding and transfer-control messages." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:286 #: b4c3c6ea61f344b8a3d3baf6b659d9c7 @@ -934,21 +922,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:471 -#: a47f9db0c3674a12ab6d3ab5d056a8c4 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Client_APIs.rst:483 #: 6740137a28b84e5dbb3f85e8d58d32b5 @@ -1736,11 +1709,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Client_APIs.rst:872 -#: 0f3b14e3c193426191323ee76959278e -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Client_APIs.rst:874 #: 5bf1072901734d3d8d0b119e5f50627c msgid "The client console commands (interactive mode):" diff --git a/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po b/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po index 553bb8d..d4ad0c7 100644 --- a/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po +++ b/docs/source/locale/zh_TW/LC_MESSAGES/Network_APIs/TCP_Server_APIs.po @@ -860,21 +860,6 @@ msgstr "" msgid "Encrypted transfer connections" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:457 -#: 8ca69d91005b459a8a56f21902eaa690 -msgid "" -"File and folder transfers, forwarded items and the forward relay run on " -"temporary TCP connections. The listening side stamps each leg with its " -"mode and a fresh nonce in the port advertisement " -"(``/server_file_transfer_port``, ``/forward_upload``); with " -"``is_enable_encrypto`` enabled both sides then frame that connection " -"exactly like the control channel, with the same session key material, so " -"file contents, names and sizes no longer cross the wire in the clear. The" -" public-key pushes themselves stay plaintext by construction: they carry " -"the keys the session is built on. A leg whose stamp promises encryption " -"is aborted when the session key material is not ready rather than " -"silently downgraded." -msgstr "" #: ../../source/Network_APIs/TCP_Server_APIs.rst:469 #: b32b4fa4788d4f9d88926958c81ad73f @@ -1082,21 +1067,7 @@ msgid "" " switch as a public API)." msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:601 -#: 7f3801d57ac34670bbe16736683db30e -msgid "" -"``/resume_trans `` and ``/forward_resume " -"``: internal transfer-reconnect messages: the receiver asks the " -"sender (or, for a forwarded transfer, the originator) to continue the " -"file from the bytes already on disk. See `resume_transfers`." -msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:605 -#: abd7d3032e15468e92f96a0b5af1f926 -msgid "" -"``/server_file_transfer_port ``: internal protocol " -"message used to coordinate file transfer ports." -msgstr "``/server_file_transfer_port ``：用於協調檔案傳輸連接埠的內部協定訊息。" #: ../../source/Network_APIs/TCP_Server_APIs.rst:607 #: 1e9e32071bb1425b8617a419a7ef87f6 @@ -1521,14 +1492,6 @@ msgid "" "``/file_folder``." msgstr "「handle_command」使用「file_transfer_server_recv_server_start_thread」啟動專用檔案伺服器執行緒。" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:899 -#: e5ef46040255421fa92c6de33d976671 -msgid "" -"The server allocates an ephemeral transfer port with ``palloc`` and sends" -" ``/server_file_transfer_port `` back to the client." -msgstr "" -"伺服器使用“palloc”分配臨時傳輸連接埠並將“/server_file_transfer_port " -"”傳回客戶端。" #: ../../source/Network_APIs/TCP_Server_APIs.rst:903 #: 6eaa70d4d985489e8f0e683baa6b52a1 @@ -1937,11 +1900,6 @@ msgstr "" msgid "`pause_trans_remote` / `start_trans_remote`" msgstr "" -#: ../../source/Network_APIs/TCP_Server_APIs.rst:1063 -#: 187f0ce4a65e41adb7ee99b021de4d2b -msgid "`resume_transfers` / `resume_trans`" -msgstr "" - #: ../../source/Network_APIs/TCP_Server_APIs.rst:1065 #: 944bb0f890764ad7a62584cc73782927 msgid "The server console commands:" diff --git a/docs/source/locale/zh_TW/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po b/docs/source/locale/zh_TW/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po index 65c3282..c9861c5 100644 --- a/docs/source/locale/zh_TW/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po +++ b/docs/source/locale/zh_TW/LC_MESSAGES/api/PyFlow.network_api.connect_tcp.po @@ -61,24 +61,6 @@ msgid "" "on the same socket and share its command namespace;" msgstr "" -#: 3488995e2a8349a58393d9f2d1155b99 PyFlow.network_api.connect_tcp:18 of -msgid "" -"an interrupted transfer is resumed after a disconnect: both sides persist" -" their transfer stores under the protocol's ``.Flow`` directory and the " -"receiver asks the sender to continue the file from the bytes already on " -"disk (``resume_transfers``; the wire lines are ``/resume_trans`` and " -"``/forward_resume``)." -msgstr "" - -#: 4cc5743bdfe54148ad38f54f10e688c9 PyFlow.network_api.connect_tcp:24 of -msgid "" -"The forwarding extensions use the module-level parsers " -"`parse_forwarded_message`, `parse_forward_items_and_addrs`, " -"`parse_forward_originator` and `forward_skip_message`; the reconnect " -"logic uses `parse_resume_command`, `parse_forward_resume_command` and " -"`_resume_offset_for`." -msgstr "" - #: 49796529cb7f49ee8131b257212b5420 PyFlow.network_api.connect_tcp:30 of msgid "" "Concepts live in ``docs/source/Network_APIs/TCP_Server_APIs.rst`` and " @@ -86,10 +68,6 @@ msgid "" " the docstrings below." msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_resume_command:1 of -msgid "Split a ``/resume_trans `` line." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 14dcda8b059b480982e3fb03399e665b 99035a24f6e94207b84545f8c71a451e @@ -97,35 +75,14 @@ msgstr "" msgid "Returns" msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_resume_command:3 of -msgid "``(fid, offset)``, or None when the line is malformed." -msgstr "" #: ../../source/api/PyFlow.network_api.connect_tcp.rst #: 19eb6fca79324b0791f8ec7488a5246c msgid "Return type" msgstr "" -#: 39274682c4bd46489ed6fe535b50ede5 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:1 of -msgid "Split a ``/forward_resume `` line." -msgstr "" -#: 09b385f0a98640c981d8563427e44d7e -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:3 of -msgid "``(target, fid, offset)``, or None when the line is malformed." -msgstr "" -#: 04a0882a3a5e4099a114728b4d1c79c0 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:5 of -msgid "``(target, fid, offset)``, or None when the line is" -msgstr "" - -#: 4ead8d8f93644adeb812150e760f7792 -#: PyFlow.network_api.connect_tcp.parse_forward_resume_command:6 of -msgid "malformed." -msgstr "" #: 39274682c4bd46489ed6fe535b50ede5 #: PyFlow.network_api.connect_tcp.parse_forwarded_message:1 of @@ -416,14 +373,6 @@ msgid "" "never the relay." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Server_Base:96 -#: f9528e6c490d4807b949f6730c6ece8e of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when a connection drops and read back on the next reconnect " -"(``resume_transfers``)." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base:110 #: PyFlow.network_api.connect_tcp.TCP_Server_Base:104 #: ed04a3bf285b4f57bb6a5de0c4f7284f of @@ -1599,51 +1548,12 @@ msgid "" "byte." msgstr "" -#: 1634dbe84a95421a92a660f7c6fdd10e -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:1 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:1 of -msgid "Continue the interrupted transfers of a peer that is back." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:3 -#: eb46142fa4e44147ad3ad8c680d5bee0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and, for " -"every download this instance was receiving from ``peer``, opens a fresh " -"transfer listener and asks the sender to continue the file from the bytes" -" already on disk. Restored uploads need no action here: the peer drives " -"their resume from its own side." -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:9 -#: b7d54977ca494a8b9968365dffd9bd3d of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer that is " -"connected right now." -msgstr "" - -#: 15c0118df8da4454a7fea8d529989237 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_transfers:13 of -msgid "Number of resumed receive legs started." -msgstr "" - #: 52cb873f015542b6a94506cebfe735a1 737ed182911443bc8cfff3c4d094b9a9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:1 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:1 of msgid "Continue one interrupted transfer this instance was sending." msgstr "" -#: 50f292ac853841c9b082eb3c3ff4d6f4 6c968a5b77b342ed891883336cfcc255 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:3 -#: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:3 of -msgid "" -"Runs when a receiver asks to continue a transfer (``/resume_trans``, see " -"`resume_transfers` on the receiving side): the leg is re-established " -"against the receiver's fresh listener and the source file is read from " -"the byte offset the receiver already holds. Also the public API for " -"resuming one transfer on demand." -msgstr "" - #: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_trans:9 #: PyFlow.network_api.connect_tcp.TCP_Server_Base.resume_trans:9 #: b541e8446e374bc8b38cd7912a1fa33c of @@ -1772,14 +1682,6 @@ msgid "" "resumes the interrupted transfers)." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base:102 -#: c194bbee7cf14d268c330e06f94ae973 of -msgid "" -"Path under the protocol's ``.Flow`` directory where the two stores are " -"written when the connection drops and read back on the next reconnect " -"(`resume_transfers`)." -msgstr "" - #: 40f2a9e1b730422db3dd58bc2a6046e3 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:1 of msgid "Create the client and, unless extended, connect and start reading." @@ -1846,16 +1748,6 @@ msgid "" "client's forward path does not read it today. Defaults to 2048." msgstr "" -#: 14c4889553fb42f7b0393a4a7ab46b4a -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:72 of -msgid "" -"Reconnect (and resume the interrupted transfers, see `resume_transfers`) " -"when the connection drops while `start_TCP_client` runs. The transfers " -"that died with the connection are persisted to the protocol's ``.Flow`` " -"directory and read back on the next connect. Defaults to False: a dropped" -" connection ends the client." -msgstr "" - #: 0f06ff3b6db940da96ec09bee3dba526 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.__init__:80 of msgid "If ``is_wait_server`` is True and ``timeout`` is not None." @@ -2224,18 +2116,6 @@ msgstr "" msgid "Dispatch one command line pushed by the server." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.handle_server_command:3 -#: a7e781fde939475cb081204adac16ded of -msgid "" -"Handles the protocol's own lines: ``/crypto_mode`` (a mismatch closes the" -" connection), ``/client_alloc_port_range``, the ``/crypto_*`` exchange " -"lines, and the transfer lines ``/file``, ``/file_folder``, " -"``/server_file_transfer_port``, ``/forward_upload``, ``/pause_trans``, " -"``/start_trans``, ``/resume_trans``, ``/forward_resume`` and " -"``/forward_error``. Any other name goes to the handlers registered for " -"the \"server\" side via `register_command`; an unknown command is only " -"reported on the console." -msgstr "" #: 91942d88960c4793b5eab8c3654c1400 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_messages:1 of @@ -2303,25 +2183,6 @@ msgid "" "there instead of its first byte." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:3 -#: a3c59554b8de41b280df7536b307bed0 of -msgid "" -"Reads the persisted stores back (`_load_transfer_progress`) and drives " -"every download that can be continued: a file the server was sending is" -msgstr "" - -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:6 -#: fae378e888be4821a14ed2483cfde1e0 of -msgid "" -"Peer ``(ip, port)`` that just connected; None resumes every peer the " -"client can reach." -msgstr "" - -#: 48061103ed9d4bcaae8a8a837b363557 -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.resume_transfers:10 of -msgid "Number of resumed transfers started or requested." -msgstr "" - #: 6dbae803a56c442588b28af664b7a0e9 #: PyFlow.network_api.connect_tcp.TCP_Client_Base.forward_file_console:1 of msgid "" @@ -2373,15 +2234,6 @@ msgid "" "while still connecting returns quietly, because that stop was asked for." msgstr "" -#: PyFlow.network_api.connect_tcp.TCP_Client_Base.start_TCP_client:9 -#: eec663f58484446ebce460e6402d7b5f of -msgid "" -"With ``is_auto_reconnect`` True a dropped connection does not end the " -"client: it reconnects (waiting for the server when ``is_wait_server`` is " -"set) and every session carries the interrupted transfers of the previous " -"one (`resume_transfers`). A close or ``/quit`` still stops the client." -msgstr "" - #~ msgid "" #~ "Concepts live in " #~ "``docs/Network_APIs/TCP_Server_APIs.rst`` and " @@ -2570,19 +2422,6 @@ msgstr "" #~ " and releases the port range." #~ msgstr "" -#~ msgid "" -#~ "Handles the protocol's own lines: " -#~ "``/crypto_mode`` (a mismatch closes the " -#~ "connection), ``/client_alloc_port_range``, the " -#~ "``/crypto_*`` exchange lines, and the " -#~ "transfer lines ``/file``, ``/file_folder``, " -#~ "``/forward_upload``, ``/pause_trans``, ``/start_trans``," -#~ " ``/forward_error``. Any other name goes" -#~ " to the handlers registered for the" -#~ " \"server\" side via `register_command`; an" -#~ " unknown command is only reported on" -#~ " the console." -#~ msgstr "" #~ msgid "" #~ "Stops the receive loop, releases the " diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 2cba9e6..a661936 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -1,6 +1,7 @@ """Shared helpers for the integration tests.""" import collections +import errno import socket import time @@ -42,6 +43,67 @@ def free_port(): raise RuntimeError("the OS kept handing out recently used ports") +def inject_wsaeacces_once(monkeypatch, *, operation, address): + """Inject one Windows access-denied socket error on a matching operation.""" + state = {"raised": False, "calls": []} + real_socket = socket.socket + + def access_denied(): + error = OSError(errno.EACCES, "simulated WSAEACCES") + error.winerror = 10013 + return error + + class WsaeaccesSocket(real_socket): + def bind(self, local_address): + state["calls"].append(("bind", local_address)) + if operation == "bind" and local_address == address and not state["raised"]: + state["raised"] = True + raise access_denied() + return super().bind(local_address) + + def connect(self, remote_address): + state["calls"].append(("connect", remote_address)) + if operation == "connect" and remote_address == address and not state["raised"]: + state["raised"] = True + raise access_denied() + return super().connect(remote_address) + + monkeypatch.setattr(socket, "socket", WsaeaccesSocket) + return state + + +def read_text_or_none(path): + """Return a file's text, or None while it is missing or unreadable. + + A receiving leg opens its target as soon as the transfer starts (truncated + for a fresh transfer), so ``exists()`` turning true says nothing about the + bytes having arrived. Polling the content must tolerate the moment between + the open and the first chunk. + """ + try: + return path.read_text() + except OSError: + return None + + +def wait_for_text(path, expected, timeout=10.0): + """Poll a file until its text equals ``expected``. + + Waiting for the content, not for ``exists()``, is what makes a receive + assertion race-free: the receiver creates (and truncates) the target when + the leg starts, before any byte is written. + + Args: + path (Path): File the transfer writes. + expected (str): Text the finished file must hold. + timeout (float): Seconds to wait. + + Returns: + bool: True once the content matched. + """ + return wait_until(lambda: read_text_or_none(path) == expected, timeout=timeout) + + def isolate_instance_files(instance, tmp_path, name): """Point one instance's logs and transfer progress file at a temp directory. @@ -53,3 +115,32 @@ def isolate_instance_files(instance, tmp_path, name): instance.messages_log_file = str(tmp_path / f"{name}_messages_log.json") instance.events_log_file = str(tmp_path / f"{name}_events_log.json") instance.file_trans_progress_file = str(tmp_path / f"{name}_file_trans_progress.json") + + +def request_transfer_listener(sender, receiver, command, *, timeout=10.0): + """Request a real listener over the control channel before low-level I/O. + + ``command`` contains its existing trailing FID; this helper registers that + socket/FID pair before sending the unchanged command. + """ + from PyFlow.network_api.connect_tcp import ( + TCP_Server_Base, + _send_transfer_request, + _transfer_handshake, + ) + + if isinstance(sender, TCP_Server_Base): + peer = receiver.client_socket.getsockname() + assert wait_until(lambda: peer in sender.clients), "receiver not registered" + control = sender.clients[peer]["socket"] + else: + control = sender.client_socket + command = command.strip() + fid = int(command.rsplit(" ", 1)[1]) + with _transfer_handshake(sender, control, fid) as record: + sent = _send_transfer_request(sender, record, command) + assert sent, ("listener request was not sent", command) + result = sender._transfer_handshakes.wait(record, timeout) + assert result is not None and result[0] == "port", ("listener did not reply", result) + assert result[1] == fid, ("listener changed the FID", result) + return result[2], result[3] diff --git a/test/integration/network_api/test_crypto_tcp.py b/test/integration/network_api/test_crypto_tcp.py index 3aa8064..9723a20 100644 --- a/test/integration/network_api/test_crypto_tcp.py +++ b/test/integration/network_api/test_crypto_tcp.py @@ -15,7 +15,7 @@ import pytest -from helpers import free_port, server_ready, wait_until +from helpers import free_port, request_transfer_listener, server_ready, wait_until from PyFlow.network_api import rsa_crypto from PyFlow.network_api.connect_tcp import ( @@ -128,18 +128,48 @@ def _server_sock(server, client): def _wait_server_reexchanged(server, client, timeout=20): - """Wait until the server dropped the socket (decode failure) and - re-flipped it (re-exchange complete), or the client died.""" + """Wait until the server counted a decode failure and framed the socket again. + + Both observations are monotonic, so neither can be missed between polls: + ``decode_failures`` survives a re-exchange (only genuine peer traffic + resets it) and framing is a settled state, not a transient one. Waiting on + the socket leaving ``_encrypted_sockets`` raced the poll interval — a + re-exchange on loopback can finish inside one 50 ms tick. + """ sock = _server_sock(server, client) - if not wait_until(lambda: sock not in server._encrypted_sockets, timeout=timeout): + + def counted(): + with server._crypto_lock: + state = server._crypto_state.get(server._crypto_sock_addr.get(sock)) + return bool(state is not None and state.get("decode_failures")) + + def framed(): + channel = server._channels.get(sock) + return bool(channel is not None and channel.secure) + + if not wait_until(counted, timeout=timeout): return False if not client.running: return False + return wait_until(lambda: framed() and client.running, timeout=timeout) + +def _server_streak(server, sock): + """Read the server's consecutive decode-failure count for one control socket. + + This is the quantity the breaker acts on, and unlike ``channel.secure`` or + ``_encrypted_sockets`` it is monotonic across a re-exchange: only genuine + peer traffic resets it (protocol lines such as the re-exchange do not). A + test that samples transient state can miss the window entirely on loopback. + """ + with server._crypto_lock: + state = server._crypto_state.get(server._crypto_sock_addr.get(sock)) + return 0 if state is None else state.get("decode_failures", 0) - return wait_until( - lambda: sock in server._encrypted_sockets and client.running, - timeout=timeout, - ) + +def _server_framed(server, sock): + """Return whether the server's channel for ``sock`` currently speaks frames.""" + channel = server._channels.get(sock) + return bool(channel is not None and channel.secure) def _roundtrip(client, message, timeout=2.0): @@ -224,38 +254,51 @@ def sender(t): def test_slow_sender_does_not_reorder_frames(tcp_pair): """A thread preempted between seq allocation and the wire write must not let a later sender overtake it (the peer would drop the earlier - frame as out-of-order). Deterministic: the first sender's write is - delayed under the per-connection send lock.""" + frame as out-of-order). Deterministic: the first sender's framed write + is held under the per-connection send lock. Only that thread's framed + write is stalled; the handshake's control lines and the ECDH publish + are written by other threads and pass straight through.""" server, client, _ = tcp_pair assert _wait_flip(client) + channel = client._channels[client.client_socket] + assert wait_until(lambda: channel.frames_allowed.is_set(), timeout=5) + assert wait_until( + lambda: channel.frames_allowed.is_set() + and server._channels[_server_sock(server, client)].frames_allowed.is_set(), + timeout=5, + ) gate = threading.Event() - stalled = [0] - - class _StallingSocket(socket.socket): - __slots__ = () # keep the exact socket layout so __class__ assignment works + stalled = threading.Event() + first_sender = [] # the thread that issues "first"; only its framed write stalls + real_sendall = channel._sendall - def sendall(self, data): - if stalled[0] == 0: - stalled[0] = 1 # stall exactly one (the first) sender after seq allocation - gate.wait() - super().sendall(data) + def stalling_sendall(data): + if threading.current_thread() is first_sender[0] and not stalled.is_set(): + stalled.set() # the first sender holds seq 0 and is inside the send lock + gate.wait() + real_sendall(data) - client.client_socket.__class__ = _StallingSocket + channel._sendall = stalling_sendall buf = io.StringIO() - with contextlib.redirect_stdout(buf): - t1 = threading.Thread(target=client.send_message, args=(client.client_socket, "first")) - t1.start() - time.sleep(0.05) # t1 allocates seq 0 and stalls inside sendall - t2 = threading.Thread(target=client.send_message, args=(client.client_socket, "second")) - t2.start() - time.sleep(0.1) # t2 attempts its send while t1 is still stalled + try: + with contextlib.redirect_stdout(buf): + t1 = threading.Thread(target=client.send_message, args=(client.client_socket, "first")) + first_sender.append(t1) + t1.start() + assert stalled.wait(timeout=5), "first sender never reached the framed write" + t2 = threading.Thread(target=client.send_message, args=(client.client_socket, "second")) + t2.start() + time.sleep(0.1) # t2 attempts its send while t1 is still stalled + gate.set() + t1.join() + t2.join() + echoed = wait_until( + lambda: "msg send: first" in buf.getvalue() and "msg send: second" in buf.getvalue(), + timeout=10, + ) + finally: gate.set() - t1.join() - t2.join() - echoed = wait_until( - lambda: "msg send: first" in buf.getvalue() and "msg send: second" in buf.getvalue(), - timeout=10, - ) + channel._sendall = real_sendall assert echoed, f"frame reordering dropped a message; echoes: {buf.getvalue()!r}" @@ -494,25 +537,25 @@ def test_decode_failure_burst_closes_connection(tcp_pair): forever. Only a frame the server still parses as framed is counted: one that - lands after it dropped framing again (mid re-exchange) is stale and is - deliberately ignored — and the framed state is the channel's ``secure`` - flag, which is set after the socket rejoins ``_encrypted_sockets``. - Waiting on that flag keeps every injection counted. + lands while it has dropped framing again (mid re-exchange) is stale and is + deliberately ignored, so each injection waits for framing first. The + observation after the injection is the breaker's own counter, not the + framed flag: on loopback the re-exchange can finish inside one poll tick, + which makes "the flag is currently down" unobservable even though the + failure was counted. """ server, client, _ = tcp_pair assert _wait_flip(client) sock = _server_sock(server, client) - def server_framed(): - channel = server._channels.get(sock) - return bool(channel is not None and channel.secure) - - for _ in range(MAX_DECODE_FAILURES): - assert wait_until(server_framed, timeout=30), "the server never framed the channel again" + for expected in range(1, MAX_DECODE_FAILURES + 1): + assert wait_until( + lambda: _server_framed(server, sock), timeout=30 + ), "the server never framed the channel again" client.client_socket.sendall(_frame(FRAME_TYPE_RSA, b"garbage")) - # the frame was parsed, counted, and framing dropped again assert wait_until( - lambda: not server_framed() or not client.running, timeout=30 + lambda: _server_streak(server, sock) >= expected or not client.running, + timeout=30, ), "the malformed frame was not counted by the breaker" assert _wait_disconnected(client) @@ -525,44 +568,56 @@ def test_stale_frame_coalesced_behind_a_plaintext_line_is_consumed(tcp_pair): frame behind it into one read; serving those frame bytes as a line echoes them as a message and resets the decode-failure streak, so the breaker never trips (that leak made the burst test above fail on Windows CI). + + The re-exchange is held from the start: on loopback it can finish before a + poll ever observes the unframed window, and then the injected "stale" + frame would be framed and counted a second time. """ server, client, _ = tcp_pair assert _wait_flip(client) sock = _server_sock(server, client) + assert wait_until(lambda: _server_framed(server, sock), timeout=30) - def server_framed(): - channel = server._channels.get(sock) - return bool(channel is not None and channel.secure) - - # one counted failure drops the server's framing and re-exchanges - assert wait_until(server_framed, timeout=30) - client.client_socket.sendall(_frame(FRAME_TYPE_RSA, b"garbage")) - assert wait_until(lambda: not server_framed(), timeout=30) - - # hold the server in the re-exchange window: what lands next is stale + dropped = [] # the channel's own stale-frame notice + channel = server._channels[sock] + real_channel_debug = channel._debug + channel._debug = dropped.append real_try_flip = server._crypto_try_server_flip - server._crypto_try_server_flip = lambda *args, **kwargs: None + gate = threading.Event() - def streak(): - with server._crypto_lock: - state = server._crypto_state.get(server._crypto_sock_addr.get(sock)) - return None if state is None else state.get("decode_failures") + def held_flip(*args, **kwargs): + # runs on the handshake worker: a raise here would surface as an + # unhandled thread exception, so the gate only delays the flip + gate.wait(timeout=30) + return real_try_flip(*args, **kwargs) - dropped = [] # the channel's own stale-frame notice - server._channels[sock]._debug = dropped.append - client.client_socket.sendall( - CRYPTO_FRAMES_START.encode("utf-8") + b"\n" + _frame(FRAME_TYPE_RSA, b"garbage") - ) - - assert wait_until( - lambda: any("dropped a stale frame" in entry for entry in dropped), timeout=5 - ), "the stale frame behind the line was not consumed as a frame" - assert streak() == 1, "frame bytes echoed as a line reset the decode-failure streak" + server._crypto_try_server_flip = held_flip + try: + # one counted failure drops the server's framing ... + client.client_socket.sendall(_frame(FRAME_TYPE_RSA, b"garbage")) + assert wait_until( + lambda: _server_streak(server, sock) == 1, timeout=30 + ), "the first malformed frame was not counted" - # let the re-exchange settle before the fixture tears down: an in-flight - # key push would race the stop (its thread reports an unhandled exception) - server._crypto_try_server_flip = real_try_flip - assert wait_until(server_framed, timeout=30), "the re-exchange never settled" + # ... and the re-exchange is held, so what lands next is stale + client.client_socket.sendall( + CRYPTO_FRAMES_START.encode("utf-8") + b"\n" + _frame(FRAME_TYPE_RSA, b"garbage") + ) + assert wait_until( + lambda: any("dropped a stale frame" in entry for entry in dropped), timeout=5 + ), "the stale frame behind the line was not consumed as a frame" + assert _server_streak(server, sock) == 1, ( + "frame bytes echoed as a line reset the decode-failure streak" + ) + finally: + # let the re-exchange settle before the fixture tears down: an in-flight + # key push would race the stop (its thread reports an unhandled exception) + server._crypto_try_server_flip = real_try_flip + gate.set() # let a flip that already reached the gate run now + channel._debug = real_channel_debug + assert wait_until(lambda: _server_framed(server, sock), timeout=30), ( + "the re-exchange never settled" + ) def test_mode_negotiation_survives_server_announcement_race(tmp_path): @@ -613,10 +668,10 @@ def test_flip_completes_even_if_push_ack_is_lost(tcp_pair, monkeypatch): server, client, _ = tcp_pair orig_send = server.send_message - def flaky_ack(client_socket, message): + def flaky_ack(client_socket, message, is_send_raw=False): if message == server.server_received_file_data_sign: raise OSError(9, "Bad file descriptor") # the flaky ack send - return orig_send(client_socket, message) + return orig_send(client_socket, message, is_send_raw=is_send_raw) monkeypatch.setattr(server, "send_message", flaky_ack) assert _wait_flip(client) @@ -811,11 +866,7 @@ def test_encrypted_file_transfer_client_to_server(tcp_pair, tmp_path, monkeypatc src = tmp_path / "payload.bin" src.write_bytes(payload) - server_sock = _server_sock(server, client) - server.file_transfer_server_recv_server_start_thread("cid", server_sock, f"/file {src} 0") - assert wait_until(lambda: list(client.file_server_port_list), timeout=10) - with client.file_transfer_server_port_lock: - port, _cid, stamp = client.file_server_port_list[0] + port, stamp = request_transfer_listener(client, server, f"/file {src} 0") assert stamp.startswith("enc:"), f"listener stamped {stamp!r}" assert client.file_transfer_mode( str(src), "127.0.0.1", port, 0, control_socket=client.client_socket, crypto=stamp @@ -844,12 +895,7 @@ def test_encrypted_file_transfer_server_to_client(tcp_pair, tmp_path, monkeypatc src = tmp_path / "big.bin" src.write_bytes(payload) - client.file_transfer_client_recv_server_start_thread( - "cid", client.client_socket, f"/file {src} 0" - ) - assert wait_until(lambda: list(server.file_server_port_list), timeout=10) - with server.file_transfer_server_port_lock: - port, _cid, stamp = server.file_server_port_list[0] + port, stamp = request_transfer_listener(server, client, f"/file {src} 0") assert stamp.startswith("enc:"), f"listener stamped {stamp!r}" assert server.file_transfer_mode( str(src), @@ -911,7 +957,7 @@ def test_encrypted_forward_between_clients(tmp_path, monkeypatch): src = tmp_path / "fwd.bin" src.write_bytes(payload) dest = receiver.client_socket.getsockname() - sender._forward_one("file", "", "fwd.bin", str(src), [dest]) + assert sender._forward_one("file", "", "fwd.bin", str(src), [dest]) is True received = recv_dir / "fwd.bin" assert wait_until( @@ -920,6 +966,9 @@ def test_encrypted_forward_between_clients(tmp_path, monkeypatch): assert received.read_bytes() == payload assert stamps and stamps[0].startswith("enc:"), "the target leg was not encrypted" assert raw and payload not in raw, "the payload appeared in the clear on the wire" + assert sender._transfer_progress_snapshot()["relays"] == {} + assert sender.file_upload_progress == {} + assert wait_until(lambda: not server._fid_routes and not server._fid_route_index) finally: for c in clients: c.close() diff --git a/test/integration/network_api/test_listener_lifecycle.py b/test/integration/network_api/test_listener_lifecycle.py new file mode 100644 index 0000000..d2b50f2 --- /dev/null +++ b/test/integration/network_api/test_listener_lifecycle.py @@ -0,0 +1,196 @@ +"""Lifecycle tests for the TCP listener delivery contract. + +``close()`` must join the client receive thread and both ``close()`` and +``stop()`` must stop new listener deliveries. A stopped instance keeps no +listener callbacks, so nothing can be invoked after the call returns. +""" + +import threading + +import pytest + +from helpers import free_port, isolate_instance_files, server_ready, wait_until + +from PyFlow.network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base + + +@pytest.fixture +def pair(tmp_path): + port = free_port() + server = TCP_Server_Base( + host="127.0.0.1", + port=port, + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + is_print_log=False, + ) + isolate_instance_files(server, tmp_path, "server") + threading.Thread(target=server.start_TCP_Server, daemon=True).start() + assert server_ready(server), "server did not start" + client = TCP_Client_Base( + host="127.0.0.1", + port=port, + client_host="127.0.0.1", + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + is_print_log=False, + ) + isolate_instance_files(client, tmp_path, "client") + assert client.connect() + yield server, client + client.close() + server.stop() + + +def _server_sock(server, client): + """Return the server-side socket for a connected client.""" + return server.clients[client.client_socket.getsockname()]["socket"] + + +def test_close_joins_receive_thread_and_clears_listeners(pair): + """close() waits for the receive thread and stops delivering events.""" + _server, client = pair + events = [] + client.add_message_listener(lambda sender, message: events.append(("msg", message))) + client.add_send_message_listener(lambda target, message: events.append(("send", message))) + client.add_file_listener(lambda *args: events.append(("file", args))) + assert client.receive_thread is not None and client.receive_thread.is_alive() + + client.close() + + assert not client.receive_thread.is_alive() + assert client._listeners_stopped is True + assert client._message_listeners == [] + assert client._send_message_listeners == [] + assert client._file_listeners == [] + + before = len(events) + client._notify_message_received("peer", "late") + client._notify_transfer_chunk(client.client_socket, b"late") + client._notify_message_sent(client.client_socket, "late") + client._notify_file_received("path", "name", 1, "/cmd") + assert len(events) == before + client.close() # idempotent + + +def test_close_from_receive_listener_does_not_self_join(pair): + """close() called on the receive thread must not join itself.""" + server, client = pair + closed = threading.Event() + + def listener(sender, message): + if message == "close-now" and not closed.is_set(): + client.close() + closed.set() + + second_called = threading.Event() + + def second_listener(sender, message): + if message == "close-now": + second_called.set() + + client.add_message_listener(listener) + client.add_message_listener(second_listener) + server.send_message(_server_sock(server, client), "close-now") + + assert closed.wait(3.0), "close() from the receive listener did not return" + assert not second_called.is_set(), "a later listener ran after close()" + assert wait_until(lambda: not client.receive_thread.is_alive(), timeout=3.0) + + +def test_close_does_not_deliver_inflight_chunk_after_return(pair): + """A chunk already read by the receive thread must not reach a listener.""" + server, client = pair + marker_events = [] + + def listener(sender, message): + if isinstance(message, bytes): + if b"RACE-MARKER" in message: + marker_events.append(("bytes", message)) + elif "RACE-MARKER" in message: + marker_events.append(("str", message)) + + client.add_message_listener(listener) + entered = threading.Event() + release = threading.Event() + original_measure = client._measure_network_speed + + def measure(payload, is_upload=False): + if isinstance(payload, (bytes, bytearray)) and b"RACE-MARKER" in payload: + entered.set() + release.wait(5.0) + return original_measure(payload, is_upload=is_upload) + + client._measure_network_speed = measure + server_sock = _server_sock(server, client) + server.send_message(server_sock, "RACE-MARKER") + assert entered.wait(2.0), "the marker chunk never reached the client receive path" + + close_thread = threading.Thread(target=client.close, daemon=True) + close_thread.start() + try: + assert wait_until(lambda: client._listeners_stopped, timeout=1.0) + assert close_thread.is_alive(), "close() returned before joining the receive thread" + finally: + release.set() + close_thread.join(3.0) + + assert not close_thread.is_alive() + assert not client.receive_thread.is_alive() + assert marker_events == [] + + +def test_close_before_connect_is_idempotent(tmp_path): + """close() without a session is a safe no-op and still stops listeners.""" + client = TCP_Client_Base( + host="127.0.0.1", + port=free_port(), + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + is_print_log=False, + ) + isolate_instance_files(client, tmp_path, "client") + events = [] + client.add_message_listener(lambda sender, message: events.append(message)) + + client.close() + client.close() + + assert client.receive_thread is None + assert client._listeners_stopped is True + assert client._message_listeners == [] + assert events == [] + + +def test_stop_clears_listeners_and_blocks_new_delivery(tmp_path): + """stop() drops every listener and refuses any later notification.""" + server = TCP_Server_Base( + host="127.0.0.1", + port=free_port(), + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + is_print_log=False, + ) + isolate_instance_files(server, tmp_path, "server") + events = [] + server.add_message_listener(lambda *args: events.append("msg")) + server.add_send_message_listener(lambda *args: events.append("send")) + server.add_file_listener(lambda *args: events.append("file")) + + server.stop() + + assert server._listeners_stopped is True + assert server._message_listeners == [] + assert server._send_message_listeners == [] + assert server._file_listeners == [] + + before = len(events) + server._notify_message_received("peer", "late") + server._notify_transfer_chunk(object(), b"late") + server._notify_message_sent(object(), "late") + server._notify_file_received("peer", "path", "name", 1, "/cmd") + assert len(events) == before diff --git a/test/integration/network_api/test_message_event_store.py b/test/integration/network_api/test_message_event_store.py index bdc0f6e..15aefa0 100644 --- a/test/integration/network_api/test_message_event_store.py +++ b/test/integration/network_api/test_message_event_store.py @@ -210,7 +210,7 @@ def test_flush_keeps_sender_key_after_connection_closed(pair): assert any(e[0] == "persist me" for e in data.get(key, [])), ( f"no entry under {key}; log keys: {list(data)}" ) - server._flush_events_dict() # nothing buffered under the socket any more + server._flush_record_stores() # the outbound stores also buffer this socket assert server_sock not in server._socket_keys diff --git a/test/integration/network_api/test_send_message_store.py b/test/integration/network_api/test_send_message_store.py new file mode 100644 index 0000000..caa81a5 --- /dev/null +++ b/test/integration/network_api/test_send_message_store.py @@ -0,0 +1,138 @@ +"""Tests for the outbound side: send listeners and the sent_* stores. + +Contract on both TCP classes: +- every payload written by ``send_message`` (plaintext) is reported to the + send listeners, text and bytes alike; +- ``is_send_raw`` protocol lines are neither reported nor recorded; +- ``sent_events_dict`` holds outbound "/" commands; +- ``sent_messages_dict`` holds outbound plain text; +- bytes payloads are never recorded in a store. +""" + +import threading + +import pytest + +from helpers import free_port, server_ready, wait_until + +from PyFlow.network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base + + +@pytest.fixture +def pair(tmp_path): + port = free_port() + server = TCP_Server_Base( + host="127.0.0.1", + port=port, + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + server.messages_log_file = str(tmp_path / "server_messages_log.json") + server.events_log_file = str(tmp_path / "server_events_log.json") + server.sent_messages_log_file = str(tmp_path / "server_sent_messages_log.json") + server.sent_events_log_file = str(tmp_path / "server_sent_events_log.json") + server.file_trans_progress_file = str(tmp_path / "server_file_trans_progress.json") + threading.Thread(target=server.start_TCP_Server, daemon=True).start() + assert server_ready(server), "server did not start" + client = TCP_Client_Base( + host="127.0.0.1", + port=port, + client_host="127.0.0.1", + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + client.messages_log_file = str(tmp_path / "client_messages_log.json") + client.events_log_file = str(tmp_path / "client_events_log.json") + client.sent_messages_log_file = str(tmp_path / "client_sent_messages_log.json") + client.sent_events_log_file = str(tmp_path / "client_sent_events_log.json") + client.file_trans_progress_file = str(tmp_path / "client_file_trans_progress.json") + assert client.connect() + yield server, client + client.close() + server.stop() + + +def _server_sock(server, client): + return server.clients[client.client_socket.getsockname()]["socket"] + + +def _client_key(client): + ip, port = client.client_socket.getsockname() + return f"{ip}:{port}" + + +def test_server_send_listener_sees_text_and_commands(pair): + server, client = pair + sent = [] + server.add_send_message_listener(lambda target, msg: sent.append((target, msg))) + sock = _server_sock(server, client) + server.send_message(sock, "hello client") + server.send_message(sock, "/client_alloc_port_range NO_LIMIT") + assert wait_until(lambda: len(sent) >= 2), f"send listener not called {sent=}" + assert (_client_key(client), "hello client") in sent + assert (_client_key(client), "/client_alloc_port_range NO_LIMIT") in sent + + +def test_server_sent_stores_split_text_and_commands(pair): + server, client = pair + sock = _server_sock(server, client) + server.send_message(sock, "plain outbound") + server.send_message(sock, "/probe 1") + assert wait_until( + lambda: any(e[0] == "/probe 1" for e in server.sent_events_dict.get(sock, [])) + ), "command not recorded in sent_events_dict" + texts = [e[0] for e in server.sent_messages_dict.get(sock, [])] + assert "plain outbound" in texts + assert all(e[0].startswith("/") for e in server.sent_events_dict[sock]) + + +def test_is_send_raw_is_not_reported_or_recorded(pair): + server, client = pair + sent = [] + server.add_send_message_listener(lambda target, msg: sent.append(msg)) + sock = _server_sock(server, client) + server.send_message(sock, "/crypto_mode 0", is_send_raw=True) + server.send_message(sock, "after raw") + assert wait_until(lambda: "after raw" in sent), f"listener never fired {sent=}" + assert "/crypto_mode 0" not in sent + assert not any(e[0] == "/crypto_mode 0" for e in server.sent_events_dict.get(sock, [])) + + +def test_bytes_are_reported_but_not_recorded(pair): + server, client = pair + sent = [] + server.add_send_message_listener(lambda target, msg: sent.append(msg)) + sock = _server_sock(server, client) + chunk = b"\x2fbinary-chunk" # starts with "/" as a byte: must not be recorded + server.send_message(sock, chunk) + server.send_message(sock, "marker") + assert wait_until(lambda: "marker" in sent), f"listener never fired {sent=}" + assert chunk in sent + recorded = [e[0] for e in server.sent_events_dict.get(sock, [])] + recorded += [e[0] for e in server.sent_messages_dict.get(sock, [])] + assert "binary-chunk" not in " ".join(recorded) + + +def test_client_send_listener_and_sent_stores(pair): + server, client = pair + sent = [] + client.add_send_message_listener(lambda target, msg: sent.append((target, msg))) + client.send_message(client.client_socket, "hello server") + client.send_message(client.client_socket, "/time") + assert wait_until(lambda: len(sent) >= 2), f"client send listener not called {sent=}" + sock = client.client_socket + assert [e[0] for e in client.sent_messages_dict.get(sock, [])] == ["hello server"] + assert [e[0] for e in client.sent_events_dict.get(sock, [])] == ["/time"] + + +def test_failed_send_is_not_reported(pair): + server, client = pair + sent = [] + client.add_send_message_listener(lambda target, msg: sent.append(msg)) + client.running = False # client refuses the write: nothing was sent + assert client.send_message(client.client_socket, "never sent") is False + assert sent == [] + recorded = client.sent_messages_dict.get(client.client_socket, []) + assert not any(e[0] == "never sent" for e in recorded) diff --git a/test/integration/network_api/test_side_channel_records.py b/test/integration/network_api/test_side_channel_records.py new file mode 100644 index 0000000..29a08db --- /dev/null +++ b/test/integration/network_api/test_side_channel_records.py @@ -0,0 +1,151 @@ +"""Side-channel reads and the control/transfer split of the message stores. + +Contract on both TCP classes: +- a control socket's inbound lines are recorded (messages_dict / events_dict); +- a side-channel (transfer) socket's chunks are reported to the message + listeners as raw bytes and are never recorded in any store, including the + ``sign`` tokens the transfer protocol exchanges on that socket; +- outbound writes on a side-channel socket are reported to the send listeners + but never recorded in sent_messages_dict / sent_events_dict, while outbound + control commands are recorded in sent_events_dict. +""" + +import os +import threading + +import pytest + +from helpers import free_port, server_ready, wait_until + +from PyFlow.network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base + + +@pytest.fixture +def pair(tmp_path): + port = free_port() + server = TCP_Server_Base( + host="127.0.0.1", + port=port, + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + server.messages_log_file = str(tmp_path / "server_messages_log.json") + server.events_log_file = str(tmp_path / "server_events_log.json") + server.sent_messages_log_file = str(tmp_path / "server_sent_messages_log.json") + server.sent_events_log_file = str(tmp_path / "server_sent_events_log.json") + server.file_trans_progress_file = str(tmp_path / "server_file_trans_progress.json") + threading.Thread(target=server.start_TCP_Server, daemon=True).start() + assert server_ready(server), "server did not start" + client = TCP_Client_Base( + host="127.0.0.1", + port=port, + client_host="127.0.0.1", + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + client.messages_log_file = str(tmp_path / "client_messages_log.json") + client.events_log_file = str(tmp_path / "client_events_log.json") + client.sent_messages_log_file = str(tmp_path / "client_sent_messages_log.json") + client.sent_events_log_file = str(tmp_path / "client_sent_events_log.json") + client.file_trans_progress_file = str(tmp_path / "client_file_trans_progress.json") + recv_dir = tmp_path / "recv" + recv_dir.mkdir() + client.file_transfer_dir = str(recv_dir) + assert client.connect() + yield server, client, tmp_path + client.close() + server.stop() + + +def _server_sock(server, client): + return server.clients[client.client_socket.getsockname()]["socket"] + + +def _push_command(client, src): + """Build the console-form push command the suite uses for a client address.""" + addr = client.client_socket.getsockname() + return '/file "{}" "({}, {})"'.format(src, repr(addr[0]), addr[1]) + + +def test_server_push_chunks_reach_client_listeners_and_no_client_store(pair): + """A large server->client push moves raw chunks over the side channel. + + The chunks reach the client's message listeners as bytes, and nothing the + transfer exchanges on that socket lands in a client store. + """ + server, client, tmp_path = pair + size = 512 * 1024 + src = tmp_path / "big.bin" + src.write_bytes(os.urandom(size)) + chunks = [] + client.add_message_listener(lambda sender, msg: chunks.append(msg)) + before = { + (id(store), key): len(entries) + for store in client._record_dicts() + for key, entries in store.items() + } + server.file_transfer_server_recv_client_start( + _push_command(client, src), file_folder_abspath=None + ) + + def raw_bytes(): + return sum(len(m) for m in chunks if isinstance(m, bytes)) + + assert wait_until(lambda: raw_bytes() >= size, timeout=30), ( + f"listeners saw {raw_bytes()} of {size} bytes" + ) + for store in client._record_dicts(): + for key, entries in store.items(): + for entry in entries[before.get((id(store), key), 0):]: + assert isinstance(entry[0], str) + assert not entry[0].startswith( + (client.send_file_header_sign, client.send_file_data_sign) + ) + + +def test_side_channel_sign_reported_but_not_recorded_on_server(pair): + """Side-channel sign writes are reported to the send listeners and never + recorded. The control push command, sent on the control socket, is + recorded in sent_events_dict under its own wire form.""" + server, client, tmp_path = pair + sent = [] + server.add_send_message_listener(lambda target, msg: sent.append(msg)) + src = tmp_path / "small.bin" + src.write_bytes(b"x" * 4096) + server.file_transfer_server_recv_client_start( + _push_command(client, src), file_folder_abspath=None + ) + server_sock = _server_sock(server, client) + assert wait_until( + lambda: any( + e[0].startswith("/file ") and str(src) in e[0] + for e in server.sent_events_dict.get(server_sock, []) + ), + timeout=15, + ), "the control push command was not recorded in sent_events_dict" + assert wait_until(lambda: server.error_sign in sent, timeout=15), ( + "the side-channel sign was never reported to the send listeners" + ) + assert not any( + e[0].startswith(server.error_sign) for e in server.sent_events_dict.get(server_sock, []) + ), "a side-channel sign was recorded in sent_events_dict" + + +def test_control_lines_still_recorded_on_both_sides(pair): + """Control traffic keeps the record stores: a plain line lands in the + messages store and a command in the events store, on each side.""" + server, client, _ = pair + client.send_message(client.client_socket, "control plain") + client.send_message(client.client_socket, "/time") + sock = _server_sock(server, client) + assert wait_until( + lambda: any(e[0] == "control plain" for e in server.messages_dict.get(sock, [])) + ), "control line not recorded on the server" + assert wait_until( + lambda: any(e[0] == "/time" for e in server.events_dict.get(sock, [])) + ), "control command not recorded on the server" + own = client.client_socket + assert any(e[0] == "control plain" for e in client.sent_messages_dict.get(own, [])) + assert any(e[0] == "/time" for e in client.sent_events_dict.get(own, [])) diff --git a/test/integration/network_api/test_tcp_async_server.py b/test/integration/network_api/test_tcp_async_server.py index 337c713..09eb9df 100644 --- a/test/integration/network_api/test_tcp_async_server.py +++ b/test/integration/network_api/test_tcp_async_server.py @@ -13,7 +13,7 @@ import time import pytest -from helpers import free_port, server_ready, wait_until +from helpers import free_port, request_transfer_listener, server_ready, wait_until from PyFlow.network_api import rsa_crypto from PyFlow.network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base @@ -216,10 +216,7 @@ def test_file_transfer_over_coroutine_control_channel(asynic, tmp_path): try: assert client.connect() assert wait_until(lambda: len(server.clients) == 1) - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - server.file_transfer_server_recv_server_start_thread("cid", server_sock, f"/file {src} 0") - port = _wait_transfer_port(client, timeout=30) - assert port is not None, "client did not advertise a transfer port" + port, _stamp = request_transfer_listener(client, server, f"/file {src} 0", timeout=30.0) sent = client.file_transfer_mode(str(src), "127.0.0.1", port, 0) @@ -234,17 +231,6 @@ def received_matches(): _stop_server(server, thread, join=asynic) -def _wait_transfer_port(instance, timeout=10.0): - """Wait until an instance advertised a file transfer port.""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - with instance.file_transfer_server_port_lock: - ports = list(instance.file_server_port_list) - if ports: - return ports[0][0] - time.sleep(0.05) - return None - def test_stop_ends_the_event_loop(asynic_server): """`stop` releases the accept loop, so the listener stops accepting. diff --git a/test/integration/network_api/test_tcp_broadcast_alloc.py b/test/integration/network_api/test_tcp_broadcast_alloc.py index 28caa01..d0ab409 100644 --- a/test/integration/network_api/test_tcp_broadcast_alloc.py +++ b/test/integration/network_api/test_tcp_broadcast_alloc.py @@ -79,6 +79,34 @@ def test_broadcast_removes_disconnected_clients(server): dead.close.assert_called_once() +def test_broadcast_listener_can_rebroadcast_without_holding_client_lock(): + """A send listener may call broadcast(); the client lock must be released.""" + server = _server() + client_socket = MagicMock() + client_socket.getpeername.return_value = ("127.0.0.1", 1) + server.running = True + server.clients = { + ("127.0.0.1", 1): {"socket": client_socket, "address": ("127.0.0.1", 1)} + } + lock_states = [] + + def listener(target, message): + lock_states.append(server.client_lock.locked()) + if message == "trigger": + server.broadcast("reply") # would deadlock if the lock were held + + server.add_send_message_listener(listener) + worker = threading.Thread(target=lambda: server.broadcast("trigger"), daemon=True) + worker.start() + worker.join(2.0) + try: + assert not worker.is_alive(), "broadcast deadlocked while a listener rebroadcast" + assert lock_states and not any(lock_states) + finally: + if not worker.is_alive(): + server.stop() + + # ---- send_msg_to_specific_client -------------------------------------------- @@ -231,6 +259,30 @@ def test_console_input_status_clients_stop(server, monkeypatch, capsys): server.stop.assert_called_once() +def test_transfers_console_prints_local_progress_without_logging(server, monkeypatch, capsys): + import json + + server.is_print_log = False + server.file_recv_progress[22] = [9, 9, None, "/recv/完整.bin", ("127.0.0.1", 50001)] + server.running = True + lines = iter(["/TrAnSfErS", "/transfers --unknown"]) + + def input_line(): + try: + return next(lines) + except StopIteration: + server.running = False + return "" + + monkeypatch.setattr("builtins.input", input_line) + server.console_input() + output = capsys.readouterr().out + view, end = json.JSONDecoder().raw_decode(output) + row, = view["memory"]["downloading"] + assert row["path"] == "/recv/完整.bin" and row["transferred"] == row["total"] == 9 + assert "/transfers" in output[end:] + + def test_console_input_help_and_unknown(server, monkeypatch, capsys): server.running = True monkeypatch.setattr(server, "stop", MagicMock()) diff --git a/test/integration/network_api/test_tcp_file_transfer.py b/test/integration/network_api/test_tcp_file_transfer.py index e2710a3..0e39754 100644 --- a/test/integration/network_api/test_tcp_file_transfer.py +++ b/test/integration/network_api/test_tcp_file_transfer.py @@ -2,14 +2,21 @@ import os import threading -import time import shlex from pathlib import Path import pytest -from helpers import free_port, isolate_instance_files, server_ready, wait_until +from helpers import ( + free_port, + inject_wsaeacces_once, + isolate_instance_files, + request_transfer_listener, + server_ready, + wait_for_text, + wait_until, +) from PyFlow.network_api.connect_tcp import ( TCP_Client_Base, @@ -61,31 +68,13 @@ def pair(tmp_path): server.stop() -def _wait_port(client, timeout=5.0): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - with client.file_transfer_server_port_lock: - ports = list(client.file_server_port_list) - if ports: - return ports[0][0] - time.sleep(0.05) - return None - - def test_client_receives_file_from_server(pair, tmp_path): server, client, recv_dir = pair src = tmp_path / "payload.bin" payload = os.urandom(2048) src.write_bytes(payload) - # the client side of the transfer: it believes the server sent a - # "/file" command, so it opens a temporary server and advertises the - # port to the server over the control channel - client.file_transfer_client_recv_server_start_thread( - "cid", client.client_socket, f"/file {src} 0" - ) - server_port = _wait_port(server) - assert server_port is not None + server_port, _stamp = request_transfer_listener(server, client, f"/file {src} 0") server.file_transfer_mode(str(src), "127.0.0.1", server_port, 0) @@ -100,12 +89,7 @@ def test_client_receives_large_file_from_server(pair, tmp_path): src = tmp_path / "big.bin" payload = os.urandom(300 * 1024) # ~300 KB, many chunks src.write_bytes(payload) - - client.file_transfer_client_recv_server_start_thread( - "cid", client.client_socket, f"/file {src} 1" - ) - server_port = _wait_port(server) - assert server_port is not None + server_port, _stamp = request_transfer_listener(server, client, f"/file {src} 1") server.file_transfer_mode(str(src), "127.0.0.1", server_port, 0) @@ -124,12 +108,7 @@ def test_server_receives_file_from_client(pair, tmp_path): payload = os.urandom(8192) src.write_bytes(payload) - # the server believes the client sent a "/file" command: it opens a - # temporary server and advertises the port back to the client - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - server.file_transfer_server_recv_server_start_thread("cid", server_sock, f"/file {src} 0") - server_port = _wait_port(client) - assert server_port is not None + server_port, _stamp = request_transfer_listener(client, server, f"/file {src} 0") client.file_transfer_mode(str(src), "127.0.0.1", server_port, 0) @@ -140,6 +119,92 @@ def test_server_receives_file_from_client(pair, tmp_path): assert files[0].read_bytes() == payload +@pytest.mark.parametrize( + "sender_role", ["server", "client"], ids=["server-sender", "client-sender"] +) +@pytest.mark.parametrize("operation", ["bind", "connect"]) +def test_sender_falls_back_when_fixed_source_port_returns_wsaeacces( + pair, tmp_path, monkeypatch, sender_role, operation +): + server, client, client_recv = pair + source = tmp_path / "wsaeacces.bin" + payload = os.urandom(8192) + source.write_bytes(payload) + fixed_port = free_port() + + if sender_role == "server": + sender, receiver = server, client + destination = client_recv / source.name + else: + sender, receiver = client, server + destination_dir = tmp_path / "server_recv" + destination_dir.mkdir() + server.file_transfer_dir = str(destination_dir) + destination = destination_dir / source.name + + transfer_port, _stamp = request_transfer_listener( + sender, receiver, f"/file {shlex.quote(str(source))} 0" + ) + address = ( + ("127.0.0.1", fixed_port) + if operation == "bind" + else ("127.0.0.1", transfer_port) + ) + injected = inject_wsaeacces_once( + monkeypatch, operation=operation, address=address + ) + assert sender.file_transfer_mode( + str(source), "127.0.0.1", transfer_port, fixed_port + ) is True + assert injected["raised"] + binds = [call[1] for call in injected["calls"] if call[0] == "bind"] + assert binds[:2] == [("127.0.0.1", fixed_port), ("127.0.0.1", 0)] + assert wait_until(destination.exists, timeout=10), "the receiver did not create the file" + assert destination.read_bytes() == payload + + +@pytest.mark.parametrize( + "receiver_role", ["server", "client"], ids=["server-listener", "client-listener"] +) +def test_receiver_listener_falls_back_when_fixed_port_returns_wsaeacces( + pair, tmp_path, monkeypatch, receiver_role +): + server, client, client_recv = pair + source = tmp_path / "listener-wsaeacces.bin" + payload = os.urandom(4096) + source.write_bytes(payload) + fixed_port = free_port() + + if receiver_role == "server": + sender, receiver = client, server + destination_dir = tmp_path / "server_recv" + destination_dir.mkdir() + server.file_transfer_dir = str(destination_dir) + listener_address = (server.host, fixed_port) + else: + sender, receiver = server, client + destination_dir = client_recv + listener_address = (client.client_host, fixed_port) + monkeypatch.setattr(receiver, "palloc", lambda: fixed_port) + injected = inject_wsaeacces_once( + monkeypatch, operation="bind", address=listener_address + ) + + transfer_port, _stamp = request_transfer_listener( + sender, receiver, f"/file {shlex.quote(str(source))} 0" + ) + assert injected["raised"] + assert transfer_port != fixed_port + binds = [call[1] for call in injected["calls"] if call[0] == "bind"] + assert binds[:2] == [listener_address, (listener_address[0], 0)] + assert sender.file_transfer_mode( + str(source), "127.0.0.1", transfer_port, 0 + ) is True + destination = destination_dir / source.name + assert wait_until(destination.exists, timeout=10), "the receiver did not create the file" + assert destination.read_bytes() == payload + + def test_transfer_thread_survives_a_stop_before_the_start_sign(pair, tmp_path): """A stop between the accept and the start sign must not kill the thread. @@ -167,22 +232,19 @@ def hook(args): stopped_first = threading.Event() transfer_thread = [] - def stop_then_send(client_socket, message): + def stop_then_send(client_socket, message, is_send_raw=False): if message is server.server_start_file_transfer_sign and not stopped_first.is_set(): # the race under test: the stop lands after the accept and before # the transfer thread's start sign transfer_thread.append(threading.current_thread()) stopped_first.set() server.stop() - return real_send(client_socket, message) + return real_send(client_socket, message, is_send_raw=is_send_raw) server.send_message = stop_then_send threading.excepthook = hook try: - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - server.file_transfer_server_recv_server_start_thread("cid", server_sock, f"/file {src} 0") - server_port = _wait_port(client) - assert server_port is not None + server_port, _stamp = request_transfer_listener(client, server, f"/file {src} 0") client.file_transfer_mode(str(src), "127.0.0.1", server_port, 0) assert wait_until(stopped_first.is_set, timeout=10), ( "the server-side transfer never reached its start sign" @@ -195,7 +257,6 @@ def stop_then_send(client_socket, message): threading.excepthook = previous_hook client.close() server.stop() - thread = transfer_thread[0] assert not thread.is_alive(), "the transfer thread did not end" failures = [args for args in escaped if args.thread is thread] @@ -241,12 +302,7 @@ def test_custom_file_transfer_chunk_len_is_honored_in_both_directions(tmp_path): payload = os.urandom(64 * 1024 + 123) # 64 chunks of 1 KiB, plus a tail src.write_bytes(payload) - # server -> client (server sends, client writes to disk) - client.file_transfer_client_recv_server_start_thread( - "cid", client.client_socket, f"/file {src} 0" - ) - server_port = _wait_port(server) - assert server_port is not None + server_port, _stamp = request_transfer_listener(server, client, f"/file {src} 0") server.file_transfer_mode(str(src), "127.0.0.1", server_port, 0) assert wait_until(lambda: any(recv_dir.iterdir()), timeout=15), ( "client did not receive the file" @@ -257,10 +313,7 @@ def test_custom_file_transfer_chunk_len_is_honored_in_both_directions(tmp_path): server_recv = tmp_path / "server_recv" server_recv.mkdir() server.file_transfer_dir = str(server_recv) - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - server.file_transfer_server_recv_server_start_thread("cid", server_sock, f"/file {src} 0") - client_port = _wait_port(client) - assert client_port is not None + client_port, _stamp = request_transfer_listener(client, server, f"/file {src} 0") client.file_transfer_mode(str(src), "127.0.0.1", client_port, 0) assert wait_until(lambda: any(server_recv.iterdir()), timeout=15), ( "server did not receive the file" @@ -306,13 +359,9 @@ def test_server_receives_file_at_destination(pair, tmp_path): dest.mkdir() # paths must be quoted: shlex treats an unquoted backslash as an escape, - # which mangles Windows paths like C:\Users\... - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - server.file_transfer_server_recv_server_start_thread( - "cid", server_sock, f'/file "{src}" "{dest}" 0' + server_port, _stamp = request_transfer_listener( + client, server, f'/file "{src}" "{dest}" 0' ) - server_port = _wait_port(client) - assert server_port is not None client.file_transfer_mode(str(src), "127.0.0.1", server_port, 0) @@ -363,17 +412,58 @@ def test_client_uploads_folder_to_destination(pair, tmp_path): # the folder keeps its own name under the destination (same structure as # the default receive directory, with the destination as its root) - assert wait_until(lambda: (dest / "data" / "a.txt").exists(), timeout=10), ( - "a.txt not at destination" - ) - assert wait_until(lambda: (dest / "data" / "sub" / "b.txt").exists(), timeout=10), ( - "b.txt not at destination" - ) - assert (dest / "data" / "a.txt").read_text() == "hello" - assert (dest / "data" / "sub" / "b.txt").read_text() == "world" + assert wait_for_text(dest / "data" / "a.txt", "hello"), "a.txt not at destination" + assert wait_for_text(dest / "data" / "sub" / "b.txt", "world"), "b.txt not at destination" assert not any(server_recv.iterdir()), "folder must not land in the default directory" +def test_client_uploads_folder_skips_empty_directories(pair, tmp_path): + """A directory with no file in its subtree is never created at the peer. + + Only a directory that receives a file (directly or deeper) needs to exist; + announcing an empty one would leave an empty directory on the far side. + """ + server, client, _ = pair + server_recv = tmp_path / "server_recv" + server_recv.mkdir() + server.file_transfer_dir = str(server_recv) + folder = tmp_path / "data" + (folder / "sub" / "deep").mkdir(parents=True) + (folder / "empty").mkdir() + (folder / "a.txt").write_text("hello") + (folder / "sub" / "b.txt").write_text("world") + + client.folder_file_transfer_client_recv_client_start(f'/file_folder "{folder}"') + + assert wait_for_text(server_recv / "data" / "a.txt", "hello"), "a.txt not received" + assert wait_for_text(server_recv / "data" / "sub" / "b.txt", "world"), "b.txt not received" + assert not (server_recv / "data" / "empty").exists(), "an empty directory was created" + assert not (server_recv / "data" / "sub" / "deep").exists(), ( + "a directory holding no file was created" + ) + + +def test_folder_creation_command_creates_directory_without_a_transfer(pair, tmp_path): + """``/file_folder []`` creates the directory on the control line. + + The command carries no client id, so it opens no transfer leg: the + directory must exist as soon as the line is processed, with no listener + or port involved. + """ + server, client, _ = pair + server_recv = tmp_path / "server_recv" + server_recv.mkdir() + server.file_transfer_dir = str(server_recv) + dest = tmp_path / "dest" + dest.mkdir() + + client.send_message(client.client_socket, "/file_folder /made/by/command") + client.send_message(client.client_socket, f'/file_folder /made/with/dest "{dest}"') + + assert wait_until(lambda: (server_recv / "made" / "by" / "command").is_dir(), timeout=10) + assert wait_until(lambda: (dest / "made" / "with" / "dest").is_dir(), timeout=10) + + def test_server_pushes_folder_to_destination(pair, tmp_path): """Server /file_folder with a destination dir: the folder structure is recreated under the destination on the client.""" @@ -391,14 +481,8 @@ def test_server_pushes_folder_to_destination(pair, tmp_path): ) server.folder_file_transfer_server_recv_client_start(cmd) - assert wait_until(lambda: (dest / "data" / "a.txt").exists(), timeout=10), ( - "a.txt not at destination" - ) - assert wait_until(lambda: (dest / "data" / "sub" / "b.txt").exists(), timeout=10), ( - "b.txt not at destination" - ) - assert (dest / "data" / "a.txt").read_text() == "hello" - assert (dest / "data" / "sub" / "b.txt").read_text() == "world" + assert wait_for_text(dest / "data" / "a.txt", "hello"), "a.txt not at destination" + assert wait_for_text(dest / "data" / "sub" / "b.txt", "world"), "b.txt not at destination" def test_client_uploads_multiple_files_to_destination(pair, tmp_path): @@ -417,8 +501,8 @@ def test_client_uploads_multiple_files_to_destination(pair, tmp_path): client.multiple_file_transfer_client_recv_client_start(f'/multiple_file "{f1}" "{f2}" "{dest}"') - assert wait_until(lambda: (dest / "one.bin").exists(), timeout=10), "one.bin not at destination" - assert wait_until(lambda: (dest / "two.bin").exists(), timeout=10), "two.bin not at destination" + assert wait_for_text(dest / "one.bin", "one", timeout=10), "one.bin incomplete at destination" + assert wait_for_text(dest / "two.bin", "two", timeout=10), "two.bin incomplete at destination" assert (dest / "one.bin").read_bytes() == b"one" assert (dest / "two.bin").read_bytes() == b"two" @@ -444,14 +528,8 @@ def test_client_uploads_multiple_folders_to_destination(pair, tmp_path): f'/multiple_file_folder "{folder_a}" "{folder_b}" "{dest}"' ) - assert wait_until(lambda: (dest / "data_a" / "a.txt").exists(), timeout=10), ( - "a.txt not at destination" - ) - assert wait_until(lambda: (dest / "data_b" / "b.txt").exists(), timeout=10), ( - "b.txt not at destination" - ) - assert (dest / "data_a" / "a.txt").read_text() == "hello" - assert (dest / "data_b" / "b.txt").read_text() == "world" + assert wait_for_text(dest / "data_a" / "a.txt", "hello"), "a.txt not at destination" + assert wait_for_text(dest / "data_b" / "b.txt", "world"), "b.txt not at destination" def test_windows_paths_need_quoting_through_shlex(): @@ -527,7 +605,6 @@ def _forward_client(server, recv_dir): def test_parse_forward_originator(): """The originator tuple in a pushed /file or /file_folder command is recovered; a direct send (the receiver's own address) is filtered out.""" - # the wire form produced by forward_target_command (tuple shlex-quoted) cmd = lambda *a: TCP_Server_Base.forward_target_command(None, *a) assert parse_forward_originator(cmd("file", "", "a.txt", ("127.0.0.1", 3000), 7)) == ( "127.0.0.1:3000" @@ -631,14 +708,12 @@ def test_forward_folder_to_client(pair, tmp_path): cmd = '/forward_folder "{}" "({}, {})"'.format(folder, repr(a[0]), a[1]) forwarder.forward_folder_console(cmd) - assert wait_until(lambda: (recv_dir / "data" / "a.txt").exists(), timeout=15), ( + assert wait_for_text(recv_dir / "data" / "a.txt", "hello", timeout=15), ( "a.txt not forwarded" ) - assert wait_until(lambda: (recv_dir / "data" / "sub" / "b.txt").exists(), timeout=15), ( + assert wait_for_text(recv_dir / "data" / "sub" / "b.txt", "world", timeout=15), ( "sub/b.txt not forwarded" ) - assert (recv_dir / "data" / "a.txt").read_text() == "hello" - assert (recv_dir / "data" / "sub" / "b.txt").read_text() == "world" forwarder.close() @@ -693,9 +768,9 @@ def test_forward_flow_control_pauses_uploader(pair, tmp_path, monkeypatch): sent = [] real_send = server.send_message - def spy(sock, message): + def spy(sock, message, is_send_raw=False): sent.append(message if isinstance(message, str) else "") - return real_send(sock, message) + return real_send(sock, message, is_send_raw=is_send_raw) monkeypatch.setattr(server, "send_message", spy) a = target.client_socket.getsockname() @@ -754,14 +829,12 @@ def test_forward_folder_with_destination(pair, tmp_path): cmd = '/forward_folder "{}" "({}, {})" "{}"'.format(folder, repr(a[0]), a[1], dest) forwarder.forward_folder_console(cmd) - assert wait_until(lambda: (dest / "data" / "a.txt").exists(), timeout=15), ( + assert wait_for_text(dest / "data" / "a.txt", "hello", timeout=15), ( "a.txt not at the destination" ) - assert wait_until(lambda: (dest / "data" / "sub" / "b.txt").exists(), timeout=15), ( + assert wait_for_text(dest / "data" / "sub" / "b.txt", "world", timeout=15), ( "sub/b.txt not at the destination" ) - assert (dest / "data" / "a.txt").read_text() == "hello" - assert (dest / "data" / "sub" / "b.txt").read_text() == "world" assert not any(recv_dir.iterdir()), "default directory must stay empty" forwarder.close() @@ -791,12 +864,18 @@ def test_multiple_file_multiple_client_console_command(pair, tmp_path): addr2 = client2.client_socket.getsockname() dest = tmp_path / "dest" dest.mkdir() + received = set() + client.add_file_listener(lambda *_: received.add("client1")) + client2.add_file_listener(lambda *_: received.add("client2")) cmd = '/multiple_file_multiple_client "{}" "({}, {})" "({}, {})" "{}"'.format( src, repr(addr1[0]), addr1[1], repr(addr2[0]), addr2[1], dest ) server.multiple_file_multiple_client_transfer_server_recv_client_start(cmd) - assert wait_until(lambda: (dest / "broadcast.bin").exists(), timeout=10), ( - "client 1 did not receive the broadcast file at the destination" + # Both legs write the same destination path and a fresh leg truncates it. + # `exists()` therefore says nothing about the bytes: wait until both + # receivers have reported the fully written file before reading it. + assert wait_until(lambda: received == {"client1", "client2"}, timeout=10), ( + "not every client received the broadcast file at the destination" ) assert not any(recv_dir.iterdir()), "default directory must stay empty on client 1" assert not any(recv2.iterdir()), "default directory must stay empty on client 2" @@ -821,14 +900,9 @@ def test_file_progress_and_speed_are_tracked(pair, tmp_path): payload = os.urandom(300 * 1024) src.write_bytes(payload) fid = 4242 + upload_key = ((client.host, client.port), fid) - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - client_id = "%s:%s" % client.client_socket.getsockname() - server.file_transfer_server_recv_server_start_thread( - client_id, server_sock, f"/file {src} {fid}" - ) - server_port = _wait_port(client) - assert server_port is not None + server_port, _stamp = request_transfer_listener(client, server, f"/file {src} {fid}") assert client.pause_trans(fid) is False # armed before the leg exists sender = threading.Thread( @@ -842,11 +916,11 @@ def test_file_progress_and_speed_are_tracked(pair, tmp_path): assert wait_until(lambda: server.file_recv_progress.get(fid), timeout=10), ( "the receive side did not register its progress" ) - assert wait_until(lambda: client.file_upload_progress.get(fid), timeout=10), ( + assert wait_until(lambda: client.file_upload_progress.get(upload_key), timeout=10), ( "the upload side did not register its progress" ) recv_entry = server.file_recv_progress[fid] - upload_entry = client.file_upload_progress[fid] + upload_entry = client.file_upload_progress[upload_key] assert recv_entry[:2] == [0, len(payload)] assert upload_entry[:2] == [0, len(payload)] # the local path of each side, and the transfer's original peer @@ -870,7 +944,7 @@ def got(): assert recv_entry[2] > 0 and upload_entry[2] > 0 # ... and were dropped when the transfer ended assert fid not in server.file_recv_progress - assert fid not in client.file_upload_progress + assert upload_key not in client.file_upload_progress # the protocol-wide measurement covers the control layer as well assert server.recv_network_speed > 0 assert client.upload_network_speed > 0 @@ -879,29 +953,24 @@ def test_download_progress_and_speed_are_tracked(pair, tmp_path): """The reverse direction is tracked too: the server uploads, the client downloads, and both progress stores follow the same fid. - The server-side pause uses the same public API as the client's: any - outbound transfer is held by the fid it runs under. + The server-side pause uses the same public API as the client's, with an + explicit receiver peer to identify the outbound transfer's namespace. """ server, client, recv_dir = pair src = tmp_path / "download.bin" payload = os.urandom(300 * 1024) src.write_bytes(payload) fid = 5150 + receiver = client.client_socket.getsockname() + upload_key = (receiver, fid) - # the client side of the transfer: it believes the server sent a - # "/file" command, so it opens a temporary server and advertises the - # port to the server over the control channel - client.file_transfer_client_recv_server_start_thread( - "cid", client.client_socket, f"/file {src} {fid}" - ) - server_port = _wait_port(server) - assert server_port is not None + server_port, _stamp = request_transfer_listener(server, client, f"/file {src} {fid}") - assert server.pause_trans(fid) is False # armed before the leg exists + assert server.pause_trans(fid, receiver) is False # armed before the leg exists sender = threading.Thread( target=server.file_transfer_mode, args=(str(src), "127.0.0.1", server_port, 0), - kwargs={"fid": fid, "peer": client.client_socket.getsockname()}, + kwargs={"fid": fid, "peer": receiver}, daemon=True, ) sender.start() @@ -909,11 +978,11 @@ def test_download_progress_and_speed_are_tracked(pair, tmp_path): assert wait_until(lambda: client.file_recv_progress.get(fid), timeout=10), ( "the client did not register its download progress" ) - assert wait_until(lambda: server.file_upload_progress.get(fid), timeout=10), ( + assert wait_until(lambda: server.file_upload_progress.get(upload_key), timeout=10), ( "the server did not register its upload progress" ) recv_entry = client.file_recv_progress[fid] - upload_entry = server.file_upload_progress[fid] + upload_entry = server.file_upload_progress[upload_key] assert recv_entry[:2] == [0, len(payload)] assert upload_entry[:2] == [0, len(payload)] assert recv_entry[3] == str(recv_dir / "download.bin") @@ -921,7 +990,7 @@ def test_download_progress_and_speed_are_tracked(pair, tmp_path): assert recv_entry[4] == (server.host, server.port) assert upload_entry[4] == client.client_socket.getsockname() finally: - server.start_trans(fid) + server.start_trans(fid, receiver) sender.join(timeout=15) assert not sender.is_alive(), "the download did not finish" @@ -934,7 +1003,7 @@ def got(): assert recv_entry[0] == len(payload) and upload_entry[0] == len(payload) assert recv_entry[2] > 0 and upload_entry[2] > 0 assert fid not in client.file_recv_progress - assert fid not in server.file_upload_progress + assert upload_key not in server.file_upload_progress assert client.recv_network_speed > 0 assert server.upload_network_speed > 0 @@ -976,12 +1045,7 @@ def test_file_progress_and_speed_can_be_disabled(tmp_path): src.write_bytes(payload) fid = 7 - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - server.file_transfer_server_recv_server_start_thread( - "cid", server_sock, f"/file {src} {fid}" - ) - server_port = _wait_port(client) - assert server_port is not None + server_port, _stamp = request_transfer_listener(client, server, f"/file {src} {fid}") client.pause_trans(fid) # the pause works with tracking off too sender = threading.Thread( diff --git a/test/integration/network_api/test_tcp_transfer_resume.py b/test/integration/network_api/test_tcp_transfer_resume.py index 3070a65..7bde561 100644 --- a/test/integration/network_api/test_tcp_transfer_resume.py +++ b/test/integration/network_api/test_tcp_transfer_resume.py @@ -15,9 +15,29 @@ import threading import time -from helpers import free_port, isolate_instance_files, server_ready, wait_until - -from PyFlow.network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base +import pytest + +from helpers import ( + free_port, + inject_wsaeacces_once, + isolate_instance_files, + request_transfer_listener, + server_ready, + wait_until, +) + +from PyFlow.network_api.connect_tcp import ( + TRANSFER_SIGN_BUFFER_LIMIT, + TCP_Client_Base, + TCP_Server_Base, + _TransferSignBuffer, + _sending_transfer, + _socket_local_endpoint, + _socket_peer_endpoint, + _start_resume_send, + _transfer_handshake, + format_resume_line, +) def _isolate(instance, tmp_path, name): @@ -25,17 +45,6 @@ def _isolate(instance, tmp_path, name): isolate_instance_files(instance, tmp_path, name) -def _wait_port(instance, timeout=10.0): - """Wait for the transfer port the instance's peer announced.""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - with instance.file_transfer_server_port_lock: - ports = list(instance.file_server_port_list) - if ports: - return ports[0][0] - time.sleep(0.05) - return None - def _redirect_crypto(crypto, tmp_path, ssh_dir, subdir="pub_key"): """Keep one instance's key material inside the test's temp directory.""" @@ -80,12 +89,43 @@ def _start_client(tmp_path, server, name="client", recv_dir=None, **kwargs): return client +def _track_socket_workers(monkeypatch): + """Retain real control, listener, relay and file workers for teardown.""" + workers = [] + real_start = threading.Thread.start + + def start(worker, *args, **kwargs): + result = real_start(worker, *args, **kwargs) + workers.append(worker) + return result + + monkeypatch.setattr(threading.Thread, "start", start) + return workers + + +def _close_socket_workers(server, clients, workers): + for client in clients: + client.close() + server.stop() + deadline = time.monotonic() + 20 + # Joining parents also lets any final child workers enter this list. + for worker in workers: + worker.join(timeout=max(0, deadline - time.monotonic())) + assert not any(worker.is_alive() for worker in workers), ( + "socket workers survived teardown", + [worker.name for worker in workers if worker.is_alive()], + ) + + def test_progress_entries_round_trip_through_the_progress_file(tmp_path): """Both stores are written with their peer tuples and read back intact.""" server = _start_server(tmp_path) + receiver = ("127.0.0.1", 6000) + upload_key = (receiver, 9) + upload_json_key = json.dumps(upload_key, separators=(",", ":")) try: server.file_recv_progress[7] = [128, 1024, None, "/tmp/part.bin", ("127.0.0.1", 5000)] - server.file_upload_progress[9] = [64, 512, None, "/tmp/src.bin", ("127.0.0.1", 6000)] + server.file_upload_progress[upload_key] = [64, 512, None, "/tmp/src.bin", receiver] server._persist_transfer_progress() with open(server.file_trans_progress_file, "r", encoding="utf-8") as f: @@ -97,7 +137,9 @@ def test_progress_entries_round_trip_through_the_progress_file(tmp_path): "/tmp/part.bin", ["127.0.0.1", 5000], ] - assert written["upload"]["9"] == [64, 512, None, "/tmp/src.bin", ["127.0.0.1", 6000]] + assert written["upload"][upload_json_key] == [ + 64, 512, None, "/tmp/src.bin", list(receiver) + ] server.file_recv_progress.clear() server.file_upload_progress.clear() @@ -109,7 +151,7 @@ def test_progress_entries_round_trip_through_the_progress_file(tmp_path): "/tmp/part.bin", ("127.0.0.1", 5000), ] - assert server.file_upload_progress[9] == [ + assert server.file_upload_progress[upload_key] == [ 64, 512, None, @@ -123,11 +165,304 @@ def test_progress_entries_round_trip_through_the_progress_file(tmp_path): server.file_recv_progress[7] = [999, 1024, None, "/tmp/live.bin", None] assert server._load_transfer_progress() == 1 assert server.file_recv_progress[7] == [999, 1024, None, "/tmp/live.bin", None] - assert server.file_upload_progress[9][0] == 64 # noqa: PLR2004 + assert server.file_upload_progress[upload_key][0] == 64 # noqa: PLR2004 finally: server.stop() +def test_legacy_upload_progress_is_rejected_without_overwriting_snapshot(tmp_path): + """Neither TCP class may silently migrate or replace a bare-fid upload file.""" + legacy_bytes = ( + b'{\n "download": {},\n' + b' "upload": {"9": [64, 512, null, "/tmp/src.bin", ["127.0.0.1", 6000]]},\n' + b' "relays": {}\n}\n\n' + ) + for name, tcp_class, persist_first in ( + ("server_load_first", TCP_Server_Base, False), + ("server_persist_first", TCP_Server_Base, True), + ("client_load_first", TCP_Client_Base, False), + ("client_persist_first", TCP_Client_Base, True), + ): + # Keep connection/resume workers out of this on-disk preservation case. + instance = tcp_class( + host="127.0.0.1", + port=free_port(), + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + _isolate(instance, tmp_path, f"legacy_{name}") + progress_path = tmp_path / f"legacy_{name}_file_trans_progress.json" + messages = [] + real_log = instance._log + + def record(message): + messages.append(message) + real_log(message) + + instance._log = record + try: + progress_path.write_bytes(legacy_bytes) + if persist_first: + instance._persist_transfer_progress() + assert progress_path.read_bytes() == legacy_bytes + assert any( + "unsupported" in message.lower() and "upload" in message.lower() + for message in messages + ), f"{name} did not log rejection before its first persistence" + assert instance._load_transfer_progress() == 0 + assert instance.file_upload_progress == {} + assert instance.file_recv_progress == {} + assert any( + "unsupported" in message.lower() and "upload" in message.lower() + for message in messages + ), f"{name} did not log rejection of the unsupported upload key" + assert progress_path.read_bytes() == legacy_bytes + + # Persisting an empty current state must not erase rejected history. + instance._persist_transfer_progress() + assert progress_path.read_bytes() == legacy_bytes + + # A new scoped transfer is not permission to overwrite that history + # either; ordinary shutdown persists the current stores once more. + receiver = ( + (instance.host, instance.port) + if tcp_class is TCP_Client_Base + else ("127.0.0.1", 6000) + ) + upload_key = (receiver, 9) + instance.file_upload_progress[upload_key] = [ + 0, 1024, None, str(tmp_path / f"{name}_new.bin"), receiver + ] + instance._persist_transfer_progress() + assert progress_path.read_bytes() == legacy_bytes + finally: + if tcp_class is TCP_Server_Base: + instance.stop() + else: + instance.close() + assert progress_path.read_bytes() == legacy_bytes + + +def test_equal_server_upload_fids_are_isolated_by_receiver(tmp_path): + """Pausing and clearing one real receiver leaves its equal-id sibling intact.""" + server = _start_server(tmp_path) + clients = [] + senders = [] + receivers = [] + fid = 602 + results = {} + try: + sources = [] + payloads = [] + outputs = [] + for index in range(2): + recv_dir = tmp_path / f"receiver_{index}" + recv_dir.mkdir() + client = _start_client( + tmp_path, server, name=f"client_{index}", recv_dir=recv_dir + ) + clients.append(client) + receiver = client.client_socket.getsockname() + receivers.append(receiver) + assert wait_until(lambda: receiver in server.clients) + src = tmp_path / f"payload_{index}.bin" + payload = os.urandom((index + 1) * 128 * 1024 + 17) + src.write_bytes(payload) + sources.append(src) + payloads.append(payload) + outputs.append(recv_dir / src.name) + assert server.pause_trans(fid, receiver) is False + results[receiver] = request_transfer_listener( + server, client, f"/file {shlex.quote(str(src))} {fid}" + ) + + # Both listeners share a numeric FID but have distinct request sockets. + ports = {receiver: results.pop(receiver) for receiver in reversed(receivers)} + assert all(port is not None for port in ports.values()) + + def send(receiver, src): + port, stamp = ports[receiver] + results[receiver] = server.file_transfer_mode( + str(src), receiver[0], port, 0, + control_socket=server.clients[receiver]["socket"], + crypto=stamp, fid=fid, peer=receiver, + ) + + for receiver, src in zip(receivers, sources): + sender = threading.Thread(target=send, args=(receiver, src), daemon=True) + senders.append(sender) + sender.start() + + keys = [(receiver, fid) for receiver in receivers] + assert wait_until( + lambda: all(key in server.file_upload_progress for key in keys), timeout=15 + ), "the equal-id uploads did not register independently" + assert wait_until( + lambda: all(fid in client.file_recv_progress for client in clients), timeout=10 + ) + rows = [server.file_upload_progress[key] for key in keys] + assert rows[0] is not rows[1] + for row, src, payload, receiver in zip(rows, sources, payloads, receivers): + assert row == [0, len(payload), None, str(src), receiver] + sockets = [server._active_transfer_sockets[key] for key in keys] + assert sockets[0] is not sockets[1] + assert all(key in server._active_transfer_fids for key in keys) + assert server.pause_trans(fid, receivers[0]) is True + + assert server._drop_stale_transfers(receivers[0]) == 1 + assert sockets[0].fileno() == -1 + assert keys[0] not in server._active_transfer_sockets + assert keys[0] not in server._active_transfer_fids + assert server._active_transfer_sockets[keys[1]] is sockets[1] + assert keys[1] in server._active_transfer_fids + + # Release only the dead leg's waiter and let its real cleanup finish. + # If start/pause or cleanup is fid-only, the other leg now sends or dies. + server.start_trans(fid, receivers[0]) + senders[0].join(timeout=15) + assert not senders[0].is_alive(), "the cleared upload did not stop" + assert results[receivers[0]] is False + assert server.file_upload_progress[keys[1]] is rows[1] + assert rows[1][0] == 0 + assert clients[1].file_recv_progress[fid][0] == 0 + assert server._transfer_pause[keys[1]] is True + assert server._active_transfer_sockets[keys[1]] is sockets[1] + assert senders[1].is_alive() + + assert server.start_trans(fid, receivers[1]) is True + senders[1].join(timeout=20) + assert not senders[1].is_alive(), "the unaffected receiver did not finish" + assert results[receivers[1]] is True + assert outputs[1].read_bytes() == payloads[1] + assert rows[1][0] == len(payloads[1]) + assert wait_until(lambda: fid not in clients[1].file_recv_progress, timeout=10) + assert keys[1] not in server.file_upload_progress + assert keys[1] not in server._active_transfer_sockets + assert keys[1] not in server._active_transfer_fids + finally: + for receiver in receivers: + server.start_trans(fid, receiver) + for client in clients: + client.close() + server.stop() + for sender in senders: + sender.join(timeout=10) + + +def test_equal_upload_fids_persist_and_reload_per_receiver(tmp_path): + """Two real client addresses keep separate compact JSON keys and five fields.""" + server = _start_server(tmp_path) + clients = [] + try: + for index in range(2): + clients.append(_start_client(tmp_path, server, name=f"client_{index}")) + receivers = [client.client_socket.getsockname() for client in clients] + assert receivers[0] != receivers[1] + fid = 604 + keys = [(receiver, fid) for receiver in receivers] + rows = [ + [128, 1024, None, str(tmp_path / "first.bin"), receivers[0]], + [256, 2048, None, str(tmp_path / "second.bin"), receivers[1]], + ] + server.file_upload_progress.update(zip(keys, rows)) + server._persist_transfer_progress() + with open(server.file_trans_progress_file, "r", encoding="utf-8") as f: + written = json.load(f) + json_keys = [json.dumps(key, separators=(",", ":")) for key in keys] + assert set(written["upload"]) == set(json_keys) + for json_key, row in zip(json_keys, rows): + assert written["upload"][json_key] == row[:4] + [list(row[4])] + + server.file_upload_progress.clear() + assert server._load_transfer_progress() == 2 + assert server.file_upload_progress == dict(zip(keys, rows)) + live = [512, 1024, None, str(tmp_path / "live.bin"), receivers[0]] + server.file_upload_progress[keys[0]] = live + del server.file_upload_progress[keys[1]] + assert server._load_transfer_progress() == 1 + assert server.file_upload_progress[keys[0]] is live + assert server.file_upload_progress[keys[1]] == rows[1] + finally: + for client in clients: + client.close() + server.stop() + + +def test_equal_inbound_and_outbound_fids_cleanup_only_their_own_sockets(tmp_path): + """Receive cleanup cannot close an equal-id upload, including a client forward.""" + server = _start_server(tmp_path) + client = None + other = None + sockets = [] + try: + client = _start_client(tmp_path, server) + other = _start_client(tmp_path, server, name="other") + client_peer = client.client_socket.getsockname() + server_peer = (client.host, client.port) + forwarded_peer = other.client_socket.getsockname() + cases = ( + (server, client_peer, forwarded_peer, forwarded_peer), + (client, forwarded_peer, server_peer, forwarded_peer), + ) + for instance, inbound_peer, receiver, logical_peer in cases: + inbound, inbound_remote = socket.socketpair() + outbound, outbound_remote = socket.socketpair() + sockets.extend((inbound, inbound_remote, outbound, outbound_remote)) + inbound_remote.settimeout(2) + outbound_remote.settimeout(2) + fid = 606 + upload_key = (receiver, fid) + recv_entry = instance._file_progress_start( + instance.file_recv_progress, fid, 1024, + str(tmp_path / "received.bin"), peer=inbound_peer, + ) + upload_entry = instance._file_progress_start( + instance.file_upload_progress, upload_key, 2048, + str(tmp_path / "source.bin"), peer=logical_peer, + ) + instance._register_transfer_socket(fid, inbound) + instance._register_transfer_socket(upload_key, outbound) + assert instance.pause_trans(fid, receiver) is True + assert len(recv_entry) == len(upload_entry) == 5 + assert upload_entry[4] == logical_peer + + assert instance._drop_stale_transfers(inbound_peer) == 1 + assert inbound_remote.recv(1) == b"" + assert fid not in instance._active_transfer_sockets + assert fid not in instance._active_transfer_fids + instance._file_progress_finish(instance.file_recv_progress, fid, recv_entry) + instance._release_transfer_socket(fid, inbound) + assert fid not in instance.file_recv_progress + assert instance.file_upload_progress[upload_key] is upload_entry + assert upload_key in instance._active_transfer_fids + assert instance._active_transfer_sockets[upload_key] is outbound + assert instance._transfer_pause[upload_key] is True + outbound.sendall(b"x") + assert outbound_remote.recv(1) == b"x" + + # The client's receiver is its server even when row[4] names the + # logical forward target; cleanup must use the scoped namespace. + assert instance._drop_stale_transfers(receiver) == 1 + assert outbound_remote.recv(1) == b"" + assert upload_key not in instance._active_transfer_sockets + assert upload_key not in instance._active_transfer_fids + instance._file_progress_finish( + instance.file_upload_progress, upload_key, upload_entry + ) + instance._release_transfer_socket(upload_key, outbound) + assert upload_key not in instance.file_upload_progress + instance.start_trans(fid, receiver) + finally: + for sock in sockets: + sock.close() + if other is not None: + other.close() + if client is not None: + client.close() + server.stop() + + def test_paused_transfer_is_persisted_when_the_connection_drops(tmp_path): """A disconnect during a transfer writes both stores for the next connect. @@ -146,19 +481,14 @@ def test_paused_transfer_is_persisted_when_the_connection_drops(tmp_path): src = tmp_path / "payload.bin" payload = os.urandom(64 * 1024) src.write_bytes(payload) - fid = 77 + fid = 76 # even low bit: a plain leg's id - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - sender_id = "%s:%s" % client.client_socket.getsockname() - server.file_transfer_server_recv_server_start_thread( - sender_id, server_sock, f"/file {src} {fid}" - ) - assert _wait_port(client) is not None + transfer_port, _stamp = request_transfer_listener(client, server, f"/file {src} {fid}") client.pause_trans(fid) # hold the leg before its first chunk sender = threading.Thread( target=client.file_transfer_mode, - args=(str(src), "127.0.0.1", _wait_port(client) or 0, 0), + args=(str(src), "127.0.0.1", transfer_port, 0), kwargs={"fid": fid, "peer": (server.host, server.port)}, daemon=True, ) @@ -176,8 +506,9 @@ def test_paused_transfer_is_persisted_when_the_connection_drops(tmp_path): with open(client.file_trans_progress_file, "r", encoding="utf-8") as f: client_state = json.load(f) - assert client_state["upload"][str(fid)][3] == str(src) - assert client_state["upload"][str(fid)][4] == [server.host, server.port] + upload_json_key = json.dumps(((client.host, client.port), fid), separators=(",", ":")) + assert client_state["upload"][upload_json_key][3] == str(src) + assert client_state["upload"][upload_json_key][4] == [server.host, server.port] assert wait_until( lambda: os.path.exists(server.file_trans_progress_file), timeout=10 @@ -213,7 +544,7 @@ def test_upload_resumes_after_a_reconnect(tmp_path): src = tmp_path / "payload.bin" payload = os.urandom(256 * 1024) src.write_bytes(payload) - fid = 101 + fid = 100 # even low bit: a plain leg's id # a fixed client port keeps the client's address stable across the # reconnect, so the server can match the interrupted transfer to it @@ -232,17 +563,18 @@ def test_upload_resumes_after_a_reconnect(tmp_path): assert wait_until(lambda: client.running and client.client_socket is not None) client_addr = client.client_socket.getsockname() - server_sock = server.clients[client_addr]["socket"] - client_id = "%s:%s" % client_addr - server.file_transfer_server_recv_server_start_thread( - client_id, server_sock, f"/file {shlex.quote(str(src))} {fid}" + # the server registers an accepted client on its own thread: wait for it + assert wait_until(lambda: client_addr in server.clients, timeout=10), ( + "server never registered the reconnecting client" + ) + transfer_port, _stamp = request_transfer_listener( + client, server, f"/file {shlex.quote(str(src))} {fid}" ) - assert _wait_port(client) is not None client.pause_trans(fid) # freeze the leg, then give it a partial file sender = threading.Thread( target=client.file_transfer_mode, - args=(str(src), "127.0.0.1", _wait_port(client) or 0, 0), + args=(str(src), "127.0.0.1", transfer_port, 0), kwargs={"fid": fid, "peer": (server.host, server.port)}, daemon=True, ) @@ -281,7 +613,10 @@ def got(): assert wait_until(got, timeout=30), "the resumed upload did not finish" assert partial.read_bytes() == payload assert wait_until(lambda: fid not in server.file_recv_progress, timeout=10) - assert wait_until(lambda: fid not in client.file_upload_progress, timeout=10) + assert wait_until( + lambda: ((client.host, client.port), fid) not in client.file_upload_progress, + timeout=10, + ) client.start_trans(fid) sender.join(timeout=10) finally: @@ -290,6 +625,90 @@ def got(): server.stop() +@pytest.mark.parametrize("kind", ["file", "folder"]) +@pytest.mark.parametrize("ack_failure", ["false", "exception"]) +def test_server_ack_failure_keeps_received_file_until_eof_confirmation( + tmp_path, monkeypatch, kind, ack_failure +): + """Failed final ACKs preserve the original receive row, even on EOF retry.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + recv_dir = tmp_path / "server_recv" + recv_dir.mkdir() + server.file_transfer_dir = str(recv_dir) + client = _start_client(tmp_path, server) + clients.append(client) + peer = client.client_socket.getsockname() + assert wait_until(lambda: peer in server.clients) + src = tmp_path / "ack_failure.bin" + payload = os.urandom(64 * 1024 + 37) + src.write_bytes(payload) + fid = 606 + output = recv_dir / "bundle/nested" / src.name if kind == "folder" else recv_dir / src.name + upload_key = ((server.host, server.port), fid) + attempts = [] + outcomes = [] + real_send = server.send_message + real_transfer = client.file_transfer_mode + + def fail_final_ack(sock, message, is_send_raw=False): + if message == server.server_received_file_data_sign: + attempts.append(sock) + if len(attempts) <= 2: + if ack_failure == "exception": + raise ConnectionError("injected Server final ACK write failure") + return False + return real_send(sock, message, is_send_raw=is_send_raw) + + def observe_transfer(*args, **kwargs): + result = real_transfer(*args, **kwargs) + outcomes.append((kwargs.get("fid"), kwargs.get("resume_offset", 0), result)) + return result + + monkeypatch.setattr(server, "send_message", fail_final_ack) + monkeypatch.setattr(client, "file_transfer_mode", observe_transfer) + command = ( + f"/file_folder bundle/nested {src.name} {fid}" if kind == "folder" + else f"/file {shlex.quote(str(src))} {fid}" + ) + port, stamp = request_transfer_listener(client, server, command) + assert client.file_transfer_mode( + str(src), server.host, port, 0, control_socket=client.client_socket, + crypto=stamp, fid=fid, peer=(server.host, server.port), + ) is False + for attempt in range(2): + assert wait_until(lambda: fid not in server._active_transfer_sockets) + assert len(attempts) == attempt + 1 + assert output.read_bytes() == payload + assert server.file_recv_progress[fid] == [ + len(payload), len(payload), None, str(output), peer + ] + assert client.file_upload_progress[upload_key][0:2] == [len(payload), len(payload)] + with open(server.file_trans_progress_file, encoding="utf-8") as stream: + snapshot = json.load(stream) + assert snapshot["download"][str(fid)] == [ + len(payload), len(payload), None, str(output), list(peer) + ] + assert server._request_resumes(peer) == 1 + assert wait_until(lambda: len(outcomes) == attempt + 2, timeout=15) + assert outcomes == [ + (fid, 0, False), (fid, len(payload), False), (fid, len(payload), True) + ] + assert len(attempts) == 3 + assert wait_until(lambda: fid not in server._active_transfer_sockets) + assert fid not in server.file_recv_progress + assert upload_key not in client.file_upload_progress + with open(server.file_trans_progress_file, encoding="utf-8") as stream: + assert json.load(stream)["download"] == {} + assert output.read_bytes() == payload + if kind == "folder": + assert not (recv_dir / src.name).exists() + finally: + _close_socket_workers(server, clients, workers) + + def test_download_resumes_from_the_partial_file(tmp_path): """The receiver drives a resume of a file the server was sending. @@ -306,7 +725,7 @@ def test_download_resumes_from_the_partial_file(tmp_path): src = tmp_path / "download.bin" payload = os.urandom(128 * 1024) src.write_bytes(payload) - fid = 202 + fid = 202 # even low bit: a plain leg's id partial = recv_dir / "download.bin" with open(partial, "wb") as f: @@ -318,7 +737,8 @@ def test_download_resumes_from_the_partial_file(tmp_path): str(partial), (server.host, server.port), ] - server.file_upload_progress[fid] = [ + upload_key = (client.client_socket.getsockname(), fid) + server.file_upload_progress[upload_key] = [ 0, len(payload), None, @@ -326,7 +746,7 @@ def test_download_resumes_from_the_partial_file(tmp_path): client.client_socket.getsockname(), ] - assert client.resume_transfers(peer=(server.host, server.port)) == 1 + client._request_resumes() # the same call connect() dispatches on reconnect def got(): return partial.stat().st_size == len(payload) @@ -334,14 +754,289 @@ def got(): assert wait_until(got, timeout=20), "the resumed download did not finish" assert partial.read_bytes() == payload assert wait_until(lambda: fid not in client.file_recv_progress, timeout=10) - assert wait_until(lambda: fid not in server.file_upload_progress, timeout=10) + assert wait_until(lambda: upload_key not in server.file_upload_progress, timeout=10) finally: if client is not None: client.close() server.stop() -def test_forwarded_file_resumes_through_the_server(tmp_path): +@pytest.mark.parametrize("kind", ["file", "folder"]) +@pytest.mark.parametrize("offset", [0, 4096], ids=["initial", "resumed"]) +def test_server_final_error_keeps_upload_until_successful_resume( + tmp_path, monkeypatch, kind, offset +): + """A real error sign must preserve a direct upload, including an EOF replay.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + recv_dir = tmp_path / "receiver" + recv_dir.mkdir() + client = _start_client(tmp_path, server, recv_dir=recv_dir) + clients.append(client) + receiver = client.client_socket.getsockname() + assert wait_until(lambda: receiver in server.clients) + src = tmp_path / "rejected.bin" + payload = os.urandom(64 * 1024 + 37) + src.write_bytes(payload) + fid = 604 + key = (receiver, fid) + output = recv_dir / "bundle/nested" / src.name if kind == "folder" else recv_dir / src.name + if offset: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(payload[:offset]) + client.file_recv_progress[fid] = [ + offset, len(payload), None, str(output), (server.host, server.port) + ] + server.file_upload_progress[key] = [ + offset, len(payload), None, str(src), receiver + ] + header_seen = threading.Event() + rejected = threading.Event() + real_receive = server.receive_message + real_send = client.send_message + + def observe_header(sock, *args, **kwargs): + data = real_receive(sock, *args, **kwargs) + if data.strip() == server.server_received_file_header_sign.encode("utf-8"): + header_seen.set() + return data + + def reject_final_ack(sock, message, is_send_raw=False): + if message == client.server_received_file_data_sign and not rejected.is_set(): + # Separate the two signs even on an unframed TCP stream. + assert header_seen.wait(timeout=10) + assert real_send(sock, client.error_sign) + rejected.set() + raise ConnectionError("injected receiver rejection after the file body") + return real_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(server, "receive_message", observe_header) + monkeypatch.setattr(client, "send_message", reject_final_ack) + command = ( + f"/file_folder bundle/nested {src.name} {fid}" if kind == "folder" + else f"/file {shlex.quote(str(src))} {fid}" + ) + port, stamp = request_transfer_listener(server, client, command) + result = server.file_transfer_mode( + str(src), receiver[0], port, 0, + control_socket=server.clients[receiver]["socket"], crypto=stamp, + fid=fid, peer=receiver, resume_offset=offset, + ) + assert rejected.is_set() + assert result is False + assert output.read_bytes() == payload + assert server.file_upload_progress[key] == [ + len(payload), len(payload), None, str(src), receiver + ] + with open(server.file_trans_progress_file, encoding="utf-8") as stream: + snapshot = json.load(stream) + json_key = json.dumps(key, separators=(",", ":")) + assert snapshot["upload"][json_key] == [ + len(payload), len(payload), None, str(src), list(receiver) + ] + assert wait_until(lambda: fid not in client._active_transfer_fids) + assert client._request_resumes() == 1 + assert wait_until(lambda: key not in server.file_upload_progress, timeout=15) + assert wait_until(lambda: fid not in client.file_recv_progress) + assert output.read_bytes() == payload + if kind == "folder": + assert not (recv_dir / src.name).exists() + finally: + _close_socket_workers(server, clients, workers) + + +def _coalesce_receiver_signs(monkeypatch, sender, control, ack): + """Deliver the real start line, then combine the real header and ACK.""" + real_receive = sender.receive_message + pending = b"" + start_delivered = False + merged = [] + + def receive(sock, length): + nonlocal pending, start_delivered + if sock is control or merged: + return real_receive(sock, length) + while True: + if not start_delivered and b"\n" in pending: + start, pending = pending.split(b"\n", 1) + start_delivered = True + return start + b"\n" + if start_delivered and pending.endswith(ack): + merged.append(pending) + pending = b"" + return merged[-1] + chunk = real_receive(sock, length) + if not chunk: + data, pending = pending, b"" + return data + pending += chunk + + monkeypatch.setattr(sender, "receive_message", receive) + return merged + + +@pytest.mark.parametrize("server_sends", [False, True], ids=["upload", "download"]) +@pytest.mark.parametrize("eof_replay", [False, True], ids=["initial", "eof"]) +def test_coalesced_receiver_signs_keep_the_final_ack( + tmp_path, monkeypatch, server_sends, eof_replay +): + """Both senders retain a final ACK sharing a read with the header sign.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + client = _start_client(tmp_path, server) + clients.append(client) + client_addr = client.client_socket.getsockname() + assert wait_until(lambda: client_addr in server.clients) + sender, receiver = (server, client) if server_sends else (client, server) + peer = client_addr if server_sends else (server.host, server.port) + origin = (server.host, server.port) if server_sends else client_addr + control = server.clients[client_addr]["socket"] if server_sends else client.client_socket + recv_dir = tmp_path / "coalesced_recv" + recv_dir.mkdir() + receiver.file_transfer_dir = str(recv_dir) + src = tmp_path / "coalesced.bin" + payload = os.urandom(48 * 1024 + 11) + src.write_bytes(payload) + output = recv_dir / src.name + fid = 707 + offset = len(payload) if eof_replay else 0 + if eof_replay: + output.write_bytes(payload) + receiver.file_recv_progress[fid] = [ + offset, len(payload), None, str(output), origin + ] + sender.file_upload_progress[(peer, fid)] = [ + offset, len(payload), None, str(src), peer + ] + header = (receiver.server_received_file_header_sign + "\n").encode("utf-8") + ack = (receiver.server_received_file_data_sign + "\n").encode("utf-8") + merged = _coalesce_receiver_signs(monkeypatch, sender, control, ack) + port, stamp = request_transfer_listener( + sender, receiver, f"/file {shlex.quote(str(src))} {fid}" + ) + assert sender.file_transfer_mode( + str(src), peer[0], port, 0, control_socket=control, + crypto=stamp, fid=fid, peer=peer, resume_offset=offset, + ) is True + assert merged == [header + ack], "the real receiver signs did not share one read" + assert output.read_bytes() == payload + assert wait_until(lambda: fid not in receiver.file_recv_progress) + assert (peer, fid) not in sender.file_upload_progress + finally: + _close_socket_workers(server, clients, workers) + + +def _watch_sign_capacity(monkeypatch, sender, receiver, overflow, capacity_handled): + """Hold body transmission while queued signs compact or a line overflows.""" + body_held = threading.Event() + retained = [] + errors = [] + compactions = [] + real_feed = _TransferSignBuffer.feed + real_scan = _TransferSignBuffer._scan_pending_locked + real_send = receiver.send_message + + def observe_scan(signs): + before = signs.pending_bytes + real_scan(signs) + if before >= TRANSFER_SIGN_BUFFER_LIMIT: + compactions.append(before) + capacity_handled.set() + + def observe_feed(signs, data): + try: + return real_feed(signs, data) + except ValueError as exc: + errors.append(str(exc)) + capacity_handled.set() + raise + finally: + with signs.lock: + retained.append(signs.pending_bytes + len(signs.partial)) + + def inject_after_header(sock, message, is_send_raw=False): + written = real_send(sock, message, is_send_raw=is_send_raw) + if message == receiver.server_received_file_header_sign: + assert body_held.wait(timeout=10) + noise = ( + b"x" * TRANSFER_SIGN_BUFFER_LIMIT if overflow == "partial" + else b"x\n" * (TRANSFER_SIGN_BUFFER_LIMIT // 2 + 1) + ) + try: + real_send(sock, noise) + except OSError: + pass # the sender may already have rejected an overlong line + return written + + def hold_body(_key, leg_socket=None): + body_held.set() + assert capacity_handled.wait(timeout=10), "the buffer never reached or rejected its limit" + + monkeypatch.setattr(_TransferSignBuffer, "feed", observe_feed) + monkeypatch.setattr(_TransferSignBuffer, "_scan_pending_locked", observe_scan) + monkeypatch.setattr(receiver, "send_message", inject_after_header) + monkeypatch.setattr(sender, "_wait_transfer_paused", hold_body) + return retained, errors, compactions + +@pytest.mark.parametrize("server_sends", [False, True], ids=["upload", "download"]) +@pytest.mark.parametrize("overflow", ["partial", "pending"]) +def test_sign_capacity_compacts_complete_rows_and_rejects_oversized_line( + tmp_path, monkeypatch, server_sends, overflow +): + """Full-line batches compact; a single unterminated line still fails.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + capacity_handled = threading.Event() + try: + client = _start_client(tmp_path, server) + clients.append(client) + client_addr = client.client_socket.getsockname() + assert wait_until(lambda: client_addr in server.clients) + sender, receiver = (server, client) if server_sends else (client, server) + peer = client_addr if server_sends else (server.host, server.port) + control = server.clients[client_addr]["socket"] if server_sends else client.client_socket + recv_dir = tmp_path / "overflow_recv" + recv_dir.mkdir() + receiver.file_transfer_dir = str(recv_dir) + src = tmp_path / "overflow.bin" + payload = b"222\n\n111 " * 1024 + src.write_bytes(payload) + fid = 709 + retained, errors, compactions = _watch_sign_capacity( + monkeypatch, sender, receiver, overflow, capacity_handled + ) + port, stamp = request_transfer_listener(sender, receiver, f"/file {src} {fid}") + result = sender.file_transfer_mode( + str(src), peer[0], port, 0, control_socket=control, + crypto=stamp, fid=fid, peer=peer, + ) + assert capacity_handled.is_set() + assert retained and max(retained) <= TRANSFER_SIGN_BUFFER_LIMIT + if overflow == "partial": + assert result is False + assert len(errors) == 1 and "unterminated transfer sign" in errors[0] + assert not compactions + row = sender.file_upload_progress[(peer, fid)] + assert row[1] == len(payload) and row[3] == str(src) and row[4] == peer + with open(sender.file_trans_progress_file, encoding="utf-8") as stream: + persisted = json.load(stream)["upload"] + assert json.dumps((peer, fid), separators=(",", ":")) in persisted + else: + assert result is True + assert not errors and compactions and max(compactions) == TRANSFER_SIGN_BUFFER_LIMIT + assert (recv_dir / src.name).read_bytes() == payload + assert (peer, fid) not in sender.file_upload_progress + finally: + capacity_handled.set() + _close_socket_workers(server, clients, workers) + + +def test_forwarded_file_resumes_through_the_server(tmp_path, monkeypatch): """A target that lost a forwarded file gets the remainder re-sent. The target asks the server to continue the transfer; the server routes @@ -350,6 +1045,7 @@ def test_forwarded_file_resumes_through_the_server(tmp_path): download entry and the originator's upload entry name the original peer, never the relay. """ + workers = _track_socket_workers(monkeypatch) server = _start_server(tmp_path) recv_dir = tmp_path / "target_recv" recv_dir.mkdir() @@ -361,8 +1057,8 @@ def test_forwarded_file_resumes_through_the_server(tmp_path): src = tmp_path / "payload.bin" payload = os.urandom(96 * 1024) src.write_bytes(payload) - fid = 303 # the forwarder's transfer id - tfid = 304 # the target's transfer id + fid = 302 + tfid = 304 # route ownership, never a low-bit kind, identifies this leg target_addr = target.client_socket.getsockname() forwarder_addr = forwarder.client_socket.getsockname() @@ -379,44 +1075,980 @@ def test_forwarded_file_resumes_through_the_server(tmp_path): str(partial), forwarder_addr, ] - forwarder.file_upload_progress[fid] = [0, len(payload), None, str(src), target_addr] - forwarder._forward_resume_records[fid] = { - "fid": fid, - "kind": "file", - "rel_dir": "", - "fname": "payload.bin", - "path": str(src), - "destination": None, - "targets": [list(target_addr)], - } - server._forward_resume_records[tfid] = { - "fid": fid, - "originator": list(forwarder_addr), - "targets": [list(target_addr)], - "tfids": [tfid], - "kind": "file", - "rel_dir": "", - "fname": "payload.bin", - "destination": None, - } - - assert target._request_forward_resume( - tfid, target.file_recv_progress[tfid], forwarder_addr + upload_key = ((forwarder.host, forwarder.port), fid) + forwarder.file_upload_progress[upload_key] = [0, len(payload), None, str(src), target_addr] + assert server._register_fid_route( + fid, forwarder_addr, {target_addr: tfid} + ) is not None + + allocated_port = free_port() + released_ports = [] + real_pfree = server.pfree + + def record_pfree(port): + released_ports.append(port) + return real_pfree(port) + + monkeypatch.setattr(server, "palloc", lambda: allocated_port) + monkeypatch.setattr(server, "pfree", record_pfree) + injected = inject_wsaeacces_once( + monkeypatch, + operation="bind", + address=(server.host, allocated_port), ) + assert target._request_resumes() == 1 + def got(): return partial.stat().st_size == len(payload) assert wait_until(got, timeout=20), "the forwarded resume did not finish" + assert injected["raised"] + assert wait_until(lambda: allocated_port in released_ports, timeout=10) + bind_addresses = [call[1] for call in injected["calls"] if call[0] == "bind"] + fixed_index = bind_addresses.index((server.host, allocated_port)) + assert (server.host, 0) in bind_addresses[fixed_index + 1:] assert partial.read_bytes() == payload assert wait_until(lambda: tfid not in target.file_recv_progress, timeout=10) - assert wait_until(lambda: fid not in forwarder.file_upload_progress, timeout=10) + assert wait_until(lambda: upload_key not in forwarder.file_upload_progress, timeout=10) finally: - if forwarder is not None: - forwarder.close() - if target is not None: - target.close() - server.stop() + clients = [client for client in (forwarder, target) if client is not None] + _close_socket_workers(server, clients, workers) + + +def test_sender_owned_fids_to_distinct_targets_survive_out_of_order_replies( + tmp_path, monkeypatch +): + """Keep source-owned upstream FIDs paired with targets after reply reordering.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + release_first = threading.Event() + first_reply_waiting = threading.Event() + first_port_blocked = threading.Event() + second_reply_sent = threading.Event() + request_lock = threading.Lock() + request_names = {} + fids_by_name = {} + route_registrations = [] + server_allocations = [] + sender_threads = [] + try: + source = _start_client(tmp_path, server, name="source") + targets = [] + target_dirs = [] + for name in ("first", "second"): + recv_dir = tmp_path / f"{name}_target" + recv_dir.mkdir() + target_dirs.append(recv_dir) + targets.append( + _start_client(tmp_path, server, name=f"{name}_target", recv_dir=recv_dir) + ) + clients.extend((source, *targets)) + source_addr = source.client_socket.getsockname() + target_addrs = [target.client_socket.getsockname() for target in targets] + payloads = { + "first.bin": os.urandom(96 * 1024 + 11), + "second.bin": os.urandom(80 * 1024 + 7), + } + paths = {} + for name, payload in payloads.items(): + path = tmp_path / name + path.write_bytes(payload) + paths[name] = path + assert wait_until(lambda: all( + addr in server.clients for addr in [source_addr, *target_addrs] + )) + + real_source_send = source.send_message + + def capture_source_request(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/forward_item "): + parts = shlex.split(message) + name = next(name for name in payloads if name in parts) + upstream_fid = int(next(part[4:] for part in parts if part.startswith("fid="))) + upload_key = ((source.host, source.port), upstream_fid) + assert source.file_upload_progress[upload_key][3] == str(paths[name]) + with request_lock: + request_names[upstream_fid] = name + fids_by_name[name] = upstream_fid + return real_source_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(source, "send_message", capture_source_request) + real_wait = source._transfer_handshakes.wait + + def observe_wait(record, timeout): + with request_lock: + name = request_names.get(record["key"][1]) + if name == "first.bin": + first_reply_waiting.set() + return real_wait(record, timeout) + + monkeypatch.setattr(source._transfer_handshakes, "wait", observe_wait) + real_register = server._register_fid_route + + def capture_route(fid, originator, route_targets, completed=()): + route_registrations.append((fid, tuple(originator), dict(route_targets))) + return real_register(fid, originator, route_targets, completed) + + monkeypatch.setattr(server, "_register_fid_route", capture_route) + real_allocate = server._fid_alloc.alloc + + def capture_allocation(*, receiver=None): + fid = real_allocate(receiver=receiver) + server_allocations.append((tuple(receiver) if receiver is not None else None, fid)) + return fid + + monkeypatch.setattr(server._fid_alloc, "alloc", capture_allocation) + real_server_send = server.send_message + + def send_second_port_first(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/server_file_transfer_port "): + parts = shlex.split(message) + with request_lock: + name = request_names.get(int(parts[2])) + if name == "first.bin": + first_port_blocked.set() + assert release_first.wait(timeout=20), "first port release was not signaled" + return real_server_send(sock, message, is_send_raw=is_send_raw) + if name == "second.bin": + sent = real_server_send(sock, message, is_send_raw=is_send_raw) + second_reply_sent.set() + return sent + return real_server_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(server, "send_message", send_second_port_first) + outcomes = {} + failures = [] + + def forward(name, target_addr): + try: + outcomes[name] = source._forward_one( + "file", "", name, str(paths[name]), [target_addr] + ) + except Exception as exc: + failures.append((name, exc)) + + first_sender = threading.Thread( + target=forward, args=("first.bin", target_addrs[0]), daemon=True + ) + sender_threads.append(first_sender) + first_sender.start() + assert first_reply_waiting.wait(timeout=15), "first upload never waited for its port" + + second_sender = threading.Thread( + target=forward, args=("second.bin", target_addrs[1]), daemon=True + ) + sender_threads.append(second_sender) + second_sender.start() + assert second_reply_sent.wait(timeout=20), "second target did not get its port first" + second_output = target_dirs[1] / "second.bin" + assert wait_until( + lambda: second_output.exists() and second_output.read_bytes() == payloads["second.bin"], + timeout=20, + ), "the second target consumed the first target's transfer" + assert first_port_blocked.wait(timeout=15), "first port response was not held" + + release_first.set() + first_output = target_dirs[0] / "first.bin" + assert wait_until( + lambda: first_output.exists() and first_output.read_bytes() == payloads["first.bin"], + timeout=20, + ), "the first target did not receive its own delayed transfer" + for sender in sender_threads: + sender.join(timeout=10) + assert not any(sender.is_alive() for sender in sender_threads) + assert failures == [] + assert outcomes == {"first.bin": True, "second.bin": True} + assert set(fids_by_name) == set(payloads) + + routes_by_fid = { + fid: (originator, route_targets) + for fid, originator, route_targets in route_registrations + } + assert len(routes_by_fid) == 2 + assert len(server_allocations) == 2 + for name, target_addr in zip(payloads, target_addrs, strict=True): + upstream_fid = fids_by_name[name] + assert routes_by_fid[upstream_fid][0] == source_addr + assert set(routes_by_fid[upstream_fid][1]) == {target_addr} + child_fids = { + next(iter(route_targets.values())) + for _, route_targets in routes_by_fid.values() + } + expected_allocations = { + (target_addr, child_fid) + for _, route_targets in routes_by_fid.values() + for target_addr, child_fid in route_targets.items() + } + assert set(server_allocations) == expected_allocations + assert {target for target, _ in server_allocations} == set(target_addrs) + assert {fid for _, fid in server_allocations} == child_fids + assert not source.file_upload_progress + assert not server._fid_routes + assert not server._forward_relays + finally: + release_first.set() + _close_socket_workers(server, clients, workers) + + +def test_forward_resume_with_zero_offset_reuses_existing_route(tmp_path, monkeypatch): + """A zero offset explicitly replays the existing route instead of allocating a new one.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + recv_dir = tmp_path / "zero_offset_target" + recv_dir.mkdir() + target = _start_client(tmp_path, server, name="zero_offset_target", recv_dir=recv_dir) + source = _start_client(tmp_path, server, name="zero_offset_source") + clients.extend((source, target)) + target_addr = target.client_socket.getsockname() + source_addr = source.client_socket.getsockname() + server_peer = (source.host, source.port) + src = tmp_path / "empty.bin" + src.write_bytes(b"") + output = recv_dir / src.name + output.write_bytes(b"") + + upstream_fid = source._fid_alloc.alloc(receiver=server_peer) + child_fid = server._fid_alloc.alloc(receiver=target_addr) + upload_key = (server_peer, upstream_fid) + source.file_upload_progress[upload_key] = [0, 0, None, str(src), target_addr] + target.file_recv_progress[child_fid] = [0, 0, None, str(output), source_addr] + route = server._register_fid_route( + upstream_fid, source_addr, {target_addr: child_fid} + ) + assert route is not None + + commands = [] + real_send = source.send_message + + def capture_command(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/forward_item "): + commands.append(shlex.split(message)) + return real_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(source, "send_message", capture_command) + registrations = [] + real_register = server._register_fid_route + + def observe_registration(fid, originator, route_targets, completed=()): + registrations.append(fid) + return real_register(fid, originator, route_targets, completed) + + monkeypatch.setattr(server, "_register_fid_route", observe_registration) + dropped_routes = [] + real_drop = server._drop_fid_route + + def observe_drop(fid, candidate): + dropped_routes.append(candidate) + return real_drop(fid, candidate) + + monkeypatch.setattr(server, "_drop_fid_route", observe_drop) + allocations = [] + real_allocate = server._fid_alloc.alloc + + def observe_allocation(*, receiver=None): + allocated = real_allocate(receiver=receiver) + allocations.append((tuple(receiver) if receiver is not None else None, allocated)) + return allocated + + monkeypatch.setattr(server._fid_alloc, "alloc", observe_allocation) + + assert source._forward_one( + "file", "", src.name, str(src), [target_addr], + resume_offset=0, upstream_fid=upstream_fid, + ) is True + + assert output.read_bytes() == b"" + assert commands and f"fid={upstream_fid}" in commands[0] + assert "resume=0" in commands[0] + assert registrations == [] + assert dropped_routes == [route] + assert allocations == [] + assert upstream_fid not in server._fid_routes + assert upload_key not in source.file_upload_progress + assert child_fid not in target.file_recv_progress + finally: + _close_socket_workers(server, clients, workers) + + +def test_duplicate_upstream_fid_is_rejected_without_replacing_existing_route( + tmp_path, monkeypatch +): + """A duplicate source FID fails without mutating the existing route or progress.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + source = _start_client(tmp_path, server, name="first_source") + duplicate = _start_client(tmp_path, server, name="duplicate_source") + target = _start_client(tmp_path, server, name="collision_target") + clients.extend((source, duplicate, target)) + target_addr = target.client_socket.getsockname() + source_addr = source.client_socket.getsockname() + duplicate_addr = duplicate.client_socket.getsockname() + server_peer = (source.host, source.port) + upstream_fid = source._fid_alloc.alloc(receiver=server_peer) + child_fid = server._fid_alloc.alloc(receiver=target_addr) + source_path = tmp_path / "original.bin" + duplicate_path = tmp_path / "duplicate.bin" + payload = os.urandom(4096) + source_path.write_bytes(payload) + duplicate_path.write_bytes(payload[::-1]) + source_upload_key = (server_peer, upstream_fid) + source.file_upload_progress[source_upload_key] = [ + 0, len(payload), None, str(source_path), target_addr + ] + route = server._register_fid_route( + upstream_fid, source_addr, {target_addr: child_fid} + ) + assert route is not None + routes_before = dict(server._fid_routes) + indexes_before = dict(server._fid_route_index) + + monkeypatch.setattr( + duplicate._fid_alloc, "alloc", lambda *, receiver=None: upstream_fid + ) + duplicate.is_enable_file_transfer_progress_bar = True + errors = [] + real_server_send = server.send_message + + def observe_error(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/forward_error "): + errors.append(shlex.split(message)) + return real_server_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(server, "send_message", observe_error) + + assert duplicate._forward_one( + "file", "", duplicate_path.name, str(duplicate_path), [target_addr] + ) is False + + assert wait_until(lambda: bool(errors)) + assert errors[0][0] == "/forward_error" + assert errors[0][2] == "relay fid is already in use" + assert server._fid_routes == routes_before + assert server._fid_route_index == indexes_before + assert server._fid_routes[upstream_fid] is route + assert source.file_upload_progress[source_upload_key] == [ + 0, len(payload), None, str(source_path), target_addr + ] + duplicate_upload_key = (server_peer, upstream_fid) + assert duplicate.file_upload_progress[duplicate_upload_key] == [ + 0, len(payload), None, str(duplicate_path), target_addr + ] + assert target.file_recv_progress == {} + assert not server._forward_relays + assert duplicate_addr != source_addr + finally: + _close_socket_workers(server, clients, workers) + + +@pytest.mark.parametrize("kind", ["file", "folder"]) +@pytest.mark.parametrize("successful_targets", [0, 1], ids=["zero-ack", "partial-ack"]) +def test_forward_ack_loss_retains_state_and_resumes_original_ids( + tmp_path, monkeypatch, kind, successful_targets +): + """Only real final ACKs complete targets, including confirmation at EOF.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + release_ack = threading.Event() + resume_ack = threading.Event() + try: + source = _start_client(tmp_path, server, name="source") + clients.append(source) + receivers = [] + outputs = [] + rel_dir = "bundle/nested" if kind == "folder" else "" + src = tmp_path / "ack_lost.bin" + payload = os.urandom(64 * 1024 + 37) + src.write_bytes(payload) + for index in range(2): + recv_dir = tmp_path / f"ack_target_{index}" + recv_dir.mkdir() + target = _start_client( + tmp_path, server, name=f"target_{index}", recv_dir=recv_dir + ) + clients.append(target) + receivers.append(target) + outputs.append(recv_dir / rel_dir / src.name) + target_addrs = [target.client_socket.getsockname() for target in receivers] + source_addr = source.client_socket.getsockname() + assert wait_until(lambda: all( + addr in server.clients for addr in [source_addr, *target_addrs] + )) + failed = target_addrs[successful_targets:] + losing_ack = {addr: True for addr in failed} + ack_attempts = {addr: 0 for addr in target_addrs} + held_target = failed[0] + + def intercept_ack(target, addr): + real_send = target.send_message + + def send(sock, message, is_send_raw=False): + if message == target.server_received_file_data_sign: + ack_attempts[addr] += 1 + if losing_ack.get(addr, False): + # Both supported failure forms must retain the row after + # all bytes reached disk, without sending a final ACK. + if successful_targets == 0: + raise ConnectionError("injected final target ACK loss") + return False + if addr == held_target and ack_attempts[addr] == 2: + resume_ack.set() + if not release_ack.wait(timeout=10): + raise TimeoutError("the resumed final ACK was not released") + return real_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(target, "send_message", send) + + for target, addr in zip(receivers, target_addrs): + intercept_ack(target, addr) + + dispatches = [] + real_dispatch = server.forward_item_to + + def observe_dispatch(target, kind, rel_dir, fname, originator_addr, tfid, + destination_path=None): + dispatches.append((target, tfid)) + return real_dispatch( + target, kind, rel_dir, fname, originator_addr, tfid, destination_path + ) + + monkeypatch.setattr(server, "forward_item_to", observe_dispatch) + calls = [] + outcomes = [] + real_forward = source._forward_one + + def observe_forward(*args, **kwargs): + call = { + "targets": list(args[4]), + "upstream_fid": kwargs.get("upstream_fid"), + "offset": kwargs.get("resume_offset", 0), + } + replay_fid = call["upstream_fid"] + if replay_fid is not None: + call["upload"] = source.file_upload_progress.get( + ((source.host, source.port), replay_fid) + ) + calls.append(call) + result = real_forward(*args, **kwargs) + outcomes.append(result) + return result + + monkeypatch.setattr(source, "_forward_one", observe_forward) + resume_requests = [] + real_resume = source.resume_trans + + def observe_resume(target, fid, offset): + result = real_resume(target, fid, offset) + resume_requests.append((target, fid, offset)) + return result + + monkeypatch.setattr(source, "resume_trans", observe_resume) + upstream_acks = [] + terminal_controls = [] + source_control = server.clients[source_addr]["socket"] + real_server_send = server.send_message + + def observe_upstream_ack(sock, message, is_send_raw=False): + marker = message.encode("utf-8") if isinstance(message, str) else message + written = real_server_send(sock, message, is_send_raw=is_send_raw) + if (sock is source_control and isinstance(message, str) + and message.startswith(("/server_file_transfer_port ", "/forward_error "))): + terminal_controls.append(shlex.split(message)[0]) + if marker == server.server_received_file_data_sign.encode("utf-8") and written: + upstream_acks.append(marker) + return written + + monkeypatch.setattr(server, "send_message", observe_upstream_ack) + sender = threading.Thread( + target=source._forward_one, + args=(kind, rel_dir, src.name, str(src), target_addrs), daemon=True, + ) + sender.start() + sender.join(timeout=20) + assert not sender.is_alive(), "the source attempt did not finish" + assert outcomes == [False] + assert wait_until(lambda: not server._forward_relays, timeout=10) + assert len(server._fid_routes) == 1 + fid, route = next(iter(server._fid_routes.items())) + original_targets = dict(route["targets"]) + assert set(original_targets) == set(target_addrs) + assert route["originator"] == source_addr + assert route["completed"] == set(target_addrs[:successful_targets]) + assert server._fid_route_index == { + (addr, original_targets[addr]): fid for addr in failed + } + assert upstream_acks == [] + assert terminal_controls == ["/server_file_transfer_port"] + upload_key = ((source.host, source.port), fid) + assert set(source.file_upload_progress) == {upload_key} + upload = source.file_upload_progress[upload_key] + assert upload[0:2] == [len(payload), len(payload)] + assert upload[3:] == [str(src), target_addrs[0]] + assert len(upload) == 5 + assert source._transfer_progress_snapshot()["relays"] == {} + for target, addr, output in zip(receivers, target_addrs, outputs): + assert output.read_bytes() == payload + child_fid = original_targets[addr] + assert wait_until(lambda: child_fid not in target._active_transfer_fids) + if addr in failed: + assert set(target.file_recv_progress) == {child_fid} + entry = target.file_recv_progress[child_fid] + assert entry == [len(payload), len(payload), None, str(output), source_addr] + else: + assert target.file_recv_progress == {} + assert dispatches == [(addr, original_targets[addr]) for addr in target_addrs] + + for index, addr in enumerate(failed): + target = receivers[target_addrs.index(addr)] + child_fid = original_targets[addr] + losing_ack[addr] = False + # The receiver discovers the disk offset using its normal reconnect + # path. A second request while its ACK is held must be coalesced. + assert target._request_resumes() == 1 + if index == 0: + assert resume_ack.wait(timeout=10) + assert len(calls) == 2 + assert source.file_upload_progress[upload_key][0:2] == [len(payload), len(payload)] + assert source.file_upload_progress[upload_key][3:] == [str(src), addr] + # The first request's data socket is still registered. Replay + # the reconnect request after clearing that stale session marker. + target._active_transfer_fids.discard(child_fid) + assert target._request_resumes() == 1 + assert wait_until(lambda: len(resume_requests) == 2, timeout=10) + assert resume_requests == [(addr, fid, len(payload))] * 2 + assert len(calls) == 2, "a duplicate resume started another upload" + release_ack.set() + assert wait_until(lambda: len(outcomes) == index + 2, timeout=10) + assert wait_until(lambda: not server._forward_relays, timeout=10) + assert outcomes[-1] is (index == len(failed) - 1) + assert len(upstream_acks) == (1 if index == len(failed) - 1 else 0) + assert terminal_controls == ["/server_file_transfer_port"] * (index + 2) + assert source.client_socket.fileno() >= 0 and source.running + assert calls[-1]["upstream_fid"] == fid + assert calls[-1]["targets"] == [addr] + assert calls[-1]["offset"] == len(payload) + assert calls[-1]["upload"] is not None + assert len(calls[-1]["upload"]) == 5 + assert calls[-1]["upload"][3:] == [str(src), target_addrs[0]] + assert source._transfer_progress_snapshot()["relays"] == {} + assert route["targets"] == original_targets + assert child_fid not in target.file_recv_progress + if index != len(failed) - 1: + assert server._fid_routes[fid] is route + assert len(source.file_upload_progress[upload_key]) == 5 + assert upload_key in source.file_upload_progress + assert route["completed"] == set( + target_addrs[:successful_targets] + failed[:index + 1] + ) + assert server._fid_route_index == { + (peer, original_targets[peer]): fid for peer in failed[index + 1:] + } + + assert dispatches == [ + (addr, original_targets[addr]) for addr in [*target_addrs, *failed] + ], "an acknowledged target was sent again or a child id changed" + assert ack_attempts == { + addr: 2 if addr in failed else 1 for addr in target_addrs + } + assert upstream_acks == [server.server_received_file_data_sign.encode("utf-8")] + assert server._fid_routes == {} + assert server._fid_route_index == {} + assert source._transfer_progress_snapshot()["relays"] == {} + assert source.file_upload_progress == {} + for target, output in zip(receivers, outputs): + assert target.file_recv_progress == {} + assert output.read_bytes() == payload + with open(source.file_trans_progress_file, encoding="utf-8") as stream: + source_snapshot = json.load(stream) + assert source_snapshot["upload"] == {} + assert source_snapshot["relays"] == {} + with open(server.file_trans_progress_file, encoding="utf-8") as stream: + assert json.load(stream)["relays"] == {} + finally: + release_ack.set() + _close_socket_workers(server, clients, workers) + + +@pytest.mark.parametrize("ack_failure", ["false", "exception"]) +def test_forward_ack_write_failure_retains_source_and_completed_route( + tmp_path, monkeypatch, ack_failure +): + """A failed original ACK write keeps the source upload and the Server route.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + source = _start_client(tmp_path, server, name="source") + clients.append(source) + recv_dir = tmp_path / "ack_write_failure_recv" + recv_dir.mkdir() + target = _start_client(tmp_path, server, name="target", recv_dir=recv_dir) + clients.append(target) + target_addr = target.client_socket.getsockname() + assert wait_until(lambda: target_addr in server.clients) + src = tmp_path / "ack_write_failure.bin" + payload = os.urandom(64 * 1024 + 37) + src.write_bytes(payload) + failed_writes = [] + real_send = server.send_message + + def fail_upstream_ack(sock, message, is_send_raw=False): + marker = message.encode("utf-8") if isinstance(message, str) else message + if marker == server.server_received_file_data_sign.encode("utf-8"): + failed_writes.append(sock) + if ack_failure == "exception": + raise ConnectionError("injected upstream final ACK write failure") + return False + return real_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(server, "send_message", fail_upstream_ack) + assert source._forward_one("file", "", src.name, str(src), [target_addr]) is False + assert len(failed_writes) == 1 + assert wait_until(lambda: not server._forward_relays, timeout=10) + assert (recv_dir / src.name).read_bytes() == payload + assert wait_until(lambda: not target.file_recv_progress, timeout=10) + assert len(source.file_upload_progress) == 1 + upload_key, upload = next(iter(source.file_upload_progress.items())) + assert upload_key[0] == (source.host, source.port) + fid = upload_key[1] + assert len(upload) == 5 and upload[3:] == [str(src), target_addr] + assert source.file_upload_progress[upload_key][0:2] == [len(payload), len(payload)] + route = server._fid_routes[fid] + assert route["completed"] == {target_addr} + assert set(route["targets"]) == {target_addr} + assert server._fid_route_index == {} + with open(source.file_trans_progress_file, encoding="utf-8") as stream: + snapshot = json.load(stream) + assert len(snapshot["upload"]) == 1 + assert snapshot["relays"] == {} + + reloaded = TCP_Client_Base( + host=source.host, port=source.port, is_extend_command=True, + is_input_command_in_console=False, is_enable_encrypto=False, + ) + _isolate(reloaded, tmp_path, "reloaded_source") + reloaded.file_trans_progress_file = source.file_trans_progress_file + try: + assert reloaded._load_transfer_progress() == 1 + assert ( + reloaded.file_upload_progress[upload_key] + == source.file_upload_progress[upload_key] + ) + assert reloaded._transfer_progress_snapshot()["relays"] == {} + finally: + reloaded.file_trans_progress_file = str(tmp_path / "inspection_progress.json") + reloaded.close() + finally: + _close_socket_workers(server, clients, workers) + + +def test_forward_waits_for_last_target_ack_before_upstream_ack(tmp_path, monkeypatch): + """The source remains pending until the last real target ACK is released.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + release_ack = threading.Event() + target_bytes_read = threading.Event() + upstream_acks = [] + try: + source = _start_client(tmp_path, server, name="source") + clients.append(source) + recv_dir = tmp_path / "held_final_ack_recv" + recv_dir.mkdir() + target = _start_client(tmp_path, server, name="target", recv_dir=recv_dir) + clients.append(target) + target_addr = target.client_socket.getsockname() + assert wait_until(lambda: target_addr in server.clients) + real_target_send = target.send_message + real_server_send = server.send_message + + def hold_target_ack(sock, message, is_send_raw=False): + if message == target.server_received_file_data_sign: + target_bytes_read.set() + if not release_ack.wait(timeout=10): + raise TimeoutError("the final target ACK was not released") + return real_target_send(sock, message, is_send_raw=is_send_raw) + + def observe_upstream_ack(sock, message, is_send_raw=False): + marker = message.encode("utf-8") if isinstance(message, str) else message + written = real_server_send(sock, message, is_send_raw=is_send_raw) + if marker == server.server_received_file_data_sign.encode("utf-8") and written: + upstream_acks.append(marker) + return written + + monkeypatch.setattr(target, "send_message", hold_target_ack) + monkeypatch.setattr(server, "send_message", observe_upstream_ack) + src = tmp_path / "held_final_ack.bin" + payload = os.urandom(64 * 1024 + 37) + src.write_bytes(payload) + result = [] + sender = threading.Thread( + target=lambda: result.append( + source._forward_one("file", "", src.name, str(src), [target_addr]) + ), daemon=True, + ) + sender.start() + assert target_bytes_read.wait(timeout=10), "target did not read all forwarded bytes" + assert wait_until(lambda: (recv_dir / src.name).exists()) + assert (recv_dir / src.name).stat().st_size == len(payload) + assert sender.is_alive(), "source returned before the final target ACK" + assert source.client_socket.fileno() >= 0 + assert upstream_acks == [] + assert len(source.file_upload_progress) == 1 + upload_key, upload = next(iter(source.file_upload_progress.items())) + assert upload_key[0] == (source.host, source.port) + fid = upload_key[1] + assert len(upload) == 5 and upload[3:] == [str(src), target_addr] + assert source.file_upload_progress[upload_key][0:2] == [len(payload), len(payload)] + route = server._fid_routes[fid] + assert route["completed"] == set() + + release_ack.set() + sender.join(timeout=15) + assert not sender.is_alive() + assert result == [True] + assert upstream_acks == [target.server_received_file_data_sign.encode("utf-8")] + assert server._fid_routes == {} + assert server._fid_route_index == {} + assert source._transfer_progress_snapshot()["relays"] == {} + assert source.file_upload_progress == {} + finally: + release_ack.set() + _close_socket_workers(server, clients, workers) + + +def test_forward_route_survives_a_refused_target_listener_and_reload(tmp_path, monkeypatch): + """One real completed target cannot erase another accepted target's route.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + restored = None + # Bound but deliberately not listening: this port cannot be stolen, and + # the relay's real connect() receives a deterministic connection refusal. + refused_listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + refused_listener.bind(("127.0.0.1", 0)) + advertised = threading.Event() + try: + good_dir = tmp_path / "good_recv" + bad_dir = tmp_path / "refused_recv" + good_dir.mkdir() + bad_dir.mkdir() + source = _start_client(tmp_path, server, name="source") + clients.append(source) + good = _start_client(tmp_path, server, name="good", recv_dir=good_dir) + clients.append(good) + bad = _start_client(tmp_path, server, name="refused", recv_dir=bad_dir) + clients.append(bad) + source_addr = source.client_socket.getsockname() + good_addr = good.client_socket.getsockname() + bad_addr = bad.client_socket.getsockname() + assert wait_until(lambda: all( + peer in server.clients for peer in (source_addr, good_addr, bad_addr) + )) + + def advertise_refused_listener(client_id, client_socket, command, + new_save_path=None, file_name=None): + parts = shlex.split(command) + tfid = int(parts[-1]) + stamp = bad._crypto_transfer_stamp(client_socket) + bad.send_message( + client_socket, + f"/server_file_transfer_port {refused_listener.getsockname()[1]} " + f"{tfid} {stamp}\n", + is_send_raw=True, + ) + advertised.set() + + monkeypatch.setattr( + bad, "file_transfer_client_recv_server_start", advertise_refused_listener + ) + src = tmp_path / "partially_forwarded.bin" + payload = os.urandom(96 * 1024) + src.write_bytes(payload) + sender = threading.Thread( + target=source._forward_one, + args=("file", "", src.name, str(src), [good_addr, bad_addr]), + daemon=True, + ) + sender.start() + assert advertised.wait(timeout=10), "the faulty target did not advertise its port" + output = good_dir / src.name + assert wait_until( + lambda: output.exists() and output.stat().st_size == len(payload), timeout=20 + ), "the healthy target did not receive the full forwarded file" + assert output.read_bytes() == payload + sender.join(timeout=20) + assert not sender.is_alive(), "the real source transfer did not finish" + assert wait_until(lambda: not server._forward_relays, timeout=10) + assert wait_until(lambda: not good.file_recv_progress, timeout=10) + assert not (bad_dir / src.name).exists() + + assert len(server._fid_routes) == 1 + fid, route = next(iter(server._fid_routes.items())) + assert route["originator"] == source_addr + assert set(route["targets"]) == {good_addr, bad_addr} + assert server._lookup_fid_route(None, fid) == (fid, route) + assert route["completed"] == {good_addr} + assert server._lookup_fid_route(good_addr, route["targets"][good_addr]) is None + found = server._lookup_fid_route(bad_addr, route["targets"][bad_addr]) + assert found[0] == fid + assert found[1] is route + expected_index = { + (bad_addr, route["targets"][bad_addr]): fid + } + assert server._fid_route_index == expected_index + + server._persist_transfer_progress() + with open(server.file_trans_progress_file, "r", encoding="utf-8") as stream: + snapshot = json.load(stream) + assert snapshot["relays"] == { + str(fid): { + "originator": list(source_addr), + "targets": [ + [list(target), tfid] for target, tfid in route["targets"].items() + ], + "completed": [list(good_addr)], + } + } + restored = TCP_Server_Base( + host="127.0.0.1", port=free_port(), is_extend_command=True, + is_input_command_in_console=False, is_enable_encrypto=False, + ) + _isolate(restored, tmp_path, "restored") + restored.file_trans_progress_file = server.file_trans_progress_file + assert restored._load_transfer_progress() == 1 + restored_route = restored._fid_routes[fid] + assert restored_route == route + assert restored._fid_route_index == expected_index + assert restored._lookup_fid_route(None, fid)[1] is restored_route + assert restored._lookup_fid_route(good_addr, route["targets"][good_addr]) is None + found = restored._lookup_fid_route(bad_addr, route["targets"][bad_addr]) + assert found[0] == fid + assert found[1] is restored_route + + # Drive both rejection and offline routing through real control sockets. + # Waiting for the real handler to return avoids timing-based negatives. + source.close() + assert wait_until(lambda: source_addr not in server.clients, timeout=10) + handled = [] + real_resume_request = server._resume_request + + def observe_resume(client_socket, peer, requested_fid, offset, origin, target): + try: + return real_resume_request( + client_socket, peer, requested_fid, offset, origin, target + ) + finally: + handled.append((peer, requested_fid, offset)) + + monkeypatch.setattr(server, "_resume_request", observe_resume) + bad_fid = route["targets"][bad_addr] + unknown_fid = max([fid, *route["targets"].values()]) + 2 + assert good.send_message( + good.client_socket, format_resume_line(bad_fid, 0, source_addr, good_addr) + ) + assert bad.send_message( + bad.client_socket, format_resume_line(unknown_fid, 0, source_addr, bad_addr) + ) + assert wait_until(lambda: len(handled) == 2, timeout=10) + assert set(handled) == {(good_addr, bad_fid, 0), (bad_addr, unknown_fid, 0)} + assert server._pending_originator_resumes == {} + assert bad.send_message( + bad.client_socket, format_resume_line(bad_fid, 0, source_addr, bad_addr) + ) + assert wait_until(lambda: len(handled) == 3, timeout=10) + assert server._pending_originator_resumes == {source_addr: {(fid, bad_addr): 0}} + assert server._fid_routes[fid] is route + finally: + refused_listener.close() + if restored is not None: + restored.stop() + _close_socket_workers(server, clients, workers) + + +def test_forward_route_is_removed_only_after_all_real_targets_complete(tmp_path, monkeypatch): + """A completed sibling leaves the route live until the held writer finishes.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + held = {} + try: + source = _start_client(tmp_path, server, name="source") + clients.append(source) + targets = [] + outputs = [] + for index in range(2): + recv_dir = tmp_path / f"forward_target_{index}" + recv_dir.mkdir() + target = _start_client( + tmp_path, server, name=f"target_{index}", recv_dir=recv_dir + ) + clients.append(target) + targets.append(target.client_socket.getsockname()) + outputs.append(recv_dir / "complete_forward.bin") + assert wait_until(lambda: all(target in server.clients for target in targets)) + real_forward_item_to = server.forward_item_to + + def hold_second_writer(target, kind, rel_dir, fname, originator_addr, tfid, + destination_path=None): + if target == targets[1]: + held["fid"] = tfid + held["paused"] = server.pause_trans(tfid, target) + return real_forward_item_to( + target, kind, rel_dir, fname, originator_addr, tfid, destination_path + ) + + monkeypatch.setattr(server, "forward_item_to", hold_second_writer) + src = tmp_path / "complete_forward.bin" + payload = os.urandom(96 * 1024) + src.write_bytes(payload) + sender = threading.Thread( + target=source._forward_one, + args=("file", "", src.name, str(src), targets), daemon=True, + ) + sender.start() + assert wait_until( + lambda: outputs[0].exists() and outputs[0].stat().st_size == len(payload), + timeout=20, + ), "the unpaused target did not finish" + assert outputs[0].read_bytes() == payload + assert wait_until(lambda: not clients[1].file_recv_progress, timeout=10) + assert held["paused"] is True + assert not outputs[1].exists() or outputs[1].stat().st_size == 0 + assert len(server._fid_routes) == 1 + fid, route = next(iter(server._fid_routes.items())) + assert set(route["targets"]) == set(targets) + assert wait_until(lambda: route["completed"] == {targets[0]}, timeout=10) + expected_index = {(targets[1], route["targets"][targets[1]]): fid} + assert server._fid_route_index == expected_index + assert server._lookup_fid_route(targets[0], route["targets"][targets[0]]) is None + assert server._lookup_fid_route(targets[1], route["targets"][targets[1]])[1] is route + + assert server.start_trans(held["fid"], targets[1]) is True + assert wait_until( + lambda: outputs[1].exists() and outputs[1].stat().st_size == len(payload), + timeout=20, + ), "the released target did not finish" + assert outputs[1].read_bytes() == payload + sender.join(timeout=20) + assert not sender.is_alive(), "the real source transfer did not finish" + assert wait_until(lambda: not server._forward_relays, timeout=10) + assert wait_until(lambda: not clients[2].file_recv_progress, timeout=10) + assert server._fid_routes == {} + assert server._fid_route_index == {} + assert server._lookup_fid_route(None, fid) is None + for target, tfid in route["targets"].items(): + assert server._lookup_fid_route(target, tfid) is None + finally: + if held: + server.start_trans(held["fid"], targets[1]) + _close_socket_workers(server, clients, workers) def test_pause_and_start_trans_control_a_plain_transfer(tmp_path): @@ -436,23 +2068,20 @@ def test_pause_and_start_trans_control_a_plain_transfer(tmp_path): src = tmp_path / "payload.bin" payload = os.urandom(256 * 1024) src.write_bytes(payload) - fid = 404 + fid = 404 # even low bit: a plain leg's id - server_sock = server.clients[client.client_socket.getsockname()]["socket"] - server.file_transfer_server_recv_server_start_thread( - "cid", server_sock, f"/file {src} {fid}" - ) - assert _wait_port(client) is not None + transfer_port, _stamp = request_transfer_listener(client, server, f"/file {src} {fid}") assert client.pause_trans(fid) is False # armed before the leg starts sender = threading.Thread( target=client.file_transfer_mode, - args=(str(src), "127.0.0.1", _wait_port(client) or 0, 0), + args=(str(src), "127.0.0.1", transfer_port, 0), kwargs={"fid": fid, "peer": (server.host, server.port)}, daemon=True, ) sender.start() - assert wait_until(lambda: client.file_upload_progress.get(fid)), ( + upload_key = ((client.host, client.port), fid) + assert wait_until(lambda: client.file_upload_progress.get(upload_key)), ( "the uploader never registered the transfer" ) time.sleep(0.5) @@ -515,7 +2144,7 @@ def test_download_resumes_over_the_encrypted_channel(tmp_path): src = tmp_path / "download.bin" payload = os.urandom(64 * 1024) src.write_bytes(payload) - fid = 909 + fid = 908 # even low bit: a plain leg's id partial = recv_dir / "download.bin" with open(partial, "wb") as f: f.write(payload[:1024]) @@ -526,7 +2155,8 @@ def test_download_resumes_over_the_encrypted_channel(tmp_path): str(partial), (server.host, server.port), ] - server.file_upload_progress[fid] = [ + upload_key = (client.client_socket.getsockname(), fid) + server.file_upload_progress[upload_key] = [ 0, len(payload), None, @@ -534,13 +2164,13 @@ def test_download_resumes_over_the_encrypted_channel(tmp_path): client.client_socket.getsockname(), ] - assert client.resume_transfers(peer=(server.host, server.port)) == 1 + client._request_resumes() # the same call connect() dispatches on reconnect assert wait_until(lambda: partial.stat().st_size == len(payload), timeout=20), ( "the encrypted resume did not finish" ) assert partial.read_bytes() == payload assert wait_until(lambda: fid not in client.file_recv_progress, timeout=10) - assert wait_until(lambda: fid not in server.file_upload_progress, timeout=10) + assert wait_until(lambda: upload_key not in server.file_upload_progress, timeout=10) finally: if client is not None: client.close() @@ -581,4 +2211,782 @@ def test_client_connects_when_its_configured_port_is_taken(tmp_path): holder.close() if client is not None: client.close() - server.stop() \ No newline at end of file + server.stop() + + +@pytest.mark.parametrize("operation", ["bind", "connect"]) +def test_client_falls_back_when_fixed_port_returns_wsaeacces( + tmp_path, monkeypatch, operation +): + server = _start_server(tmp_path) + client_port = free_port() + client = TCP_Client_Base( + host="127.0.0.1", + port=server.port, + client_host="127.0.0.1", + client_port=client_port, + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + _isolate(client, tmp_path, "client") + address = ( + ("127.0.0.1", client_port) + if operation == "bind" + else ("127.0.0.1", server.port) + ) + injected = inject_wsaeacces_once( + monkeypatch, operation=operation, address=address + ) + try: + assert client.connect(), "the client did not connect after WSAEACCES" + assert injected["raised"] + assert client.client_socket.getsockname()[1] != client_port + assert wait_until( + lambda: server.clients.get(client.client_socket.getsockname()) is not None + ), "the server never saw the client" + finally: + client.close() + server.stop() + + +@pytest.mark.parametrize("client_port", [None, 0], ids=["unbound", "os-assigned"]) +def test_client_does_not_fallback_on_wsaeacces_without_fixed_port( + tmp_path, monkeypatch, client_port +): + server = _start_server(tmp_path) + client = TCP_Client_Base( + host="127.0.0.1", + port=server.port, + client_host="127.0.0.1", + client_port=client_port, + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + is_wait_server=False, + ) + _isolate(client, tmp_path, "client") + server_address = ("127.0.0.1", server.port) + injected = inject_wsaeacces_once( + monkeypatch, operation="connect", address=server_address + ) + try: + assert client.connect() is False + assert injected["raised"] + attempts = [ + call + for call in injected["calls"] + if call[:2] == ("connect", server_address) + ] + assert len(attempts) == 1 + bind_attempts = [call for call in injected["calls"] if call[0] == "bind"] + if client_port is None: + assert not bind_attempts + else: + assert [call[1] for call in bind_attempts] == [("127.0.0.1", 0)] + finally: + client.close() + server.stop() + + +def test_resume_with_a_different_total_is_refused_and_keeps_the_partial_file(tmp_path): + """A continued leg whose size disagrees with the progress entry is refused. + + The sender answers ``/resume_trans`` with a file whose length differs from + the one the interrupted entry recorded. The receiver must refuse the leg and + leave the partial bytes and the progress entry untouched, so a later + reconnect still has what it needs to decide. + """ + server = _start_server(tmp_path) + client = None + try: + recv_dir = tmp_path / "client_recv" + recv_dir.mkdir() + client = _start_client(tmp_path, server, recv_dir=recv_dir) + src = tmp_path / "download.bin" + payload = os.urandom(128 * 1024) + src.write_bytes(payload) + fid = 302 # even low bit: a plain leg's id + kept = payload[:4096] + partial = recv_dir / "download.bin" + with open(partial, "wb") as f: + f.write(kept) + stale_total = len(payload) + 1 # the entry recorded a different file of this name + client.file_recv_progress[fid] = [ + len(kept), + stale_total, + None, + str(partial), + (server.host, server.port), + ] + upload_key = (client.client_socket.getsockname(), fid) + server.file_upload_progress[upload_key] = [ + 0, + len(payload), + None, + str(src), + client.client_socket.getsockname(), + ] + messages = [] + log = client._log + + def record(message): + messages.append(message) + log(message) + + client._log = record + client._request_resumes() + assert wait_until(lambda: any("resume rejected" in m for m in messages), timeout=20), ( + "the mismatched resume was not refused" + ) + assert partial.read_bytes() == kept + assert client.file_recv_progress[fid][1] == stale_total + finally: + if client is not None: + client.close() + server.stop() + + +def test_download_resumes_after_connect_without_a_manual_call(tmp_path): + """A download interrupted by a disconnect resumes via the client's own reconnect. + + Nothing calls the resume driver by hand: the client reconnects + (``is_auto_reconnect``) and its connect path asks the server to continue the + file from the bytes already on disk. This covers the trigger itself, which + the partial-file tests inject directly. + """ + server = _start_server(tmp_path) + recv_dir = tmp_path / "client_recv" + recv_dir.mkdir() + client = None + try: + src = tmp_path / "payload.bin" + payload = os.urandom(32 * 1024 * 1024) # still in flight when the fault hits + src.write_bytes(payload) + + # a fixed client port keeps its address stable, so the server can match + # the interrupted download to the reconnecting peer + client = TCP_Client_Base( + host="127.0.0.1", + port=server.port, + client_host="127.0.0.1", + client_port=free_port(), + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + is_auto_reconnect=True, + ) + _isolate(client, tmp_path, "client") + client.file_transfer_dir = str(recv_dir) + threading.Thread(target=client.start_TCP_client, daemon=True).start() + assert wait_until(lambda: client.running and client.client_socket is not None) + client_addr = client.client_socket.getsockname() + assert wait_until(lambda: client_addr in server.clients, timeout=10), ( + "server never registered the client" + ) + + partial = recv_dir / src.name + # Predict the next server-allocated id and arm this receiver's pause + # before the leg starts, so the fault lands while it is incomplete. + alloc = server._fid_alloc + with alloc._lock: + fid = ((alloc._counter + 1) << 32) | (alloc._epoch << 1) + upload_key = (client_addr, fid) + assert server.pause_trans(fid, client_addr) is False, "the leg was already running" + + # the server pushes the file: the client opens its listener and the + # server connects from a send leg + server.file_transfer_server_recv_client_start_thread( + "/file {} {}".format(shlex.quote(str(src)), shlex.quote(repr(client_addr))) + ) + assert wait_until(lambda: server.file_upload_progress.get(upload_key), timeout=30), ( + "the server never registered the outgoing transfer" + ) + # the leg is parked before its first chunk, so the receiver holds an + # empty target; give it the bytes an interrupted transfer would have + # left behind (the leg's own writes would only be the same file) + assert wait_until(lambda: partial.exists(), timeout=20), ( + "the receiver never created the target file" + ) + half = len(payload) // 2 + partial.write_bytes(payload[:half]) + + # a fault drops the control connection from the server side; the parked + # leg cannot finish the file, so the partial stays on disk + dead_leg = server._active_transfer_sockets.get(upload_key) + assert dead_leg is not None, "the send leg was not registered" + dead_socket = client.client_socket + server_sock = server.clients[client_addr]["socket"] + server_sock.shutdown(2) + server_sock.close() + assert wait_until(lambda: dead_socket.fileno() == -1, timeout=20), ( + "the reconnect did not release the dead session's socket" + ) + assert wait_until(lambda: dead_leg.fileno() == -1, timeout=10), ( + "the disconnected control session left its paused data socket open" + ) + # the reconnecting client sweeps the dead session's legs and then asks + # for the remainder; only the swept leg may be released, or it would + # write from its own offset into the file the resumed leg continues + assert wait_until( + lambda: server._active_transfer_sockets.get(upload_key) is not dead_leg, timeout=20 + ), "the dead session's leg was never cleared" + server.start_trans(fid, client_addr) # the test, not the protocol, parked this leg + assert wait_until(lambda: server.clients.get(client_addr) is not None, timeout=20), ( + "the client did not come back" + ) + assert client.client_socket.getsockname() == client_addr, ( + "the client came back from a different port" + ) + + assert wait_until(lambda: partial.stat().st_size == len(payload), timeout=30), ( + "the resumed download did not finish" + ) + assert partial.read_bytes() == payload, "the resumed download did not continue the partial" + finally: + if client is not None: + client.close() + server.stop() + + +def test_unified_resume_serializes_distinct_offsets_without_local_fid_fallback(tmp_path, monkeypatch): + """One upload serves queued targets once, preserving paths and endpoint scope.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + held = threading.Event() + release = threading.Event() + try: + source = _start_client(tmp_path, server, name="queued_source") + clients.append(source) + receivers, outputs = [], [] + payload = os.urandom(96 * 1024 + 19) + src = tmp_path / "queued.bin" + src.write_bytes(payload) + source_addr = source.client_socket.getsockname() + offsets = [12345, 54321] + for index, offset in enumerate(offsets): + recv_dir = tmp_path / f"queued_{index}" + recv_dir.mkdir() + target = _start_client(tmp_path, server, name=f"queued_{index}", recv_dir=recv_dir) + clients.append(target) + receivers.append(target) + output = recv_dir / "original/nested" / src.name + output.parent.mkdir(parents=True) + output.write_bytes(payload[:offset]) + target.file_recv_progress[502 + index * 2] = [ + offset, len(payload), None, str(output), source_addr, + ] + outputs.append(output) + first, second = receivers + first_addr, second_addr = [target.client_socket.getsockname() for target in receivers] + assert wait_until(lambda: all(peer in server.clients for peer in ( + source_addr, first_addr, second_addr + ))) + route = server._register_fid_route(500, source_addr, {first_addr: 502, second_addr: 504}) + key = ((source.host, source.port), 500) + source.file_upload_progress[key] = [0, len(payload), None, str(src), first_addr] + local_path = tmp_path / "unrelated.bin" + local_path.write_bytes(b"unrelated local upload") + local_row = [0, local_path.stat().st_size, None, str(local_path), first_addr] + server.file_upload_progress[(first_addr, 502)] = local_row + original_send = first.send_message + original_mode = source.file_transfer_mode + transfers = [] + def hold_ack(sock, message, is_send_raw=False): + if message == first.server_received_file_data_sign: + held.set() + assert release.wait(timeout=10) + return original_send(sock, message, is_send_raw=is_send_raw) + + def observe_mode(*args, **kwargs): + transfers.append((kwargs.get("fid"), kwargs.get("resume_offset", 0))) + return original_mode(*args, **kwargs) + + monkeypatch.setattr(first, "send_message", hold_ack) + monkeypatch.setattr(source, "file_transfer_mode", observe_mode) + assert first._request_resumes() == 1 + assert held.wait(timeout=10), "the first target never reached its ACK barrier" + assert second._request_resumes() == 1 + assert first.send_message( + first.client_socket, format_resume_line(502, 999, source_addr, first_addr) + ) + assert wait_until( + lambda: source._resume_sends.get(key, {}).get("targets") == {first_addr, second_addr} + ) + assert wait_until(lambda: transfers == [(500, offsets[0])]) + release.set() + assert wait_until(lambda: key not in source.file_upload_progress, timeout=15) + assert wait_until(lambda: not source._resume_sends and not server._fid_routes) + assert transfers == [(500, offset) for offset in offsets] + assert route["targets"] == {first_addr: 502, second_addr: 504} + assert route["completed"] == {first_addr, second_addr} + assert server.file_upload_progress[(first_addr, 502)] == local_row + assert source._transfer_progress_snapshot()["relays"] == {} + for target, child, output in zip(receivers, (502, 504), outputs): + assert wait_until(lambda: child not in target.file_recv_progress) + assert output.read_bytes() == payload + assert not (output.parents[2] / src.name).exists() + finally: + release.set() + _close_socket_workers(server, clients, workers) + +def test_concurrent_forward_upload_handshakes_match_out_of_order_replies(tmp_path, monkeypatch): + """A later relay reply cannot hand its port/FID to the first waiting upload.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + release_first = threading.Event() + first_reply_waiting = threading.Event() + first_port_blocked = threading.Event() + second_reply_sent = threading.Event() + request_names = {} + request_lock = threading.Lock() + sender_threads = [] + try: + source = _start_client(tmp_path, server, name="ordered_source") + target_dir = tmp_path / "ordered_target" + target_dir.mkdir() + target = _start_client( + tmp_path, server, name="ordered_target", recv_dir=target_dir + ) + clients.extend((source, target)) + source_addr = source.client_socket.getsockname() + target_addr = target.client_socket.getsockname() + payloads = { + "first.bin": os.urandom(96 * 1024 + 11), + "second.bin": os.urandom(80 * 1024 + 7), + } + paths = {} + for name, payload in payloads.items(): + path = tmp_path / name + path.write_bytes(payload) + paths[name] = path + assert wait_until(lambda: source_addr in server.clients and target_addr in server.clients) + + original_client_send = source.send_message + + def track_forward_request(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/forward_item "): + parts = shlex.split(message) + fid = int(next(part[4:] for part in parts if part.startswith("fid="))) + name = next(item for item in payloads if item in parts) + with request_lock: + request_names[fid] = name + return original_client_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(source, "send_message", track_forward_request) + original_wait = source._transfer_handshakes.wait + + def observe_wait(record, timeout): + with request_lock: + name = request_names.get(record["key"][1]) + if name == "first.bin": + first_reply_waiting.set() + return original_wait(record, timeout) + + monkeypatch.setattr(source._transfer_handshakes, "wait", observe_wait) + original_server_send = server.send_message + + def send_second_port_first(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/server_file_transfer_port "): + parts = shlex.split(message) + with request_lock: + name = request_names.get(int(parts[2])) + if name == "first.bin": + first_port_blocked.set() + assert release_first.wait(timeout=20), "first port response was not held" + return original_server_send(sock, message, is_send_raw=is_send_raw) + if name == "second.bin": + sent = original_server_send(sock, message, is_send_raw=is_send_raw) + second_reply_sent.set() + return sent + return original_server_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(server, "send_message", send_second_port_first) + outcomes = {} + failures = [] + + def forward(name): + try: + outcomes[name] = source._forward_one( + "file", "", name, str(paths[name]), [target_addr] + ) + except Exception as exc: + failures.append((name, exc)) + + first_sender = threading.Thread(target=forward, args=("first.bin",), daemon=True) + sender_threads.append(first_sender) + first_sender.start() + assert first_reply_waiting.wait(timeout=15), "first upload never waited for its port" + + second_sender = threading.Thread(target=forward, args=("second.bin",), daemon=True) + sender_threads.append(second_sender) + second_sender.start() + assert second_reply_sent.wait(timeout=20), "second upload did not get its port first" + second_output = target_dir / "second.bin" + assert wait_until( + lambda: second_output.exists() and second_output.read_bytes() == payloads["second.bin"], + timeout=20, + ), "second transfer consumed the first request's port/FID" + assert first_port_blocked.wait(timeout=15), "first port response was not held" + + release_first.set() + first_output = target_dir / "first.bin" + assert wait_until( + lambda: first_output.exists() and first_output.read_bytes() == payloads["first.bin"], + timeout=20, + ), "first transfer did not resume on its own delayed port/FID" + for sender in sender_threads: + sender.join(timeout=10) + assert not any(sender.is_alive() for sender in sender_threads) + assert failures == [] + assert outcomes == {"first.bin": True, "second.bin": True} + assert set(request_names.values()) == {"first.bin", "second.bin"} + assert len(request_names) == 2 + assert not source.file_upload_progress + assert not server._fid_routes + assert not server._forward_relays + finally: + release_first.set() + _close_socket_workers(server, clients, workers) + + +@pytest.mark.parametrize("server_sends", [False, True], ids=["upload", "download"]) +def test_unresolved_listener_isolates_same_session_fid(tmp_path, monkeypatch, server_sends): + """A late terminal response cannot reopen an uncertain same-session FID.""" + from PyFlow.network_api.connect_tcp import _send_transfer_request, _transfer_handshake + + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + old_reply = [] + try: + recv_dir = tmp_path / "late_ports" + recv_dir.mkdir() + client = _start_client(tmp_path, server, name="late_ports", recv_dir=recv_dir) + clients.append(client) + server.file_transfer_dir = str(recv_dir) + peer = client.client_socket.getsockname() + assert wait_until(lambda: peer in server.clients) + sender, receiver = (server, client) if server_sends else (client, server) + control = server.clients[peer]["socket"] if server_sends else client.client_socket + actual_send = receiver.send_message + fid = 806 + held = threading.Event() + + orphan_src = tmp_path / "orphan.bin" + orphan_src.write_bytes(b"orphan listener payload") + + def hold_first_listener_reply(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/server_file_transfer_port "): + if not old_reply: + old_reply.append(message) + held.set() + return True + return actual_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(receiver, "send_message", hold_first_listener_reply) + with _transfer_handshake(sender, control, fid) as record: + assert _send_transfer_request( + sender, record, f"/file {shlex.quote(str(orphan_src))} {fid}" + ) + assert held.wait(timeout=10), "the original listener response was not held" + assert sender._transfer_handshakes.wait(record, 0) is None + assert old_reply + with _transfer_handshake(sender, control, fid) as blocked: + assert blocked is None + assert not _send_transfer_request(sender, blocked, f"/file retry.bin {fid}") + + late_port = old_reply[0] + late_error = f"/forward_error {fid} {shlex.quote('late listener error')}" + if server_sends: + server.handle_command(control, peer, late_port) + assert not sender._transfer_handshakes.resolve_error( + control, fid, "late listener error" + ) + else: + client.handle_server_command(late_port, control_socket=control) + client.handle_server_command(late_error, control_socket=control) + + src = tmp_path / "fresh.bin" + payload = os.urandom(96 * 1024 + 13) + src.write_bytes(payload) + port, stamp = request_transfer_listener(sender, receiver, f"/file {src} {fid + 1}") + assert sender.file_transfer_mode( + str(src), "127.0.0.1", port, 0, control_socket=control, crypto=stamp, + fid=fid + 1, peer=peer if server_sends else (server.host, server.port), + ) + assert (recv_dir / src.name).read_bytes() == payload + command_dir = recv_dir / "made" / "by" / "command" + assert sender.send_message(control, "/file_folder /made/by/command") + assert wait_until(command_dir.is_dir) + assert sender._transfer_handshakes.register(control, fid) is None + finally: + _close_socket_workers(server, clients, workers) + + +@pytest.mark.parametrize("server_sends", [False, True], ids=["upload", "download"]) +def test_reconnected_handshake_rejects_reply_from_old_control_socket(tmp_path, monkeypatch, server_sends): + """An old-session callback cannot resolve a new-session listener request.""" + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + recv_dir = tmp_path / "session_ports" + recv_dir.mkdir() + client = _start_client( + tmp_path, server, name="session_ports", recv_dir=recv_dir, client_port=free_port() + ) + clients.append(client) + server.file_transfer_dir = str(recv_dir) + old_client = client.client_socket + old_peer = old_client.getsockname() + assert wait_until(lambda: old_peer in server.clients) + old_server = server.clients[old_peer]["socket"] + old_sender = server if server_sends else client + old_receiver = client if server_sends else server + old_control = old_server if server_sends else old_client + old_receiver_control = old_client if server_sends else old_server + old_src = tmp_path / "old_session.bin" + old_src.write_bytes(b"old session listener") + old_port, old_stamp = request_transfer_listener( + old_sender, old_receiver, f"/file {shlex.quote(str(old_src))} 808" + ) + old_callback = f"/server_file_transfer_port {old_port} 808 {old_stamp}\n" + old_resources = tuple( + old_receiver._transfer_handshakes._resources[old_receiver_control] + ) + old_client.shutdown(socket.SHUT_RDWR) + assert wait_until(lambda: not client.running and old_peer not in server.clients) + client.receive_thread.join(timeout=10) + client._release_client_socket() + assert old_resources and all(sock.fileno() == -1 for sock in old_resources) + assert client.connect() + peer = client.client_socket.getsockname() + assert wait_until(lambda: peer in server.clients) + sender, receiver = (server, client) if server_sends else (client, server) + control = server.clients[peer]["socket"] if server_sends else client.client_socket + assert control is not old_control + actual_send = receiver.send_message + rejected = [] + + def replay_old_session_callback(sock, message, is_send_raw=False): + if isinstance(message, str) and message.startswith("/server_file_transfer_port "): + if server_sends: + server.handle_command(old_server, old_peer, old_callback) + else: + client.handle_server_command(old_callback, control_socket=old_client) + record = sender._transfer_handshakes._pending[(control, 808)] + assert record["result"] is None + rejected.append(record) + return actual_send(sock, message, is_send_raw=is_send_raw) + + monkeypatch.setattr(receiver, "send_message", replay_old_session_callback) + src = tmp_path / "new_session.bin" + payload = os.urandom(32781) + src.write_bytes(payload) + port, stamp = request_transfer_listener( + sender, receiver, f"/file {shlex.quote(str(src))} 808" + ) + assert len(rejected) == 1 + assert sender.file_transfer_mode( + str(src), "127.0.0.1", port, 0, control_socket=control, crypto=stamp, fid=808, + peer=peer if server_sends else (server.host, server.port), + ) + assert (recv_dir / src.name).read_bytes() == payload + assert not sender._transfer_handshakes._pending + finally: + _close_socket_workers(server, clients, workers) + + +@pytest.mark.parametrize( + "server_host,client_host", + [("127.0.0.1", "localhost"), ("0.0.0.0", "127.0.0.1")], + ids=["localhost-alias", "wildcard-bind"], +) +@pytest.mark.parametrize( + "recorded_peer", ["configured", "connected"], ids=["legacy-row", "new-row"] +) +def test_resume_identity_uses_the_connected_endpoint( + tmp_path, server_host, client_host, recorded_peer +): + """Resume works when the configured host differs from the connected one. + + The configured address can name a wildcard bind or a host alias; the + concrete endpoint the kernel selected for the control connection is what + both sides can compare. A legacy direct row still records the configured + server address and is migrated in memory, without touching the file format. + """ + port = free_port() + server = TCP_Server_Base( + host=server_host, + port=port, + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + _isolate(server, tmp_path, f"server_{server_host}_{recorded_peer}") + server_recv = tmp_path / f"server_recv_{server_host}_{recorded_peer}" + server_recv.mkdir() + server.file_transfer_dir = str(server_recv) + threading.Thread(target=server.start_TCP_Server, daemon=True).start() + assert server_ready(server) + + client = TCP_Client_Base( + host=client_host, + port=port, + client_host="127.0.0.1", + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + ) + _isolate(client, tmp_path, f"client_{client_host}_{recorded_peer}") + client_recv = tmp_path / f"client_recv_{client_host}_{recorded_peer}" + client_recv.mkdir() + client.file_transfer_dir = str(client_recv) + try: + assert client.connect() + peer = client.client_socket.getsockname() + assert wait_until(lambda: peer in server.clients, timeout=10) + src = tmp_path / f"identity_{recorded_peer}.bin" + payload = os.urandom(64 * 1024) + src.write_bytes(payload) + half = len(payload) // 2 + configured_peer = (client.host, client.port) + connected_peer = client.client_socket.getpeername() + + # client -> server: the server asks the reconnected client to continue + partial = server_recv / src.name + partial.write_bytes(payload[:half]) + upload_key = (configured_peer, 701) + client.file_upload_progress[upload_key] = [ + half, len(payload), None, str(src), configured_peer, + ] + server.file_recv_progress[701] = [ + half, len(payload), None, str(partial), peer, + ] + assert server._request_resumes(peer) == 1 + assert wait_until(lambda: 701 not in server.file_recv_progress, timeout=20) + assert partial.read_bytes() == payload + assert wait_until(lambda: upload_key not in client.file_upload_progress, timeout=10) + + # server -> client: the client asks the connected server to continue + partial2 = client_recv / src.name + partial2.write_bytes(payload[:half]) + recorded = configured_peer if recorded_peer == "configured" else connected_peer + server.file_upload_progress[(peer, 702)] = [ + half, len(payload), None, str(src), peer, + ] + client.file_recv_progress[702] = [ + half, len(payload), None, str(partial2), recorded, + ] + assert client._request_resumes() == 1 + assert wait_until(lambda: 702 not in client.file_recv_progress, timeout=20) + assert partial2.read_bytes() == payload + assert wait_until(lambda: (peer, 702) not in server.file_upload_progress, timeout=10) + finally: + client.close() + server.stop() + + +def test_closed_control_socket_falls_back_to_the_configured_endpoint(tmp_path): + """A socket that is already gone falls back to the configured identity.""" + server = _start_server(tmp_path) + dead = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + dead.close() + try: + assert _socket_local_endpoint(dead) is None + assert _socket_peer_endpoint(dead) is None + assert server._control_local_endpoint(dead) == (server.host, server.port) + assert server._is_control_local_endpoint((server.host, server.port), dead) + assert not server._is_control_local_endpoint(("127.0.0.1", 1), dead) + finally: + server.stop() + + +@pytest.mark.parametrize("sender_side", ["server", "client"]) +def test_paused_leg_is_released_by_stale_sweep(tmp_path, monkeypatch, sender_side): + """A reconnect sweep releases a paused leg and unblocks its queued resume. + + The sweep removes the leg's socket from the registry before closing it, so + the pause wait must also observe the leg's own socket; otherwise the leg + keeps the per-fid send lock and a resume request never runs. + """ + workers = _track_socket_workers(monkeypatch) + server = _start_server(tmp_path) + clients = [] + try: + server_recv = tmp_path / "server_recv" + server_recv.mkdir() + server.file_transfer_dir = str(server_recv) + client_recv = tmp_path / "client_recv" + client_recv.mkdir() + client = _start_client(tmp_path, server, name="client", recv_dir=client_recv) + clients.append(client) + peer = client.client_socket.getsockname() + assert wait_until(lambda: peer in server.clients) + + src = tmp_path / "paused.bin" + src.write_bytes(os.urandom(64 * 1024)) + fid = 4322 + if sender_side == "server": + sender, receiver = server, client + command = ( + f"/file {shlex.quote(str(src))} {shlex.quote(repr(peer))} {fid}" + ) + key = (peer, fid) + target = peer + assert sender.pause_trans(fid, peer) is False + else: + sender, receiver = client, server + command = f"/file {shlex.quote(str(src))} {fid}" + key = ((client.host, client.port), fid) + target = (client.host, client.port) + assert sender.pause_trans(fid) is False + port, stamp = request_transfer_listener(sender, receiver, command) + control = ( + server.clients[peer]["socket"] if sender is server else client.client_socket + ) + result = {} + + def send_leg(): + with _sending_transfer(sender, key), _transfer_handshake( + sender, control, fid + ): + return sender.file_transfer_mode( + str(src), + "127.0.0.1", + port, + 0, + control_socket=control, + crypto=stamp, + fid=fid, + peer=target, + ) + + leg = threading.Thread( + target=lambda: result.setdefault("value", send_leg()), daemon=True + ) + leg.start() + assert wait_until( + lambda: key in sender._active_transfer_sockets + and key in sender.file_upload_progress, + timeout=10, + ), "the paused leg never registered" + assert sender._transfer_pause.get(key) is True + swept = sender._drop_stale_transfers(peer=peer if sender is server else None) + assert swept == 1 + assert wait_until(lambda: not leg.is_alive(), timeout=10), ( + "the swept leg stayed paused" + ) + assert result.get("value") is False + assert key not in sender._transfer_pause + assert key in sender.file_upload_progress + + calls = [] + assert _start_resume_send(sender, key, target, calls.append) + assert wait_until(lambda: calls, timeout=10), "the queued resume never ran" + assert calls[0] is sender.file_upload_progress[key] + finally: + _close_socket_workers(server, clients, workers) \ No newline at end of file diff --git a/test/integration/test_forward_msg_attribution.py b/test/integration/test_forward_msg_attribution.py index e674f21..f73204b 100644 --- a/test/integration/test_forward_msg_attribution.py +++ b/test/integration/test_forward_msg_attribution.py @@ -212,9 +212,7 @@ def test_forward_records_keyed_by_originator(trio): def test_public_forward_api(trio): - """The packaged forward API for extension authors: forward_message_to - tags the envelope, forward_target_command builds the tagged transfer - command, and unreachable targets are reported.""" + """The packaged forward API tags messages and builds target commands.""" server, a, b = trio received = [] b.add_message_listener(lambda sender, text: received.append((sender, text))) @@ -224,7 +222,7 @@ def test_public_forward_api(trio): ) cmd = server.forward_target_command("file", "", "a.txt", _addr(a), 7) assert parse_forward_originator(cmd) == _client_key(a) - assert server.forward_message_to(("127.0.0.1", 1), "x", _addr(a)) is False # unreachable + assert server.forward_message_to(("127.0.0.1", 1), "x", _addr(a)) is False def test_extension_command_cannot_hijack_envelope(trio): diff --git a/test/unit/network_api/test_TCP_client.py b/test/unit/network_api/test_TCP_client.py index 2cb837c..a10f2da 100644 --- a/test/unit/network_api/test_TCP_client.py +++ b/test/unit/network_api/test_TCP_client.py @@ -21,6 +21,7 @@ def test_tcp_client_init(): assert client.upload_network_speed is None assert client.recv_network_speed is None +import errno # noqa: E402 import socket # noqa: E402 import threading # noqa: E402 import time # noqa: E402 @@ -36,6 +37,18 @@ def _client(**kwargs): return TCP_Client_Base(is_extend_command=True, **options) +def test_eacces_is_a_local_port_conflict_only_on_windows(monkeypatch): + error = OSError(errno.EACCES, "access denied") + assert _connect_tcp._is_local_port_conflict(error) is (_connect_tcp.os.name == "nt") + assert not _connect_tcp._should_fallback_local_port(None, error) + assert not _connect_tcp._should_fallback_local_port(0, error) + + with monkeypatch.context() as patch: + patch.setattr(_connect_tcp.os, "name", "nt") + assert _connect_tcp._is_local_port_conflict(error) + assert _connect_tcp._should_fallback_local_port(49152, error) + + def test_console_custom_command_matches_the_whole_first_token(monkeypatch): """Reach a console handler registered as ``/mycmd``. @@ -69,6 +82,35 @@ def fake_input(): assert "/mycmd hello" not in sent # not forwarded to the server as chat +def test_transfers_console_prints_local_records_with_logging_disabled(tmp_path, monkeypatch, capsys): + import json + + client = _client(is_print_log=False, is_enable_encrypto=False) + for attr in ("messages_log_file", "events_log_file", "sent_messages_log_file", + "sent_events_log_file", "file_trans_progress_file"): + setattr(client, attr, str(tmp_path / f"{attr}.json")) + client.file_recv_progress[22] = [9, 9, None, "/recv/完整.bin", ("127.0.0.1", 65004)] + client.running = True + lines = iter([" /TrAnSfErS ", '/transfers "']) + + def input_line(): + try: + return next(lines) + except StopIteration: + client.running = False + return "" + + monkeypatch.setattr("builtins.input", input_line) + client.interactive_mode() + output = capsys.readouterr().out + view, end = json.JSONDecoder().raw_decode(output) + row, = view["memory"]["downloading"] + assert row["path"] == "/recv/完整.bin" and row["transferred"] == row["total"] == 9 + assert view["persisted"]["status"] == "missing" + # Unsupported arguments, including an unmatched quote, remain local usage. + assert "/transfers" in output[end:] + + def test_multiple_file_keeps_the_semaphore_slot_during_the_transfer(monkeypatch): """Cap concurrent ``/multiple_file`` sends at ``max_thread_num``. diff --git a/test/unit/network_api/test_delete_config.py b/test/unit/network_api/test_delete_config.py new file mode 100644 index 0000000..d1d0d17 --- /dev/null +++ b/test/unit/network_api/test_delete_config.py @@ -0,0 +1,283 @@ +"""Contracts for named local state deletion and discarded-progress lifetime.""" + +import json +import threading +from pathlib import Path + +import pytest + +from PyFlow.network_api import connect_tcp +from PyFlow.network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base + +FILES = ("file_trans_progress_file", "messages_log_file", "events_log_file", + "sent_messages_log_file", "sent_events_log_file") +PEER = ("127.0.0.1", 50001) + + +@pytest.fixture(params=[TCP_Server_Base, TCP_Client_Base], ids=["server", "client"]) +def instance(request, tmp_path): + owner = request.param( + host="127.0.0.1", port=0, is_extend_command=True, + is_input_command_in_console=False, is_enable_encrypto=False, is_print_log=False, + ) + for attr in FILES: + setattr(owner, attr, str(tmp_path / Path(getattr(owner, attr)).name)) + yield owner + owner._active_transfer_fids.clear() + owner._active_transfer_sockets.clear() + close(owner) + + +def close(owner): + owner.stop() if isinstance(owner, TCP_Server_Base) else owner.close() + + +def populate(owner): + for attr in FILES: + Path(getattr(owner, attr)).write_text("{}", encoding="utf-8") + + +def test_list_is_read_only_and_takes_precedence_over_name(instance, tmp_path): + populate(instance) + extra = tmp_path / "setup.json" + extra.write_text("do not delete", encoding="utf-8") + before = {p: p.read_bytes() for p in tmp_path.iterdir()} + instance.running = True + try: + names = instance.delete_config([], is_list_config=True) + assert names == sorted(Path(getattr(instance, attr)).name for attr in FILES) + assert instance.running is True + assert {p: p.read_bytes() for p in before} == before + with pytest.raises(RuntimeError): + instance.delete_config(names[0]) + finally: + instance.running = False + + +@pytest.mark.parametrize("attr", FILES) +def test_named_delete_leaves_every_other_file_and_repeated_close_intact(instance, attr): + populate(instance) + close(instance) + paths = [Path(getattr(instance, field)) for field in FILES] + selected = Path(getattr(instance, attr)) + untouched = {p: p.read_bytes() for p in paths if p != selected} + assert instance.delete_config(selected.name) is True + assert instance.delete_config(selected.name) is False + close(instance) + assert not selected.exists() + assert {p: p.read_bytes() for p in untouched} == untouched + + +@pytest.mark.parametrize("name", [None, "", ".", "..", "../messages_log.json", + "/messages_log.json", "C:\\messages_log.json", + "*.json", "setup.json", "decode_command_table.json"]) +def test_invalid_names_never_resolve_to_files(instance, name): + populate(instance) + before = {attr: Path(getattr(instance, attr)).read_bytes() for attr in FILES} + with pytest.raises(ValueError): + instance.delete_config(name) + assert {attr: Path(getattr(instance, attr)).read_bytes() for attr in FILES} == before + + +def test_non_string_delete_name_is_rejected(instance): + with pytest.raises(TypeError): + instance.delete_config(["messages_log.json"]) + + +@pytest.mark.parametrize("kind", ["directory", "symlink"]) +def test_nonregular_targets_are_not_listed_or_followed(instance, tmp_path, kind): + path = Path(instance.messages_log_file) + other = tmp_path / "protected.txt" + other.write_bytes(b"protected content") + if kind == "directory": + path.mkdir() + else: + path.symlink_to(other) + assert path.name not in instance.delete_config(is_list_config=True) + with pytest.raises(ValueError): + instance.delete_config(path.name) + assert other.read_bytes() == b"protected content" + assert path.is_dir() if kind == "directory" else path.is_symlink() + + +def test_redirected_names_are_supported_but_ambiguous_names_are_refused(instance, tmp_path): + path = tmp_path / "custom" / "progress.json" + path.parent.mkdir() + path.write_bytes(b"unreadable old progress is still removable") + instance.file_trans_progress_file = str(path) + assert instance.delete_config(is_list_config=True) == [path.name] + assert instance.delete_config(path.name) is True + first = tmp_path / "left" / "same.json" + second = tmp_path / "right" / "same.json" + for candidate in (first, second): + candidate.parent.mkdir() + candidate.write_bytes(b"keep") + instance.messages_log_file, instance.events_log_file = str(first), str(second) + with pytest.raises(ValueError): + instance.delete_config("same.json") + with pytest.raises(ValueError): + instance.delete_config(is_list_config=True) + assert first.read_bytes() == second.read_bytes() == b"keep" + + +def test_failed_unlink_does_not_disable_normal_progress_persistence(instance, monkeypatch): + path = Path(instance.file_trans_progress_file) + path.write_text("{}", encoding="utf-8") + + def denied(candidate): + raise PermissionError("deletion denied") + + with monkeypatch.context() as patch: + patch.setattr(connect_tcp.os, "unlink", denied) + with pytest.raises(PermissionError, match="deletion denied"): + instance.delete_config(path.name) + assert path.read_text(encoding="utf-8") == "{}" + instance.file_recv_progress[6] = [3, 10, None, "/recv/partial.bin", PEER] + close(instance) + assert json.loads(path.read_text())["download"]["6"] == [ + 3, 10, None, "/recv/partial.bin", list(PEER) + ] + + +def test_discard_cannot_be_undone_by_late_finish_reload_or_start(instance, monkeypatch): + row = [3, 10, None, "/recv/partial.bin", PEER] + instance.file_recv_progress[6] = row + close(instance) + path = Path(instance.file_trans_progress_file) + assert instance.delete_config(path.name) is True + instance._file_progress_finish(instance.file_recv_progress, 6, row, keep=True) + close(instance) + assert not path.exists() + # A different writer's new snapshot must not be loaded or overwritten by this old object. + other = '{"download":{"8":[1,2,null,"/other.bin",["127.0.0.1",50001]]}}' + path.write_text(other, encoding="utf-8") + assert instance._load_transfer_progress() == 0 + instance._persist_transfer_progress() + assert 8 not in instance.file_recv_progress and path.read_text() == other + + def no_network(*args, **kwargs): + raise AssertionError("discarded object attempted network I/O") + + monkeypatch.setattr(connect_tcp.socket, "socket", no_network) + entries = [instance.start_TCP_Server] if isinstance(instance, TCP_Server_Base) else [ + instance.connect, instance.start_TCP_client + ] + for start in entries: + with pytest.raises(RuntimeError, match="new instance"): + start() + + +def test_missing_progress_is_an_explicit_discard_too(instance): + path = Path(instance.file_trans_progress_file) + assert instance.delete_config(path.name) is False + instance.file_recv_progress[6] = [3, 10, None, "/recv/partial.bin", PEER] + close(instance) + assert not path.exists() + + +def test_closing_in_progress_rejects_delete_until_successful_completion(instance, monkeypatch): + populate(instance) + entered, release = threading.Event(), threading.Event() + real_free = instance.free_port + + def held_free(): + entered.set() + assert release.wait(timeout=5) + return real_free() + + monkeypatch.setattr(instance, "free_port", held_free) + worker = threading.Thread(target=close, args=(instance,)) + worker.start() + try: + assert entered.wait(timeout=5) + with pytest.raises(RuntimeError): + instance.delete_config(Path(instance.file_trans_progress_file).name) + finally: + release.set() + worker.join(timeout=5) + assert not worker.is_alive() + assert instance.delete_config(Path(instance.file_trans_progress_file).name) is True + + +def test_active_transfers_relays_and_pending_logs_prevent_deletion(instance): + populate(instance) + close(instance) + name = Path(instance.file_trans_progress_file).name + instance._active_transfer_fids.add(6) + with pytest.raises(RuntimeError): + instance.delete_config(name) + instance._active_transfer_fids.clear() + instance._active_transfer_sockets[6] = object() + with pytest.raises(RuntimeError): + instance.delete_config(name) + instance._active_transfer_sockets.clear() + if isinstance(instance, TCP_Server_Base): + instance._forward_relays[6] = {} + else: + instance._resume_sends[(PEER, 6)] = { + "lock": threading.RLock(), "users": 1, "targets": {PEER}, + } + try: + with pytest.raises(RuntimeError): + instance.delete_config(name) + finally: + if isinstance(instance, TCP_Server_Base): + instance._forward_relays.clear() + else: + instance._resume_sends.clear() + instance.messages_dict["peer"] = [["late message", "now"]] + with pytest.raises(RuntimeError): + instance.delete_config(Path(instance.messages_log_file).name) + close(instance) + assert instance.delete_config(Path(instance.messages_log_file).name) is True + + +@pytest.mark.parametrize("users,targets", [(0, set()), (1, set()), (0, {PEER})]) +def test_transient_resume_state_prevents_deletion_until_released(instance, users, targets): + populate(instance) + close(instance) + path = Path(instance.file_trans_progress_file) + before = path.read_bytes() + key = (PEER, 6) + state = {"lock": threading.RLock(), "users": users, "targets": targets} + instance._resume_sends[key] = state + try: + with pytest.raises(RuntimeError): + instance.delete_config(path.name) + assert path.read_bytes() == before + assert instance._resume_sends[key] is state + finally: + instance._resume_sends.clear() + assert instance.delete_config(path.name) is True + + +def test_delete_waits_for_inflight_progress_publication(instance, monkeypatch): + instance.file_recv_progress[6] = [3, 10, None, "/recv/partial.bin", PEER] + close(instance) + path = Path(instance.file_trans_progress_file) + entered, release = threading.Event(), threading.Event() + real_replace = connect_tcp.os.replace + outcomes = [] + + def held_replace(source, destination): + if destination == str(path): + entered.set() + assert release.wait(timeout=5) + return real_replace(source, destination) + + monkeypatch.setattr(connect_tcp.os, "replace", held_replace) + writer = threading.Thread(target=instance._persist_transfer_progress) + deleter = threading.Thread(target=lambda: outcomes.append(instance.delete_config(path.name))) + writer.start() + try: + assert entered.wait(timeout=5) + deleter.start() + finally: + release.set() + writer.join(timeout=5) + if deleter.ident is not None: + deleter.join(timeout=5) + assert not writer.is_alive() and not deleter.is_alive() + assert outcomes == [True] + close(instance) + assert not path.exists() diff --git a/test/unit/network_api/test_fid_allocator.py b/test/unit/network_api/test_fid_allocator.py new file mode 100644 index 0000000..3c80cb8 --- /dev/null +++ b/test/unit/network_api/test_fid_allocator.py @@ -0,0 +1,66 @@ +"""Receiver-scoped allocator occupancy must not overwrite a live upload.""" + +from PyFlow.network_api.connect_tcp import FID_LOW_BITS, _FidAllocator, format_fid + + +def test_allocator_skips_a_live_upload_in_the_same_receiver_namespace(): + receiver = ("127.0.0.1", 40001) + uploads = {} + allocator = _FidAllocator((uploads, {}, {})) + candidate = (1 << FID_LOW_BITS) | (allocator._epoch << 1) + entry = [0, 1024, None, "/source.bin", receiver] + uploads[(receiver, candidate)] = entry + + allocated = allocator.alloc(receiver=receiver) + + assert allocated != candidate + assert (receiver, allocated) not in uploads + assert uploads[(receiver, candidate)] is entry + + +def test_another_receiver_does_not_occupy_the_candidate_namespace(): + receiver, other = ("127.0.0.1", 40001), ("127.0.0.1", 40002) + uploads = {} + allocator = _FidAllocator((uploads, {}, {})) + candidate = (1 << FID_LOW_BITS) | (allocator._epoch << 1) + uploads[(other, candidate)] = [0, 1024, None, "/other.bin", other] + + assert allocator.alloc(receiver=receiver) == candidate + assert (receiver, candidate) not in uploads + +def test_allocator_skips_restored_route_primary_and_scoped_downstream_ids(): + receiver = ("127.0.0.1", 40001) + other_receiver = ("127.0.0.1", 40002) + uploads, downloads = {}, {} + routes, route_index = {}, {} + allocator = _FidAllocator((uploads, downloads, routes, route_index)) + primary = (1 << FID_LOW_BITS) | (allocator._epoch << 1) + downstream = (2 << FID_LOW_BITS) | (allocator._epoch << 1) + route = {"originator": ("127.0.0.1", 40000), "targets": {receiver: downstream}} + routes[primary] = route + route_index[(receiver, downstream)] = primary + + allocated = allocator.alloc(receiver=receiver) + assert allocated not in (primary, downstream) + + # The child id is occupied only in its receiver scope; the upstream route + # remains globally occupied regardless of the requested receiver. + allocator._counter = 0 + assert allocator.alloc(receiver=other_receiver) == downstream + + +def test_all_allocations_have_reserved_zero_lowbit_and_kindless_log_format(): + allocator = _FidAllocator(({}, {}, {}, {})) + for counter, receiver in enumerate((None, ("127.0.0.1", 40001), None), start=1): + fid = allocator.alloc(receiver=receiver) + assert fid == (counter << FID_LOW_BITS) | (allocator._epoch << 1) + assert fid & 1 == 0 + assert format_fid(fid) == f"e{allocator._epoch}.{counter}" + + +def test_allocator_no_longer_accepts_a_kind_selector(): + import pytest + + allocator = _FidAllocator(({}, {}, {}, {})) + with pytest.raises(TypeError): + allocator.alloc(kind=1) diff --git a/test/unit/network_api/test_fid_routes.py b/test/unit/network_api/test_fid_routes.py new file mode 100644 index 0000000..88d99b5 --- /dev/null +++ b/test/unit/network_api/test_fid_routes.py @@ -0,0 +1,411 @@ +"""Unit tests for the server's authoritative, receiver-scoped FID routes.""" + +import json +import socket + +import pytest + +from PyFlow.network_api.connect_tcp import TCP_Server_Base, parse_resume_command + + +# The server's control side as the unit tests drive `_resume_request` by hand. +SERVER_LOCAL = ("127.0.0.1", 65432) +ORIGINATOR = ("127.0.0.1", 50001) +TARGET_1 = ("127.0.0.1", 50002) +TARGET_2 = ("127.0.0.1", 50003) +TARGET_3 = ("127.0.0.1", 50004) + + +def _server(tmp_path, name="server"): + """Build an unstarted server whose progress file belongs to this test.""" + server = TCP_Server_Base( + host="127.0.0.1", + port=0, + is_extend_command=True, + is_input_command_in_console=False, + is_enable_encrypto=False, + is_print_log=False, + ) + server.messages_log_file = str(tmp_path / f"{name}_messages_log.json") + server.events_log_file = str(tmp_path / f"{name}_events_log.json") + server.sent_messages_log_file = str(tmp_path / f"{name}_sent_messages_log.json") + server.sent_events_log_file = str(tmp_path / f"{name}_sent_events_log.json") + server.file_trans_progress_file = str(tmp_path / f"{name}_file_trans_progress.json") + return server + + +@pytest.fixture +def server(tmp_path): + instance = _server(tmp_path) + try: + yield instance + finally: + instance.stop() + + +def test_lookup_returns_the_same_authoritative_route_from_upstream_and_targets(server): + route = server._register_fid_route( + 303, + ORIGINATOR, + {TARGET_1: 305, TARGET_2: 306, TARGET_3: 307}, + ) + assert route == {"originator": ORIGINATOR, "targets": { + TARGET_1: 305, + TARGET_2: 306, + TARGET_3: 307, + }, "completed": set()} + assert server._lookup_fid_route(None, 303) == (303, route) + for target, child_fid in route["targets"].items(): + upstream_fid, found = server._lookup_fid_route(target, child_fid) + assert upstream_fid == 303 + assert found is route + + +def test_completion_is_identity_safe_and_removes_only_one_reverse_index(server): + route = server._register_fid_route( + 303, ORIGINATOR, {TARGET_1: 305, TARGET_2: 306} + ) + assert route is not None + + assert server._complete_fid_target(303, route, TARGET_3, 305) is False + assert server._complete_fid_target(303, route, TARGET_1, 306) is False + assert server._complete_fid_target(303, {**route}, TARGET_1, 305) is False + assert server._complete_fid_target(303, route, TARGET_1, 305) is True + assert server._complete_fid_target(303, route, TARGET_1, 305) is False + assert route["completed"] == {TARGET_1} + assert server._lookup_fid_route(TARGET_1, 305) is None + assert server._lookup_fid_route(TARGET_2, 306) == (303, route) + + +def test_registration_rejects_completed_outside_targets_without_mutation(server): + assert server._register_fid_route( + 303, ORIGINATOR, {TARGET_1: 305}, completed=(TARGET_2,) + ) is None + assert server._fid_routes == {} + assert server._fid_route_index == {} + + +def test_equal_child_fids_are_scoped_by_receiver_even_across_routes(server): + first = server._register_fid_route(303, ORIGINATOR, {TARGET_1: 305, TARGET_2: 305}) + second = server._register_fid_route(304, ORIGINATOR, {TARGET_3: 305}) + + assert first is not None + assert second is not None + assert server._lookup_fid_route(TARGET_1, 305) == (303, first) + assert server._lookup_fid_route(TARGET_2, 305) == (303, first) + assert server._lookup_fid_route(TARGET_3, 305) == (304, second) + assert server._lookup_fid_route(TARGET_3, 306) is None + + +def test_lookup_rejects_unknown_and_wrong_peers_without_falling_back_to_source(server): + route = server._register_fid_route(303, ORIGINATOR, {TARGET_1: 305}) + + assert route is not None + assert server._lookup_fid_route(None, 305) is None + assert server._lookup_fid_route(ORIGINATOR, 305) is None + assert server._lookup_fid_route(("127.0.0.1", 59999), 305) is None + assert server._lookup_fid_route(TARGET_1, 303) is None + assert server._lookup_fid_route(None, 999) is None + + +def test_registration_collision_is_atomic_for_primary_and_downstream_indexes(server): + original = server._register_fid_route(303, ORIGINATOR, {TARGET_1: 305, TARGET_2: 306}) + assert original is not None + primary_before = dict(server._fid_routes) + index_before = dict(server._fid_route_index) + + assert server._register_fid_route(303, TARGET_3, {TARGET_3: 307}) is None + assert server._fid_routes == primary_before + assert server._fid_route_index == index_before + + assert server._register_fid_route(304, TARGET_3, {TARGET_2: 306, TARGET_3: 308}) is None + assert server._fid_routes == primary_before + assert server._fid_route_index == index_before + + +@pytest.mark.parametrize( + "owner", + [ + "receive", "upload", "active_fid", "active_receive_fid", + "active_socket", "active_receive_socket", "pause", "relay", "route_index", + ], +) +def test_registration_rejects_upstream_fid_that_collides_with_live_transfer_identity( + server, owner +): + fid = 303 + target_key = (TARGET_1, fid) + if owner == "receive": + server.file_recv_progress[fid] = [0, 1, None, "/tmp/live", ORIGINATOR] + elif owner == "upload": + server.file_upload_progress[target_key] = [0, 1, None, "/tmp/live", TARGET_1] + elif owner == "active_fid": + server._active_transfer_fids.add(target_key) + elif owner == "active_receive_fid": + server._active_transfer_fids.add(fid) + elif owner == "active_socket": + server._active_transfer_sockets[target_key] = object() + elif owner == "active_receive_socket": + server._active_transfer_sockets[fid] = object() + elif owner == "pause": + server._transfer_pause[target_key] = True + elif owner == "relay": + server._forward_relays[target_key] = object() + else: + assert server._register_fid_route(404, ORIGINATOR, {TARGET_1: fid}) is not None + + routes_before = dict(server._fid_routes) + indexes_before = dict(server._fid_route_index) + + assert server._register_fid_route(303, ORIGINATOR, {TARGET_1: 306}) is None + assert server._fid_routes == routes_before + assert server._fid_route_index == indexes_before + + +def test_registration_scopes_live_downstream_upload_conflicts_by_receiver(server): + fid = 303 + server.file_upload_progress[(TARGET_2, fid)] = [0, 1, None, "/tmp/live", TARGET_2] + + route = server._register_fid_route(fid, ORIGINATOR, {TARGET_1: 305}) + + assert route is not None + assert server._lookup_fid_route(TARGET_1, 305) == (fid, route) + + +def test_route_removal_is_identity_safe_and_clears_all_owned_indexes(server): + route = server._register_fid_route(303, ORIGINATOR, {TARGET_1: 305, TARGET_2: 306}) + assert route is not None + + forged_equal_route = { + "originator": route["originator"], + "targets": dict(route["targets"]), + } + assert server._drop_fid_route(303, forged_equal_route) is False + assert server._lookup_fid_route(None, 303) == (303, route) + assert server._lookup_fid_route(TARGET_1, 305) == (303, route) + + assert server._drop_fid_route(303, route) is True + assert server._lookup_fid_route(None, 303) is None + assert server._lookup_fid_route(TARGET_1, 305) is None + assert server._lookup_fid_route(TARGET_2, 306) is None + + +def test_route_persists_and_round_trips_through_a_fresh_server(tmp_path): + first = _server(tmp_path, "first") + second = None + try: + route = first._register_fid_route(303, ORIGINATOR, {TARGET_1: 305, TARGET_2: 306}) + assert route is not None + first._persist_transfer_progress() + + with open(first.file_trans_progress_file, "r", encoding="utf-8") as stream: + snapshot = json.load(stream) + assert snapshot["relays"] == { + "303": { + "originator": list(ORIGINATOR), + "targets": [[list(TARGET_1), 305], [list(TARGET_2), 306]], + "completed": [], + } + } + + second = _server(tmp_path, "second") + second.file_trans_progress_file = first.file_trans_progress_file + assert second._load_transfer_progress() == 1 + restored = second._fid_routes[303] + assert restored["originator"] == ORIGINATOR + assert restored["targets"] == {TARGET_1: 305, TARGET_2: 306} + assert restored["completed"] == set() + assert second._lookup_fid_route(TARGET_1, 305) == (303, restored) + assert second._lookup_fid_route(TARGET_2, 306) == (303, restored) + finally: + first.stop() + if second is not None: + second.stop() + + +def test_reload_does_not_clobber_live_primary_or_downstream_index_owners(tmp_path): + persisted = _server(tmp_path, "persisted") + live = None + try: + assert persisted._register_fid_route(303, ORIGINATOR, {TARGET_1: 305}) is not None + assert persisted._register_fid_route(304, ORIGINATOR, {TARGET_2: 306}) is not None + persisted._persist_transfer_progress() + + live = _server(tmp_path, "live") + live.file_trans_progress_file = persisted.file_trans_progress_file + primary_owner = live._register_fid_route(303, TARGET_3, {TARGET_3: 307}) + index_owner = live._register_fid_route(305, TARGET_3, {TARGET_2: 306}) + assert primary_owner is not None + assert index_owner is not None + + assert live._load_transfer_progress() == 0 + assert live._fid_routes == {303: primary_owner, 305: index_owner} + assert live._fid_route_index == { + (TARGET_3, 307): 303, + (TARGET_2, 306): 305, + } + finally: + persisted.stop() + if live is not None: + live.stop() + + +def test_unsupported_old_server_relay_snapshot_is_preserved_on_write_and_stop(tmp_path): + server = _server(tmp_path) + try: + legacy = { + "upload": {}, + "download": {}, + "relays": { + "303": { + "fid": 303, + "originator": list(ORIGINATOR), + "targets": [list(TARGET_1)], + "tfids": [305], + "kind": 1, + } + }, + } + with open(server.file_trans_progress_file, "w", encoding="utf-8") as stream: + json.dump(legacy, stream, indent=2) + before = open(server.file_trans_progress_file, "rb").read() + + assert server._load_transfer_progress() == 0 + assert server._fid_routes == {} + assert server._register_fid_route(303, ORIGINATOR, {TARGET_1: 305}) is not None + server._persist_transfer_progress() + assert open(server.file_trans_progress_file, "rb").read() == before + server.stop() + assert open(server.file_trans_progress_file, "rb").read() == before + finally: + # stop() is intentionally idempotent; the explicit call above checks the + # shutdown persistence path while this cleanup handles assertion failures. + server.stop() + + +def test_completed_targets_survive_reload_without_recreating_their_indexes(tmp_path): + first = _server(tmp_path, "first") + second = _server(tmp_path, "second") + try: + route = first._register_fid_route(303, ORIGINATOR, {TARGET_1: 305, TARGET_2: 306}) + assert first._complete_fid_target(303, route, TARGET_1, 305) + second.file_trans_progress_file = first.file_trans_progress_file + assert second._load_transfer_progress() == 1 + restored = second._fid_routes[303] + assert restored["targets"] == {TARGET_1: 305, TARGET_2: 306} + assert restored["completed"] == {TARGET_1} + assert second._lookup_fid_route(None, 303)[1] is restored + assert second._lookup_fid_route(TARGET_1, 305) is None + assert second._lookup_fid_route(TARGET_2, 306)[1] is restored + assert second._fid_route_index == {(TARGET_2, 306): 303} + assert second._drop_fid_route(303, restored) + replacement = second._register_fid_route(303, TARGET_3, {TARGET_1: 305}) + assert not second._complete_fid_target(303, restored, TARGET_2, 306) + assert second._lookup_fid_route(TARGET_1, 305)[1] is replacement + finally: + first.stop() + second.stop() + + +@pytest.mark.parametrize("completed", [None, [list(TARGET_2)], [list(TARGET_1)] * 2]) +def test_invalid_completed_snapshot_is_preserved_on_load_write_and_stop(tmp_path, completed): + instance = _server(tmp_path) + snapshot = {"relays": {"303": { + "originator": list(ORIGINATOR), "targets": [[list(TARGET_1), 305]], + "completed": completed, + }}} + with open(instance.file_trans_progress_file, "w", encoding="utf-8") as stream: + json.dump(snapshot, stream) + with open(instance.file_trans_progress_file, "rb") as stream: + before = stream.read() + try: + assert instance._load_transfer_progress() == 0 + assert instance._fid_routes == {} + assert instance._register_fid_route(303, ORIGINATOR, {TARGET_1: 305}) is not None + instance._persist_transfer_progress() + instance.stop() + with open(instance.file_trans_progress_file, "rb") as stream: + assert stream.read() == before + finally: + instance.stop() + + +def test_canonical_snapshot_without_completed_has_no_acknowledged_targets(tmp_path): + instance = _server(tmp_path) + with open(instance.file_trans_progress_file, "w", encoding="utf-8") as stream: + json.dump({"relays": {"303": { + "originator": list(ORIGINATOR), "targets": [[list(TARGET_1), 305]], + }}}, stream) + try: + assert instance._load_transfer_progress() == 1 + route = instance._fid_routes[303] + assert route["completed"] == set() + assert instance._lookup_fid_route(TARGET_1, 305)[1] is route + finally: + instance.stop() + + +@pytest.mark.parametrize("downstream", [304, 305]) +def test_resume_uses_scoped_route_not_id_lowbit_and_rejects_forged_endpoints(server, downstream): + route = server._register_fid_route(302, ORIGINATOR, {TARGET_1: downstream}) + for peer, origin, target, fid in ( + (TARGET_1, TARGET_2, TARGET_1, downstream), + (TARGET_1, ORIGINATOR, TARGET_2, downstream), + (TARGET_2, ORIGINATOR, TARGET_2, downstream), + (TARGET_1, ORIGINATOR, TARGET_1, downstream + 2), + ): + assert not server._resume_request(SERVER_LOCAL, peer, fid, 7, origin, target) + assert server._pending_originator_resumes == {} + assert server._fid_routes[302] is route + # Offline is not success, but a verified receiver's request is retained. + assert not server._resume_request(SERVER_LOCAL, TARGET_1, downstream, 7, ORIGINATOR, TARGET_1) + assert not server._resume_request(SERVER_LOCAL, TARGET_1, downstream, 19, ORIGINATOR, TARGET_1) + assert server._pending_originator_resumes == {ORIGINATOR: {(302, TARGET_1): 19}} + assert server._fid_route_index == {(TARGET_1, downstream): 302} + + +def test_pending_resume_survives_failed_send_and_prunes_completed_or_missing_routes(server): + route = server._register_fid_route(302, ORIGINATOR, {TARGET_1: 304, TARGET_2: 306}) + for target, child in route["targets"].items(): + assert not server._resume_request(SERVER_LOCAL, target, child, 13, ORIGINATOR, target) + broken, peer = socket.socketpair() + peer.close() + # A closed peer does not make the next write fail on every platform: Windows + # accepts the first send into the local buffer and only reports the reset + # later. Shut the writing half down as well, so "the send failed" is a + # property of the socket here, not of the platform's timing. + broken.shutdown(socket.SHUT_WR) + server.running = True + server.clients[ORIGINATOR] = {"socket": broken} + try: + assert server._flush_pending_originator_resumes(ORIGINATOR) == 0 + assert server._pending_originator_resumes == { + ORIGINATOR: {(302, TARGET_1): 13, (302, TARGET_2): 13} + } + finally: + broken.close() + assert server._complete_fid_target(302, route, TARGET_1, 304) + writer, reader = socket.socketpair() + reader.settimeout(2) + server.clients[ORIGINATOR] = {"socket": writer} + try: + assert server._flush_pending_originator_resumes(ORIGINATOR) == 1 + assert parse_resume_command(reader.recv(4096).decode()) == ( + 302, 13, ORIGINATOR, TARGET_2 + ) + assert server._pending_originator_resumes == {} + assert not server._resume_request(SERVER_LOCAL, TARGET_1, 304, 0, ORIGINATOR, TARGET_1) + # An accepted request must not outlive its owning route. + assert server._queue_resume_to_originator(302, route, TARGET_2, 29) + reader.recv(4096) + server.clients.pop(ORIGINATOR) + assert not server._queue_resume_to_originator(302, route, TARGET_2, 31) + assert server._drop_fid_route(302, route) + server.clients[ORIGINATOR] = {"socket": writer} + assert server._flush_pending_originator_resumes(ORIGINATOR) == 0 + assert server._pending_originator_resumes == {} + finally: + server.running = False + server.clients.clear() + writer.close() + reader.close() \ No newline at end of file diff --git a/test/unit/network_api/test_resume_matching.py b/test/unit/network_api/test_resume_matching.py new file mode 100644 index 0000000..588feb4 --- /dev/null +++ b/test/unit/network_api/test_resume_matching.py @@ -0,0 +1,82 @@ +"""Unit tests for the receiver's ``/file`` resume matching (``_resumable_entry``). + +A ``/file `` line continues an interrupted download only when the +fid, the sender and the file name all match the progress entry. Anything else is +a new transfer and must never be matched to an existing partial file. +""" + +import pytest + +from PyFlow.network_api.connect_tcp import ( + _resumable_entry, + format_resume_line, + parse_resume_command, +) + +PEER = ("127.0.0.1", 6000) + + +def _progress(peer=PEER, path="/recv/download.bin"): + return {7: [1024, 4096, None, path, peer]} + + +def test_matching_fid_peer_and_name_resumes(): + progress = _progress() + entry = progress[7] + assert _resumable_entry(progress, 7, PEER, ["/file", "download.bin", "7"]) is entry + + +def test_name_is_compared_by_basename_of_the_partial_path(): + progress = _progress(path="/recv/nested/dir/download.bin") + assert _resumable_entry(progress, 7, PEER, ["/file", "download.bin", "7"]) is progress[7] + + +def test_persisted_peer_list_matches_a_peer_tuple(): + # the progress file stores the peer as a JSON list; the live line carries a tuple + progress = _progress(peer=["127.0.0.1", 6000]) + assert _resumable_entry(progress, 7, PEER, ["/file", "download.bin", "7"]) is progress[7] + + +@pytest.mark.parametrize( + ("progress", "fid", "peer", "command_part"), + [ + pytest.param(_progress(), 8, PEER, ["/file", "download.bin", "8"], id="unknown-fid"), + pytest.param( + _progress(), 7, ("10.0.0.2", 6000), ["/file", "download.bin", "7"], id="other-peer" + ), + pytest.param(_progress(), 7, PEER, ["/file", "other.bin", "7"], id="other-file-name"), + pytest.param( + _progress(peer=None), 7, PEER, ["/file", "download.bin", "7"], id="no-recorded-peer" + ), + pytest.param(_progress(), None, PEER, ["/file", "download.bin"], id="no-fid"), + pytest.param(_progress(), 7, None, ["/file", "download.bin", "7"], id="no-peer"), + pytest.param( + _progress(), 7, PEER, ["/file_folder", "download.bin", "7"], id="folder-command" + ), + pytest.param(_progress(), 7, PEER, ["/file", "7"], id="too-few-tokens"), + ], +) +def test_anything_else_is_a_new_transfer(progress, fid, peer, command_part): + assert _resumable_entry(progress, fid, peer, command_part) is None + + +def test_resume_command_preserves_shell_quoted_endpoint_identity(): + origin = ("sender's host", 50001) + target = ("target host", 50002) + line = format_resume_line(304, 12345, origin, target) + assert parse_resume_command(line) == (304, 12345, origin, target) + + +@pytest.mark.parametrize("line", [ + "/resume_trans 304 1", + "/forward_resume '(\"host\", 5)' 304 1", + "/resume_trans 304 -1 '(\"host\", 5)' '(\"target\", 6)'", + "/resume_trans -1 1 '(\"host\", 5)' '(\"target\", 6)'", + "/resume_trans 304 1 '[\"host\", 5]' '(\"target\", 6)'", + "/resume_trans 304 1 '(\"host\", \"5\")' '(\"target\", 6)'", + "/resume_trans 304 1 '(\"host\", 5)' '(\"target\",)'", + "/resume_trans 304 1 '(\"host\", 5)' '(\"target\", 6)' extra", + "/resume_trans 304 1 'unterminated", +]) +def test_resume_command_rejects_ambiguous_or_malformed_identity(line): + assert parse_resume_command(line) is None diff --git a/test/unit/network_api/test_transfer_handshakes.py b/test/unit/network_api/test_transfer_handshakes.py new file mode 100644 index 0000000..b93d7b0 --- /dev/null +++ b/test/unit/network_api/test_transfer_handshakes.py @@ -0,0 +1,158 @@ +"""Contract tests for transient transfer-handshake correlation.""" + +import socket + +import pytest + +from PyFlow.network_api.connect_tcp import ( + _TransferHandshakeRegistry, + _parse_transfer_port_announcement, +) + + + +class _Control: + pass + + + +def test_port_replies_match_socket_and_fid_independently_of_order(): + registry = _TransferHandshakeRegistry() + control, other_control = _Control(), _Control() + first, second = registry.register(control, 17), registry.register(control, 18) + assert first["key"] == (control, 17) + assert second["key"] == (control, 18) + assert registry.mark_sent(first) + assert not registry.mark_sent(first) + assert registry.mark_sent(second) + assert registry.resolve_port(control, 18, 41002, "enc:second") + assert not registry.resolve_port(other_control, 17, 41001, "enc:wrong-socket") + assert not registry.resolve_port(control, 19, 41001, "enc:wrong-fid") + assert registry.resolve_port(control, 17, 41001, "enc:first") + assert not registry.resolve_port(control, 17, 41003, "enc:duplicate") + assert registry.wait_many((first, second), 0) == { + (control, 17): ("port", 17, 41001, "enc:first"), + (control, 18): ("port", 18, 41002, "enc:second"), + } + assert not registry.resolve_port(control, 17, 41003, "enc:late") + assert registry._pending == {} + + +def test_pre_send_cancellation_does_not_make_fid_unresolved(): + registry = _TransferHandshakeRegistry() + control = object() + cancelled = registry.register(control, 17) + registry.cancel(cancelled) + assert not registry.mark_sent(cancelled) + reusable = registry.register(control, 17) + assert reusable is not None + assert registry.mark_sent(reusable) + assert registry.resolve_error(control, 17, "listener refused") + assert registry.wait(reusable, 0) == ("error", "listener refused") + + +def test_sent_cancellation_and_timeout_leave_fid_unresolved_until_disconnect(): + registry = _TransferHandshakeRegistry() + control = _Control() + cancelled = registry.register(control, 17) + assert registry.mark_sent(cancelled) + registry.cancel(cancelled) + assert registry.register(control, 17) is None + assert not registry.resolve_port(control, 17, 42001, "plain") + + timed_out = registry.register(control, 18) + assert registry.mark_sent(timed_out) + assert registry.wait(timed_out, 0) is None + assert registry.register(control, 18) is None + assert not registry.resolve_error(control, 18, "late error") + registry.cancel_socket(control) + assert (control, 17) not in registry._unresolved + assert (control, 18) not in registry._unresolved + assert registry.register(control, 17) is None + + +def test_terminal_reply_wins_atomically_over_timeout_or_cancel(): + registry = _TransferHandshakeRegistry() + control = _Control() + record = registry.register(control, 17) + assert registry.mark_sent(record) + assert registry.resolve_port(control, 17, 43001, "enc:pair") + assert registry.wait(record, 0) == ("port", 17, 43001, "enc:pair") + registry.cancel(record) + assert registry.register(control, 17) is not None + + +def test_old_record_cannot_cancel_a_later_same_key_registration(): + registry = _TransferHandshakeRegistry() + control = _Control() + old = registry.register(control, 17) + registry.cancel(old) + new = registry.register(control, 17) + assert new is not None and new is not old + registry.cancel(old) + assert registry.mark_sent(new) + assert registry.resolve_error(control, 17, "new attempt") + assert registry.wait(new, 0) == ("error", "new attempt") + + +def test_wait_many_classifies_each_target_and_keeps_completed_result_reusable(): + registry = _TransferHandshakeRegistry() + control = _Control() + done = registry.register(control, 17) + pending = registry.register(control, 18) + assert registry.mark_sent(done) + assert registry.mark_sent(pending) + assert registry.resolve_error(control, 17, "one target refused") + assert registry.wait_many((done, pending), 0) is None + assert registry.register(control, 17) is not None + assert registry.register(control, 18) is None + + +def test_disconnect_cancels_only_its_socket_requests_and_resources(): + registry = _TransferHandshakeRegistry() + old_control, new_control = _Control(), _Control() + first = registry.register(old_control, 17) + second = registry.register(new_control, 17) + assert registry.mark_sent(first) + assert registry.mark_sent(second) + left, right = socket.socketpair() + assert registry.track_socket(old_control, left) + registry.cancel_socket(old_control) + assert left.fileno() == -1 + right.close() + assert not registry.resolve_port(old_control, 17, 43001, "plain") + assert registry.resolve_port(new_control, 17, 43002, "plain") + assert registry.wait(first, 0) is None + assert registry.wait(second, 0) == ("port", 17, 43002, "plain") + + +def test_reconnect_socket_allows_same_fid_after_old_session_is_retired(): + registry = _TransferHandshakeRegistry() + old_control, new_control = _Control(), _Control() + old = registry.register(old_control, 17) + assert registry.mark_sent(old) + assert registry.wait(old, 0) is None + registry.cancel_socket(old_control) + new = registry.register(new_control, 17) + assert new is not None + assert registry.mark_sent(new) + assert not registry.resolve_port(old_control, 17, 44001, "old") + assert registry.resolve_port(new_control, 17, 44002, "new") + assert registry.wait(new, 0) == ("port", 17, 44002, "new") + + +def test_port_announcement_keeps_port_and_stamp_pair(): + assert _parse_transfer_port_announcement( + "/server_file_transfer_port 40000 17 enc:stamp" + ) == (40000, 17, "enc:stamp") + +@pytest.mark.parametrize("line", [ + "/server_file_transfer_port 40000 17", + "/server_file_transfer_port 0 17 plain", + "/forward_upload 17 40000 plain", + "/server_file_transfer_port 40000 17 plain " + "a" * 32, + "/server_file_transfer_port 40000 -1 plain", + "/server_file_transfer_port 40000 17 plain invalid", +]) +def test_old_or_malformed_listener_replies_are_rejected(line): + assert _parse_transfer_port_announcement(line) is None diff --git a/test/unit/network_api/test_transfer_progress_query.py b/test/unit/network_api/test_transfer_progress_query.py new file mode 100644 index 0000000..452b21a --- /dev/null +++ b/test/unit/network_api/test_transfer_progress_query.py @@ -0,0 +1,187 @@ +"""Behavioral contracts for detached, local-only transfer inspection.""" + +import json +from pathlib import Path + +import pytest + +from PyFlow.network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base + +PEER_A = ("127.0.0.1", 50001) +PEER_B = ("127.0.0.1", 50002) +ORIGIN = ("127.0.0.1", 50003) + + +@pytest.fixture(params=[TCP_Server_Base, TCP_Client_Base], ids=["server", "client"]) +def instance(request, tmp_path): + owner = request.param( + host="127.0.0.1", port=0, is_extend_command=True, + is_input_command_in_console=False, is_enable_encrypto=False, is_print_log=False, + ) + for attr in ("messages_log_file", "events_log_file", "sent_messages_log_file", + "sent_events_log_file", "file_trans_progress_file"): + setattr(owner, attr, str(tmp_path / f"{attr}.json")) + try: + yield owner + finally: + owner.file_trans_progress_file = str(tmp_path / "teardown_progress.json") + owner.stop() if isinstance(owner, TCP_Server_Base) else owner.close() + + +def test_scoped_rows_keep_all_local_states_separate(instance): + instance.file_upload_progress.update({ + (PEER_A, 8): [4, 10, 2.0, "/send/a.bin", ORIGIN], + (PEER_B, 8): [10, 10, None, "/send/b.bin", PEER_B], + }) + instance.file_recv_progress[8] = [3, 20, None, "/recv/a.bin", PEER_A] + instance._active_transfer_fids.add((PEER_A, 8)) + instance._active_transfer_sockets[8] = object() + instance._transfer_pause[(PEER_A, 8)] = True + view = instance.get_transfer_progress(include_persisted=False) + rows = {tuple(row["receiver"]): row for row in view["memory"]["uploading"]} + assert set(rows) == {PEER_A, PEER_B} + assert rows[PEER_A]["peer"] == list(ORIGIN) + assert rows[PEER_A]["active"] is True and rows[PEER_A]["paused"] is True + assert rows[PEER_A]["socket_registered"] is False + assert rows[PEER_B]["transferred"] == rows[PEER_B]["total"] == 10 + assert rows[PEER_B]["active"] is False and rows[PEER_B]["paused"] is False + download, = view["memory"]["downloading"] + assert download["fid"] == 8 and download["transferred"] == 3 + assert download["receiver"] is None and download["paused"] is None + assert download["active"] is False and download["socket_registered"] is True + assert instance.running is False + + +def test_memory_disk_and_returned_mutations_do_not_change_recovery_state(instance): + instance.file_recv_progress[9] = [100, 100, None, "/recv/current.bin", PEER_A] + if isinstance(instance, TCP_Server_Base): + instance._register_fid_route(11, ORIGIN, {PEER_A: 12, PEER_B: 12}, (PEER_A,)) + else: + instance.file_upload_progress[(PEER_A, 11)] = [ + 100, 100, None, "/send/source.bin", PEER_B, + ] + original = instance._transfer_progress_snapshot() + disk = {"upload": {}, "download": {"9": [20, 100, None, "/recv/old.bin", list(PEER_A)], + "10": [5, 50, None, "/recv/disk_only.bin", list(PEER_B)]}, + "relays": original["relays"]} + path = Path(instance.file_trans_progress_file) + path.write_text(json.dumps(disk), encoding="utf-8") + before = path.read_bytes() + view = instance.get_transfer_progress() + current, = view["memory"]["downloading"] + assert current["path"] == "/recv/current.bin" and current["transferred"] == 100 + saved = {row["fid"]: row for row in view["persisted"]["snapshot"]["downloading"]} + assert set(saved) == {9, 10} and saved[9]["transferred"] == 20 + assert saved[10]["path"] == "/recv/disk_only.bin" + assert all(saved[9][field] is None for field in ("active", "socket_registered", "paused")) + if isinstance(instance, TCP_Server_Base): + relay, = view["memory"]["relays"] + assert relay["fid"] == 11 + assert relay["completed"] == [list(PEER_A)] + assert relay["targets"] == [[list(PEER_A), 12], [list(PEER_B), 12]] + assert "path" not in relay + relay["targets"].clear() + else: + assert view["memory"]["relays"] == [] + assert view["persisted"]["snapshot"]["relays"] == [] + assert original["relays"] == {} + upload, = view["memory"]["uploading"] + assert upload["fid"] == 11 + assert upload["receiver"] == list(PEER_A) + assert upload["peer"] == list(PEER_B) + assert upload["path"] == "/send/source.bin" + upload["path"] = "changed" + json.dumps(view) + current["peer"][0] = "changed" + saved[10]["path"] = "changed" + assert instance._transfer_progress_snapshot() == original + assert path.read_bytes() == before + assert 10 not in instance.file_recv_progress + assert not instance._active_transfer_fids and not instance._active_transfer_sockets + + +def test_absent_empty_and_unrequested_disk_are_distinct(instance, monkeypatch): + path = Path(instance.file_trans_progress_file) + assert instance.get_transfer_progress()["persisted"]["status"] == "missing" + path.write_text('{"upload":{},"download":{},"relays":{}}', encoding="utf-8") + empty = instance.get_transfer_progress()["persisted"] + assert empty["status"] == "ok" and empty["snapshot"] == { + "uploading": [], "downloading": [], "relays": [] + } + + def reject_io(*args, **kwargs): + raise AssertionError("include_persisted=False performed file I/O") + + monkeypatch.setattr("builtins.open", reject_io) + skipped = instance.get_transfer_progress(include_persisted=False)["persisted"] + assert skipped["status"] == "not_requested" + assert skipped["snapshot"] is None and skipped["error"] is None + assert skipped["path"] == str(path) + + +@pytest.mark.parametrize("text", [ + "{", "[]", '{"upload":{"3":[]}}', '{"download":{"3":[1]}}', + '{"download":[1]}', '{"relays":{"3":[]}}', +]) +def test_bad_snapshots_remain_intact_and_do_not_hide_memory(instance, text): + path = Path(instance.file_trans_progress_file) + path.write_text(text, encoding="utf-8") + instance.file_recv_progress[2] = [7, 9, None, "/recv/kept.bin", PEER_A] + view = instance.get_transfer_progress() + assert view["persisted"]["status"] == "error" + assert view["persisted"]["snapshot"] is None and view["persisted"]["error"] + assert view["memory"]["downloading"][0]["path"] == "/recv/kept.bin" + assert path.read_text(encoding="utf-8") == text + + +def test_read_failure_is_visible_without_touching_live_records(instance, monkeypatch): + instance.file_recv_progress[2] = [7, 9, None, "/recv/kept.bin", PEER_A] + + def denied(*args, **kwargs): + raise PermissionError("inspection access denied") + + monkeypatch.setattr("builtins.open", denied) + view = instance.get_transfer_progress() + assert view["persisted"]["status"] == "error" + assert "inspection access denied" in view["persisted"]["error"] + assert view["memory"]["downloading"][0]["transferred"] == 7 + + +def test_disabled_tracking_reports_existing_records_but_no_live_speed(instance): + instance.file_recv_progress[2] = [9, 9, 100.0, "/recv/retained.bin", PEER_A] + instance.is_enable_file_transfer_progress_bar = False + instance.is_enable_real_time_network_speed = False + view = instance.get_transfer_progress(include_persisted=False) + assert view["progress_enabled"] is False and view["speed_enabled"] is False + row, = view["memory"]["downloading"] + assert row["path"] == "/recv/retained.bin" and row["total"] == row["transferred"] == 9 + assert row["speed"] is None and row["active"] is False + assert instance.file_recv_progress[2][2] == 100.0 + + +def test_client_legacy_relay_file_is_rejected_without_losing_original_bytes(tmp_path): + client = TCP_Client_Base( + host="127.0.0.1", port=0, is_extend_command=True, + is_input_command_in_console=False, is_enable_encrypto=False, is_print_log=False, + ) + for attr in ("messages_log_file", "events_log_file", "sent_messages_log_file", + "sent_events_log_file", "file_trans_progress_file"): + setattr(client, attr, str(tmp_path / f"legacy_{attr}.json")) + path = Path(client.file_trans_progress_file) + original = json.dumps({"upload": {}, "download": {}, "relays": { + "302": {"path": "/old/source.bin", "targets": [list(PEER_A)]}, + }}).encode() + path.write_bytes(original) + try: + assert client._load_transfer_progress() == 0 + assert client._transfer_progress_snapshot() == { + "upload": {}, "download": {}, "relays": {}, + } + assert client.get_transfer_progress()["persisted"]["status"] == "error" + client.file_recv_progress[304] = [1, 2, None, "/recv/current.bin", PEER_B] + client._persist_transfer_progress() + assert path.read_bytes() == original + client.close() + assert path.read_bytes() == original + finally: + client.close() diff --git a/test/unit/network_api/test_wire_lines.py b/test/unit/network_api/test_wire_lines.py new file mode 100644 index 0000000..2a0cd38 --- /dev/null +++ b/test/unit/network_api/test_wire_lines.py @@ -0,0 +1,256 @@ +"""Transfer signs survive read boundaries without retaining consumed history.""" + +import socket +import threading + +import pytest + +from PyFlow.network_api.connect_tcp import ( + TRANSFER_SIGN_BUFFER_LIMIT, + TCP_Server_Base, + _TransferSignBuffer, +) + +START = "/FILE_SEND_SERVER_START_FILE_TRANSFER" +HEADER = "/FILE_RECEIVE_CLIENT_HEADER" +ACK = "/FILE_RECEIVE_CLIENT_DATA" +ERROR = "/FER" + + +def _buffer(): + return _TransferSignBuffer(START, ACK, ERROR) + + +def test_coalesced_signs_and_unterminated_tail_survive_every_read_boundary(): + wire = f"\r\n {START} \r\n\n{HEADER}\n{ACK}\n{ERROR}".encode() + for boundary in range(len(wire) + 1): + signs = _buffer() + signs.feed(wire[:boundary]) + signs.consume() + signs.feed(wire[boundary:]) + signs.consume() + assert signs.start_seen and signs.ack_seen, boundary + assert not signs.error_seen, boundary + assert signs.pending == [] and signs.pending_bytes == 0, boundary + assert signs.partial == ERROR.encode(), boundary + assert signs.feed(b"\n") is True + signs.consume() + assert signs.error_seen, boundary + + +def test_fragmented_ack_is_not_delivered_before_its_newline(): + signs = _buffer() + for byte in ACK.encode(): + signs.feed(bytes([byte])) + signs.consume() + assert not signs.ack_seen + assert signs.partial == ACK.encode() + signs.feed(b"\n") + signs.consume() + assert signs.ack_seen and not signs.partial + assert signs.pending == [] and signs.pending_bytes == 0 + + +def test_scanning_records_at_capacity_latches_results_before_deleting_them(): + signs = _buffer() + signs.feed(f"{HEADER}\n{ACK}\n{ERROR}\n".encode()) + signs.consume() + assert signs.ack_seen and signs.error_seen + assert signs.pending == [] and signs.pending_bytes == 0 + + +@pytest.mark.parametrize("suffix", [b"", b"\n", b"\n" + ACK.encode() + b"\n"]) +def test_oversize_line_is_rejected_without_reinterpreting_its_tail(suffix): + signs = _buffer() + signs.feed(b"x" * (TRANSFER_SIGN_BUFFER_LIMIT - 1)) + with pytest.raises(ValueError, match="64 KiB"): + signs.feed(b"x" + suffix) + signs.consume() + assert not signs.ack_seen + assert len(signs.partial) == TRANSFER_SIGN_BUFFER_LIMIT - 1 + assert signs.pending_bytes == 0 + + +def test_pending_quota_batch_scans_complete_lines_and_preserves_flags(): + signs = _buffer() + start_line = START.encode() + b"\n" + filler_size = TRANSFER_SIGN_BUFFER_LIMIT - len(start_line) + assert filler_size % 2 == 0 + signs.feed(start_line + b"x\n" * (filler_size // 2)) + assert signs.pending == [] and signs.pending_bytes == 0 and not signs.partial + assert signs.start_seen + signs.feed(ACK.encode() + b"\n") + _, ack_seen, _ = signs.consume() + assert ack_seen + + +def test_partial_tail_at_total_quota_scans_all_complete_records(): + signs = _buffer() + queued_size = TRANSFER_SIGN_BUFFER_LIMIT - 5 + queued_line = START.encode() + b" " * (queued_size - len(START) - 1) + b"\n" + signs.feed(queued_line + b"tail!") + assert signs.start_seen + assert signs.pending == [] and signs.pending_bytes == 0 + assert signs.partial == b"tail!" + +def test_pending_quota_compacts_before_appending_to_an_incomplete_tail(): + signs = _buffer() + first_line = b" x " + b" " * (TRANSFER_SIGN_BUFFER_LIMIT - 8) + b"\n" + signs.feed(first_line + b"abc") + assert signs.pending_bytes + len(signs.partial) == TRANSFER_SIGN_BUFFER_LIMIT - 1 + signs.feed(b"de") + assert signs.pending_bytes == 0 and signs.partial == b"abcde" + signs.feed(b"\n" + ACK.encode() + b"\n") + _, ack_seen, _ = signs.consume() + assert ack_seen + + +def test_complete_short_line_batches_compact_without_failing_or_waiting(): + signs = _buffer() + signs.feed(b"x\n" * TRANSFER_SIGN_BUFFER_LIMIT) + assert signs.pending_bytes == 0 and signs.pending == [] + assert not signs.ack_seen + signs.feed(ACK.encode() + b"\n") + _, ack_seen, _ = signs.consume() + assert ack_seen and signs.pending_bytes == 0 + + +def test_exact_limit_complete_line_is_accepted_and_scanned(): + signs = _buffer() + full_line = START.encode() + b" " * (TRANSFER_SIGN_BUFFER_LIMIT - len(START) - 1) + b"\n" + signs.feed(full_line) + assert signs.pending_bytes == 0 and signs.pending == [] + assert signs.start_seen + +def test_many_consumed_batches_do_not_exhaust_the_pending_quota(): + signs = _buffer() + for _ in range(4): + signs.feed(b"x\n" * (TRANSFER_SIGN_BUFFER_LIMIT // 2)) + signs.consume() + assert signs.pending_bytes == 0 and signs.pending == [] + signs.feed(f"{START}\n{ACK}\n".encode()) + signs.consume() + assert signs.start_seen and signs.ack_seen + + +def test_large_read_of_empty_lines_does_not_become_a_single_oversize_line(): + signs = _buffer() + signs.feed(b"\n" * (2 * TRANSFER_SIGN_BUFFER_LIMIT) + ACK.encode() + b"\n") + signs.consume() + assert signs.ack_seen + assert not signs.partial and signs.pending_bytes == 0 + + +def test_producer_and_consumer_do_not_lose_ack_across_prefix_deletion(): + signs = _buffer() + batches = [HEADER, START, ACK, ERROR] + handoff = threading.Barrier(2, timeout=5) + failures = [] + + def produce(): + try: + for line in batches: + signs.feed((line + "\n").encode()) + handoff.wait() + handoff.wait() + except Exception as exc: + failures.append(exc) + handoff.abort() + + worker = threading.Thread(target=produce) + worker.start() + try: + for _ in batches: + handoff.wait() + signs.consume() + assert signs.pending == [] and signs.pending_bytes == 0 + handoff.wait() + except BaseException: + handoff.abort() + raise + finally: + worker.join(timeout=5) + assert not worker.is_alive() and not failures + assert signs.start_seen and signs.ack_seen and signs.error_seen + + +@pytest.fixture +def relay_server(tmp_path): + server = TCP_Server_Base( + host="127.0.0.1", port=0, is_extend_command=True, + is_input_command_in_console=False, is_enable_encrypto=False, is_print_log=False, + ) + server.file_trans_progress_file = str(tmp_path / "relay_progress.json") + try: + yield server + finally: + server.stop() + + +def _relay_exchange(server, monkeypatch, wire, marker, read_size): + reader, writer = socket.socketpair() + previous_timeout = 1.25 + reader.settimeout(previous_timeout) + real_receive = server.receive_message + + def write_peer(): + try: + writer.sendall(wire) + except OSError: + pass # an error marker may make the relay close before the tail + finally: + writer.close() + + def coalesce(sock, length): + data = bytearray() + while len(data) < length: + chunk = real_receive(sock, length - len(data)) + if not chunk: + break + data.extend(chunk) + return bytes(data) + + worker = threading.Thread(target=write_peer) + worker.start() + try: + with monkeypatch.context() as patch: + patch.setattr(server, "recv_messages_chunk_len", read_size) + patch.setattr(server, "receive_message", coalesce) + result = server._forward_read_sign(reader, marker, timeout=2) + assert reader.gettimeout() == previous_timeout + return result + finally: + reader.close() + worker.join(timeout=5) + assert not worker.is_alive() + + +@pytest.mark.parametrize("marker", [START, ACK]) +def test_relay_markers_survive_every_read_split(relay_server, monkeypatch, marker): + wire = marker.encode() + for split in range(1, len(wire)): + assert _relay_exchange(relay_server, monkeypatch, wire, marker, split) == wire + + +@pytest.mark.parametrize("marker", [START, ACK]) +def test_relay_markers_survive_noise_and_scan_window_boundaries( + relay_server, monkeypatch, marker +): + wire = ( + b"x" * (3 * TRANSFER_SIGN_BUFFER_LIMIT - len(marker) // 2) + + marker.encode() + b"\n" + ) + assert _relay_exchange(relay_server, monkeypatch, wire, marker, len(wire)) == marker.encode() + + +def test_relay_error_later_in_the_same_large_read_still_rejects_ack( + relay_server, monkeypatch +): + wire = ACK.encode() + b"x" * (2 * TRANSFER_SIGN_BUFFER_LIMIT) + ERROR.encode() + assert _relay_exchange(relay_server, monkeypatch, wire, ACK, len(wire)) is None + + +def test_relay_error_split_across_reads_rejects_ack(relay_server, monkeypatch): + wire = ERROR.encode() + ACK.encode() + for split in range(1, len(ERROR)): + assert _relay_exchange(relay_server, monkeypatch, wire, ACK, split) is None diff --git a/test/unit/transfer_web/test_web_transfers.py b/test/unit/transfer_web/test_web_transfers.py index 9117fa5..0a5da90 100644 --- a/test/unit/transfer_web/test_web_transfers.py +++ b/test/unit/transfer_web/test_web_transfers.py @@ -85,6 +85,37 @@ def _logged_in_flask_client(web): return flask_client +@pytest.mark.parametrize("backend", ["server", "client"]) +def test_transfers_keep_receiver_scoped_uploads_separate(backend, request, tmp_path): + """Equal numeric fids in different receiver scopes survive Flask serialization.""" + if backend == "server": + app = request.getfixturevalue("web") + tcp = app.server + flask_client = _logged_in_flask_client(app) + else: + app, flask_client = request.getfixturevalue("client_web") + tcp = app.client + + first_receiver = ("127.0.0.1", 6000) + second_receiver = ("127.0.0.1", 6001) + tcp.file_upload_progress[(second_receiver, 9)] = [ + 3072, 8192, 512.5, str(tmp_path / "second" / "zeta.bin"), second_receiver + ] + tcp.file_upload_progress[(first_receiver, 9)] = [ + 512, 2048, 1024.0, str(tmp_path / "first" / "alpha.bin"), first_receiver + ] + + response = flask_client.get("/api/transfers") + + assert response.status_code == 200 + body = response.get_json() + assert body["uploading"] == [ + {"fid": 9, "name": "alpha.bin", "transferred": 512, "total": 2048, "speed": 1024.0}, + {"fid": 9, "name": "zeta.bin", "transferred": 3072, "total": 8192, "speed": 512.5}, + ] + assert body["downloading"] == [] + + # ---- server backend ---------------------------------------------------------- @@ -94,8 +125,9 @@ def test_server_transfers_report_both_directions(web): A row carries the fid, the file name, the bytes and the live speed; the two speed attributes ride along for the topbar readout. """ - web.server.file_recv_progress[7] = [512, 2048, 1024.0, "alpha.bin"] - web.server.file_upload_progress[9] = [10, 100, None, "beta.bin"] + peer = ("127.0.0.1", 40000) + web.server.file_recv_progress[7] = [512, 2048, 1024.0, "alpha.bin", peer] + web.server.file_upload_progress[(peer, 9)] = [10, 100, None, "beta.bin", peer] web.server.upload_network_speed = 2048.0 web.server.recv_network_speed = 4096.0 @@ -116,8 +148,9 @@ def test_server_transfers_report_both_directions(web): def test_server_rows_are_ordered_by_name(web): """Concurrent transfers of a folder list in a stable, readable order.""" - web.server.file_upload_progress[2] = [0, 10, None, "zeta.bin"] - web.server.file_upload_progress[1] = [0, 10, None, "alpha.bin"] + peer = ("127.0.0.1", 40000) + web.server.file_upload_progress[(peer, 2)] = [0, 10, None, "zeta.bin", peer] + web.server.file_upload_progress[(peer, 1)] = [0, 10, None, "alpha.bin", peer] body = _logged_in_flask_client(web).get("/api/transfers").get_json() @@ -149,17 +182,24 @@ def test_server_transfers_are_empty_while_the_tcp_server_is_down(web): # ---- client backend ---------------------------------------------------------- -def test_client_transfers_report_the_running_transfer(client_web): - """The client page reads its own instance's download row and speed.""" +def test_client_transfers_report_both_directions(client_web): + """Forwarded uploads retain their local fid; download rows stay unchanged.""" app, flask_client = client_web - app.client.file_recv_progress[11] = [1024, 4096, 2048.0, "pack.bin"] + receiver = (app.client.host, app.client.port) + logical_target = ("127.0.0.1", 40001) + app.client.file_recv_progress[11] = [1024, 4096, 2048.0, "pack.bin", receiver] + app.client.file_upload_progress[(receiver, 9)] = [ + 10, 100, None, "beta.bin", logical_target + ] app.client.recv_network_speed = 2048.0 body = flask_client.get("/api/transfers").get_json() assert body["progress_enabled"] is True assert body["speed_enabled"] is True - assert body["uploading"] == [] + assert body["uploading"] == [ + {"fid": 9, "name": "beta.bin", "transferred": 10, "total": 100, "speed": None} + ] assert body["downloading"] == [ {"fid": 11, "name": "pack.bin", "transferred": 1024, "total": 4096, "speed": 2048.0} ]