diff --git a/setup.py b/setup.py index ee6d1b01f5..3905956c2a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ def main(): dep_sdk = "tencentcloud-sdk-python >= %s" % __version__.rsplit(".", 1)[0] setup( name='tccli', - install_requires=[dep_sdk, "jmespath==0.10.0", "six==1.16.0"], + install_requires=[dep_sdk, "jmespath==0.10.0", "six==1.16.0", "cos-python-sdk-v5>=1.9.0"], version=__version__, packages=find_packages(), include_package_data=True, diff --git a/tccli/plugins/cos/__init__.py b/tccli/plugins/cos/__init__.py new file mode 100644 index 0000000000..b6512a1ad7 --- /dev/null +++ b/tccli/plugins/cos/__init__.py @@ -0,0 +1,810 @@ +# -*- coding: utf-8 -*- +""" +COS 命令行工具插件 +将 COS 的所有命令集成到 tencentcloud-cli 中 +""" +from .list_object import list_object +from .upload_object import upload_object +from .download_object import download_object +from .delete_object import delete_object +from .copy_object import copy_object +from .move_object import move_object +from .list_buckets import list_buckets +from .create_bucket import create_bucket +from .delete_bucket import delete_bucket +from .head_object import head_object +from .restore_object import restore_object +from .sync_upload_object import sync_upload_object +from .sync_download_object import sync_download_object +from .sync_copy_object import sync_copy_object +from .signurl_object import signurl_object +from .acl_object import get_bucket_acl, put_bucket_acl, get_object_acl, put_object_acl +from .abort_multipart import abort_multipart +from .hash_object import hash_object +from .tagging_object import get_object_tagging, put_object_tagging, delete_object_tagging +from .du_object import du_object +from .cat_object import cat_object +from .lsparts_object import lsparts_object + +service_name = "cos" +service_version = "2021-02-24" + +_spec = { + "metadata": { + "serviceShortName": service_name, + "apiVersion": service_version, + "description": "COS (Cloud Object Storage) command line tool", + }, + "actions": { + # ===== 文件操作 ===== + "list": { + "name": "列出文件", + "document": "列出 COS 存储桶中的文件，支持按前缀过滤和分页", + "input": "listRequest", + "output": "listResponse", + "action_caller": list_object, + }, + "upload": { + "name": "上传文件", + "document": "上传本地文件到 COS，自动根据文件大小选择简单上传或分片上传", + "input": "uploadRequest", + "output": "uploadResponse", + "action_caller": upload_object, + }, + "download": { + "name": "下载文件", + "document": "从 COS 下载文件到本地，自动根据文件大小选择简单下载或分片下载", + "input": "downloadRequest", + "output": "downloadResponse", + "action_caller": download_object, + }, + "delete": { + "name": "删除文件", + "document": "删除 COS 存储桶中的指定文件，支持递归批量删除", + "input": "deleteRequest", + "output": "deleteResponse", + "action_caller": delete_object, + }, + "copy": { + "name": "复制文件", + "document": "复制 COS 上的文件到另一个位置，支持跨存储桶和跨地域复制", + "input": "copyRequest", + "output": "copyResponse", + "action_caller": copy_object, + }, + "move": { + "name": "移动/重命名文件", + "document": "移动或重命名 COS 上的文件（通过复制+删除实现），支持跨存储桶移动", + "input": "moveRequest", + "output": "moveResponse", + "action_caller": move_object, + }, + # ===== 存储桶操作 ===== + "list_buckets": { + "name": "列出存储桶", + "document": "列出当前账号下的所有存储桶", + "input": "list_bucketsRequest", + "output": "list_bucketsResponse", + "action_caller": list_buckets, + }, + "create_bucket": { + "name": "创建存储桶", + "document": "创建一个新的 COS 存储桶", + "input": "create_bucketRequest", + "output": "create_bucketResponse", + "action_caller": create_bucket, + }, + "delete_bucket": { + "name": "删除存储桶", + "document": "删除指定的 COS 存储桶，使用 --force 可强制清空后删除", + "input": "delete_bucketRequest", + "output": "delete_bucketResponse", + "action_caller": delete_bucket, + }, + # ===== 对象元信息操作 ===== + "head": { + "name": "查询对象元信息", + "document": "查询 COS 对象的元数据信息，包括大小、类型、修改时间、CRC64 等", + "input": "headRequest", + "output": "headResponse", + "action_caller": head_object, + }, + "restore": { + "name": "恢复归档文件", + "document": "恢复归档存储类型（ARCHIVE/DEEP_ARCHIVE）的 COS 对象，使其可被下载", + "input": "restoreRequest", + "output": "restoreResponse", + "action_caller": restore_object, + }, + # ===== 同步操作（拆分为三个独立接口） ===== + "sync_upload": { + "name": "同步上传", + "document": "同步本地目录到 COS，增量上传（通过比较文件大小判断），支持删除目标端多余文件", + "input": "sync_uploadRequest", + "output": "sync_uploadResponse", + "action_caller": sync_upload_object, + }, + "sync_download": { + "name": "同步下载", + "document": "同步 COS 到本地目录，增量下载（通过比较文件大小判断），支持删除本地多余文件", + "input": "sync_downloadRequest", + "output": "sync_downloadResponse", + "action_caller": sync_download_object, + }, + "sync_copy": { + "name": "同步复制", + "document": "同步 COS 到另一个 COS 位置，增量复制（通过比较文件大小判断），支持删除目标端多余文件", + "input": "sync_copyRequest", + "output": "sync_copyResponse", + "action_caller": sync_copy_object, + }, + # ===== 预签名 URL ===== + "signurl": { + "name": "生成预签名URL", + "document": "生成 COS 对象的预签名 URL，可用于临时授权访问", + "input": "signurlRequest", + "output": "signurlResponse", + "action_caller": signurl_object, + }, + # ===== ACL 操作 ===== + "get_bucket_acl": { + "name": "获取存储桶ACL", + "document": "获取存储桶的访问控制列表（ACL）", + "input": "get_bucket_aclRequest", + "output": "get_bucket_aclResponse", + "action_caller": get_bucket_acl, + }, + "put_bucket_acl": { + "name": "设置存储桶ACL", + "document": "设置存储桶的访问控制策略", + "input": "put_bucket_aclRequest", + "output": "put_bucket_aclResponse", + "action_caller": put_bucket_acl, + }, + "get_object_acl": { + "name": "获取对象ACL", + "document": "获取 COS 对象的访问控制列表（ACL）", + "input": "get_object_aclRequest", + "output": "get_object_aclResponse", + "action_caller": get_object_acl, + }, + "put_object_acl": { + "name": "设置对象ACL", + "document": "设置 COS 对象的访问控制策略", + "input": "put_object_aclRequest", + "output": "put_object_aclResponse", + "action_caller": put_object_acl, + }, + # ===== 分片上传管理 ===== + "abort": { + "name": "清理分片上传", + "document": "清理存储桶中未完成的分片上传任务", + "input": "abortRequest", + "output": "abortResponse", + "action_caller": abort_multipart, + }, + # ===== 哈希计算 ===== + "hash": { + "name": "计算哈希值", + "document": "计算本地文件的哈希值（md5/sha1/sha256/crc64），或获取 COS 对象的 ETag/CRC64 信息", + "input": "hashRequest", + "output": "hashResponse", + "action_caller": hash_object, + }, + # ===== 标签管理 ===== + "get_object_tagging": { + "name": "获取对象标签", + "document": "获取 COS 对象的标签信息", + "input": "get_object_taggingRequest", + "output": "get_object_taggingResponse", + "action_caller": get_object_tagging, + }, + "put_object_tagging": { + "name": "设置对象标签", + "document": "设置 COS 对象的标签，格式为 key1=value1,key2=value2", + "input": "put_object_taggingRequest", + "output": "put_object_taggingResponse", + "action_caller": put_object_tagging, + }, + "delete_object_tagging": { + "name": "删除对象标签", + "document": "删除 COS 对象的所有标签", + "input": "delete_object_taggingRequest", + "output": "delete_object_taggingResponse", + "action_caller": delete_object_tagging, + }, + # ===== 统计操作 ===== + "du": { + "name": "统计大小", + "document": "统计存储桶或指定前缀下的对象总大小和数量，按存储类型分类统计", + "input": "duRequest", + "output": "duResponse", + "action_caller": du_object, + }, + # ===== 查看文件内容 ===== + "cat": { + "name": "查看文件内容", + "document": "查看 COS 对象的文本内容，默认最大读取 10MB，支持指定范围读取", + "input": "catRequest", + "output": "catResponse", + "action_caller": cat_object, + }, + # ===== 列出分片上传 ===== + "lsparts": { + "name": "列出分片上传", + "document": "列出存储桶中未完成的分片上传任务", + "input": "lspartsRequest", + "output": "lspartsResponse", + "action_caller": lsparts_object, + }, + }, + "objects": { + # ===== list 接口参数 ===== + "listRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "prefix", "member": "string", "type": "string", "required": False, + "document": "对象键前缀，用于过滤列出的对象"}, + {"name": "marker", "member": "string", "type": "string", "required": False, + "document": "分页标记，从该标记之后开始列出对象"}, + {"name": "max_keys", "member": "int64", "type": "int64", "required": False, + "document": "最大返回数量，默认1000，最大1000"}, + {"name": "delimiter", "member": "string", "type": "string", "required": False, + "document": "分隔符，通常设为 / 以模拟目录结构"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归列出所有对象（忽略 delimiter），默认 false"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式，支持通配符，如 *.txt"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式，支持通配符，如 *.log"}, + ], + }, + "listResponse": { + "members": [], + }, + # ===== upload 接口参数 ===== + "uploadRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "目标存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "local_path", "member": "string", "type": "string", "required": True, + "document": "本地文件或目录路径"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "COS 上的目标对象键（Key），递归上传时作为前缀"}, + {"name": "storage_class", "member": "string", "type": "string", "required": False, + "document": "存储类型: STANDARD(默认), STANDARD_IA, ARCHIVE, DEEP_ARCHIVE, INTELLIGENT_TIERING, MAZ_STANDARD, MAZ_STANDARD_IA"}, + {"name": "content_type", "member": "string", "type": "string", "required": False, + "document": "文件内容类型（MIME），如 text/plain, image/jpeg"}, + {"name": "meta", "member": "string", "type": "string", "required": False, + "document": "自定义元数据，格式: key1=value1#key2=value2"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归上传目录，默认 false"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式（递归上传时生效），支持通配符"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式（递归上传时生效），支持通配符"}, + {"name": "thread_num", "member": "int64", "type": "int64", "required": False, + "document": "单文件分片上传并发线程数，默认 5"}, + {"name": "routines", "member": "int64", "type": "int64", "required": False, + "document": "文件间并发数（同时传输的文件数），默认 3"}, + {"name": "part_size", "member": "int64", "type": "int64", "required": False, + "document": "分片大小（MB），默认 20"}, + {"name": "rate_limiting", "member": "int64", "type": "int64", "required": False, + "document": "单链接限速（MB/s），0 表示不限速"}, + {"name": "retry", "member": "int64", "type": "int64", "required": False, + "document": "失败重试次数，默认 3，设为 0 表示不重试"}, + {"name": "log_file", "member": "string", "type": "string", "required": False, + "document": "失败日志文件路径，指定后将失败记录写入该文件，默认不输出日志"}, + ], + }, + "uploadResponse": { + "members": [], + }, + # ===== download 接口参数 ===== + "downloadRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "源存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "COS 上的源对象键（Key），递归下载时作为前缀"}, + {"name": "local_path", "member": "string", "type": "string", "required": True, + "document": "本地保存路径，递归下载时为目标目录"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归下载前缀下所有对象，默认 false"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式（递归下载时生效），支持通配符"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式（递归下载时生效），支持通配符"}, + {"name": "thread_num", "member": "int64", "type": "int64", "required": False, + "document": "单文件分片下载并发线程数，默认 5"}, + {"name": "routines", "member": "int64", "type": "int64", "required": False, + "document": "文件间并发数（同时传输的文件数），默认 3"}, + {"name": "part_size", "member": "int64", "type": "int64", "required": False, + "document": "分片大小（MB），默认 20"}, + {"name": "rate_limiting", "member": "int64", "type": "int64", "required": False, + "document": "单链接限速（MB/s），0 表示不限速"}, + {"name": "version_id", "member": "string", "type": "string", "required": False, + "document": "指定下载的对象版本 ID（开启版本控制时使用）"}, + {"name": "retry", "member": "int64", "type": "int64", "required": False, + "document": "失败重试次数，默认 3，设为 0 表示不重试"}, + {"name": "log_file", "member": "string", "type": "string", "required": False, + "document": "失败日志文件路径，指定后将失败记录写入该文件，默认不输出日志"}, + ], + }, + "downloadResponse": { + "members": [], + }, + # ===== delete 接口参数 ===== + "deleteRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "要删除的对象键（Key），递归删除时作为前缀"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归删除前缀下所有对象，默认 false"}, + {"name": "force", "member": "bool", "type": "bool", "required": False, + "document": "递归删除时跳过确认提示，默认 false"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式（递归删除时生效），支持通配符"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式（递归删除时生效），支持通配符"}, + {"name": "version_id", "member": "string", "type": "string", "required": False, + "document": "指定删除的对象版本 ID（开启版本控制时使用）"}, + ], + }, + "deleteResponse": { + "members": [], + }, + # ===== copy 接口参数 ===== + "copyRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "源存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "源对象键（Key），递归复制时作为前缀"}, + {"name": "dest_bucket", "member": "string", "type": "string", "required": False, + "document": "目标存储桶名称，不填则与源存储桶相同"}, + {"name": "dest_key", "member": "string", "type": "string", "required": True, + "document": "目标对象键（Key），递归复制时作为目标前缀"}, + {"name": "dest_region", "member": "string", "type": "string", "required": False, + "document": "目标地域，不填则与当前地域相同"}, + {"name": "storage_class", "member": "string", "type": "string", "required": False, + "document": "目标存储类型: STANDARD, STANDARD_IA, ARCHIVE, DEEP_ARCHIVE, INTELLIGENT_TIERING, MAZ_STANDARD, MAZ_STANDARD_IA"}, + {"name": "meta", "member": "string", "type": "string", "required": False, + "document": "自定义元数据，格式: key1=value1#key2=value2（设置后使用 Replaced 模式）"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归复制前缀下所有对象，默认 false"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式（递归复制时生效），支持通配符"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式（递归复制时生效），支持通配符"}, + {"name": "routines", "member": "int64", "type": "int64", "required": False, + "document": "文件间并发数（同时复制的文件数），默认 3"}, + {"name": "retry", "member": "int64", "type": "int64", "required": False, + "document": "失败重试次数，默认 3，设为 0 表示不重试"}, + {"name": "log_file", "member": "string", "type": "string", "required": False, + "document": "失败日志文件路径，指定后将失败记录写入该文件，默认不输出日志"}, + ], + }, + "copyResponse": { + "members": [], + }, + # ===== move 接口参数 ===== + "moveRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "源存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "源对象键（Key），递归移动时作为前缀"}, + {"name": "dest_bucket", "member": "string", "type": "string", "required": False, + "document": "目标存储桶名称，不填则与源存储桶相同"}, + {"name": "dest_key", "member": "string", "type": "string", "required": True, + "document": "目标对象键（Key），递归移动时作为目标前缀"}, + {"name": "dest_region", "member": "string", "type": "string", "required": False, + "document": "目标地域，不填则与当前地域相同"}, + {"name": "storage_class", "member": "string", "type": "string", "required": False, + "document": "目标存储类型: STANDARD, STANDARD_IA, ARCHIVE, DEEP_ARCHIVE, INTELLIGENT_TIERING, MAZ_STANDARD, MAZ_STANDARD_IA"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归移动前缀下所有对象，默认 false"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式（递归移动时生效），支持通配符"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式（递归移动时生效），支持通配符"}, + {"name": "routines", "member": "int64", "type": "int64", "required": False, + "document": "文件间并发数（同时移动的文件数），默认 3"}, + {"name": "retry", "member": "int64", "type": "int64", "required": False, + "document": "失败重试次数，默认 3，设为 0 表示不重试"}, + {"name": "log_file", "member": "string", "type": "string", "required": False, + "document": "失败日志文件路径，指定后将失败记录写入该文件，默认不输出日志"}, + ], + }, + "moveResponse": { + "members": [], + }, + # ===== list_buckets 接口参数 ===== + "list_bucketsRequest": { + "members": [ + {"name": "filter_region", "member": "string", "type": "string", "required": False, + "document": "按地域过滤存储桶，如 ap-guangzhou"}, + ], + }, + "list_bucketsResponse": { + "members": [], + }, + # ===== create_bucket 接口参数 ===== + "create_bucketRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "acl", "member": "string", "type": "string", "required": False, + "document": "访问控制策略，可选值: private(默认), public-read, public-read-write"}, + ], + }, + "create_bucketResponse": { + "members": [], + }, + # ===== delete_bucket 接口参数 ===== + "delete_bucketRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "要删除的存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "force", "member": "bool", "type": "bool", "required": False, + "document": "强制删除：先清空存储桶中所有对象、版本对象和未完成的分片上传，再删除存储桶，默认 false"}, + ], + }, + "delete_bucketResponse": { + "members": [], + }, + # ===== head 接口参数 ===== + "headRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "要查询的对象键（Key）"}, + {"name": "version_id", "member": "string", "type": "string", "required": False, + "document": "指定查询的对象版本 ID（开启版本控制时使用）"}, + ], + }, + "headResponse": { + "members": [], + }, + # ===== restore 接口参数 ===== + "restoreRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "要恢复的归档对象键（Key），递归恢复时作为前缀"}, + {"name": "days", "member": "int64", "type": "int64", "required": False, + "document": "恢复后的有效天数，默认 7 天"}, + {"name": "tier", "member": "string", "type": "string", "required": False, + "document": "恢复模式: Standard(标准，3-5小时), Expedited(极速，1-5分钟), Bulk(批量，5-12小时)，默认 Standard"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归恢复前缀下所有归档对象，默认 false"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式（递归恢复时生效），支持通配符"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式（递归恢复时生效），支持通配符"}, + ], + }, + "restoreResponse": { + "members": [], + }, + # ===== sync_upload 接口参数 ===== + "sync_uploadRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "目标存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "local_path", "member": "string", "type": "string", "required": True, + "document": "本地文件或目录路径"}, + {"name": "cos_key", "member": "string", "type": "string", "required": False, + "document": "COS 上的目标对象键（Key），作为前缀"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归同步目录，默认 false"}, + {"name": "delete", "member": "bool", "type": "bool", "required": False, + "document": "是否删除 COS 上多余的文件（本地不存在的），默认 false"}, + {"name": "ignore_existing", "member": "bool", "type": "bool", "required": False, + "document": "目标已存在即跳过，默认 false。与 --update 互斥，优先级高于 --update"}, + {"name": "update", "member": "bool", "type": "bool", "required": False, + "document": "仅在源文件比目标文件新时更新（按 Last-Modified 比较），默认 false。未指定 --ignore-existing 和 --update 时使用 CRC64 校验判断是否跳过"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式，支持通配符，如 *.txt"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式，支持通配符，如 *.log"}, + {"name": "storage_class", "member": "string", "type": "string", "required": False, + "document": "上传时的存储类型: STANDARD, STANDARD_IA, ARCHIVE, DEEP_ARCHIVE, INTELLIGENT_TIERING, MAZ_STANDARD, MAZ_STANDARD_IA"}, + {"name": "content_type", "member": "string", "type": "string", "required": False, + "document": "文件内容类型（MIME），如 text/plain, image/jpeg"}, + {"name": "meta", "member": "string", "type": "string", "required": False, + "document": "自定义元数据，格式: key1=value1#key2=value2"}, + {"name": "thread_num", "member": "int64", "type": "int64", "required": False, + "document": "单文件分片上传并发线程数，默认 5"}, + {"name": "routines", "member": "int64", "type": "int64", "required": False, + "document": "文件间并发数（同时传输的文件数），默认 3"}, + {"name": "part_size", "member": "int64", "type": "int64", "required": False, + "document": "分片大小（MB），默认 20"}, + {"name": "rate_limiting", "member": "int64", "type": "int64", "required": False, + "document": "单链接限速（MB/s），0 表示不限速"}, + {"name": "retry", "member": "int64", "type": "int64", "required": False, + "document": "失败重试次数，默认 3，设为 0 表示不重试"}, + {"name": "log_file", "member": "string", "type": "string", "required": False, + "document": "失败日志文件路径，指定后将失败记录写入该文件，默认不输出日志"}, + ], + }, + "sync_uploadResponse": { + "members": [], + }, + # ===== sync_download 接口参数 ===== + "sync_downloadRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "源存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "local_path", "member": "string", "type": "string", "required": True, + "document": "本地目标目录路径"}, + {"name": "cos_key", "member": "string", "type": "string", "required": False, + "document": "COS 上的源对象键（Key），作为前缀"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归同步目录，默认 false"}, + {"name": "delete", "member": "bool", "type": "bool", "required": False, + "document": "是否删除本地多余的文件（COS 上不存在的），默认 false"}, + {"name": "ignore_existing", "member": "bool", "type": "bool", "required": False, + "document": "本地已存在即跳过，默认 false。与 --update 互斥，优先级高于 --update"}, + {"name": "update", "member": "bool", "type": "bool", "required": False, + "document": "仅在源文件比目标文件新时更新（按 Last-Modified 比较），默认 false。未指定 --ignore-existing 和 --update 时使用 CRC64 校验判断是否跳过"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式，支持通配符，如 *.txt"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式，支持通配符，如 *.log"}, + {"name": "thread_num", "member": "int64", "type": "int64", "required": False, + "document": "单文件分片下载并发线程数，默认 5"}, + {"name": "routines", "member": "int64", "type": "int64", "required": False, + "document": "文件间并发数（同时传输的文件数），默认 3"}, + {"name": "part_size", "member": "int64", "type": "int64", "required": False, + "document": "分片大小（MB），默认 20"}, + {"name": "rate_limiting", "member": "int64", "type": "int64", "required": False, + "document": "单链接限速（MB/s），0 表示不限速"}, + {"name": "retry", "member": "int64", "type": "int64", "required": False, + "document": "失败重试次数，默认 3，设为 0 表示不重试"}, + {"name": "log_file", "member": "string", "type": "string", "required": False, + "document": "失败日志文件路径，指定后将失败记录写入该文件，默认不输出日志"}, + ], + }, + "sync_downloadResponse": { + "members": [], + }, + # ===== sync_copy 接口参数 ===== + "sync_copyRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "源存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": False, + "document": "源 COS 对象键（Key），作为前缀"}, + {"name": "dest_bucket", "member": "string", "type": "string", "required": False, + "document": "目标存储桶名称，不填则与源存储桶相同"}, + {"name": "dest_key", "member": "string", "type": "string", "required": False, + "document": "目标 COS 对象键（Key），作为目标前缀"}, + {"name": "dest_region", "member": "string", "type": "string", "required": False, + "document": "目标地域，不填则与当前地域相同"}, + {"name": "recursive", "member": "bool", "type": "bool", "required": False, + "document": "是否递归同步复制，默认 false"}, + {"name": "delete", "member": "bool", "type": "bool", "required": False, + "document": "是否删除目标端多余的文件（源端不存在的），默认 false"}, + {"name": "ignore_existing", "member": "bool", "type": "bool", "required": False, + "document": "目标已存在即跳过，默认 false。与 --update 互斥，优先级高于 --update"}, + {"name": "update", "member": "bool", "type": "bool", "required": False, + "document": "仅在源对象比目标对象新时更新（按 Last-Modified 比较），默认 false。未指定 --ignore-existing 和 --update 时使用 CRC64 校验判断是否跳过"}, + {"name": "include", "member": "string", "type": "string", "required": False, + "document": "包含匹配模式，支持通配符，如 *.txt"}, + {"name": "exclude", "member": "string", "type": "string", "required": False, + "document": "排除匹配模式，支持通配符，如 *.log"}, + {"name": "storage_class", "member": "string", "type": "string", "required": False, + "document": "目标存储类型: STANDARD, STANDARD_IA, ARCHIVE, DEEP_ARCHIVE, INTELLIGENT_TIERING, MAZ_STANDARD, MAZ_STANDARD_IA"}, + {"name": "meta", "member": "string", "type": "string", "required": False, + "document": "自定义元数据，格式: key1=value1#key2=value2（设置后使用 Replaced 模式）"}, + {"name": "routines", "member": "int64", "type": "int64", "required": False, + "document": "文件间并发数（同时复制的文件数），默认 3"}, + {"name": "retry", "member": "int64", "type": "int64", "required": False, + "document": "失败重试次数，默认 3，设为 0 表示不重试"}, + {"name": "log_file", "member": "string", "type": "string", "required": False, + "document": "失败日志文件路径，指定后将失败记录写入该文件，默认不输出日志"}, + ], + }, + "sync_copyResponse": { + "members": [], + }, + # ===== signurl 接口参数 ===== + "signurlRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "对象键（Key）"}, + {"name": "expired", "member": "int64", "type": "int64", "required": False, + "document": "URL 有效期（秒），默认 3600"}, + {"name": "method", "member": "string", "type": "string", "required": False, + "document": "HTTP 方法: GET(下载，默认), PUT(上传)"}, + ], + }, + "signurlResponse": { + "members": [], + }, + # ===== get_bucket_acl 接口参数 ===== + "get_bucket_aclRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + ], + }, + "get_bucket_aclResponse": { + "members": [], + }, + # ===== put_bucket_acl 接口参数 ===== + "put_bucket_aclRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "acl", "member": "string", "type": "string", "required": False, + "document": "访问控制策略: private(默认), public-read, public-read-write"}, + {"name": "grant_read", "member": "string", "type": "string", "required": False, + "document": "授予读权限的用户，格式: id=\"账号ID\""}, + {"name": "grant_write", "member": "string", "type": "string", "required": False, + "document": "授予写权限的用户，格式: id=\"账号ID\""}, + {"name": "grant_full_control", "member": "string", "type": "string", "required": False, + "document": "授予完全控制权限的用户，格式: id=\"账号ID\""}, + ], + }, + "put_bucket_aclResponse": { + "members": [], + }, + # ===== get_object_acl 接口参数 ===== + "get_object_aclRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "对象键（Key）"}, + ], + }, + "get_object_aclResponse": { + "members": [], + }, + # ===== put_object_acl 接口参数 ===== + "put_object_aclRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "对象键（Key）"}, + {"name": "acl", "member": "string", "type": "string", "required": False, + "document": "访问控制策略: private(默认), public-read"}, + {"name": "grant_read", "member": "string", "type": "string", "required": False, + "document": "授予读权限的用户，格式: id=\"账号ID\""}, + {"name": "grant_full_control", "member": "string", "type": "string", "required": False, + "document": "授予完全控制权限的用户，格式: id=\"账号ID\""}, + ], + }, + "put_object_aclResponse": { + "members": [], + }, + # ===== abort 接口参数 ===== + "abortRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "prefix", "member": "string", "type": "string", "required": False, + "document": "对象键前缀，用于过滤要清理的分片上传"}, + {"name": "cos_key", "member": "string", "type": "string", "required": False, + "document": "对象键（指定 upload_id 时必填）"}, + {"name": "upload_id", "member": "string", "type": "string", "required": False, + "document": "指定要取消的分片上传 ID，不填则清理所有未完成的分片上传"}, + ], + }, + "abortResponse": { + "members": [], + }, + # ===== hash 接口参数 ===== + "hashRequest": { + "members": [ + {"name": "local_path", "member": "string", "type": "string", "required": False, + "document": "本地文件路径（计算本地文件哈希时使用）"}, + {"name": "bucket", "member": "string", "type": "string", "required": False, + "document": "存储桶名称（获取 COS 对象哈希时使用）"}, + {"name": "cos_key", "member": "string", "type": "string", "required": False, + "document": "对象键（获取 COS 对象哈希时使用）"}, + {"name": "hash_type", "member": "string", "type": "string", "required": False, + "document": "哈希类型: md5(默认), sha1, sha256, crc64"}, + ], + }, + "hashResponse": { + "members": [], + }, + # ===== get_object_tagging 接口参数 ===== + "get_object_taggingRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "对象键（Key）"}, + ], + }, + "get_object_taggingResponse": { + "members": [], + }, + # ===== put_object_tagging 接口参数 ===== + "put_object_taggingRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "对象键（Key）"}, + {"name": "tags", "member": "string", "type": "string", "required": True, + "document": "标签列表，格式: key1=value1,key2=value2"}, + ], + }, + "put_object_taggingResponse": { + "members": [], + }, + # ===== delete_object_tagging 接口参数 ===== + "delete_object_taggingRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "对象键（Key）"}, + ], + }, + "delete_object_taggingResponse": { + "members": [], + }, + # ===== du 接口参数 ===== + "duRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "prefix", "member": "string", "type": "string", "required": False, + "document": "对象键前缀，用于统计指定目录的大小"}, + ], + }, + "duResponse": { + "members": [], + }, + # ===== cat 接口参数 ===== + "catRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "cos_key", "member": "string", "type": "string", "required": True, + "document": "对象键（Key）"}, + {"name": "range", "member": "string", "type": "string", "required": False, + "document": "指定读取范围，格式: bytes=0-1023"}, + {"name": "max_size", "member": "int64", "type": "int64", "required": False, + "document": "最大读取大小（MB），超过此大小仅显示部分内容，默认 10"}, + ], + }, + "catResponse": { + "members": [], + }, + # ===== lsparts 接口参数 ===== + "lspartsRequest": { + "members": [ + {"name": "bucket", "member": "string", "type": "string", "required": True, + "document": "存储桶名称，格式如 my-bucket-1250000000"}, + {"name": "prefix", "member": "string", "type": "string", "required": False, + "document": "对象键前缀，用于过滤列出的分片上传"}, + ], + }, + "lspartsResponse": { + "members": [], + }, + }, + "version": "1.0", +} + + +def register_service(specs): + specs[service_name] = { + service_version: _spec, + } diff --git a/tccli/plugins/cos/abort_multipart.py b/tccli/plugins/cos/abort_multipart.py new file mode 100644 index 0000000000..5d50b680b0 --- /dev/null +++ b/tccli/plugins/cos/abort_multipart.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +abort 操作：清理未完成的分片上传 +对齐 coscli abort 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def abort_multipart(args, parsed_globals): + """清理存储桶中未完成的分片上传""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + prefix = args.get("prefix", "") or "" + upload_id = args.get("upload_id", "") or "" + + try: + # 如果指定了 upload_id，则直接取消指定的分片上传 + if upload_id: + cos_key = args.get("cos_key", "") or "" + if not cos_key: + print("Error: 指定 upload_id 时必须同时指定 cos_key") + return + client.abort_multipart_upload( + Bucket=bucket, + Key=cos_key, + UploadId=upload_id, + ) + print("已取消分片上传: cos://%s/%s (UploadId: %s)" % (bucket, cos_key, upload_id)) + return + + # 否则列出并清理所有未完成的分片上传 + key_marker = "" + upload_id_marker = "" + aborted = 0 + + while True: + response = client.list_multipart_uploads( + Bucket=bucket, + Prefix=prefix, + KeyMarker=key_marker, + UploadIdMarker=upload_id_marker, + MaxUploads=1000, + ) + + uploads = response.get("Upload", []) + if not isinstance(uploads, list): + uploads = [uploads] + + for upload in uploads: + if not upload: + continue + key = upload.get("Key", "") + uid = upload.get("UploadId", "") + initiated = upload.get("Initiated", "") + + client.abort_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=uid, + ) + aborted += 1 + print("已取消: cos://%s/%s (UploadId: %s, Initiated: %s)" % (bucket, key, uid, initiated)) + + if response.get("IsTruncated") == "true": + key_marker = response.get("NextKeyMarker", "") + upload_id_marker = response.get("NextUploadIdMarker", "") + else: + break + + if aborted == 0: + print("没有未完成的分片上传") + else: + print("\n共清理 %d 个未完成的分片上传" % aborted) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) \ No newline at end of file diff --git a/tccli/plugins/cos/acl_object.py b/tccli/plugins/cos/acl_object.py new file mode 100644 index 0000000000..3ea43caa63 --- /dev/null +++ b/tccli/plugins/cos/acl_object.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +""" +ACL 操作：获取和设置存储桶/对象的访问控制 +对齐 coscli 的 ACL 管理能力 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def get_bucket_acl(args, parsed_globals): + """获取存储桶的访问控制列表""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + + try: + response = client.get_bucket_acl(Bucket=bucket) + + owner = response.get("Owner", {}) + print("存储桶 ACL: %s" % bucket) + print("-" * 60) + print("Owner ID: %s" % owner.get("ID", "")) + print("Owner DisplayName: %s" % owner.get("DisplayName", "")) + + grants = response.get("AccessControlList", {}).get("Grant", []) + if not isinstance(grants, list): + grants = [grants] + + print("\n授权列表:") + for grant in grants: + grantee = grant.get("Grantee", {}) + permission = grant.get("Permission", "") + grantee_id = grantee.get("ID", "") + grantee_type = grantee.get("type", "") + grantee_uri = grantee.get("URI", "") + if grantee_uri: + print(" - URI: %-40s Permission: %s" % (grantee_uri, permission)) + else: + print(" - ID: %-40s Type: %-15s Permission: %s" % (grantee_id, grantee_type, permission)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def put_bucket_acl(args, parsed_globals): + """设置存储桶的访问控制""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + acl = args.get("acl", "") or "" + grant_read = args.get("grant_read", "") or "" + grant_write = args.get("grant_write", "") or "" + grant_full_control = args.get("grant_full_control", "") or "" + + try: + kwargs = {"Bucket": bucket} + if acl: + kwargs["ACL"] = acl + if grant_read: + kwargs["GrantRead"] = grant_read + if grant_write: + kwargs["GrantWrite"] = grant_write + if grant_full_control: + kwargs["GrantFullControl"] = grant_full_control + + client.put_bucket_acl(**kwargs) + print("存储桶 ACL 设置成功: %s" % bucket) + if acl: + print("ACL: %s" % acl) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def get_object_acl(args, parsed_globals): + """获取对象的访问控制列表""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + + try: + response = client.get_object_acl(Bucket=bucket, Key=cos_key) + + owner = response.get("Owner", {}) + print("对象 ACL: cos://%s/%s" % (bucket, cos_key)) + print("-" * 60) + print("Owner ID: %s" % owner.get("ID", "")) + print("Owner DisplayName: %s" % owner.get("DisplayName", "")) + + grants = response.get("AccessControlList", {}).get("Grant", []) + if not isinstance(grants, list): + grants = [grants] + + print("\n授权列表:") + for grant in grants: + grantee = grant.get("Grantee", {}) + permission = grant.get("Permission", "") + grantee_id = grantee.get("ID", "") + grantee_type = grantee.get("type", "") + grantee_uri = grantee.get("URI", "") + if grantee_uri: + print(" - URI: %-40s Permission: %s" % (grantee_uri, permission)) + else: + print(" - ID: %-40s Type: %-15s Permission: %s" % (grantee_id, grantee_type, permission)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def put_object_acl(args, parsed_globals): + """设置对象的访问控制""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + acl = args.get("acl", "") or "" + grant_read = args.get("grant_read", "") or "" + grant_full_control = args.get("grant_full_control", "") or "" + + try: + kwargs = {"Bucket": bucket, "Key": cos_key} + if acl: + kwargs["ACL"] = acl + if grant_read: + kwargs["GrantRead"] = grant_read + if grant_full_control: + kwargs["GrantFullControl"] = grant_full_control + + client.put_object_acl(**kwargs) + print("对象 ACL 设置成功: cos://%s/%s" % (bucket, cos_key)) + if acl: + print("ACL: %s" % acl) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) diff --git a/tccli/plugins/cos/cat_object.py b/tccli/plugins/cos/cat_object.py new file mode 100644 index 0000000000..30e33365a1 --- /dev/null +++ b/tccli/plugins/cos/cat_object.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" +cat 操作：查看 COS 对象内容 +对齐 coscli cat 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client, format_size + + +def cat_object(args, parsed_globals): + """查看 COS 对象的内容""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + byte_range = args.get("range", "") or "" + max_size = args.get("max_size", 10) or 10 # 默认最大 10MB + + try: + # 先获取对象大小 + head_response = client.head_object( + Bucket=bucket, + Key=cos_key, + ) + content_length = int(head_response.get("Content-Length", 0)) + + # 检查文件大小限制 + max_bytes = int(max_size) * 1024 * 1024 + if content_length > max_bytes and not byte_range: + print("Warning: 文件大小 %s 超过限制 (%dMB)，仅显示前 %dMB" % ( + format_size(content_length), max_size, max_size)) + byte_range = "bytes=0-%d" % (max_bytes - 1) + + # 获取对象内容 + kwargs = { + "Bucket": bucket, + "Key": cos_key, + } + if byte_range: + kwargs["Range"] = byte_range + + response = client.get_object(**kwargs) + body = response["Body"] + + # 读取并输出内容 + content = body.get_raw_stream().read() + try: + text = content.decode("utf-8") + print(text) + except UnicodeDecodeError: + print("Warning: 文件内容不是 UTF-8 编码的文本，显示为十六进制") + hex_str = content[:1024].hex() + for i in range(0, len(hex_str), 64): + print(hex_str[i:i + 64]) + if len(content) > 1024: + print("... (共 %d 字节)" % len(content)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + except Exception as e: + print("Error: %s" % str(e)) \ No newline at end of file diff --git a/tccli/plugins/cos/copy_object.py b/tccli/plugins/cos/copy_object.py new file mode 100644 index 0000000000..5daf37269e --- /dev/null +++ b/tccli/plugins/cos/copy_object.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +""" +copy 操作：复制 COS 上的文件 +对齐 coscli cp (COS->COS) 命令 +- routines: 文件间并发数（同时复制的文件数） +""" +from concurrent.futures import ThreadPoolExecutor, as_completed +from qcloud_cos import CosServiceError +from .utils import init_cos_client, match_filters, parse_meta, build_cos_key, TransferProgressMonitor + + +def copy_object(args, parsed_globals): + """复制 COS 上的文件""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + dest_bucket = args.get("dest_bucket", bucket) or bucket + dest_key = args["dest_key"] + dest_region = args.get("dest_region", region) or region + storage_class = args.get("storage_class", "") or "" + meta = args.get("meta", "") or "" + recursive = args.get("recursive", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + routines = args.get("routines", 3) or 3 + log_file = args.get("log_file", "") or "" + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + + # 解析自定义元数据 + metadata = parse_meta(meta) + + try: + if recursive: + _copy_by_prefix(client, bucket, cos_key, dest_bucket, dest_key, + region, dest_region, storage_class, metadata, include, exclude, routines, log_file, retry) + else: + _copy_single(client, bucket, cos_key, dest_bucket, dest_key, + region, storage_class, metadata, log_file, retry) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def _copy_single(client, bucket, cos_key, dest_bucket, dest_key, + region, storage_class, metadata, log_file="", retry=3): + """复制单个文件（带进度监控）""" + monitor = TransferProgressMonitor("copy") + + # 获取源文件大小 + try: + head_resp = client.head_object(Bucket=bucket, Key=cos_key) + file_size = int(head_resp.get("Content-Length", 0)) + except Exception: + file_size = 0 + + monitor.set_scan_info(1, file_size) + monitor.start() + + last_err = None + for attempt in range(max(1, retry + 1)): + try: + source = { + "Bucket": bucket, + "Key": cos_key, + "Region": region, + } + kwargs = { + "Bucket": dest_bucket, + "Key": dest_key, + "CopySource": source, + } + if storage_class: + kwargs["StorageClass"] = storage_class + if metadata: + kwargs["Metadata"] = metadata + kwargs["CopyStatus"] = "Replaced" + + client.copy(**kwargs) + monitor.update_ok(file_size) + last_err = None + break + except CosServiceError as e: + last_err = e + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, cos_key), + dest_path="cos://%s/%s" % (dest_bucket, dest_key), + reason=err_reason, + request_id=last_err.get_request_id()) + monitor.stop(log_file=log_file) + if last_err is not None: + raise last_err + + +def _copy_by_prefix(client, bucket, prefix, dest_bucket, dest_prefix, + src_region, dest_region, storage_class, metadata, include, exclude, routines, log_file="", retry=3): + """递归复制指定前缀下的所有对象 + - routines: 文件间并发（同时复制的文件数） + """ + monitor = TransferProgressMonitor("copy") + monitor.start() + + # 先收集所有待复制的文件任务 + tasks = [] + empty_dir_keys = [] # COS 上 / 结尾的空目录对象，需在目标 COS 上同步创建 + total_size = 0 + skip_count = 0 + marker = "" + + while True: + response = client.list_objects( + Bucket=bucket, + Prefix=prefix, + Marker=marker, + MaxKeys=1000, + ) + + if "Contents" in response: + for content in response["Contents"]: + src_key = content["Key"] + rel_key = src_key[len(prefix):].lstrip("/") if prefix else src_key + + # 处理 COS 上的空目录对象（以 / 结尾，Size=0），在目标 COS 上同步创建 + if src_key.endswith("/") and int(content.get("Size", 0)) == 0: + if rel_key: + # include/exclude 过滤目录 + dir_rel = rel_key.rstrip("/") + if not match_filters(dir_rel, include, exclude): + skip_count += 1 + continue + d_key = build_cos_key(dest_prefix, rel_key) + if not d_key.endswith("/"): + d_key += "/" + empty_dir_keys.append(d_key) + continue + + # include/exclude 过滤 + if not match_filters(rel_key, include, exclude): + skip_count += 1 + continue + + file_size = int(content.get("Size", 0)) + d_key = build_cos_key(dest_prefix, rel_key) + total_size += file_size + tasks.append((src_key, d_key, file_size)) + + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + else: + break + + # 设置扫描结果（文件数 + 空目录数 + 跳过数） + monitor.set_scan_info(len(tasks) + len(empty_dir_keys) + skip_count, total_size) + for _ in range(skip_count): + monitor.update_skip(0) + + def _do_copy(src_key, d_key, file_size): + """单个文件复制任务（含重试）""" + last_err = None + for attempt in range(max(1, retry + 1)): + try: + source = { + "Bucket": bucket, + "Key": src_key, + "Region": src_region, + } + kwargs = { + "Bucket": dest_bucket, + "Key": d_key, + "CopySource": source, + } + if storage_class: + kwargs["StorageClass"] = storage_class + if metadata: + kwargs["Metadata"] = metadata + kwargs["MetadataDirective"] = "Replaced" + + client.copy(**kwargs) + monitor.update_ok(file_size) + last_err = None + break + except CosServiceError as e: + last_err = e + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, src_key), + dest_path="cos://%s/%s" % (dest_bucket, d_key), + reason=err_reason, + request_id=last_err.get_request_id()) + + # 使用线程池并发复制多个文件，routines 控制文件间并发 + if tasks: + max_workers = min(routines, len(tasks)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for src_key, d_key, file_size in tasks: + futures.append(executor.submit(_do_copy, src_key, d_key, file_size)) + for future in as_completed(futures): + future.result() + + # 在目标 COS 上创建空目录标记 + for d_key in empty_dir_keys: + try: + client.put_object(Bucket=dest_bucket, Key=d_key, Body=b"") + monitor.update_ok(0) + except CosServiceError as e: + monitor.update_err(src_path="cos://%s/%s" % (dest_bucket, d_key), + reason="创建空目录失败: %s (Code: %s)" % (e.get_error_msg(), e.get_error_code()), + request_id=e.get_request_id()) + + monitor.stop(log_file=log_file) \ No newline at end of file diff --git a/tccli/plugins/cos/create_bucket.py b/tccli/plugins/cos/create_bucket.py new file mode 100644 index 0000000000..72e7d7d890 --- /dev/null +++ b/tccli/plugins/cos/create_bucket.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" +create_bucket 操作：创建存储桶 +对齐 coscli mb 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def create_bucket(args, parsed_globals): + """创建存储桶""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + acl = args.get("acl", "private") or "private" + + try: + client.create_bucket( + Bucket=bucket, + ACL=acl, + ) + print("存储桶创建成功: %s (Region: %s, ACL: %s)" % (bucket, region, acl)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) \ No newline at end of file diff --git a/tccli/plugins/cos/delete_bucket.py b/tccli/plugins/cos/delete_bucket.py new file mode 100644 index 0000000000..798d0d760e --- /dev/null +++ b/tccli/plugins/cos/delete_bucket.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +""" +delete_bucket 操作：删除存储桶 +对齐 coscli rb 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def delete_bucket(args, parsed_globals): + """删除存储桶""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + force = args.get("force", False) + + try: + if force: + print("正在清空存储桶 %s ..." % bucket) + _clear_bucket(client, bucket) + + client.delete_bucket(Bucket=bucket) + print("存储桶删除成功: %s" % bucket) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def _clear_bucket(client, bucket): + """ + 清空存储桶中的所有对象和未完成的分片上传。 + 按照 coscli rb --force 的行为： + 1. 删除所有普通对象 + 2. 删除所有版本对象和删除标记 + 3. 清理所有未完成的分片上传 + """ + # 1. 删除所有普通对象 + deleted_objects = 0 + marker = "" + while True: + response = client.list_objects( + Bucket=bucket, + Marker=marker, + MaxKeys=1000, + ) + if "Contents" in response: + objects = [{"Key": obj["Key"]} for obj in response["Contents"]] + if objects: + client.delete_objects( + Bucket=bucket, + Delete={"Object": objects, "Quiet": "true"}, + ) + deleted_objects += len(objects) + + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + else: + break + + if deleted_objects > 0: + print(" 已删除 %d 个对象" % deleted_objects) + + # 2. 删除所有版本对象和删除标记（如果开启了版本控制） + deleted_versions = 0 + key_marker = "" + version_id_marker = "" + while True: + try: + response = client.list_objects_versions( + Bucket=bucket, + KeyMarker=key_marker, + VersionIdMarker=version_id_marker, + MaxKeys=1000, + ) + objects = [] + if "Version" in response: + versions = response["Version"] + if not isinstance(versions, list): + versions = [versions] + for v in versions: + objects.append({"Key": v["Key"], "VersionId": v["VersionId"]}) + + if "DeleteMarker" in response: + markers = response["DeleteMarker"] + if not isinstance(markers, list): + markers = [markers] + for m in markers: + objects.append({"Key": m["Key"], "VersionId": m["VersionId"]}) + + if objects: + client.delete_objects( + Bucket=bucket, + Delete={"Object": objects, "Quiet": "true"}, + ) + deleted_versions += len(objects) + + if response.get("IsTruncated") == "true": + key_marker = response.get("NextKeyMarker", "") + version_id_marker = response.get("NextVersionIdMarker", "") + else: + break + except CosServiceError: + # 如果未开启版本控制，忽略错误 + break + + if deleted_versions > 0: + print(" 已删除 %d 个版本对象" % deleted_versions) + + # 3. 清理未完成的分片上传 + aborted = 0 + key_marker = "" + upload_id_marker = "" + while True: + response = client.list_multipart_uploads( + Bucket=bucket, + KeyMarker=key_marker, + UploadIdMarker=upload_id_marker, + MaxUploads=1000, + ) + uploads = response.get("Upload", []) + if not isinstance(uploads, list): + uploads = [uploads] + + for upload in uploads: + if not upload: + continue + client.abort_multipart_upload( + Bucket=bucket, + Key=upload["Key"], + UploadId=upload["UploadId"], + ) + aborted += 1 + + if response.get("IsTruncated") == "true": + key_marker = response.get("NextKeyMarker", "") + upload_id_marker = response.get("NextUploadIdMarker", "") + else: + break + + if aborted > 0: + print(" 已清理 %d 个未完成的分片上传" % aborted) \ No newline at end of file diff --git a/tccli/plugins/cos/delete_object.py b/tccli/plugins/cos/delete_object.py new file mode 100644 index 0000000000..423b001f09 --- /dev/null +++ b/tccli/plugins/cos/delete_object.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +""" +delete 操作：删除 COS 上的文件 +对齐 coscli rm 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client, match_filters, list_all_objects_with_dirs, TransferProgressMonitor + + +def delete_object(args, parsed_globals): + """删除 COS 上的文件""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + recursive = args.get("recursive", False) + force = args.get("force", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + version_id = args.get("version_id", "") or "" + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + log_file = args.get("log_file", "") or "" + + try: + if recursive: + _delete_by_prefix(client, bucket, cos_key, include, exclude, force, retry, log_file) + else: + kwargs = {"Bucket": bucket, "Key": cos_key} + if version_id: + kwargs["VersionId"] = version_id + + client.delete_object(**kwargs) + print("删除成功: cos://%s/%s" % (bucket, cos_key)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def _delete_by_prefix(client, bucket, prefix, include, exclude, force, retry=3, log_file=""): + """递归删除指定前缀下的所有对象（含 / 结尾的空目录对象）""" + # 收集所有待删除对象（含空目录） + all_objects = list_all_objects_with_dirs(client, bucket, prefix) + file_keys = [] # 普通文件 + dir_keys = [] # / 结尾的空目录对象 + skip_count = 0 + + for key, obj_info in all_objects.items(): + rel_key = key[len(prefix):].lstrip("/") if prefix else key + + if obj_info.get("IsDir"): + dir_rel = rel_key.rstrip("/") if rel_key else key.rstrip("/").split("/")[-1] + if not match_filters(dir_rel, include, exclude): + skip_count += 1 + continue + dir_keys.append(key) + continue + + if not match_filters(rel_key, include, exclude): + skip_count += 1 + continue + + file_keys.append(key) + + total_count = len(file_keys) + len(dir_keys) + if total_count == 0: + print("没有匹配的对象需要删除") + return + + # 非 force 模式下提示确认 + if not force: + print("即将删除 %d 个对象（文件: %d，文件夹: %d，前缀: cos://%s/%s）" % ( + total_count, len(file_keys), len(dir_keys), bucket, prefix)) + print("提示: 使用 --force true 跳过确认") + try: + confirm = input("确认删除? (y/N): ").strip().lower() + if confirm != "y": + print("已取消删除") + return + except (EOFError, KeyboardInterrupt): + print("\n已取消删除") + return + + monitor = TransferProgressMonitor("delete") + monitor.set_scan_info(total_count + skip_count, 0) + for _ in range(skip_count): + monitor.update_skip(0) + monitor.start() + + # 删除普通文件（批量删除，每次最多 1000 个，含重试） + for i in range(0, len(file_keys), 1000): + batch_keys = file_keys[i:i + 1000] + last_err = None + for attempt in range(max(1, retry + 1)): + try: + client.delete_objects( + Bucket=bucket, + Delete={"Object": [{"Key": k} for k in batch_keys], "Quiet": "true"}, + ) + for k in batch_keys: + monitor.update_ok(0) + last_err = None + break + except CosServiceError as e: + last_err = e + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + for k in batch_keys: + monitor.update_err(src_path="cos://%s/%s" % (bucket, k), + reason=err_reason, + request_id=last_err.get_request_id()) + + # 删除空目录对象（逐个删除，含重试） + for key in dir_keys: + last_err = None + for attempt in range(max(1, retry + 1)): + try: + client.delete_object(Bucket=bucket, Key=key) + monitor.update_ok(0) + last_err = None + break + except CosServiceError as e: + last_err = e + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, key), + reason=err_reason, + request_id=last_err.get_request_id()) + + monitor.stop(log_file=log_file) \ No newline at end of file diff --git a/tccli/plugins/cos/doc/README.md b/tccli/plugins/cos/doc/README.md new file mode 100644 index 0000000000..5c9d7770ce --- /dev/null +++ b/tccli/plugins/cos/doc/README.md @@ -0,0 +1,1068 @@ +# tccli cos 插件使用文档 + +腾讯云 COS（对象存储）命令行工具插件，集成于 `tccli` 中，提供完整的 COS 文件管理能力。 + +## 目录 + +- [全局参数](#全局参数) +- [文件操作](#文件操作) + - [list - 列出文件](#list---列出文件) + - [upload - 上传文件](#upload---上传文件) + - [download - 下载文件](#download---下载文件) + - [delete - 删除文件](#delete---删除文件) + - [copy - 复制文件](#copy---复制文件) + - [move - 移动/重命名文件](#move---移动重命名文件) + - [cat - 查看文件内容](#cat---查看文件内容) + - [head - 查询对象元信息](#head---查询对象元信息) + - [hash - 计算哈希值](#hash---计算哈希值) +- [同步操作](#同步操作) + - [sync_upload - 同步上传](#sync_upload---同步上传) + - [sync_download - 同步下载](#sync_download---同步下载) + - [sync_copy - 同步复制](#sync_copy---同步复制) +- [存储桶操作](#存储桶操作) + - [list_buckets - 列出存储桶](#list_buckets---列出存储桶) + - [create_bucket - 创建存储桶](#create_bucket---创建存储桶) + - [delete_bucket - 删除存储桶](#delete_bucket---删除存储桶) +- [统计操作](#统计操作) + - [du - 统计大小](#du---统计大小) +- [归档恢复](#归档恢复) + - [restore - 恢复归档文件](#restore---恢复归档文件) +- [预签名 URL](#预签名-url) + - [signurl - 生成预签名URL](#signurl---生成预签名url) +- [ACL 权限管理](#acl-权限管理) + - [get_bucket_acl - 获取存储桶ACL](#get_bucket_acl---获取存储桶acl) + - [put_bucket_acl - 设置存储桶ACL](#put_bucket_acl---设置存储桶acl) + - [get_object_acl - 获取对象ACL](#get_object_acl---获取对象acl) + - [put_object_acl - 设置对象ACL](#put_object_acl---设置对象acl) +- [标签管理](#标签管理) + - [get_object_tagging - 获取对象标签](#get_object_tagging---获取对象标签) + - [put_object_tagging - 设置对象标签](#put_object_tagging---设置对象标签) + - [delete_object_tagging - 删除对象标签](#delete_object_tagging---删除对象标签) +- [分片上传管理](#分片上传管理) + - [lsparts - 列出分片上传](#lsparts---列出分片上传) + - [abort - 清理分片上传](#abort---清理分片上传) +- [附录](#附录) + - [存储类型说明](#存储类型说明) + - [失败日志格式](#失败日志格式) + +--- + +## 全局参数 + +所有命令均支持以下全局参数（通过 `tccli` 统一配置）： + +| 参数 | 说明 | +|------|------| +| `--region` | 地域，如 `ap-guangzhou`、`ap-beijing` | +| `--secretId` | 腾讯云 SecretId | +| `--secretKey` | 腾讯云 SecretKey | +| `--profile` | 使用指定的配置文件 | + +--- + +## 文件操作 + +### list - 列出文件 + +列出 COS 存储桶中的文件，默认只列出当前层级（非递归），支持按前缀过滤和 include/exclude 过滤。 + +**命令格式：** +```bash +tccli cos list [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称，格式如 `my-bucket-1250000000` | +| `--prefix` | string | ❌ | 空 | 对象键前缀，用于过滤列出的对象 | +| `--marker` | string | ❌ | 空 | 分页标记，从该标记之后开始列出 | +| `--max_keys` | int | ❌ | 1000 | 最大返回数量，最大 1000 | +| `--delimiter` | string | ❌ | `/` | 分隔符，默认 `/` 模拟目录结构 | +| `--recursive` | bool | ❌ | false | 是否递归列出所有对象 | +| `--include` | string | ❌ | 空 | 包含匹配模式，支持通配符，如 `*.txt` | +| `--exclude` | string | ❌ | 空 | 排除匹配模式，支持通配符，如 `*.log` | + +**示例：** +```bash +# 列出存储桶根目录（只显示当前层级） +tccli cos list --bucket my-bucket-1250000000 + +# 列出指定前缀下的所有文件（递归） +tccli cos list --bucket my-bucket-1250000000 --prefix data/ --recursive true + +# 只列出 txt 文件 +tccli cos list --bucket my-bucket-1250000000 --prefix logs/ --recursive true --include "*.txt" + +# 排除日志文件 +tccli cos list --bucket my-bucket-1250000000 --recursive true --exclude "*.log" +``` + +--- + +### upload - 上传文件 + +上传本地文件或目录到 COS，自动根据文件大小选择简单上传或分片上传（默认分片阈值 20MB）。 + +**命令格式：** +```bash +tccli cos upload [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 目标存储桶名称 | +| `--local_path` | string | ✅ | - | 本地文件或目录路径 | +| `--cos_key` | string | ✅ | - | COS 上的目标对象键（Key），递归上传时作为前缀 | +| `--storage_class` | string | ❌ | STANDARD | 存储类型，见[存储类型说明](#存储类型说明) | +| `--content_type` | string | ❌ | 空 | 文件内容类型（MIME），如 `text/plain` | +| `--meta` | string | ❌ | 空 | 自定义元数据，格式：`key1=value1#key2=value2` | +| `--recursive` | bool | ❌ | false | 是否递归上传目录 | +| `--include` | string | ❌ | 空 | 包含匹配模式（递归时生效），支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式（递归时生效），支持通配符 | +| `--thread_num` | int | ❌ | 5 | 单文件分片上传并发线程数 | +| `--routines` | int | ❌ | 3 | 文件间并发数（同时传输的文件数） | +| `--part_size` | int | ❌ | 20 | 分片大小（MB） | +| `--rate_limiting` | int | ❌ | 0 | 单链接限速（MB/s），0 表示不限速 | +| `--retry` | int | ❌ | 3 | 失败重试次数，0 表示不重试 | +| `--log_file` | string | ❌ | 空 | 失败日志文件路径，指定后将失败记录写入该文件 | + +**目录上传行为说明：** +- `local_path` 以 `/` 结尾（如 `/tmp/dir/`）：不保留目录名，直接映射内容到 `cos_key` 前缀下 +- `local_path` 不以 `/` 结尾（如 `/tmp/dir`）：保留目录名，映射为 `cos_key/dir/` 前缀下 + +**示例：** +```bash +# 上传单个文件 +tccli cos upload --bucket my-bucket-1250000000 --local_path /tmp/test.txt --cos_key data/test.txt + +# 上传目录（保留目录名） +tccli cos upload --bucket my-bucket-1250000000 --local_path /tmp/mydir --cos_key backup/ --recursive true + +# 上传目录（不保留目录名，直接映射内容） +tccli cos upload --bucket my-bucket-1250000000 --local_path /tmp/mydir/ --cos_key backup/ --recursive true + +# 只上传 jpg 图片，使用低频存储，限速 10MB/s +tccli cos upload --bucket my-bucket-1250000000 --local_path /tmp/photos --cos_key images/ \ + --recursive true --include "*.jpg" --storage_class STANDARD_IA --rate_limiting 10 + +# 上传并记录失败日志 +tccli cos upload --bucket my-bucket-1250000000 --local_path /tmp/data --cos_key data/ \ + --recursive true --retry 5 --log_file /tmp/upload_fail.log +``` + +--- + +### download - 下载文件 + +从 COS 下载文件到本地，自动根据文件大小选择简单下载或分片下载。 + +**命令格式：** +```bash +tccli cos download [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 源存储桶名称 | +| `--cos_key` | string | ✅ | - | COS 上的源对象键（Key），递归下载时作为前缀 | +| `--local_path` | string | ✅ | - | 本地保存路径，递归下载时为目标目录 | +| `--recursive` | bool | ❌ | false | 是否递归下载前缀下所有对象 | +| `--include` | string | ❌ | 空 | 包含匹配模式（递归时生效），支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式（递归时生效），支持通配符 | +| `--thread_num` | int | ❌ | 5 | 单文件分片下载并发线程数 | +| `--routines` | int | ❌ | 3 | 文件间并发数（同时下载的文件数） | +| `--part_size` | int | ❌ | 20 | 分片大小（MB） | +| `--rate_limiting` | int | ❌ | 0 | 单链接限速（MB/s），0 表示不限速 | +| `--version_id` | string | ❌ | 空 | 指定下载的对象版本 ID（开启版本控制时使用） | +| `--retry` | int | ❌ | 3 | 失败重试次数，0 表示不重试 | +| `--log_file` | string | ❌ | 空 | 失败日志文件路径 | + +**示例：** +```bash +# 下载单个文件 +tccli cos download --bucket my-bucket-1250000000 --cos_key data/test.txt --local_path /tmp/test.txt + +# 递归下载整个目录 +tccli cos download --bucket my-bucket-1250000000 --cos_key data/ --local_path /tmp/data --recursive true + +# 只下载 txt 文件，限速 5MB/s +tccli cos download --bucket my-bucket-1250000000 --cos_key logs/ --local_path /tmp/logs \ + --recursive true --include "*.txt" --rate_limiting 5 + +# 下载指定版本的文件 +tccli cos download --bucket my-bucket-1250000000 --cos_key data/test.txt \ + --local_path /tmp/test.txt --version_id MTg0NDUxNTc1NjIzMTQ1MDAwODg + +# 下载并记录失败日志 +tccli cos download --bucket my-bucket-1250000000 --cos_key data/ --local_path /tmp/data \ + --recursive true --retry 5 --log_file /tmp/download_fail.log +``` + +--- + +### delete - 删除文件 + +删除 COS 存储桶中的指定文件，支持递归批量删除。 + +**命令格式：** +```bash +tccli cos delete [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--cos_key` | string | ✅ | - | 要删除的对象键（Key），递归删除时作为前缀 | +| `--recursive` | bool | ❌ | false | 是否递归删除前缀下所有对象 | +| `--force` | bool | ❌ | false | 递归删除时跳过确认提示 | +| `--include` | string | ❌ | 空 | 包含匹配模式（递归时生效），支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式（递归时生效），支持通配符 | +| `--version_id` | string | ❌ | 空 | 指定删除的对象版本 ID（开启版本控制时使用） | + +> ⚠️ 递归删除时，默认会提示确认，使用 `--force true` 可跳过确认。 + +**示例：** +```bash +# 删除单个文件 +tccli cos delete --bucket my-bucket-1250000000 --cos_key data/test.txt + +# 递归删除目录（会提示确认） +tccli cos delete --bucket my-bucket-1250000000 --cos_key data/ --recursive true + +# 递归删除目录（跳过确认） +tccli cos delete --bucket my-bucket-1250000000 --cos_key data/ --recursive true --force true + +# 只删除 log 文件 +tccli cos delete --bucket my-bucket-1250000000 --cos_key logs/ --recursive true --force true --include "*.log" + +# 删除指定版本的文件 +tccli cos delete --bucket my-bucket-1250000000 --cos_key data/test.txt --version_id MTg0NDUxNTc1NjIzMTQ1MDAwODg +``` + +--- + +### copy - 复制文件 + +复制 COS 上的文件到另一个位置，支持跨存储桶和跨地域复制，支持并发和失败重试。 + +**命令格式：** +```bash +tccli cos copy [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 源存储桶名称 | +| `--cos_key` | string | ✅ | - | 源对象键（Key），递归复制时作为前缀 | +| `--dest_key` | string | ✅ | - | 目标对象键（Key），递归复制时作为目标前缀 | +| `--dest_bucket` | string | ❌ | 同源桶 | 目标存储桶名称 | +| `--dest_region` | string | ❌ | 同当前地域 | 目标地域 | +| `--storage_class` | string | ❌ | 空 | 目标存储类型，见[存储类型说明](#存储类型说明) | +| `--meta` | string | ❌ | 空 | 自定义元数据，格式：`key1=value1#key2=value2`（设置后使用 Replaced 模式） | +| `--recursive` | bool | ❌ | false | 是否递归复制前缀下所有对象 | +| `--include` | string | ❌ | 空 | 包含匹配模式（递归时生效），支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式（递归时生效），支持通配符 | +| `--routines` | int | ❌ | 3 | 文件间并发数（同时复制的文件数） | +| `--retry` | int | ❌ | 3 | 失败重试次数，0 表示不重试 | +| `--log_file` | string | ❌ | 空 | 失败日志文件路径 | + +**示例：** +```bash +# 同桶内复制单个文件 +tccli cos copy --bucket my-bucket-1250000000 --cos_key data/test.txt --dest_key backup/test.txt + +# 跨存储桶复制 +tccli cos copy --bucket src-bucket-1250000000 --cos_key data/test.txt \ + --dest_bucket dst-bucket-1250000000 --dest_key data/test.txt + +# 跨地域复制整个目录 +tccli cos copy --bucket src-bucket-1250000000 --cos_key data/ \ + --dest_bucket dst-bucket-1250000000 --dest_key data/ \ + --dest_region ap-beijing --recursive true + +# 复制并修改存储类型 +tccli cos copy --bucket my-bucket-1250000000 --cos_key data/ --dest_key archive/ \ + --recursive true --storage_class ARCHIVE +``` + +--- + +### move - 移动/重命名文件 + +移动或重命名 COS 上的文件（通过复制+删除实现），支持跨存储桶移动，支持并发和失败重试。 + +**命令格式：** +```bash +tccli cos move [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 源存储桶名称 | +| `--cos_key` | string | ✅ | - | 源对象键（Key），递归移动时作为前缀 | +| `--dest_key` | string | ✅ | - | 目标对象键（Key），递归移动时作为目标前缀 | +| `--dest_bucket` | string | ❌ | 同源桶 | 目标存储桶名称 | +| `--dest_region` | string | ❌ | 同当前地域 | 目标地域 | +| `--storage_class` | string | ❌ | 空 | 目标存储类型，见[存储类型说明](#存储类型说明) | +| `--recursive` | bool | ❌ | false | 是否递归移动前缀下所有对象 | +| `--include` | string | ❌ | 空 | 包含匹配模式（递归时生效），支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式（递归时生效），支持通配符 | +| `--routines` | int | ❌ | 3 | 文件间并发数（同时移动的文件数） | +| `--retry` | int | ❌ | 3 | 失败重试次数，0 表示不重试 | +| `--log_file` | string | ❌ | 空 | 失败日志文件路径 | + +**示例：** +```bash +# 重命名单个文件 +tccli cos move --bucket my-bucket-1250000000 --cos_key data/old.txt --dest_key data/new.txt + +# 移动整个目录 +tccli cos move --bucket my-bucket-1250000000 --cos_key data/ --dest_key archive/ --recursive true + +# 跨存储桶移动 +tccli cos move --bucket src-bucket-1250000000 --cos_key data/ \ + --dest_bucket dst-bucket-1250000000 --dest_key data/ --recursive true + +# 移动并记录失败日志 +tccli cos move --bucket my-bucket-1250000000 --cos_key data/ --dest_key backup/ \ + --recursive true --retry 5 --log_file /tmp/move_fail.log +``` + +--- + +### cat - 查看文件内容 + +查看 COS 对象的文本内容，默认最大读取 10MB，支持指定字节范围读取。 + +**命令格式：** +```bash +tccli cos cat [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--cos_key` | string | ✅ | - | 对象键（Key） | +| `--range` | string | ❌ | 空 | 指定读取范围，格式：`bytes=0-1023` | +| `--max_size` | int | ❌ | 10 | 最大读取大小（MB），超过此大小仅显示前 N MB | + +**示例：** +```bash +# 查看文件内容 +tccli cos cat --bucket my-bucket-1250000000 --cos_key logs/app.log + +# 查看文件的前 1KB +tccli cos cat --bucket my-bucket-1250000000 --cos_key logs/app.log --range "bytes=0-1023" + +# 查看大文件（最多显示 50MB） +tccli cos cat --bucket my-bucket-1250000000 --cos_key data/large.txt --max_size 50 +``` + +--- + +### head - 查询对象元信息 + +查询 COS 对象的元数据信息，包括大小、类型、修改时间、ETag、CRC64 等。 + +**命令格式：** +```bash +tccli cos head [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--cos_key` | string | ✅ | - | 要查询的对象键（Key） | +| `--version_id` | string | ❌ | 空 | 指定查询的对象版本 ID（开启版本控制时使用） | + +**示例：** +```bash +# 查询对象元信息 +tccli cos head --bucket my-bucket-1250000000 --cos_key data/test.txt + +# 查询指定版本的元信息 +tccli cos head --bucket my-bucket-1250000000 --cos_key data/test.txt --version_id MTg0NDUxNTc1NjIzMTQ1MDAwODg +``` + +**输出示例：** +``` +对象元信息: cos://my-bucket-1250000000/data/test.txt +-------------------------------------------------- +Content-Length: 1024 +Content-Type: text/plain +ETag: "d41d8cd98f00b204e9800998ecf8427e" +Last-Modified: Mon, 07 Apr 2025 10:00:00 GMT +Storage-Class: STANDARD +CRC64: 1234567890123456789 +``` + +--- + +### hash - 计算哈希值 + +计算本地文件的哈希值，或获取 COS 对象的 ETag/CRC64 信息。 + +**命令格式：** +```bash +tccli cos hash [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--local_path` | string | ❌ | 空 | 本地文件路径（计算本地文件哈希时使用） | +| `--bucket` | string | ❌ | 空 | 存储桶名称（获取 COS 对象哈希时使用） | +| `--cos_key` | string | ❌ | 空 | 对象键（获取 COS 对象哈希时使用） | +| `--hash_type` | string | ❌ | md5 | 哈希类型：`md5`、`sha1`、`sha256`、`crc64` | + +> 注意：`--local_path` 和 `--bucket + --cos_key` 至少指定一组，也可同时指定两组进行对比。 + +**示例：** +```bash +# 计算本地文件 MD5 +tccli cos hash --local_path /tmp/test.txt + +# 计算本地文件 SHA256 +tccli cos hash --local_path /tmp/test.txt --hash_type sha256 + +# 获取 COS 对象的 ETag 和 CRC64 +tccli cos hash --bucket my-bucket-1250000000 --cos_key data/test.txt + +# 同时计算本地和 COS 对象的哈希（用于校验一致性） +tccli cos hash --local_path /tmp/test.txt --bucket my-bucket-1250000000 --cos_key data/test.txt +``` + +--- + +## 同步操作 + +同步操作对齐 `coscli sync` 命令的跳过逻辑（优先级从高到低）： + +- `--ignore_existing true`：目标已存在即跳过，不做内容比较 +- `--update true`：按 `Last-Modified` 时间比较，目标 ≥ 源时跳过（用于只向新版本推送） +- 默认：**CRC64 校验**（对比 COS `x-cos-hash-crc64ecma` 与对端 CRC64），相同则跳过 + +> 目标不存在时一律不跳过。CRC64 计算依赖 `crcmod` 库（已包含在项目依赖中）。 + +### sync_upload - 同步上传 + +同步本地目录到 COS，增量上传，支持删除 COS 上多余的文件。 + +**命令格式：** +```bash +tccli cos sync_upload [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 目标存储桶名称 | +| `--local_path` | string | ✅ | - | 本地目录路径 | +| `--cos_key` | string | ❌ | 空 | COS 上的目标前缀 | +| `--recursive` | bool | ❌ | false | 是否递归同步目录 | +| `--delete` | bool | ❌ | false | 是否删除 COS 上多余的文件（本地不存在的） | +| `--include` | string | ❌ | 空 | 包含匹配模式，支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式，支持通配符 | +| `--storage_class` | string | ❌ | STANDARD | 上传时的存储类型 | +| `--content_type` | string | ❌ | 空 | 文件内容类型（MIME） | +| `--meta` | string | ❌ | 空 | 自定义元数据，格式：`key1=value1#key2=value2` | +| `--thread_num` | int | ❌ | 5 | 单文件分片上传并发线程数 | +| `--routines` | int | ❌ | 3 | 文件间并发数 | +| `--part_size` | int | ❌ | 20 | 分片大小（MB） | +| `--rate_limiting` | int | ❌ | 0 | 单链接限速（MB/s） | +| `--retry` | int | ❌ | 3 | 失败重试次数 | +| `--log_file` | string | ❌ | 空 | 失败日志文件路径 | + +**示例：** +```bash +# 同步本地目录到 COS（增量上传） +tccli cos sync_upload --bucket my-bucket-1250000000 --local_path /data/backup --cos_key backup/ --recursive true + +# 同步并删除 COS 上多余的文件（镜像同步） +tccli cos sync_upload --bucket my-bucket-1250000000 --local_path /data/backup --cos_key backup/ \ + --recursive true --delete true + +# 只同步 txt 和 csv 文件 +tccli cos sync_upload --bucket my-bucket-1250000000 --local_path /data --cos_key data/ \ + --recursive true --include "*.txt" --include "*.csv" + +# 同步并记录失败日志 +tccli cos sync_upload --bucket my-bucket-1250000000 --local_path /data --cos_key data/ \ + --recursive true --retry 5 --log_file /tmp/sync_fail.log +``` + +--- + +### sync_download - 同步下载 + +同步 COS 到本地目录，增量下载，支持删除本地多余的文件。 + +**命令格式：** +```bash +tccli cos sync_download [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 源存储桶名称 | +| `--local_path` | string | ✅ | - | 本地目标目录路径 | +| `--cos_key` | string | ❌ | 空 | COS 上的源前缀 | +| `--recursive` | bool | ❌ | false | 是否递归同步目录 | +| `--delete` | bool | ❌ | false | 是否删除本地多余的文件（COS 上不存在的） | +| `--include` | string | ❌ | 空 | 包含匹配模式，支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式，支持通配符 | +| `--thread_num` | int | ❌ | 5 | 单文件分片下载并发线程数 | +| `--routines` | int | ❌ | 3 | 文件间并发数 | +| `--part_size` | int | ❌ | 20 | 分片大小（MB） | +| `--rate_limiting` | int | ❌ | 0 | 单链接限速（MB/s） | +| `--retry` | int | ❌ | 3 | 失败重试次数 | +| `--log_file` | string | ❌ | 空 | 失败日志文件路径 | + +**示例：** +```bash +# 同步 COS 目录到本地（增量下载） +tccli cos sync_download --bucket my-bucket-1250000000 --cos_key data/ --local_path /tmp/data --recursive true + +# 同步并删除本地多余的文件（镜像同步） +tccli cos sync_download --bucket my-bucket-1250000000 --cos_key data/ --local_path /tmp/data \ + --recursive true --delete true + +# 只同步图片文件 +tccli cos sync_download --bucket my-bucket-1250000000 --cos_key images/ --local_path /tmp/images \ + --recursive true --include "*.jpg" --include "*.png" +``` + +--- + +### sync_copy - 同步复制 + +同步 COS 到另一个 COS 位置，增量复制，支持删除目标端多余的文件。 + +**命令格式：** +```bash +tccli cos sync_copy [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 源存储桶名称 | +| `--cos_key` | string | ❌ | 空 | 源 COS 前缀 | +| `--dest_bucket` | string | ❌ | 同源桶 | 目标存储桶名称 | +| `--dest_key` | string | ❌ | 空 | 目标 COS 前缀 | +| `--dest_region` | string | ❌ | 同当前地域 | 目标地域 | +| `--recursive` | bool | ❌ | false | 是否递归同步复制 | +| `--delete` | bool | ❌ | false | 是否删除目标端多余的文件（源端不存在的） | +| `--include` | string | ❌ | 空 | 包含匹配模式，支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式，支持通配符 | +| `--storage_class` | string | ❌ | 空 | 目标存储类型 | +| `--meta` | string | ❌ | 空 | 自定义元数据，格式：`key1=value1#key2=value2` | +| `--routines` | int | ❌ | 3 | 文件间并发数 | +| `--retry` | int | ❌ | 3 | 失败重试次数 | +| `--log_file` | string | ❌ | 空 | 失败日志文件路径 | + +**示例：** +```bash +# 同步复制到同桶不同前缀 +tccli cos sync_copy --bucket my-bucket-1250000000 --cos_key data/ --dest_key backup/ --recursive true + +# 跨桶同步复制（镜像同步） +tccli cos sync_copy --bucket src-bucket-1250000000 --cos_key data/ \ + --dest_bucket dst-bucket-1250000000 --dest_key data/ \ + --recursive true --delete true + +# 跨地域同步复制 +tccli cos sync_copy --bucket src-bucket-1250000000 --cos_key data/ \ + --dest_bucket dst-bucket-1250000000 --dest_key data/ \ + --dest_region ap-beijing --recursive true +``` + +--- + +## 存储桶操作 + +### list_buckets - 列出存储桶 + +列出当前账号下的所有存储桶，支持按地域过滤。 + +**命令格式：** +```bash +tccli cos list_buckets [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--filter_region` | string | ❌ | 空 | 按地域过滤，如 `ap-guangzhou` | + +**示例：** +```bash +# 列出所有存储桶 +tccli cos list_buckets + +# 列出广州地域的存储桶 +tccli cos list_buckets --filter_region ap-guangzhou +``` + +--- + +### create_bucket - 创建存储桶 + +创建一个新的 COS 存储桶。 + +**命令格式：** +```bash +tccli cos create_bucket [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称，格式如 `my-bucket-1250000000` | +| `--acl` | string | ❌ | private | 访问控制策略：`private`、`public-read`、`public-read-write` | + +**示例：** +```bash +# 创建私有存储桶 +tccli cos create_bucket --bucket my-bucket-1250000000 + +# 创建公开读存储桶 +tccli cos create_bucket --bucket my-bucket-1250000000 --acl public-read +``` + +--- + +### delete_bucket - 删除存储桶 + +删除指定的 COS 存储桶，使用 `--force` 可强制清空后删除。 + +**命令格式：** +```bash +tccli cos delete_bucket [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 要删除的存储桶名称 | +| `--force` | bool | ❌ | false | 强制删除：先清空所有对象、版本对象和未完成的分片上传，再删除存储桶 | + +> ⚠️ `--force true` 会**不可逆地**删除存储桶内所有数据，请谨慎使用。 + +**示例：** +```bash +# 删除空存储桶 +tccli cos delete_bucket --bucket my-bucket-1250000000 + +# 强制清空并删除存储桶 +tccli cos delete_bucket --bucket my-bucket-1250000000 --force true +``` + +--- + +## 统计操作 + +### du - 统计大小 + +统计存储桶或指定前缀下的对象总大小、文件数量和文件夹数量，按存储类型分类统计。 + +**命令格式：** +```bash +tccli cos du [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--prefix` | string | ❌ | 空 | 对象键前缀，用于统计指定目录的大小 | + +**示例：** +```bash +# 统计整个存储桶 +tccli cos du --bucket my-bucket-1250000000 + +# 统计指定目录 +tccli cos du --bucket my-bucket-1250000000 --prefix data/ +``` + +**输出示例：** +``` +统计: cos://my-bucket-1250000000/data/ +------------------------------------------------------------ +总文件数: 100 +总文件夹数: 5 +总大小: 1.23 GB (1321205760 字节) + +按存储类型统计: + STANDARD 95 个对象, 1.20 GB + STANDARD_IA 5 个对象, 30.00 MB +``` + +--- + +## 归档恢复 + +### restore - 恢复归档文件 + +恢复归档存储类型（ARCHIVE/DEEP_ARCHIVE）的 COS 对象，使其可被下载。 + +**命令格式：** +```bash +tccli cos restore [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--cos_key` | string | ✅ | - | 要恢复的归档对象键（Key），递归恢复时作为前缀 | +| `--days` | int | ❌ | 7 | 恢复后的有效天数 | +| `--tier` | string | ❌ | Standard | 恢复模式：`Standard`（3-5小时）、`Expedited`（1-5分钟）、`Bulk`（5-12小时） | +| `--recursive` | bool | ❌ | false | 是否递归恢复前缀下所有归档对象 | +| `--include` | string | ❌ | 空 | 包含匹配模式（递归时生效），支持通配符 | +| `--exclude` | string | ❌ | 空 | 排除匹配模式（递归时生效），支持通配符 | + +**示例：** +```bash +# 恢复单个归档文件（标准模式，有效期 7 天） +tccli cos restore --bucket my-bucket-1250000000 --cos_key archive/data.zip + +# 极速恢复，有效期 30 天 +tccli cos restore --bucket my-bucket-1250000000 --cos_key archive/data.zip --tier Expedited --days 30 + +# 递归恢复整个归档目录 +tccli cos restore --bucket my-bucket-1250000000 --cos_key archive/ --recursive true --days 7 +``` + +--- + +## 预签名 URL + +### signurl - 生成预签名URL + +生成 COS 对象的预签名 URL，可用于临时授权访问，无需鉴权即可访问。 + +**命令格式：** +```bash +tccli cos signurl [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--cos_key` | string | ✅ | - | 对象键（Key） | +| `--expired` | int | ❌ | 3600 | URL 有效期（秒），默认 1 小时 | +| `--method` | string | ❌ | GET | HTTP 方法：`GET`（下载）、`PUT`（上传） | + +**示例：** +```bash +# 生成下载链接（有效期 1 小时） +tccli cos signurl --bucket my-bucket-1250000000 --cos_key data/test.txt + +# 生成下载链接（有效期 24 小时） +tccli cos signurl --bucket my-bucket-1250000000 --cos_key data/test.txt --expired 86400 + +# 生成上传链接（有效期 10 分钟） +tccli cos signurl --bucket my-bucket-1250000000 --cos_key data/upload.txt --method PUT --expired 600 +``` + +--- + +## ACL 权限管理 + +### get_bucket_acl - 获取存储桶ACL + +获取存储桶的访问控制列表（ACL）。 + +**命令格式：** +```bash +tccli cos get_bucket_acl --bucket <存储桶名称> +``` + +**示例：** +```bash +tccli cos get_bucket_acl --bucket my-bucket-1250000000 +``` + +--- + +### put_bucket_acl - 设置存储桶ACL + +设置存储桶的访问控制策略。 + +**命令格式：** +```bash +tccli cos put_bucket_acl [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--acl` | string | ❌ | 空 | 访问控制策略：`private`、`public-read`、`public-read-write` | +| `--grant_read` | string | ❌ | 空 | 授予读权限的用户，格式：`id="账号ID"` | +| `--grant_write` | string | ❌ | 空 | 授予写权限的用户，格式：`id="账号ID"` | +| `--grant_full_control` | string | ❌ | 空 | 授予完全控制权限的用户，格式：`id="账号ID"` | + +**示例：** +```bash +# 设置存储桶为公开读 +tccli cos put_bucket_acl --bucket my-bucket-1250000000 --acl public-read + +# 授予指定账号读权限 +tccli cos put_bucket_acl --bucket my-bucket-1250000000 --grant_read 'id="100000000001"' +``` + +--- + +### get_object_acl - 获取对象ACL + +获取 COS 对象的访问控制列表（ACL）。 + +**命令格式：** +```bash +tccli cos get_object_acl --bucket <存储桶名称> --cos_key <对象键> +``` + +**示例：** +```bash +tccli cos get_object_acl --bucket my-bucket-1250000000 --cos_key data/test.txt +``` + +--- + +### put_object_acl - 设置对象ACL + +设置 COS 对象的访问控制策略。 + +**命令格式：** +```bash +tccli cos put_object_acl [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--cos_key` | string | ✅ | - | 对象键（Key） | +| `--acl` | string | ❌ | 空 | 访问控制策略：`private`、`public-read` | +| `--grant_read` | string | ❌ | 空 | 授予读权限的用户，格式：`id="账号ID"` | +| `--grant_full_control` | string | ❌ | 空 | 授予完全控制权限的用户，格式：`id="账号ID"` | + +**示例：** +```bash +# 设置对象为公开读 +tccli cos put_object_acl --bucket my-bucket-1250000000 --cos_key data/test.txt --acl public-read + +# 授予指定账号完全控制权限 +tccli cos put_object_acl --bucket my-bucket-1250000000 --cos_key data/test.txt \ + --grant_full_control 'id="100000000001"' +``` + +--- + +## 标签管理 + +### get_object_tagging - 获取对象标签 + +获取 COS 对象的标签信息。 + +**命令格式：** +```bash +tccli cos get_object_tagging --bucket <存储桶名称> --cos_key <对象键> +``` + +**示例：** +```bash +tccli cos get_object_tagging --bucket my-bucket-1250000000 --cos_key data/test.txt +``` + +--- + +### put_object_tagging - 设置对象标签 + +设置 COS 对象的标签，格式为 `key1=value1,key2=value2`。 + +**命令格式：** +```bash +tccli cos put_object_tagging [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--cos_key` | string | ✅ | - | 对象键（Key） | +| `--tags` | string | ✅ | - | 标签列表，格式：`key1=value1,key2=value2` | + +**示例：** +```bash +# 设置单个标签 +tccli cos put_object_tagging --bucket my-bucket-1250000000 --cos_key data/test.txt --tags "env=prod" + +# 设置多个标签 +tccli cos put_object_tagging --bucket my-bucket-1250000000 --cos_key data/test.txt \ + --tags "env=prod,project=myapp,owner=team1" +``` + +--- + +### delete_object_tagging - 删除对象标签 + +删除 COS 对象的所有标签。 + +**命令格式：** +```bash +tccli cos delete_object_tagging --bucket <存储桶名称> --cos_key <对象键> +``` + +**示例：** +```bash +tccli cos delete_object_tagging --bucket my-bucket-1250000000 --cos_key data/test.txt +``` + +--- + +## 分片上传管理 + +### lsparts - 列出分片上传 + +列出存储桶中未完成的分片上传任务。 + +**命令格式：** +```bash +tccli cos lsparts [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--prefix` | string | ❌ | 空 | 对象键前缀，用于过滤列出的分片上传 | + +**示例：** +```bash +# 列出所有未完成的分片上传 +tccli cos lsparts --bucket my-bucket-1250000000 + +# 列出指定前缀下的未完成分片上传 +tccli cos lsparts --bucket my-bucket-1250000000 --prefix data/ +``` + +--- + +### abort - 清理分片上传 + +清理存储桶中未完成的分片上传任务，释放存储空间。 + +**命令格式：** +```bash +tccli cos abort [参数] +``` + +**参数说明：** + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `--bucket` | string | ✅ | - | 存储桶名称 | +| `--prefix` | string | ❌ | 空 | 对象键前缀，用于过滤要清理的分片上传 | +| `--cos_key` | string | ❌ | 空 | 对象键（指定 `upload_id` 时必填） | +| `--upload_id` | string | ❌ | 空 | 指定要取消的分片上传 ID，不填则清理所有未完成的分片上传 | + +**示例：** +```bash +# 清理所有未完成的分片上传 +tccli cos abort --bucket my-bucket-1250000000 + +# 清理指定前缀下的未完成分片上传 +tccli cos abort --bucket my-bucket-1250000000 --prefix data/ + +# 取消指定的分片上传 +tccli cos abort --bucket my-bucket-1250000000 --cos_key data/large.zip \ + --upload_id 1585130821cbb7df1d11846c073ad7cf9d27a33 +``` + +--- + +## 附录 + +### 存储类型说明 + +| 存储类型 | 说明 | 适用场景 | +|----------|------|----------| +| `STANDARD` | 标准存储（默认） | 高频访问数据 | +| `STANDARD_IA` | 低频存储 | 低频访问，存储 30 天以上 | +| `ARCHIVE` | 归档存储 | 极少访问，需要恢复后才能下载 | +| `DEEP_ARCHIVE` | 深度归档存储 | 长期保存，访问频率极低 | +| `INTELLIGENT_TIERING` | 智能分层存储 | 访问模式不固定的数据 | +| `MAZ_STANDARD` | 多 AZ 标准存储 | 高可用要求的高频访问数据 | +| `MAZ_STANDARD_IA` | 多 AZ 低频存储 | 高可用要求的低频访问数据 | + +### 失败日志格式 + +当指定 `--log_file` 参数时，失败的操作会以 JSON Lines 格式记录到日志文件中，每行一条记录： + +```json +{ + "time": "2025-04-07 10:00:00", + "operation": "upload", + "src": "/local/path/to/file.txt", + "dest": "cos://my-bucket-1250000000/data/file.txt", + "reason": "AccessDenied (Code: AccessDenied)", + "request_id": "NjBhMzYyMDdfOTBmYTUwNjRfMjI4NV8xNjA=" +} +``` + +**字段说明：** + +| 字段 | 说明 | +|------|------| +| `time` | 失败发生的时间 | +| `operation` | 操作类型（upload/download/copy/move/delete） | +| `src` | 源路径（本地路径或 `cos://` 路径） | +| `dest` | 目标路径（本地路径或 `cos://` 路径） | +| `reason` | 失败原因（包含错误信息和错误码） | +| `request_id` | COS 请求 ID，用于排查问题 | + +**使用示例：** +```bash +# 上传并记录失败日志 +tccli cos upload --bucket my-bucket-1250000000 --local_path /data --cos_key data/ \ + --recursive true --log_file /tmp/upload_fail.log + +# 查看失败日志 +cat /tmp/upload_fail.log +``` diff --git a/tccli/plugins/cos/download_object.py b/tccli/plugins/cos/download_object.py new file mode 100644 index 0000000000..487f077dc6 --- /dev/null +++ b/tccli/plugins/cos/download_object.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +""" +download 操作：从 COS 下载文件到本地 +对齐 coscli cp (COS->本地) 命令 +- thread_num: 单文件分块下载并发线程数（传给 SDK 的 MAXThread） +- routines: 文件间并发数（同时下载的文件数） +""" +import os +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed +from qcloud_cos import CosServiceError +from .utils import init_cos_client, match_filters, TransferProgressMonitor + + +def download_object(args, parsed_globals): + """从 COS 下载文件到本地""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + local_path = args["local_path"] + recursive = args.get("recursive", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + thread_num = args.get("thread_num", 5) or 5 + routines = args.get("routines", 3) or 3 + part_size = args.get("part_size", 20) or 20 + rate_limiting = args.get("rate_limiting", 0) or 0 + version_id = args.get("version_id", "") or "" + log_file = args.get("log_file", "") or "" + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + + try: + if recursive: + _download_directory(client, bucket, cos_key, local_path, include, exclude, + thread_num, routines, part_size, rate_limiting, version_id, log_file, retry) + else: + # 确保本地目录存在 + local_dir = os.path.dirname(local_path) + if local_dir and not os.path.exists(local_dir): + os.makedirs(local_dir) + + _download_single(client, bucket, cos_key, local_path, + thread_num, part_size, rate_limiting, version_id, log_file, retry) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + except Exception as e: + print("Error: %s" % str(e)) + + +def _build_download_kwargs(bucket, cos_key, local_path, thread_num, part_size, rate_limiting, version_id): + """构造 download_file 的参数""" + kwargs = { + "Bucket": bucket, + "Key": cos_key, + "DestFilePath": local_path, + "PartSize": part_size, + "MAXThread": thread_num, + } + if rate_limiting: + kwargs["TrafficLimit"] = str(int(rate_limiting) * 1024 * 1024 * 8) + if version_id: + kwargs["VersionId"] = version_id + return kwargs + + +def _download_single(client, bucket, cos_key, local_path, + thread_num, part_size, rate_limiting, version_id, log_file="", retry=3): + """下载单个文件（带进度监控）""" + monitor = TransferProgressMonitor("download") + + # 先获取文件大小 + try: + head_resp = client.head_object(Bucket=bucket, Key=cos_key) + file_size = int(head_resp.get("Content-Length", 0)) + except Exception: + file_size = 0 + + monitor.set_scan_info(1, file_size) + monitor.start() + + progress_cb, file_id = monitor.create_progress_callback(file_size) + last_err = None + for attempt in range(max(1, retry + 1)): + try: + kwargs = _build_download_kwargs(bucket, cos_key, local_path, + thread_num, part_size, rate_limiting, version_id) + kwargs["progress_callback"] = progress_cb + client.download_file(**kwargs) + monitor.update_ok(file_size, file_id) + last_err = None + break + except CosServiceError as e: + last_err = e + if attempt < retry: + progress_cb, file_id = monitor.create_progress_callback(file_size) + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(file_id, + src_path="cos://%s/%s" % (bucket, cos_key), + dest_path=local_path, + reason=err_reason, + request_id=last_err.get_request_id()) + monitor.stop(log_file=log_file) + if last_err is not None: + raise last_err + + +def _download_directory(client, bucket, prefix, local_dir, include, exclude, + thread_num, routines, part_size, rate_limiting, version_id, log_file="", retry=3): + """递归下载 COS 前缀下的所有对象 + - thread_num: 单文件分块并发（传给 SDK MAXThread） + - routines: 文件间并发（同时下载的文件数） + """ + monitor = TransferProgressMonitor("download") + monitor.start() + + # 先收集所有待下载的文件任务 + tasks = [] + empty_local_dirs = [] # COS 上 / 结尾的空目录对象，需在本地创建对应目录 + total_size = 0 + skip_count = 0 + marker = "" + + while True: + response = client.list_objects( + Bucket=bucket, + Prefix=prefix, + Marker=marker, + MaxKeys=1000, + ) + + if "Contents" in response: + for content in response["Contents"]: + key = content["Key"] + rel_key = key[len(prefix):].lstrip("/") if prefix else key + + # 处理 COS 上的空目录对象（以 / 结尾，Size=0） + if key.endswith("/") and int(content.get("Size", 0)) == 0: + if rel_key: + # include/exclude 过滤目录 + dir_rel = rel_key.rstrip("/") + if not match_filters(dir_rel, include, exclude): + skip_count += 1 + continue + local_subdir = os.path.join(local_dir, dir_rel.replace("/", os.sep)) + empty_local_dirs.append(local_subdir) + continue + + # include/exclude 过滤 + if not match_filters(rel_key, include, exclude): + skip_count += 1 + continue + + file_size = int(content.get("Size", 0)) + local_file = os.path.join(local_dir, rel_key.replace("/", os.sep)) + total_size += file_size + tasks.append((key, local_file, file_size)) + + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + else: + break + + # 设置扫描结果（文件数 + 空目录数 + 跳过数） + monitor.set_scan_info(len(tasks) + len(empty_local_dirs) + skip_count, total_size) + for _ in range(skip_count): + monitor.update_skip(0) + + # 预先创建所有需要的本地目录（避免并发时目录创建冲突） + dir_lock = threading.Lock() + created_dirs = set() + + def _ensure_dir(file_path): + """线程安全地创建目录""" + file_dir = os.path.dirname(file_path) + if file_dir and file_dir not in created_dirs: + with dir_lock: + if file_dir not in created_dirs: + if not os.path.exists(file_dir): + os.makedirs(file_dir) + created_dirs.add(file_dir) + + def _do_download(key, local_file, file_size): + """单个文件下载任务（含重试）""" + last_err = None + progress_cb, file_id = monitor.create_progress_callback(file_size) + for attempt in range(max(1, retry + 1)): + try: + _ensure_dir(local_file) + kwargs = _build_download_kwargs(bucket, key, local_file, + thread_num, part_size, rate_limiting, version_id) + kwargs["progress_callback"] = progress_cb + client.download_file(**kwargs) + monitor.update_ok(file_size, file_id) + last_err = None + break + except CosServiceError as e: + last_err = e + if attempt < retry: + progress_cb, file_id = monitor.create_progress_callback(file_size) + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(file_id, + src_path="cos://%s/%s" % (bucket, key), + dest_path=local_file, + reason=err_reason, + request_id=last_err.get_request_id()) + + # 使用线程池并发下载多个文件，routines 控制文件间并发 + if tasks: + max_workers = min(routines, len(tasks)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for key, local_file, file_size in tasks: + futures.append(executor.submit(_do_download, key, local_file, file_size)) + for future in as_completed(futures): + future.result() + + # 在本地创建 COS 上的空目录 + for local_subdir in empty_local_dirs: + if local_subdir and not os.path.exists(local_subdir): + os.makedirs(local_subdir, exist_ok=True) + monitor.update_ok(0) + + monitor.stop(log_file=log_file) \ No newline at end of file diff --git a/tccli/plugins/cos/du_object.py b/tccli/plugins/cos/du_object.py new file mode 100644 index 0000000000..6b153c8084 --- /dev/null +++ b/tccli/plugins/cos/du_object.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" +du 操作：统计存储桶或目录的大小 +对齐 coscli du 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client, format_size + + +def du_object(args, parsed_globals): + """统计存储桶或指定前缀下的对象总大小和数量""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + prefix = args.get("prefix", "") or "" + + try: + total_size = 0 + total_count = 0 + total_dir_count = 0 + storage_stats = {} + marker = "" + + while True: + response = client.list_objects( + Bucket=bucket, + Prefix=prefix, + Marker=marker, + MaxKeys=1000, + ) + + if "Contents" in response: + for content in response["Contents"]: + key = content["Key"] + # 目录标记单独统计 + if key.endswith("/"): + total_dir_count += 1 + continue + size = int(content.get("Size", 0)) + storage_class = content.get("StorageClass", "STANDARD") + + total_size += size + total_count += 1 + + if storage_class not in storage_stats: + storage_stats[storage_class] = {"count": 0, "size": 0} + storage_stats[storage_class]["count"] += 1 + storage_stats[storage_class]["size"] += size + + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + else: + break + + # 输出统计结果 + target = "cos://%s/%s" % (bucket, prefix) if prefix else "cos://%s" % bucket + print("统计: %s" % target) + print("-" * 60) + print("总文件数: %d" % total_count) + print("总文件夹数: %d" % total_dir_count) + print("总大小: %s (%d 字节)" % (format_size(total_size), total_size)) + + if storage_stats: + print("\n按存储类型统计:") + for sc, stats in sorted(storage_stats.items()): + print(" %-20s %d 个对象, %s" % (sc, stats["count"], format_size(stats["size"]))) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) \ No newline at end of file diff --git a/tccli/plugins/cos/hash_object.py b/tccli/plugins/cos/hash_object.py new file mode 100644 index 0000000000..f8e9489969 --- /dev/null +++ b/tccli/plugins/cos/hash_object.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +hash 操作：计算本地文件或 COS 对象的哈希值 +对齐 coscli hash 命令 +""" +import os +import hashlib +from qcloud_cos import CosServiceError +from .utils import init_cos_client, calculate_local_crc64 + + +def _calculate_local_hash(file_path, hash_type="md5"): + """计算本地文件的哈希值""" + if hash_type == "crc64": + result = calculate_local_crc64(file_path) + if result is None: + return "Error: 计算 CRC64 需要安装 crcmod 库: pip install crcmod" + return result + + if hash_type == "md5": + h = hashlib.md5() + elif hash_type == "sha1": + h = hashlib.sha1() + elif hash_type == "sha256": + h = hashlib.sha256() + else: + return "Error: 不支持的哈希类型: %s (可选: md5, sha1, sha256, crc64)" % hash_type + + with open(file_path, "rb") as f: + while True: + data = f.read(8192) + if not data: + break + h.update(data) + return h.hexdigest() + + +def hash_object(args, parsed_globals): + """计算本地文件或 COS 对象的哈希值""" + client, region = init_cos_client(parsed_globals) + + hash_type = (args.get("hash_type", "md5") or "md5").lower() + local_path = args.get("local_path", "") or "" + bucket = args.get("bucket", "") or "" + cos_key = args.get("cos_key", "") or "" + + if not local_path and not (bucket and cos_key): + print("Error: 请指定 --local_path 或 --bucket + --cos_key") + return + + # 计算本地文件哈希 + if local_path: + if not os.path.exists(local_path): + print("Error: 本地文件不存在: %s" % local_path) + return + if not os.path.isfile(local_path): + print("Error: 指定路径不是文件: %s" % local_path) + return + + result = _calculate_local_hash(local_path, hash_type) + print("本地文件: %s" % local_path) + print("%s: %s" % (hash_type.upper(), result)) + + # 获取 COS 对象哈希信息 + if bucket and cos_key: + try: + response = client.head_object( + Bucket=bucket, + Key=cos_key, + ) + etag = response.get("ETag", "").strip('"') + crc64 = response.get("x-cos-hash-crc64ecma", "") + content_length = response.get("Content-Length", "") + + print("COS 对象: cos://%s/%s" % (bucket, cos_key)) + print("大小: %s 字节" % content_length) + print("ETag (MD5): %s" % etag) + if crc64: + print("CRC64: %s" % crc64) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) \ No newline at end of file diff --git a/tccli/plugins/cos/head_object.py b/tccli/plugins/cos/head_object.py new file mode 100644 index 0000000000..4bc0e91541 --- /dev/null +++ b/tccli/plugins/cos/head_object.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" +head 操作：查询 COS 对象的元信息 +对齐 coscli head 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def head_object(args, parsed_globals): + """查询 COS 对象的元信息""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + version_id = args.get("version_id", "") or "" + + try: + kwargs = { + "Bucket": bucket, + "Key": cos_key, + } + if version_id: + kwargs["VersionId"] = version_id + + response = client.head_object(**kwargs) + + print("对象元信息: cos://%s/%s" % (bucket, cos_key)) + print("-" * 50) + print("Content-Length: %s" % response.get("Content-Length", "")) + print("Content-Type: %s" % response.get("Content-Type", "")) + print("ETag: %s" % response.get("ETag", "")) + print("Last-Modified: %s" % response.get("Last-Modified", "")) + print("Storage-Class: %s" % response.get("x-cos-storage-class", "STANDARD")) + + # CRC64 + crc64 = response.get("x-cos-hash-crc64ecma", "") + if crc64: + print("CRC64: %s" % crc64) + + # 版本 ID + vid = response.get("x-cos-version-id", "") + if vid: + print("Version-Id: %s" % vid) + + # 恢复状态 + restore = response.get("x-cos-restore", "") + if restore: + print("Restore: %s" % restore) + + # 输出自定义元数据 + for key, value in response.items(): + if key.startswith("x-cos-meta-"): + print("%-16s %s" % (key + ":", value)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) diff --git a/tccli/plugins/cos/list_buckets.py b/tccli/plugins/cos/list_buckets.py new file mode 100644 index 0000000000..cf73d09999 --- /dev/null +++ b/tccli/plugins/cos/list_buckets.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +list_buckets 操作：列出所有存储桶 +对齐 coscli ls（无参数）命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def list_buckets(args, parsed_globals): + """列出所有存储桶""" + client, region = init_cos_client(parsed_globals) + + filter_region = args.get("filter_region", "") or "" + + try: + response = client.list_buckets() + + buckets = response.get("Buckets", {}).get("Bucket", []) + if not isinstance(buckets, list): + buckets = [buckets] + + if not buckets: + print("当前账号下没有存储桶") + return + + # 按 region 过滤 + if filter_region: + buckets = [b for b in buckets if b.get("Location", "") == filter_region] + if not buckets: + print("在 %s 地域下没有存储桶" % filter_region) + return + + # 表头 + print("%-50s %-15s %-25s" % ("BucketName", "Region", "CreationDate")) + print("-" * 95) + + for bucket in buckets: + name = bucket.get("Name", "") + location = bucket.get("Location", "") + creation_date = bucket.get("CreationDate", "") + print("%-50s %-15s %-25s" % (name, location, creation_date)) + + print("\n共 %d 个存储桶" % len(buckets)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) diff --git a/tccli/plugins/cos/list_object.py b/tccli/plugins/cos/list_object.py new file mode 100644 index 0000000000..c9ece0dbdf --- /dev/null +++ b/tccli/plugins/cos/list_object.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +""" +list 操作：列出 COS 存储桶中的文件 +对齐 coscli ls 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client, format_size, match_filters + + +def list_object(args, parsed_globals): + """列出 COS 存储桶中的文件""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + prefix = args.get("prefix", "") or "" + marker = args.get("marker", "") or "" + max_keys = args.get("max_keys", 1000) or 1000 + delimiter = args.get("delimiter", "") or "" + recursive = args.get("recursive", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + + # 递归模式下不使用 delimiter，以列出所有层级的对象 + # 非递归模式下若未指定 delimiter，默认使用 / 只列出当前层级 + if recursive: + delimiter = "" + elif not delimiter: + delimiter = "/" + + try: + total_count = 0 + total_size = 0 + + while True: + response = client.list_objects( + Bucket=bucket, + Prefix=prefix, + Marker=marker, + MaxKeys=max_keys, + Delimiter=delimiter, + ) + + # 输出公共前缀（目录） + if "CommonPrefixes" in response: + for common_prefix in response["CommonPrefixes"]: + print("DIR %s" % common_prefix["Prefix"]) + + # 输出文件列表 + if "Contents" in response: + for content in response["Contents"]: + key = content["Key"] + + # include/exclude 过滤 + if not match_filters(key, include, exclude): + continue + + size = int(content.get("Size", 0)) + last_modified = content.get("LastModified", "") + storage_class = content.get("StorageClass", "STANDARD") + total_count += 1 + total_size += size + print("%-12s %-20s %-25s %s" % ( + format_size(size), storage_class, last_modified, key)) + + # 分页处理 + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + if not recursive: + print("\n结果已截断，下一页 Marker: %s" % marker) + break + else: + break + + # 输出统计信息 + if recursive or total_count > 0: + print("\n共 %d 个对象, 总大小: %s" % (total_count, format_size(total_size))) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) \ No newline at end of file diff --git a/tccli/plugins/cos/lsparts_object.py b/tccli/plugins/cos/lsparts_object.py new file mode 100644 index 0000000000..cebeafcc12 --- /dev/null +++ b/tccli/plugins/cos/lsparts_object.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +""" +lsparts 操作：列出未完成的分片上传 +对齐 coscli lsparts 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def lsparts_object(args, parsed_globals): + """列出存储桶中未完成的分片上传""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + prefix = args.get("prefix", "") or "" + + try: + key_marker = "" + upload_id_marker = "" + total = 0 + + print("%-60s %-40s %-25s" % ("Key", "UploadId", "Initiated")) + print("-" * 130) + + while True: + response = client.list_multipart_uploads( + Bucket=bucket, + Prefix=prefix, + KeyMarker=key_marker, + UploadIdMarker=upload_id_marker, + MaxUploads=1000, + ) + + uploads = response.get("Upload", []) + if not isinstance(uploads, list): + uploads = [uploads] + + for upload in uploads: + if not upload: + continue + key = upload.get("Key", "") + upload_id = upload.get("UploadId", "") + initiated = upload.get("Initiated", "") + total += 1 + print("%-60s %-40s %-25s" % (key, upload_id, initiated)) + + if response.get("IsTruncated") == "true": + key_marker = response.get("NextKeyMarker", "") + upload_id_marker = response.get("NextUploadIdMarker", "") + else: + break + + if total == 0: + print("没有未完成的分片上传") + else: + print("\n共 %d 个未完成的分片上传" % total) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) \ No newline at end of file diff --git a/tccli/plugins/cos/move_object.py b/tccli/plugins/cos/move_object.py new file mode 100644 index 0000000000..e08351fff8 --- /dev/null +++ b/tccli/plugins/cos/move_object.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +""" +move 操作：移动/重命名 COS 上的文件 +对齐 coscli cp --move (COS->COS) 命令 +- routines: 文件间并发数（同时移动的文件数） +""" +from concurrent.futures import ThreadPoolExecutor, as_completed +from qcloud_cos import CosServiceError +from .utils import init_cos_client, match_filters, build_cos_key, list_all_objects_with_dirs, TransferProgressMonitor + + +def move_object(args, parsed_globals): + """移动/重命名 COS 上的文件（先复制后删除）""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + dest_bucket = args.get("dest_bucket", bucket) or bucket + dest_key = args["dest_key"] + dest_region = args.get("dest_region", region) or region + storage_class = args.get("storage_class", "") or "" + recursive = args.get("recursive", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + routines = args.get("routines", 3) or 3 + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + log_file = args.get("log_file", "") or "" + + try: + if recursive: + _move_by_prefix(client, bucket, cos_key, dest_bucket, dest_key, + region, dest_region, storage_class, include, exclude, routines, log_file, retry) + else: + _move_single(client, bucket, cos_key, dest_bucket, dest_key, + region, storage_class, log_file, retry) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def _move_single(client, bucket, cos_key, dest_bucket, dest_key, + region, storage_class, log_file="", retry=3): + """移动单个文件（带进度监控，含重试）""" + monitor = TransferProgressMonitor("move") + + # 获取源文件大小 + try: + head_resp = client.head_object(Bucket=bucket, Key=cos_key) + file_size = int(head_resp.get("Content-Length", 0)) + except Exception: + file_size = 0 + + monitor.set_scan_info(1, file_size) + monitor.start() + + last_err = None + for attempt in range(max(1, retry + 1)): + try: + source = { + "Bucket": bucket, + "Key": cos_key, + "Region": region, + } + kwargs = { + "Bucket": dest_bucket, + "Key": dest_key, + "CopySource": source, + } + if storage_class: + kwargs["StorageClass"] = storage_class + + # 步骤1: 复制文件到目标位置 + client.copy(**kwargs) + # 步骤2: 删除源文件 + client.delete_object(Bucket=bucket, Key=cos_key) + monitor.update_ok(file_size) + last_err = None + break + except CosServiceError as e: + last_err = e + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, cos_key), + dest_path="cos://%s/%s" % (dest_bucket, dest_key), + reason=err_reason, + request_id=last_err.get_request_id()) + monitor.stop(log_file=log_file) + if last_err is not None: + raise last_err + + +def _move_by_prefix(client, bucket, prefix, dest_bucket, dest_prefix, + src_region, dest_region, storage_class, include, exclude, routines, log_file="", retry=3): + """递归移动指定前缀下的所有对象 + - routines: 文件间并发（同时移动的文件数） + """ + monitor = TransferProgressMonitor("move") + monitor.start() + + # 先收集所有待移动的文件任务（含 / 结尾的空目录对象） + all_objects = list_all_objects_with_dirs(client, bucket, prefix) + tasks = [] + empty_dir_keys = [] # COS 上 / 结尾的空目录对象，需在目标 COS 上同步创建后删除源 + total_size = 0 + skip_count = 0 + + for src_key, obj_info in all_objects.items(): + rel_key = src_key[len(prefix):].lstrip("/") if prefix else src_key + + # 处理 / 结尾的空目录对象 + if obj_info.get("IsDir"): + if rel_key: + dir_rel = rel_key.rstrip("/") + if not match_filters(dir_rel, include, exclude): + skip_count += 1 + continue + d_key = build_cos_key(dest_prefix, rel_key) + if not d_key.endswith("/"): + d_key += "/" + empty_dir_keys.append((src_key, d_key)) + continue + + # include/exclude 过滤 + if not match_filters(rel_key, include, exclude): + skip_count += 1 + continue + + file_size = obj_info.get("Size", 0) + d_key = build_cos_key(dest_prefix, rel_key) + total_size += file_size + tasks.append((src_key, d_key, file_size)) + + # 设置扫描结果（文件数 + 空目录数 + 跳过数） + monitor.set_scan_info(len(tasks) + len(empty_dir_keys) + skip_count, total_size) + for _ in range(skip_count): + monitor.update_skip(0) + + def _do_move(src_key, d_key, file_size): + """单个文件移动任务（先复制后删除，含重试）""" + last_err = None + for attempt in range(max(1, retry + 1)): + try: + source = { + "Bucket": bucket, + "Key": src_key, + "Region": src_region, + } + kwargs = { + "Bucket": dest_bucket, + "Key": d_key, + "CopySource": source, + } + if storage_class: + kwargs["StorageClass"] = storage_class + + client.copy(**kwargs) + client.delete_object(Bucket=bucket, Key=src_key) + monitor.update_ok(file_size) + last_err = None + break + except CosServiceError as e: + last_err = e + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, src_key), + dest_path="cos://%s/%s" % (dest_bucket, d_key), + reason=err_reason, + request_id=last_err.get_request_id()) + + # 使用线程池并发移动多个文件，routines 控制文件间并发 + if tasks: + max_workers = min(routines, len(tasks)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for src_key, d_key, file_size in tasks: + futures.append(executor.submit(_do_move, src_key, d_key, file_size)) + for future in as_completed(futures): + future.result() + + # 移动空目录对象：在目标 COS 上创建，然后删除源 + for src_key, d_key in empty_dir_keys: + try: + client.put_object(Bucket=dest_bucket, Key=d_key, Body=b"") + except CosServiceError as e: + err_reason = "创建目标文件夹失败: %s (Code: %s)" % (e.get_error_msg(), e.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, src_key), + dest_path="cos://%s/%s" % (dest_bucket, d_key), + reason=err_reason, + request_id=e.get_request_id()) + continue + try: + client.delete_object(Bucket=bucket, Key=src_key) + monitor.update_ok(0) + except CosServiceError as e: + err_reason = "删除源文件夹失败: %s (Code: %s)" % (e.get_error_msg(), e.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, src_key), + reason=err_reason, + request_id=e.get_request_id()) + + monitor.stop(log_file=log_file) \ No newline at end of file diff --git a/tccli/plugins/cos/restore_object.py b/tccli/plugins/cos/restore_object.py new file mode 100644 index 0000000000..8a00632e6d --- /dev/null +++ b/tccli/plugins/cos/restore_object.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +""" +restore 操作：恢复归档存储类型的 COS 对象 +对齐 coscli restore 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client, match_filters + + +def restore_object(args, parsed_globals): + """恢复归档存储类型的 COS 对象""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + days = args.get("days", 7) or 7 + tier = args.get("tier", "Standard") or "Standard" + recursive = args.get("recursive", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + + try: + if recursive: + _restore_by_prefix(client, bucket, cos_key, days, tier, include, exclude) + else: + client.restore_object( + Bucket=bucket, + Key=cos_key, + RestoreRequest={ + "Days": days, + "CASJobParameters": { + "Tier": tier, + }, + }, + ) + print("恢复请求已提交: cos://%s/%s" % (bucket, cos_key)) + print("恢复天数: %d, 恢复模式: %s" % (days, tier)) + + except CosServiceError as e: + if e.get_error_code() == "RestoreAlreadyInProgress": + print("恢复进行中: cos://%s/%s" % (bucket, cos_key)) + else: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def _restore_by_prefix(client, bucket, prefix, days, tier, include, exclude): + """递归恢复指定前缀下的所有归档对象""" + restored = 0 + skipped = 0 + failed = 0 + marker = "" + + while True: + response = client.list_objects( + Bucket=bucket, + Prefix=prefix, + Marker=marker, + MaxKeys=1000, + ) + + if "Contents" in response: + for content in response["Contents"]: + key = content["Key"] + if key.endswith("/"): + continue + + storage_class = content.get("StorageClass", "STANDARD") + # 只恢复归档类型的对象 + if storage_class not in ("ARCHIVE", "DEEP_ARCHIVE"): + skipped += 1 + continue + + rel_key = key[len(prefix):].lstrip("/") if prefix else key + + # include/exclude 过滤 + if not match_filters(rel_key, include, exclude): + skipped += 1 + continue + + try: + client.restore_object( + Bucket=bucket, + Key=key, + RestoreRequest={ + "Days": days, + "CASJobParameters": { + "Tier": tier, + }, + }, + ) + restored += 1 + print("已提交恢复: cos://%s/%s" % (bucket, key)) + except CosServiceError as e: + if e.get_error_code() == "RestoreAlreadyInProgress": + print("恢复进行中: cos://%s/%s" % (bucket, key)) + skipped += 1 + else: + failed += 1 + print("恢复失败: %s (%s)" % (key, e.get_error_msg())) + + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + else: + break + + print("\n恢复完成: 提交 %d, 跳过 %d, 失败 %d" % (restored, skipped, failed)) \ No newline at end of file diff --git a/tccli/plugins/cos/signurl_object.py b/tccli/plugins/cos/signurl_object.py new file mode 100644 index 0000000000..7972e39602 --- /dev/null +++ b/tccli/plugins/cos/signurl_object.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +signurl 操作：生成预签名 URL +对齐 coscli signurl 命令 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def signurl_object(args, parsed_globals): + """生成 COS 对象的预签名 URL""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + expired = args.get("expired", 3600) or 3600 + method = (args.get("method", "GET") or "GET").upper() + + try: + if method == "GET": + url = client.get_presigned_download_url( + Bucket=bucket, + Key=cos_key, + Expired=expired, + ) + elif method == "PUT": + url = client.get_presigned_url( + Method="PUT", + Bucket=bucket, + Key=cos_key, + Expired=expired, + ) + else: + url = client.get_presigned_url( + Method=method, + Bucket=bucket, + Key=cos_key, + Expired=expired, + ) + + print("预签名 URL (有效期 %d 秒):" % expired) + print(url) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + except Exception as e: + print("Error: %s" % str(e)) diff --git a/tccli/plugins/cos/sync_copy_object.py b/tccli/plugins/cos/sync_copy_object.py new file mode 100644 index 0000000000..bb8b3e3f29 --- /dev/null +++ b/tccli/plugins/cos/sync_copy_object.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +""" +sync_copy 操作：COS -> COS 同步复制 +对齐 coscli sync (COS->COS) 命令 +- routines: 文件间并发数（同时复制的文件数） +""" +from concurrent.futures import ThreadPoolExecutor, as_completed +from qcloud_cos import CosServiceError +from .utils import (init_cos_client, match_filters, build_cos_key, parse_meta, + list_all_objects, list_all_objects_with_dirs, TransferProgressMonitor, + should_skip_sync_copy) + + +def sync_copy_object(args, parsed_globals): + """同步复制：COS -> COS""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_prefix = args.get("cos_key", "") or "" + dest_bucket = args.get("dest_bucket", bucket) or bucket + dest_prefix = args.get("dest_key", "") or "" + dest_region = args.get("dest_region", region) or region + recursive = args.get("recursive", False) + delete = args.get("delete", False) + ignore_existing = args.get("ignore_existing", False) + update = args.get("update", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + storage_class = args.get("storage_class", "") or "" + meta = args.get("meta", "") or "" + routines = args.get("routines", 3) or 3 + log_file = args.get("log_file", "") or "" + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + + # 解析自定义元数据 + metadata = parse_meta(meta) + + try: + src_objects = list_all_objects_with_dirs(client, bucket, cos_prefix) + dest_objects = list_all_objects(client, dest_bucket, dest_prefix) + + monitor = TransferProgressMonitor("copy") + monitor.start() + + # 收集待复制的文件任务 + tasks = [] + empty_dir_keys = [] # COS 上 / 结尾的空目录对象，需在目标 COS 上同步创建 + total_size = 0 + skip_count = 0 + skip_size = 0 + for src_key, obj_info in src_objects.items(): + rel_key = src_key[len(cos_prefix):].lstrip("/") if cos_prefix else src_key + + # 处理 / 结尾的空目录对象，在目标 COS 上同步创建 + if obj_info.get("IsDir"): + if rel_key: + # include/exclude 过滤目录 + dir_rel = rel_key.rstrip("/") + if not match_filters(dir_rel, include, exclude): + skip_count += 1 + continue + d_key = build_cos_key(dest_prefix, rel_key) + if not d_key.endswith("/"): + d_key += "/" + if d_key not in dest_objects: + empty_dir_keys.append(d_key) + continue + + # include/exclude 过滤 + if not match_filters(rel_key, include, exclude): + skip_count += 1 + continue + + dest_key = build_cos_key(dest_prefix, rel_key) + + # 增量同步：对齐 coscli sync 跳过逻辑 + # - 默认：对比源 CRC64 与目标 CRC64（x-cos-hash-crc64ecma） + # - --ignore-existing：目标存在即跳过 + # - --update：按 Last-Modified 时间比较 + if dest_key in dest_objects and should_skip_sync_copy( + client, bucket, src_key, dest_bucket, dest_key, + ignore_existing=ignore_existing, update=update): + skip_count += 1 + skip_size += obj_info["Size"] + continue + + total_size += obj_info["Size"] + tasks.append((src_key, dest_key, obj_info["Size"])) + + # 设置扫描结果（文件数 + 空目录数 + 跳过数） + monitor.set_scan_info(len(tasks) + len(empty_dir_keys) + skip_count, total_size + skip_size) + for i in range(skip_count): + monitor.update_skip(skip_size // skip_count if skip_count > 0 else 0) + + def _do_copy(src_key, dest_key, file_size): + """单个文件复制任务（含重试）""" + last_err = None + for attempt in range(max(1, retry + 1)): + try: + source = { + "Bucket": bucket, + "Key": src_key, + "Region": region, + } + kwargs = { + "Bucket": dest_bucket, + "Key": dest_key, + "CopySource": source, + } + if storage_class: + kwargs["StorageClass"] = storage_class + if metadata: + kwargs["Metadata"] = metadata + kwargs["CopyStatus"] = "Replaced" + + client.copy(**kwargs) + monitor.update_ok(file_size) + last_err = None + break + except CosServiceError as e: + last_err = e + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(src_path="cos://%s/%s" % (bucket, src_key), + dest_path="cos://%s/%s" % (dest_bucket, dest_key), + reason=err_reason, + request_id=last_err.get_request_id()) + + # 使用线程池并发复制多个文件，routines 控制文件间并发 + if tasks: + max_workers = min(routines, len(tasks)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for src_key, dest_key, file_size in tasks: + futures.append(executor.submit(_do_copy, src_key, dest_key, file_size)) + for future in as_completed(futures): + future.result() + + # 在目标 COS 上创建空目录标记 + for d_key in empty_dir_keys: + try: + client.put_object(Bucket=dest_bucket, Key=d_key, Body=b"") + monitor.update_ok(0) + except CosServiceError as e: + monitor.update_err(src_path="cos://%s/%s" % (dest_bucket, d_key), + reason="创建空目录失败: %s (Code: %s)" % (e.get_error_msg(), e.get_error_code()), + request_id=e.get_request_id()) + + # 删除目标多余的文件和文件夹 + deleted = 0 + if delete: + # 重新获取目标端包含目录对象的完整列表 + dest_all_objects = list_all_objects_with_dirs(client, dest_bucket, dest_prefix) + for dest_key, obj_info in dest_all_objects.items(): + rel_key = dest_key[len(dest_prefix):].lstrip("/") if dest_prefix else dest_key + src_key = build_cos_key(cos_prefix, rel_key) + if obj_info.get("IsDir"): + # 目录对象：检查源端是否存在对应目录对象 + src_dir_key = src_key if src_key.endswith("/") else src_key + "/" + if src_dir_key not in src_objects: + client.delete_object(Bucket=dest_bucket, Key=dest_key) + deleted += 1 + else: + if src_key not in src_objects: + client.delete_object(Bucket=dest_bucket, Key=dest_key) + deleted += 1 + + monitor.stop(log_file=log_file) + + if deleted > 0: + print("已删除目标端多余文件: %d" % deleted) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) diff --git a/tccli/plugins/cos/sync_download_object.py b/tccli/plugins/cos/sync_download_object.py new file mode 100644 index 0000000000..4a6615f4c2 --- /dev/null +++ b/tccli/plugins/cos/sync_download_object.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +""" +sync_download 操作：COS -> 本地同步下载 +对齐 coscli sync (COS->本地) 命令 +- thread_num: 单文件分块下载并发线程数（传给 SDK 的 MAXThread） +- routines: 文件间并发数（同时下载的文件数） +""" +import os +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed +from qcloud_cos import CosServiceError +from .utils import (init_cos_client, match_filters, build_cos_key, + list_all_objects, list_all_objects_with_dirs, list_local_files, TransferProgressMonitor, + should_skip_sync_download) + + +def sync_download_object(args, parsed_globals): + """同步下载：COS -> 本地目录""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + local_path = args["local_path"] + cos_prefix = args.get("cos_key", "") or "" + recursive = args.get("recursive", False) + delete = args.get("delete", False) + ignore_existing = args.get("ignore_existing", False) + update = args.get("update", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + thread_num = args.get("thread_num", 5) or 5 + routines = args.get("routines", 3) or 3 + part_size = args.get("part_size", 20) or 20 + rate_limiting = args.get("rate_limiting", 0) or 0 + log_file = args.get("log_file", "") or "" + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + + if not os.path.exists(local_path): + os.makedirs(local_path) + + try: + cos_objects = list_all_objects_with_dirs(client, bucket, cos_prefix) + local_files = list_local_files(local_path) + + monitor = TransferProgressMonitor("download") + monitor.start() + + # 收集待下载的文件任务，同时识别 COS 上的空目录对象 + tasks = [] + empty_dirs = [] # COS 上以 / 结尾的空目录对象，需在本地创建对应目录 + total_size = 0 + skip_count = 0 + skip_size = 0 + for cos_key, obj_info in cos_objects.items(): + rel_key = cos_key[len(cos_prefix):].lstrip("/") if cos_prefix else cos_key + + # 处理 COS 上的空目录对象（以 / 结尾） + if obj_info.get("IsDir"): + if rel_key: + # include/exclude 过滤目录 + dir_rel = rel_key.rstrip("/") + if not match_filters(dir_rel, include, exclude): + skip_count += 1 + continue + local_dir = os.path.join(local_path, dir_rel.replace("/", os.sep)) + if local_dir and not os.path.exists(local_dir): + empty_dirs.append(local_dir) + continue + + # include/exclude 过滤 + if not match_filters(rel_key, include, exclude): + skip_count += 1 + continue + + local_file = os.path.join(local_path, rel_key.replace("/", os.sep)) + + # 增量同步：对齐 coscli sync 跳过逻辑 + # - 默认：对比本地 CRC64 与 COS CRC64（x-cos-hash-crc64ecma） + # - --ignore-existing：本地存在即跳过 + # - --update：按 Last-Modified 时间比较 + if rel_key in local_files and should_skip_sync_download( + client, bucket, cos_key, obj_info, local_file, + ignore_existing=ignore_existing, update=update): + skip_count += 1 + skip_size += obj_info["Size"] + continue + + total_size += obj_info["Size"] + tasks.append((cos_key, local_file, obj_info["Size"])) + + # 设置扫描结果（文件数 + 空目录数 + 跳过数） + monitor.set_scan_info(len(tasks) + len(empty_dirs) + skip_count, total_size + skip_size) + for i in range(skip_count): + monitor.update_skip(skip_size // skip_count if skip_count > 0 else 0) + + # 预先创建所有需要的本地目录（避免并发时目录创建冲突） + dir_lock = threading.Lock() + created_dirs = set() + + def _ensure_dir(file_path): + """线程安全地创建目录""" + file_dir = os.path.dirname(file_path) + if file_dir and file_dir not in created_dirs: + with dir_lock: + if file_dir not in created_dirs: + if not os.path.exists(file_dir): + os.makedirs(file_dir) + created_dirs.add(file_dir) + + def _do_download(cos_key, local_file, file_size): + """单个文件下载任务（含重试）""" + last_err = None + progress_cb, file_id = monitor.create_progress_callback(file_size) + for attempt in range(max(1, retry + 1)): + try: + _ensure_dir(local_file) + kwargs = { + "Bucket": bucket, + "Key": cos_key, + "DestFilePath": local_file, + "PartSize": part_size, + "MAXThread": thread_num, + "progress_callback": progress_cb, + } + if rate_limiting: + kwargs["TrafficLimit"] = str(int(rate_limiting) * 1024 * 1024 * 8) + + client.download_file(**kwargs) + monitor.update_ok(file_size, file_id) + last_err = None + break + except CosServiceError as e: + last_err = e + if attempt < retry: + progress_cb, file_id = monitor.create_progress_callback(file_size) + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(file_id, + src_path="cos://%s/%s" % (bucket, cos_key), + dest_path=local_file, + reason=err_reason, + request_id=last_err.get_request_id()) + + # 使用线程池并发下载多个文件，routines 控制文件间并发 + if tasks: + max_workers = min(routines, len(tasks)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for cos_key, local_file, file_size in tasks: + futures.append(executor.submit(_do_download, cos_key, local_file, file_size)) + for future in as_completed(futures): + future.result() + + # 在本地创建 COS 上的空目录 + for local_dir in empty_dirs: + os.makedirs(local_dir, exist_ok=True) + monitor.update_ok(0) + + # 删除本地多余的文件和空目录 + deleted = 0 + if delete: + # 第一步：删除多余的文件 + for rel_path, file_info in local_files.items(): + cos_key = build_cos_key(cos_prefix, rel_path) + if cos_key not in cos_objects: + os.remove(file_info["FullPath"]) + deleted += 1 + # 第二步：删除多余的本地目录（从最深层开始，文件删除后目录可能已变空） + for root, dirs, files in os.walk(local_path, topdown=False): + if root == local_path: + continue + rel_dir = os.path.relpath(root, local_path).replace(os.sep, "/") + cos_dir_key = build_cos_key(cos_prefix, rel_dir) + "/" + # 目录在 COS 上不存在（既无目录对象也无该前缀下的文件） + dir_exists_in_cos = (cos_dir_key in cos_objects or + any(k.startswith(cos_dir_key) for k in cos_objects)) + if not dir_exists_in_cos: + # 文件已在第一步删除，此时目录若为空则删除 + if not os.listdir(root): + os.rmdir(root) + deleted += 1 + + monitor.stop(log_file=log_file) + + if deleted > 0: + print("已删除本地多余文件: %d" % deleted) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) diff --git a/tccli/plugins/cos/sync_upload_object.py b/tccli/plugins/cos/sync_upload_object.py new file mode 100644 index 0000000000..d3b2d8df20 --- /dev/null +++ b/tccli/plugins/cos/sync_upload_object.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +""" +sync_upload 操作：本地 -> COS 同步上传 +对齐 coscli sync (本地->COS) 命令 +- thread_num: 单文件分块上传并发线程数（传给 SDK 的 MAXThread） +- routines: 文件间并发数（同时上传的文件数） +""" +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from qcloud_cos import CosServiceError +from .utils import (init_cos_client, match_filters, build_cos_key, parse_meta, + list_all_objects, list_local_files, TransferProgressMonitor, + should_skip_sync_upload) + + +def sync_upload_object(args, parsed_globals): + """同步上传：本地目录 -> COS""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + local_path = args["local_path"] + cos_prefix = args.get("cos_key", "") or "" + recursive = args.get("recursive", False) + delete = args.get("delete", False) + ignore_existing = args.get("ignore_existing", False) + update = args.get("update", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + storage_class = args.get("storage_class", "") or "" + content_type = args.get("content_type", "") or "" + meta = args.get("meta", "") or "" + thread_num = args.get("thread_num", 5) or 5 + routines = args.get("routines", 3) or 3 + part_size = args.get("part_size", 20) or 20 + rate_limiting = args.get("rate_limiting", 0) or 0 + log_file = args.get("log_file", "") or "" + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + + # 解析自定义元数据 + metadata = parse_meta(meta) + + if not os.path.isdir(local_path): + print("Error: 本地路径不是目录: %s" % local_path) + return + + try: + local_files = list_local_files(local_path) + cos_objects = list_all_objects(client, bucket, cos_prefix) + + # 收集本地空目录 + empty_dir_keys = [] + for root, dirs, files in os.walk(local_path): + if not files and not dirs: + rel_dir = os.path.relpath(root, local_path).replace(os.sep, "/") + if rel_dir == ".": + dir_key = cos_prefix.rstrip("/") + "/" if cos_prefix else "" + else: + # include/exclude 过滤目录 + if not match_filters(rel_dir, include, exclude): + continue + dir_key = build_cos_key(cos_prefix, rel_dir) + "/" + if dir_key and dir_key not in cos_objects: + empty_dir_keys.append(dir_key) + + monitor = TransferProgressMonitor("upload") + monitor.start() + + # 收集待上传的文件任务 + tasks = [] + total_size = 0 + skip_count = 0 + skip_size = 0 + for rel_path, file_info in local_files.items(): + # include/exclude 过滤 + if not match_filters(rel_path, include, exclude): + skip_count += 1 + continue + + cos_key = build_cos_key(cos_prefix, rel_path) + + # 增量同步：对齐 coscli sync 跳过逻辑 + # - 默认：对比本地 CRC64 与 COS CRC64（x-cos-hash-crc64ecma） + # - --ignore-existing：目标存在即跳过 + # - --update：按 Last-Modified 时间比较 + if cos_key in cos_objects and should_skip_sync_upload( + client, bucket, cos_key, + file_info["FullPath"], file_info.get("MTime", 0), + ignore_existing=ignore_existing, update=update): + skip_count += 1 + skip_size += file_info["Size"] + continue + + total_size += file_info["Size"] + tasks.append((file_info, cos_key)) + + # 设置扫描结果（文件数 + 空目录数 + 跳过数） + monitor.set_scan_info(len(tasks) + len(empty_dir_keys) + skip_count, total_size + skip_size) + for i in range(skip_count): + monitor.update_skip(skip_size // skip_count if skip_count > 0 else 0) + + def _do_upload(file_info, cos_key): + """单个文件上传任务（含重试）""" + last_err = None + progress_cb, file_id = monitor.create_progress_callback(file_info["Size"]) + for attempt in range(max(1, retry + 1)): + try: + kwargs = { + "Bucket": bucket, + "LocalFilePath": file_info["FullPath"], + "Key": cos_key, + "PartSize": part_size, + "MAXThread": thread_num, + "progress_callback": progress_cb, + } + if storage_class: + kwargs["StorageClass"] = storage_class + if content_type: + kwargs["ContentType"] = content_type + if metadata: + kwargs["Metadata"] = metadata + if rate_limiting: + kwargs["TrafficLimit"] = str(int(rate_limiting) * 1024 * 1024 * 8) + + client.upload_file(**kwargs) + monitor.update_ok(file_info["Size"], file_id) + last_err = None + break + except CosServiceError as e: + last_err = e + if attempt < retry: + progress_cb, file_id = monitor.create_progress_callback(file_info["Size"]) + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(file_id, + src_path=file_info["FullPath"], + dest_path="cos://%s/%s" % (bucket, cos_key), + reason=err_reason, + request_id=last_err.get_request_id()) + + # 使用线程池并发上传多个文件，routines 控制文件间并发 + if tasks: + max_workers = min(routines, len(tasks)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for file_info, cos_key in tasks: + futures.append(executor.submit(_do_upload, file_info, cos_key)) + for future in as_completed(futures): + future.result() + + # 在 COS 上创建空目录标记 + for dir_key in empty_dir_keys: + try: + client.put_object(Bucket=bucket, Key=dir_key, Body=b"") + monitor.update_ok(0) + except CosServiceError as e: + monitor.update_err(src_path=dir_key, + reason="创建空目录失败: %s (Code: %s)" % (e.get_error_msg(), e.get_error_code()), + request_id=e.get_request_id()) + + # 删除 COS 上多余的文件和文件夹 + deleted = 0 + if delete: + # 重新获取包含目录对象的完整列表 + from .utils import list_all_objects_with_dirs + cos_all_objects = list_all_objects_with_dirs(client, bucket, cos_prefix) + for cos_key, obj_info in cos_all_objects.items(): + rel_key = cos_key[len(cos_prefix):].lstrip("/") if cos_prefix else cos_key + if obj_info.get("IsDir"): + # 目录对象：检查本地是否存在对应目录 + dir_rel = rel_key.rstrip("/") + if dir_rel and not os.path.isdir(os.path.join(local_path, dir_rel.replace("/", os.sep))): + client.delete_object(Bucket=bucket, Key=cos_key) + deleted += 1 + else: + if rel_key not in local_files: + client.delete_object(Bucket=bucket, Key=cos_key) + deleted += 1 + + monitor.stop(log_file=log_file) + + if deleted > 0: + print("已删除 COS 上多余文件: %d" % deleted) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) diff --git a/tccli/plugins/cos/tagging_object.py b/tccli/plugins/cos/tagging_object.py new file mode 100644 index 0000000000..432da885e9 --- /dev/null +++ b/tccli/plugins/cos/tagging_object.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +""" +tagging 操作：对象标签管理 +对齐 coscli 的标签管理能力 +""" +from qcloud_cos import CosServiceError +from .utils import init_cos_client + + +def get_object_tagging(args, parsed_globals): + """获取对象的标签""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + + try: + response = client.get_object_tagging( + Bucket=bucket, + Key=cos_key, + ) + + tag_set = (response.get("TagSet") or {}).get("Tag", []) + if not isinstance(tag_set, list): + tag_set = [tag_set] + + print("对象标签: cos://%s/%s" % (bucket, cos_key)) + print("-" * 50) + + if not tag_set or (len(tag_set) == 1 and not tag_set[0]): + print("(无标签)") + else: + for tag in tag_set: + if tag: + print(" %s = %s" % (tag.get("Key", ""), tag.get("Value", ""))) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def put_object_tagging(args, parsed_globals): + """设置对象的标签""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + tags_str = args.get("tags", "") or "" + + # 解析标签字符串，格式: key1=value1,key2=value2 + tag_list = [] + if tags_str: + for pair in tags_str.split(","): + pair = pair.strip() + if "=" in pair: + k, v = pair.split("=", 1) + tag_list.append({"Key": k.strip(), "Value": v.strip()}) + else: + print("Error: 标签格式错误，应为 key=value 格式: %s" % pair) + return + + if not tag_list: + print("Error: 请指定至少一个标签，格式: key1=value1,key2=value2") + return + + try: + tagging = { + "TagSet": { + "Tag": tag_list, + } + } + client.put_object_tagging( + Bucket=bucket, + Key=cos_key, + Tagging=tagging, + ) + print("对象标签设置成功: cos://%s/%s" % (bucket, cos_key)) + for tag in tag_list: + print(" %s = %s" % (tag["Key"], tag["Value"])) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + + +def delete_object_tagging(args, parsed_globals): + """删除对象的所有标签""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + cos_key = args["cos_key"] + + try: + client.delete_object_tagging( + Bucket=bucket, + Key=cos_key, + ) + print("对象标签删除成功: cos://%s/%s" % (bucket, cos_key)) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) \ No newline at end of file diff --git a/tccli/plugins/cos/tests/README.md b/tccli/plugins/cos/tests/README.md new file mode 100644 index 0000000000..c13f47fc28 --- /dev/null +++ b/tccli/plugins/cos/tests/README.md @@ -0,0 +1,62 @@ +# tccli cos 插件单元测试 + +## 运行 + +```bash +# 全量 +pytest tccli/plugins/cos/tests/ -v + +# 单个命令 +pytest tccli/plugins/cos/tests/test_head_object.py -v + +# 关键字筛选 +pytest tccli/plugins/cos/tests/ -v -k "sync_upload and crc64" +``` + +## 打桩模式（对齐 coscli） + +| coscli（Go + gomonkey + goconvey） | tccli cos 插件（Python + pytest + unittest.mock） | +|---|---| +| `setupTestConfig()` 写 fake yaml（含假 secretId/secretKey） | [conftest.py](conftest.py) 的 `MOCK_GLOBALS` 常量 | +| `gomonkey.ApplyMethodFunc(reflect.TypeOf(b), "Put", ...)` 打桩 SDK 方法 | `mock_client.put_object.return_value = ...` / `side_effect = make_cos_error(...)` | +| `gomonkey.ApplyFunc(util.NewClient, ...)` 打桩工厂 | `patch_init_client("", mock_client)` fixture | +| `Convey("xxx success", ...)` 用例命名 | `def test_xxx_success(...):` 用例命名 | +| `So(e, ShouldBeNil/Error)` 断言 | `mock_client.xxx.assert_called_once()` + `capsys` 断言 stdout | +| `fmt.Errorf("test xx error")` 构造 SDK 错误 | `make_cos_error(code, msg, req_id, method, status_code)` | + +## 关键约束 + +1. **只 mock `qcloud_cos.CosS3Client` 方法和 `utils.init_cos_client`**，禁止产生真实 HTTP 请求； +2. 纯逻辑函数（`match_filters` / `build_cos_key` / `parse_meta` / `calculate_local_crc64` / `format_size`）**不 mock**，直接调用真实实现； +3. 因 `tccli.plugins.cos.__init__` 中 `from .head_object import head_object` 会把**模块名覆盖为函数**，故所有测试必须通过 [conftest.py](conftest.py) 的 `cos_module("head_object")` 或 `patch_init_client("head_object", mc)` 来获取真实模块对象再打桩； +4. `CosServiceError` 的**真实签名**是 `__init__(method, message, status_code)`，其中 `message` 必须是 COS 返回的 XML 错误体。**统一通过 [conftest.py](conftest.py) 的 `make_cos_error(...)` 构造**。 + +## 文件组织（按命令拆分，对齐 coscli `cmd/*_test.go`） + +| 测试文件 | 覆盖命令 | 对齐 coscli 的文件 | +|---|---|---| +| [conftest.py](conftest.py) | 公共 fixture（`mock_client`、`patch_init_client`、`MOCK_GLOBALS`、`make_cos_error`、`cos_module`） | `testconfig_test.go` | +| [test_head_object.py](test_head_object.py) | `head_object` | `stat_test.go` | +| [test_list_object.py](test_list_object.py) | `list_object` | `ls_test.go` | +| [test_buckets.py](test_buckets.py) | `list_buckets` / `create_bucket` / `delete_bucket` | `ls_test.go` / `mb_test.go` / `rb_test.go` | +| [test_delete_object.py](test_delete_object.py) | `delete_object`（单文件 + recursive + 确认提示 + 重试） | `rm_test.go` | +| [test_upload_object.py](test_upload_object.py) | `upload_object`（单文件 + recursive + storage_class + meta + 限速 + 重试 + 空目录标记） | `cp_test.go` Upload | +| [test_download_object.py](test_download_object.py) | `download_object`（单文件 + recursive + version_id + 限速 + 重试） | `cp_test.go` Download | +| [test_copy_move_object.py](test_copy_move_object.py) | `copy_object` / `move_object`（单文件 + recursive + meta + 跨域） | `cp_test.go` CosCopy | +| [test_sync_objects.py](test_sync_objects.py) | `sync_upload_object` / `sync_download_object` / `sync_copy_object`（CRC64 / ignore_existing / update / delete） | `sync_test.go` | +| [test_simple_ops.py](test_simple_ops.py) | `signurl` / `acl`（桶+对象）/ `tagging` / `du` / `cat` / `hash` / `lsparts` / `abort` / `restore` | `signurl_test.go` 等 | + +## 覆盖率矩阵（对齐 coscli Convey 分支） + +| 覆盖项 | coscli 对齐 | tccli 覆盖文件 | +|---|---|---| +| 成功路径 | `xxx success` | 全部 | +| SDK 返回 `CosServiceError` | `fmt.Errorf("test ... error")` | 全部 | +| 客户端工厂失败 | `NewClient error` / `CreateClient error` | `test_head_object.py` / `test_upload_object.py` | +| 参数非法 | `invalid arguments` | `test_delete_object.py`（确认 cancel） | +| 本地路径不存在 | `FormatUploadPath error` | `test_upload_object.py` / `test_sync_objects.py` | +| 重试成功 / 耗尽 | `err-retry-num` | `test_upload_object.py` / `test_copy_move_object.py` / `test_delete_object.py` | +| 递归空目标 | `... success (no objects)` | `test_download_object.py` / `test_copy_move_object.py` / `test_sync_objects.py` | +| 递归带文件 | `... single object success` | 同上 | +| `--ignore-existing` / `--update` / `--delete` | 各同步子 Convey | `test_sync_objects.py` | +| 空目录标记（/ 结尾） | coscli 同名分支 | `test_upload_object.py` / `test_download_object.py` / `test_copy_move_object.py` | diff --git a/tccli/plugins/cos/tests/conftest.py b/tccli/plugins/cos/tests/conftest.py new file mode 100644 index 0000000000..f3708bfe33 --- /dev/null +++ b/tccli/plugins/cos/tests/conftest.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +tccli cos 插件单元测试公共 fixture 与工具。 + +打桩风格对齐 coscli（Go + gomonkey）： +- coscli 用 gomonkey.ApplyMethodFunc 打桩 *cos.BucketService/ObjectService 方法； +- tccli 这边用 unittest.mock.MagicMock 对 CosS3Client 的方法做等价打桩； +- coscli 用 ApplyFunc(util.NewClient, ...) 打桩工厂函数； +- tccli 这边用 patch.object(, "init_cos_client") 做等价打桩。 + +关键注意：`tccli.plugins.cos` 的 __init__ 中有 + from .head_object import head_object +这会让 `tccli.plugins.cos.head_object` 被解析为**函数**而不是模块， +所以 @patch("tccli.plugins.cos.head_object.init_cos_client") 会失败（属性不存在）。 +统一使用 cos_module(name) 加载底层子模块，然后对模块对象打桩。 +""" +import importlib + +import pytest +from unittest.mock import MagicMock +from qcloud_cos import CosServiceError + + +# ========= 全局参数：对齐 coscli 的 setupTestConfig 中假 secret ========= +MOCK_GLOBALS = { + "secretId": "test-secret-id", + "secretKey": "test-secret-key", + "token": None, + "region": "ap-guangzhou", + "endpoint": None, + "profile": "default", +} + + +def cos_module(name): + """加载 tccli.plugins.cos 下的**子模块对象**（绕过 __init__ 的同名函数覆盖）。 + + 用法： + mod = cos_module("head_object") + with patch.object(mod, "init_cos_client", return_value=(mock_client, "ap-guangzhou")): + mod.head_object(args, MOCK_GLOBALS) + """ + return importlib.import_module("tccli.plugins.cos.%s" % name) + + +def make_cos_error(code="InternalError", msg="mock error", req_id="mock-req-id", + method="GET", status_code=500): + """ + 构造真实可用的 CosServiceError（满足 get_error_code/msg/request_id/status_code 断言）。 + CosServiceError 的真实签名为 __init__(method, message, status_code)， + 其中 message 必须是 COS 服务端返回的 XML 错误体。 + """ + xml = ( + "" + "" + "%s" + "%s" + "bucket/key" + "%s" + "mock-trace-id" + "" % (code, msg, req_id) + ) + return CosServiceError(method, xml, status_code) + + +def build_mock_client(): + """ + 构造一个 qcloud_cos.CosS3Client 的 MagicMock，并为最常见的方法预设合理返回值。 + 单测中可以进一步覆盖个别方法的 return_value/side_effect。 + """ + mc = MagicMock() + + # head_object —— 默认返回最小可用头 + mc.head_object.return_value = { + "Content-Length": "1024", + "Content-Type": "text/plain", + "ETag": '"etag-abc"', + "Last-Modified": "Mon, 07 Apr 2026 06:00:00 GMT", + "x-cos-storage-class": "STANDARD", + "x-cos-hash-crc64ecma": "", # 默认留空以避开 CRC 校验分支 + } + + # list_objects（注意 IsTruncated 是字符串 "false"） + mc.list_objects.return_value = { + "Contents": [], + "CommonPrefixes": [], + "IsTruncated": "false", + } + + # list_multipart_uploads + mc.list_multipart_uploads.return_value = {"Upload": [], "IsTruncated": "false"} + + # list_buckets + mc.list_buckets.return_value = {"Buckets": {"Bucket": []}} + + # list_objects_versions + mc.list_objects_versions.return_value = { + "Version": [], "DeleteMarker": [], "IsTruncated": "false", + } + + # ACL 默认值 + mc.get_bucket_acl.return_value = { + "Owner": {"ID": "owner-id", "DisplayName": "owner"}, + "AccessControlList": {"Grant": []}, + } + mc.get_object_acl.return_value = { + "Owner": {"ID": "owner-id", "DisplayName": "owner"}, + "AccessControlList": {"Grant": []}, + } + + # Tagging 默认值 + mc.get_object_tagging.return_value = {"TagSet": {"Tag": []}} + + # 预签名 URL + mc.get_presigned_url.return_value = "https://example.com/signed-url" + mc.get_presigned_download_url.return_value = "https://example.com/signed-download-url" + + # get_object（流式） + body_stream = MagicMock() + body_stream.get_raw_stream.return_value.read.return_value = b"hello world" + mc.get_object.return_value = {"Body": body_stream, "Content-Length": "11"} + + # upload_file / download_file / copy / put_object / delete_object 等 —— 默认成功 + mc.upload_file.return_value = None + mc.download_file.return_value = None + mc.copy.return_value = {} + mc.create_bucket.return_value = {} + mc.delete_bucket.return_value = {} + mc.put_object.return_value = {} + mc.delete_object.return_value = {} + mc.delete_objects.return_value = {"Deleted": []} + mc.put_bucket_acl.return_value = {} + mc.put_object_acl.return_value = {} + mc.put_object_tagging.return_value = {} + mc.delete_object_tagging.return_value = {} + mc.abort_multipart_upload.return_value = {} + mc.restore_object.return_value = {} + + return mc + + +# ========= pytest fixture ========= + +@pytest.fixture +def mock_client(): + """提供一个预设好的 CosS3Client MagicMock。""" + return build_mock_client() + + +@pytest.fixture +def globals_param(): + """提供一份不可变的 MOCK_GLOBALS 副本（避免用例互相污染）。""" + return dict(MOCK_GLOBALS) + + +@pytest.fixture +def patch_init_client(monkeypatch): + """ + 返回一个辅助函数：`patch_init_client(module_name, client, region="ap-guangzhou")` + 会在指定子模块上打桩 init_cos_client，使其返回 (client, region)。 + + 用法： + def test_xxx(mock_client, patch_init_client): + mod = patch_init_client("head_object", mock_client) + mod.head_object(args, MOCK_GLOBALS) + """ + def _do(module_name, client, region="ap-guangzhou"): + mod = cos_module(module_name) + monkeypatch.setattr(mod, "init_cos_client", + lambda _pg: (client, region)) + return mod + return _do \ No newline at end of file diff --git a/tccli/plugins/cos/tests/test_buckets.py b/tccli/plugins/cos/tests/test_buckets.py new file mode 100644 index 0000000000..fa8d445c90 --- /dev/null +++ b/tccli/plugins/cos/tests/test_buckets.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +""" +list_buckets / create_bucket / delete_bucket 单测 +—— 对齐 coscli cmd/ls_test.go（list buckets 分支）/ mb_test.go / rb_test.go。 +""" +from conftest import MOCK_GLOBALS, make_cos_error + + +# ========== list_buckets ========== + +class TestListBuckets: + + def test_success(self, mock_client, patch_init_client, capsys): + """Convey: list buckets success""" + mod = patch_init_client("list_buckets", mock_client) + mock_client.list_buckets.return_value = { + "Buckets": {"Bucket": [ + {"Name": "b1-125000", "Location": "ap-guangzhou", + "CreationDate": "2026-04-01T00:00:00Z"}, + {"Name": "b2-125000", "Location": "ap-beijing", + "CreationDate": "2026-04-02T00:00:00Z"}, + ]}, + } + + mod.list_buckets({}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "b1-125000" in out and "b2-125000" in out + assert "共 2 个存储桶" in out + + def test_filter_region(self, mock_client, patch_init_client, capsys): + """Convey: filter_region 过滤""" + mod = patch_init_client("list_buckets", mock_client) + mock_client.list_buckets.return_value = { + "Buckets": {"Bucket": [ + {"Name": "b1-125000", "Location": "ap-guangzhou", + "CreationDate": "2026-04-01T00:00:00Z"}, + {"Name": "b2-125000", "Location": "ap-beijing", + "CreationDate": "2026-04-02T00:00:00Z"}, + ]}, + } + + mod.list_buckets({"filter_region": "ap-beijing"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "b2-125000" in out + assert "b1-125000" not in out + + def test_empty(self, mock_client, patch_init_client, capsys): + """空存储桶列表""" + mod = patch_init_client("list_buckets", mock_client) + mock_client.list_buckets.return_value = {"Buckets": {"Bucket": []}} + + mod.list_buckets({}, MOCK_GLOBALS) + + assert "当前账号下没有存储桶" in capsys.readouterr().out + + def test_filter_region_no_match(self, mock_client, patch_init_client, capsys): + """filter_region 无匹配""" + mod = patch_init_client("list_buckets", mock_client) + mock_client.list_buckets.return_value = { + "Buckets": {"Bucket": [ + {"Name": "b1-125000", "Location": "ap-guangzhou", + "CreationDate": ""}, + ]}, + } + + mod.list_buckets({"filter_region": "ap-shanghai"}, MOCK_GLOBALS) + + assert "在 ap-shanghai 地域下没有存储桶" in capsys.readouterr().out + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + """Convey: list buckets error""" + mod = patch_init_client("list_buckets", mock_client) + mock_client.list_buckets.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "GET", 403, + ) + + mod.list_buckets({}, MOCK_GLOBALS) + + assert "Error:" in capsys.readouterr().out + + +# ========== create_bucket ========== + +class TestCreateBucket: + + def test_success_default_acl(self, mock_client, patch_init_client, capsys): + """Convey: create bucket success""" + mod = patch_init_client("create_bucket", mock_client) + + mod.create_bucket({"bucket": "test-bucket-1250000000"}, MOCK_GLOBALS) + + kwargs = mock_client.create_bucket.call_args.kwargs + assert kwargs["Bucket"] == "test-bucket-1250000000" + assert kwargs["ACL"] == "private" # 默认 + + out = capsys.readouterr().out + assert "存储桶创建成功" in out + assert "ap-guangzhou" in out + + def test_success_custom_acl(self, mock_client, patch_init_client): + """Convey: 指定 acl""" + mod = patch_init_client("create_bucket", mock_client) + + mod.create_bucket( + {"bucket": "b-1250000000", "acl": "public-read"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.create_bucket.call_args.kwargs + assert kwargs["ACL"] == "public-read" + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + """Convey: create bucket error""" + mod = patch_init_client("create_bucket", mock_client) + mock_client.create_bucket.side_effect = make_cos_error( + "BucketAlreadyExists", "exists", "req-1", "PUT", 409, + ) + + mod.create_bucket({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "Error:" in out + assert "BucketAlreadyExists" in out + + +# ========== delete_bucket ========== + +class TestDeleteBucket: + + def test_success(self, mock_client, patch_init_client, capsys): + """Convey: delete bucket success""" + mod = patch_init_client("delete_bucket", mock_client) + + mod.delete_bucket({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + mock_client.delete_bucket.assert_called_once() + assert "存储桶删除成功" in capsys.readouterr().out + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + """Convey: delete bucket error""" + mod = patch_init_client("delete_bucket", mock_client) + mock_client.delete_bucket.side_effect = make_cos_error( + "BucketNotEmpty", "not empty", "req-1", "DELETE", 409, + ) + + mod.delete_bucket({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "Error:" in out + assert "BucketNotEmpty" in out + + def test_force_clear_and_delete(self, mock_client, patch_init_client, capsys): + """Convey: force delete success —— 清空对象、版本、分片后删桶""" + mod = patch_init_client("delete_bucket", mock_client) + # 1. 普通对象（一页就结束） + mock_client.list_objects.return_value = { + "Contents": [{"Key": "a"}, {"Key": "b"}], + "IsTruncated": "false", + } + # 2. 版本对象 + mock_client.list_objects_versions.return_value = { + "Version": [{"Key": "a", "VersionId": "v1"}], + "DeleteMarker": [{"Key": "b", "VersionId": "v2"}], + "IsTruncated": "false", + } + # 3. 分片上传 + mock_client.list_multipart_uploads.return_value = { + "Upload": [{"Key": "u1", "UploadId": "up1"}], + "IsTruncated": "false", + } + + mod.delete_bucket({"bucket": "b-1250000000", "force": True}, MOCK_GLOBALS) + + # 普通对象 + 版本对象各调用一次 delete_objects + assert mock_client.delete_objects.call_count == 2 + # 分片 abort + mock_client.abort_multipart_upload.assert_called_once() + mock_client.delete_bucket.assert_called_once() + + out = capsys.readouterr().out + assert "存储桶删除成功" in out + assert "已删除 2 个对象" in out + assert "已删除 2 个版本对象" in out + assert "已清理 1 个未完成的分片上传" in out diff --git a/tccli/plugins/cos/tests/test_copy_move_object.py b/tccli/plugins/cos/tests/test_copy_move_object.py new file mode 100644 index 0000000000..6b995bf940 --- /dev/null +++ b/tccli/plugins/cos/tests/test_copy_move_object.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- +""" +copy_object / move_object 单测 —— 对齐 coscli cmd/cp_test.go CosCopy 分支 + move 行为。 +""" +from conftest import MOCK_GLOBALS, make_cos_error + + +# ========== copy ========== + +class TestCopySingle: + + def test_success(self, mock_client, patch_init_client, capsys): + """Convey: CosCopy single object success""" + mod = patch_init_client("copy_object", mock_client) + + mod.copy_object( + {"bucket": "src-b-1250000000", "cos_key": "a.txt", + "dest_bucket": "dst-b-1250000000", "dest_key": "b.txt"}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_called_once() + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["Bucket"] == "dst-b-1250000000" + assert kwargs["Key"] == "b.txt" + assert kwargs["CopySource"] == { + "Bucket": "src-b-1250000000", + "Key": "a.txt", + "Region": "ap-guangzhou", + } + + def test_storage_class_and_meta(self, mock_client, patch_init_client): + mod = patch_init_client("copy_object", mock_client) + + mod.copy_object( + {"bucket": "b", "cos_key": "a", "dest_key": "b", + "storage_class": "ARCHIVE", + "meta": "env=prod"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["StorageClass"] == "ARCHIVE" + assert kwargs["Metadata"] == {"x-cos-meta-env": "prod"} + assert kwargs["CopyStatus"] == "Replaced" # 单文件复制用 CopyStatus + + def test_dest_bucket_defaults_to_src(self, mock_client, patch_init_client): + """未指定 dest_bucket 时默认与 src 相同""" + mod = patch_init_client("copy_object", mock_client) + + mod.copy_object( + {"bucket": "same-b-125", "cos_key": "a", "dest_key": "b"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["Bucket"] == "same-b-125" + assert kwargs["CopySource"]["Bucket"] == "same-b-125" + + def test_retry_then_success(self, mock_client, patch_init_client): + mod = patch_init_client("copy_object", mock_client) + mock_client.copy.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "PUT", 500), + {}, + ] + + mod.copy_object( + {"bucket": "b", "cos_key": "a", "dest_key": "b", "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_count == 2 + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + """Convey: CosCopy error""" + mod = patch_init_client("copy_object", mock_client) + mock_client.copy.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "PUT", 404, + ) + + mod.copy_object( + {"bucket": "b", "cos_key": "a", "dest_key": "b", "retry": 0}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out + + +class TestCopyRecursive: + + def test_recursive_empty(self, mock_client, patch_init_client): + """Convey: CosCopy recursive (no objects)""" + mod = patch_init_client("copy_object", mock_client) + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_key": "dst/", "recursive": True}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_not_called() + + def test_recursive_with_files(self, mock_client, patch_init_client): + """Convey: CosCopy recursive success with files""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/empty/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.copy_object( + {"bucket": "src-b-1250000000", "cos_key": "pre/", + "dest_bucket": "dst-b-1250000000", "dest_key": "dst/", + "recursive": True}, + MOCK_GLOBALS, + ) + + # 无 include → a.txt / b.log 都复制 + assert mock_client.copy.call_count == 2 + # 空目录通过 put_object 在目标桶创建 + mock_client.put_object.assert_called() + put_kwargs = mock_client.put_object.call_args.kwargs + assert put_kwargs["Bucket"] == "dst-b-1250000000" + assert put_kwargs["Key"].endswith("empty/") + + def test_recursive_include_filter(self, mock_client, patch_init_client): + """include 过滤只保留 a.txt""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.copy_object( + {"bucket": "src-b-1250000000", "cos_key": "pre/", + "dest_bucket": "dst-b-1250000000", "dest_key": "dst/", + "recursive": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_count == 1 + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["Bucket"] == "dst-b-1250000000" + assert kwargs["Key"] == "dst/a.txt" + + +# ========== move ========== + +class TestMoveSingle: + + def test_success(self, mock_client, patch_init_client): + """单文件移动 = copy + delete 源""" + mod = patch_init_client("move_object", mock_client) + + mod.move_object( + {"bucket": "b-125", "cos_key": "a.txt", + "dest_bucket": "b-125", "dest_key": "b.txt"}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_called_once() + mock_client.delete_object.assert_called_once() + del_kwargs = mock_client.delete_object.call_args.kwargs + assert del_kwargs["Bucket"] == "b-125" + assert del_kwargs["Key"] == "a.txt" + + def test_retry_then_success(self, mock_client, patch_init_client): + mod = patch_init_client("move_object", mock_client) + mock_client.copy.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "PUT", 500), + {}, + ] + + mod.move_object( + {"bucket": "b", "cos_key": "a", "dest_key": "b", "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_count == 2 + # 成功后才调用 delete_object（1 次） + assert mock_client.delete_object.call_count == 1 + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("move_object", mock_client) + mock_client.copy.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "PUT", 404, + ) + + mod.move_object( + {"bucket": "b", "cos_key": "a", "dest_key": "b", "retry": 0}, + MOCK_GLOBALS, + ) + + # copy 失败后 delete 不应被调用 + mock_client.delete_object.assert_not_called() + assert "Error:" in capsys.readouterr().out + + +class TestMoveRecursive: + + def test_recursive_with_files(self, mock_client, patch_init_client): + """Convey: move recursive —— 每个文件 copy + delete""" + mod = patch_init_client("move_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.txt", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.move_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "dst/", + "recursive": True}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_count == 2 + assert mock_client.delete_object.call_count == 2 diff --git a/tccli/plugins/cos/tests/test_coverage_branches.py b/tccli/plugins/cos/tests/test_coverage_branches.py new file mode 100644 index 0000000000..873363056a --- /dev/null +++ b/tccli/plugins/cos/tests/test_coverage_branches.py @@ -0,0 +1,809 @@ +# -*- coding: utf-8 -*- +""" +补充覆盖率测试 —— 针对各命令文件中覆盖率偏低的分支。 + +按命令分组组织；用例命名对齐 coscli Convey 子场景。 +""" +import os + +from conftest import MOCK_GLOBALS, make_cos_error + + +# ============================================================= +# move_object 递归空目录 +# ============================================================= + +class TestMoveRecursiveEmptyDir: + + def _setup(self, mock_client, patch_init_client, include="", exclude=""): + mod = patch_init_client("move_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/sub/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/excluded/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + return mod + + def test_success_move_dir_markers(self, mock_client, patch_init_client): + """Convey: move 空目录对象 —— 创建目标 + 删除源""" + mod = self._setup(mock_client, patch_init_client) + + mod.move_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True}, + MOCK_GLOBALS, + ) + + # a.txt 的 copy + delete + # sub/、excluded/ 两个空目录各一次 put_object + # 源文件 + 两个源目录共 3 次 delete_object + assert mock_client.put_object.call_count == 2 + assert mock_client.delete_object.call_count == 3 + + def test_put_object_fails(self, mock_client, patch_init_client): + """Convey: 创建目标文件夹失败 —— 不继续 delete 源""" + mod = self._setup(mock_client, patch_init_client) + mock_client.put_object.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "PUT", 403, + ) + + mod.move_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True}, + MOCK_GLOBALS, + ) + + # 2 次尝试创建目录都失败 → 不应调用源目录 delete + # 只有 a.txt 的 delete_object（由 _do_move 内部发起） + assert mock_client.put_object.call_count == 2 + assert mock_client.delete_object.call_count == 1 + + def test_delete_src_dir_fails(self, mock_client, patch_init_client): + """Convey: 删除源文件夹失败 —— 记录 err 但继续""" + mod = self._setup(mock_client, patch_init_client) + + # delete_object 在处理 pre/sub/ 时抛 CosServiceError + def _delete_side(**kwargs): + if kwargs.get("Key", "").endswith("sub/"): + raise make_cos_error("AccessDenied", "denied", "r1", "DELETE", 403) + return {} + + mock_client.delete_object.side_effect = _delete_side + + mod.move_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True}, + MOCK_GLOBALS, + ) + + # put_object 成功 2 次（sub、excluded 两个空目录） + assert mock_client.put_object.call_count == 2 + + def test_recursive_include_filter(self, mock_client, patch_init_client): + """include 过滤 —— 只移动 *.txt，目录 skip""" + mod = patch_init_client("move_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.move_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # 只有 a.txt 被复制/删除 + assert mock_client.copy.call_count == 1 + assert mock_client.copy.call_args.kwargs["Key"] == "d/a.txt" + + +# ============================================================= +# sync_upload_object 补充分支 +# ============================================================= + +class TestSyncUploadExtraBranches: + + def test_storage_class_meta_content_type_rate_limiting(self, mock_client, patch_init_client, tmp_path): + """Convey: 透传 storage_class/meta/content_type/rate_limiting""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "storage_class": "STANDARD_IA", "content_type": "text/plain", + "meta": "env=prod", "rate_limiting": 2}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_called_once() + kwargs = mock_client.upload_file.call_args.kwargs + assert kwargs["StorageClass"] == "STANDARD_IA" + assert kwargs["ContentType"] == "text/plain" + assert kwargs["Metadata"] == {"x-cos-meta-env": "prod"} + assert kwargs["TrafficLimit"] == str(2 * 1024 * 1024 * 8) + + def test_retry_resets_progress_callback(self, mock_client, patch_init_client, tmp_path): + """Convey: retry 后必须重置 progress_callback 并成功""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + mock_client.upload_file.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "PUT", 500), + None, + ] + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.upload_file.call_count == 2 + + def test_empty_dir_upload_error_recorded(self, mock_client, patch_init_client, tmp_path): + """Convey: 空目录标记创建失败时记录 err，不中断流程""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "empty_dir").mkdir() + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + mock_client.put_object.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "PUT", 403, + ) + + # 不应抛异常 + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + mock_client.put_object.assert_called() + + def test_delete_extra_dir_and_file(self, mock_client, patch_init_client, tmp_path, capsys): + """Convey: --delete 删除 COS 上的多余文件 + 多余目录""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + + # list_all_objects（不含目录）→ 用于扫描+跳过判断 + # list_all_objects_with_dirs（含目录）→ 用于 delete 多余扫描 + listing_no_dirs = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/orphan.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + listing_with_dirs = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/orphan.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/orphan_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + # list_objects 依次用于 list_all_objects(含跳过)、list_all_objects_with_dirs(delete) + mock_client.list_objects.side_effect = [listing_no_dirs, listing_with_dirs] + # head_object 不返回 CRC —— 本地 a.txt 与 COS 一致性判断返回"不跳过" + mock_client.head_object.return_value = {"x-cos-hash-crc64ecma": ""} + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "delete": True}, + MOCK_GLOBALS, + ) + + delete_keys = [c.kwargs["Key"] for c in mock_client.delete_object.call_args_list] + # orphan.txt 与 orphan_dir/ 都应被删除 + assert "pre/orphan.txt" in delete_keys + assert "pre/orphan_dir/" in delete_keys + assert "已删除 COS 上多余文件" in capsys.readouterr().out + + def test_top_level_cos_error_captured(self, mock_client, patch_init_client, tmp_path, capsys): + """Convey: list_all_objects 抛 CosServiceError —— 顶层 except 捕获并打印""" + mod = patch_init_client("sync_upload_object", mock_client) + mock_client.list_objects.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "GET", 403, + ) + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out + + +# ============================================================= +# sync_download_object 补充分支 +# ============================================================= + +class TestSyncDownloadExtraBranches: + + def test_cos_empty_dir_include_filter(self, mock_client, patch_init_client, tmp_path): + """Convey: COS 空目录被 include/exclude 过滤掉""" + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + # 空目录标记 —— 不匹配 include *.txt 被跳过 + {"Key": "pre/excluded_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + # 正常文件 + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # 仅 a.txt 被下载 + assert mock_client.download_file.call_count == 1 + # excluded_dir 被跳过 → 本地不会出现 + assert not (tmp_path / "excluded_dir").exists() + + def test_retry_then_success(self, mock_client, patch_init_client, tmp_path): + """Convey: download 重试后成功""" + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.download_file.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "GET", 500), + None, + ] + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.download_file.call_count == 2 + + def test_rate_limiting_passed(self, mock_client, patch_init_client, tmp_path): + """Convey: rate_limiting 参数被透传为 TrafficLimit (bit/s)""" + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "rate_limiting": 3}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.download_file.call_args.kwargs + assert kwargs["TrafficLimit"] == str(3 * 1024 * 1024 * 8) + + def test_delete_extra_removes_empty_local_dirs(self, mock_client, patch_init_client, tmp_path): + """Convey: --delete 删除本地多余文件后，空目录也被清理""" + mod = patch_init_client("sync_download_object", mock_client) + # 本地有 orphan 子目录 + 孤儿文件；COS 上全无 + orphan_dir = tmp_path / "orphan_dir" + orphan_dir.mkdir() + (orphan_dir / "x.txt").write_text("x") + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "delete": True}, + MOCK_GLOBALS, + ) + + # 孤儿文件已删除，空目录也被 rmdir + assert not orphan_dir.exists() + + +# ============================================================= +# sync_copy_object 补充分支 +# ============================================================= + +class TestSyncCopyExtraBranches: + + def test_storage_class_and_meta(self, mock_client, patch_init_client): + """Convey: storage_class + meta 透传（使用 CopyStatus=Replaced）""" + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + {"Contents": [], "IsTruncated": "false"}, + ] + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/", + "storage_class": "ARCHIVE", + "meta": "env=prod"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["StorageClass"] == "ARCHIVE" + assert kwargs["Metadata"] == {"x-cos-meta-env": "prod"} + assert kwargs["CopyStatus"] == "Replaced" + + def test_empty_dir_put_object_fails(self, mock_client, patch_init_client): + """Convey: 空目录 put_object 失败 —— 记录 err 不中断""" + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "p/sub/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + {"Contents": [], "IsTruncated": "false"}, + ] + mock_client.put_object.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "PUT", 403, + ) + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/"}, + MOCK_GLOBALS, + ) + + mock_client.put_object.assert_called_once() + + def test_retry_copy_success(self, mock_client, patch_init_client): + """Convey: copy 重试后成功""" + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + {"Contents": [], "IsTruncated": "false"}, + ] + mock_client.copy.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "PUT", 500), + {}, + ] + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/", "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_count == 2 + + +# ============================================================= +# cat_object 非文本分支 +# ============================================================= + +class TestCatObjectBinary: + + def test_non_utf8_content(self, mock_client, patch_init_client, capsys): + """Convey: 非 UTF-8 编码 —— 以十六进制显示""" + mod = patch_init_client("cat_object", mock_client) + mock_client.head_object.return_value = {"Content-Length": "4"} + + body_stream = mock_client.get_object.return_value["Body"] + # 非 UTF-8 字节 + body_stream.get_raw_stream.return_value.read.return_value = b"\xff\xfe\xfd\xfc" + + mod.cat_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "十六进制" in out + assert "fffefdfc" in out + + def test_binary_long_content(self, mock_client, patch_init_client, capsys): + """Convey: >1024 字节的二进制文件，尾部显示剩余字节数""" + mod = patch_init_client("cat_object", mock_client) + mock_client.head_object.return_value = {"Content-Length": "2000"} + + body_stream = mock_client.get_object.return_value["Body"] + body_stream.get_raw_stream.return_value.read.return_value = b"\xff" * 2000 + + mod.cat_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "共 2000 字节" in out + + def test_generic_exception(self, mock_client, patch_init_client, capsys): + """Convey: 其他 Exception 被 Error: 打印（比如 head_object 返回无效 Content-Length）""" + mod = patch_init_client("cat_object", mock_client) + mock_client.head_object.side_effect = RuntimeError("boom") + + mod.cat_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "Error:" in capsys.readouterr().out + + +# ============================================================= +# restore_object 补充分支 +# ============================================================= + +class TestRestoreObjectExtraBranches: + + def test_recursive_already_in_progress(self, mock_client, patch_init_client, capsys): + """Convey: 递归 —— 子对象已在进行中""" + mod = patch_init_client("restore_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a", "Size": "1", "StorageClass": "ARCHIVE"}, + ], + "IsTruncated": "false", + } + mock_client.restore_object.side_effect = make_cos_error( + "RestoreAlreadyInProgress", "in progress", "r1", "POST", 409, + ) + + mod.restore_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True}, + MOCK_GLOBALS, + ) + + out = capsys.readouterr().out + assert "恢复进行中" in out + + def test_recursive_other_error(self, mock_client, patch_init_client, capsys): + """Convey: 递归 —— 普通失败计数""" + mod = patch_init_client("restore_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a", "Size": "1", "StorageClass": "ARCHIVE"}, + ], + "IsTruncated": "false", + } + mock_client.restore_object.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "POST", 403, + ) + + mod.restore_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True}, + MOCK_GLOBALS, + ) + + assert "恢复失败" in capsys.readouterr().out + + def test_recursive_skip_by_include(self, mock_client, patch_init_client, capsys): + """Convey: include 过滤不匹配的归档对象被跳过""" + mod = patch_init_client("restore_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.log", "Size": "1", "StorageClass": "ARCHIVE"}, + {"Key": "pre/b.txt", "Size": "2", "StorageClass": "ARCHIVE"}, + ], + "IsTruncated": "false", + } + + mod.restore_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # 只 b.txt 触发 restore_object + assert mock_client.restore_object.call_count == 1 + + def test_recursive_pagination(self, mock_client, patch_init_client): + """Convey: 递归分页""" + mod = patch_init_client("restore_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "pre/a", "Size": "1", "StorageClass": "ARCHIVE"}], + "IsTruncated": "true", "NextMarker": "pre/a"}, + {"Contents": [{"Key": "pre/b", "Size": "1", "StorageClass": "ARCHIVE"}], + "IsTruncated": "false"}, + ] + + mod.restore_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True}, + MOCK_GLOBALS, + ) + + assert mock_client.restore_object.call_count == 2 + + +# ============================================================= +# delete_bucket 补充分支 +# ============================================================= + +class TestDeleteBucketExtraBranches: + + def test_force_versions_list_error_is_ignored(self, mock_client, patch_init_client, capsys): + """Convey: list_objects_versions 抛 CosServiceError —— 忽略，继续清理分片""" + mod = patch_init_client("delete_bucket", mock_client) + + mock_client.list_objects.return_value = {"Contents": [], "IsTruncated": "false"} + mock_client.list_objects_versions.side_effect = make_cos_error( + "VersioningNotEnabled", "n/a", "r1", "GET", 400, + ) + mock_client.list_multipart_uploads.return_value = { + "Upload": [], "IsTruncated": "false", + } + + mod.delete_bucket({"bucket": "b", "force": True}, MOCK_GLOBALS) + + mock_client.delete_bucket.assert_called_once() + assert "存储桶删除成功" in capsys.readouterr().out + + def test_force_multi_page(self, mock_client, patch_init_client): + """Convey: force 清理分页场景（IsTruncated=true → false）""" + mod = patch_init_client("delete_bucket", mock_client) + + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "a"}], "IsTruncated": "true", "NextMarker": "a"}, + {"Contents": [{"Key": "b"}], "IsTruncated": "false"}, + ] + mock_client.list_objects_versions.return_value = { + "Version": [], "DeleteMarker": [], "IsTruncated": "false", + } + mock_client.list_multipart_uploads.side_effect = [ + {"Upload": [{"Key": "u1", "UploadId": "up1"}], + "IsTruncated": "true", "NextKeyMarker": "u1", "NextUploadIdMarker": "up1"}, + {"Upload": [{"Key": "u2", "UploadId": "up2"}], + "IsTruncated": "false"}, + ] + + mod.delete_bucket({"bucket": "b", "force": True}, MOCK_GLOBALS) + + # 两批对象各一次 delete_objects + assert mock_client.delete_objects.call_count == 2 + # 两次 abort + assert mock_client.abort_multipart_upload.call_count == 2 + + +# ============================================================= +# hash_object 补充分支 +# ============================================================= + +class TestHashObjectExtraBranches: + + def test_local_crc64(self, mock_client, patch_init_client, capsys, tmp_path): + """Convey: 计算本地 crc64""" + mod = patch_init_client("hash_object", mock_client) + f = tmp_path / "a.txt" + f.write_text("hello") + + mod.hash_object( + {"local_path": str(f), "hash_type": "crc64"}, + MOCK_GLOBALS, + ) + + out = capsys.readouterr().out + assert "CRC64" in out + + def test_local_sha1(self, mock_client, patch_init_client, capsys, tmp_path): + mod = patch_init_client("hash_object", mock_client) + f = tmp_path / "a.txt" + f.write_bytes(b"") # 空文件的 sha1 + + mod.hash_object( + {"local_path": str(f), "hash_type": "sha1"}, + MOCK_GLOBALS, + ) + # 空文件 sha1 = da39a3ee5e6b4b0d3255bfef95601890afd80709 + assert "da39a3ee5e6b4b0d3255bfef95601890afd80709" in capsys.readouterr().out + + def test_local_unsupported_hash_type(self, mock_client, patch_init_client, capsys, tmp_path): + """Convey: 不支持的哈希类型""" + mod = patch_init_client("hash_object", mock_client) + f = tmp_path / "a.txt" + f.write_text("hi") + + mod.hash_object( + {"local_path": str(f), "hash_type": "sha512"}, + MOCK_GLOBALS, + ) + + assert "不支持的哈希类型" in capsys.readouterr().out + + def test_local_path_is_dir(self, mock_client, patch_init_client, capsys, tmp_path): + mod = patch_init_client("hash_object", mock_client) + + mod.hash_object({"local_path": str(tmp_path)}, MOCK_GLOBALS) + + assert "指定路径不是文件" in capsys.readouterr().out + + +# ============================================================= +# acl_object 补充分支 +# ============================================================= + +class TestAclObjectExtraBranches: + + def test_get_bucket_acl_uri_grantee(self, mock_client, patch_init_client, capsys): + """Convey: URI grantee 输出分支""" + mod = patch_init_client("acl_object", mock_client) + mock_client.get_bucket_acl.return_value = { + "Owner": {"ID": "o", "DisplayName": "d"}, + "AccessControlList": {"Grant": { + "Grantee": {"URI": "http://cam.qcloud.com/groups/global/AllUsers"}, + "Permission": "READ", + }}, + } + + mod.get_bucket_acl({"bucket": "b"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "URI" in out + assert "AllUsers" in out + + def test_get_object_acl_uri_grantee(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + mock_client.get_object_acl.return_value = { + "Owner": {"ID": "o", "DisplayName": "d"}, + "AccessControlList": {"Grant": { + "Grantee": {"URI": "http://cam.qcloud.com/groups/global/AllUsers"}, + "Permission": "READ", + }}, + } + + mod.get_object_acl({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "URI" in capsys.readouterr().out + + +# ============================================================= +# abort_multipart 补充：abort_multipart_upload 指定单个 upload_id 时出错 +# ============================================================= + +class TestAbortMultipartExtraBranches: + + def test_abort_single_upload_id_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("abort_multipart", mock_client) + mock_client.abort_multipart_upload.side_effect = make_cos_error( + "NoSuchUpload", "no upload", "r1", "DELETE", 404, + ) + + mod.abort_multipart( + {"bucket": "b", "cos_key": "k", "upload_id": "up1"}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out + + def test_abort_pagination(self, mock_client, patch_init_client, capsys): + """Convey: abort 分页列举""" + mod = patch_init_client("abort_multipart", mock_client) + mock_client.list_multipart_uploads.side_effect = [ + {"Upload": [{"Key": "a", "UploadId": "u1", "Initiated": "t1"}], + "IsTruncated": "true", "NextKeyMarker": "a", "NextUploadIdMarker": "u1"}, + {"Upload": [{"Key": "b", "UploadId": "u2", "Initiated": "t2"}], + "IsTruncated": "false"}, + ] + + mod.abort_multipart({"bucket": "b"}, MOCK_GLOBALS) + + assert mock_client.abort_multipart_upload.call_count == 2 + + +# ============================================================= +# list_object / list_buckets 补充 +# ============================================================= + +class TestListExtraBranches: + + def test_list_non_recursive_pagination(self, mock_client, patch_init_client, capsys): + """Convey: 非递归模式下分页被截断提示""" + mod = patch_init_client("list_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "a", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "true", + "NextMarker": "a", + } + + mod.list_object({"bucket": "b"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "结果已截断" in out + + def test_list_buckets_sdk_error(self, mock_client, patch_init_client, capsys): + """list_buckets SDK 错误分支（再次覆盖通用 except 路径）""" + mod = patch_init_client("list_buckets", mock_client) + mock_client.list_buckets.side_effect = make_cos_error( + "InternalError", "mock", "r1", "GET", 500, + ) + mod.list_buckets({}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + +# ============================================================= +# signurl 其它 HTTP 方法分支 +# ============================================================= + +class TestSignurlExtraBranches: + + def test_other_method(self, mock_client, patch_init_client, capsys): + """Convey: 除 GET/PUT 外的其他 method（如 HEAD/DELETE）""" + mod = patch_init_client("signurl_object", mock_client) + mock_client.get_presigned_url.return_value = "https://signed-delete" + + mod.signurl_object( + {"bucket": "b", "cos_key": "k", "method": "DELETE", "expired": 60}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.get_presigned_url.call_args.kwargs + assert kwargs["Method"] == "DELETE" + assert "https://signed-delete" in capsys.readouterr().out + + def test_generic_exception(self, mock_client, patch_init_client, capsys): + """Convey: 非 CosServiceError 异常（Exception 分支）""" + mod = patch_init_client("signurl_object", mock_client) + mock_client.get_presigned_download_url.side_effect = RuntimeError("boom") + + mod.signurl_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "Error: boom" in capsys.readouterr().out + + +# ============================================================= +# du_object 分页 +# ============================================================= + +class TestDuObjectExtraBranches: + + def test_pagination(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("du_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "a", "Size": "100", "StorageClass": "STANDARD"}], + "IsTruncated": "true", "NextMarker": "a"}, + {"Contents": [{"Key": "b", "Size": "200", "StorageClass": "STANDARD"}], + "IsTruncated": "false"}, + ] + + mod.du_object({"bucket": "b"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "总文件数: 2" in out + assert "300 字节" in out + + +# ============================================================= +# lsparts 分页 +# ============================================================= + +class TestLspartsExtraBranches: + + def test_pagination(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("lsparts_object", mock_client) + mock_client.list_multipart_uploads.side_effect = [ + {"Upload": [{"Key": "a", "UploadId": "u1", "Initiated": "t1"}], + "IsTruncated": "true", "NextKeyMarker": "a", "NextUploadIdMarker": "u1"}, + {"Upload": [{"Key": "b", "UploadId": "u2", "Initiated": "t2"}], + "IsTruncated": "false"}, + ] + + mod.lsparts_object({"bucket": "b"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "u1" in out and "u2" in out + assert "共 2 个" in out diff --git a/tccli/plugins/cos/tests/test_delete_object.py b/tccli/plugins/cos/tests/test_delete_object.py new file mode 100644 index 0000000000..e9b8279e09 --- /dev/null +++ b/tccli/plugins/cos/tests/test_delete_object.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +""" +delete_object 单测 —— 对齐 coscli cmd/rm_test.go。 +""" +from unittest.mock import patch + +from conftest import MOCK_GLOBALS, make_cos_error + + +class TestDeleteObjectSingle: + + def test_success(self, mock_client, patch_init_client, capsys): + """Convey: delete object success""" + mod = patch_init_client("delete_object", mock_client) + + mod.delete_object( + {"bucket": "b-1250000000", "cos_key": "k"}, + MOCK_GLOBALS, + ) + + mock_client.delete_object.assert_called_once() + kwargs = mock_client.delete_object.call_args.kwargs + assert kwargs["Bucket"] == "b-1250000000" + assert kwargs["Key"] == "k" + assert "删除成功" in capsys.readouterr().out + + def test_with_version_id(self, mock_client, patch_init_client): + mod = patch_init_client("delete_object", mock_client) + + mod.delete_object( + {"bucket": "b", "cos_key": "k", "version_id": "v1"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.delete_object.call_args.kwargs + assert kwargs["VersionId"] == "v1" + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + """Convey: delete object error""" + mod = patch_init_client("delete_object", mock_client) + mock_client.delete_object.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "DELETE", 404, + ) + + mod.delete_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "Error:" in capsys.readouterr().out + + +class TestDeleteObjectRecursive: + + def _setup_list(self, mock_client, files, dirs=None): + """预设 list_objects 返回（list_all_objects_with_dirs 会用）""" + contents = [] + for key, size in files: + contents.append({ + "Key": key, "Size": str(size), + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"', + }) + for d in dirs or []: + contents.append({ + "Key": d, "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"', + }) + mock_client.list_objects.return_value = { + "Contents": contents, "IsTruncated": "false", + } + + def test_recursive_empty(self, mock_client, patch_init_client, capsys): + """Convey: recursive 空前缀""" + mod = patch_init_client("delete_object", mock_client) + self._setup_list(mock_client, []) + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, "force": True}, + MOCK_GLOBALS, + ) + + mock_client.delete_objects.assert_not_called() + assert "没有匹配的对象需要删除" in capsys.readouterr().out + + def test_recursive_force_success(self, mock_client, patch_init_client): + """Convey: recursive force 删除成功（文件 + 目录对象）""" + mod = patch_init_client("delete_object", mock_client) + self._setup_list( + mock_client, + files=[("pre/a.txt", 10), ("pre/b.txt", 20)], + dirs=["pre/sub/"], + ) + + mod.delete_object( + {"bucket": "b-1250000000", "cos_key": "pre/", + "recursive": True, "force": True}, + MOCK_GLOBALS, + ) + + # 文件批量删除 1 次 + mock_client.delete_objects.assert_called_once() + batch = mock_client.delete_objects.call_args.kwargs["Delete"]["Object"] + assert {o["Key"] for o in batch} == {"pre/a.txt", "pre/b.txt"} + # 目录对象逐个 delete_object + mock_client.delete_object.assert_called_once() + assert mock_client.delete_object.call_args.kwargs["Key"] == "pre/sub/" + + def test_recursive_include_filter(self, mock_client, patch_init_client): + """include 过滤生效""" + mod = patch_init_client("delete_object", mock_client) + self._setup_list( + mock_client, + files=[("pre/a.txt", 1), ("pre/b.log", 2)], + ) + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, + "force": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + batch = mock_client.delete_objects.call_args.kwargs["Delete"]["Object"] + assert [o["Key"] for o in batch] == ["pre/a.txt"] + + def test_recursive_confirm_cancel(self, mock_client, patch_init_client, capsys, monkeypatch): + """非 force 模式下用户输入非 y —— 取消删除""" + mod = patch_init_client("delete_object", mock_client) + self._setup_list(mock_client, files=[("pre/a.txt", 1)]) + monkeypatch.setattr("builtins.input", lambda *_a, **_k: "n") + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, "force": False}, + MOCK_GLOBALS, + ) + + mock_client.delete_objects.assert_not_called() + assert "已取消删除" in capsys.readouterr().out + + def test_recursive_confirm_yes(self, mock_client, patch_init_client, monkeypatch): + """非 force 模式下用户输入 y —— 继续删除""" + mod = patch_init_client("delete_object", mock_client) + self._setup_list(mock_client, files=[("pre/a.txt", 1)]) + monkeypatch.setattr("builtins.input", lambda *_a, **_k: "y") + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, "force": False}, + MOCK_GLOBALS, + ) + + mock_client.delete_objects.assert_called_once() + + def test_recursive_retry_then_success(self, mock_client, patch_init_client): + """Convey: err-retry-num 分支 —— 第一次失败，第二次成功""" + mod = patch_init_client("delete_object", mock_client) + self._setup_list(mock_client, files=[("pre/a.txt", 1), ("pre/b.txt", 2)]) + mock_client.delete_objects.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "POST", 500), + {"Deleted": []}, + ] + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, + "force": True, "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.delete_objects.call_count == 2 + + def test_recursive_retry_exhausted(self, mock_client, patch_init_client): + """重试耗尽仍失败，不抛异常""" + mod = patch_init_client("delete_object", mock_client) + self._setup_list(mock_client, files=[("pre/a.txt", 1)]) + mock_client.delete_objects.side_effect = make_cos_error( + "InternalError", "mock", "r1", "POST", 500, + ) + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, + "force": True, "retry": 2}, + MOCK_GLOBALS, + ) + + # retry=2 -> 一共 3 次尝试 + assert mock_client.delete_objects.call_count == 3 diff --git a/tccli/plugins/cos/tests/test_download_object.py b/tccli/plugins/cos/tests/test_download_object.py new file mode 100644 index 0000000000..999142db31 --- /dev/null +++ b/tccli/plugins/cos/tests/test_download_object.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +""" +download_object 单测 —— 对齐 coscli cmd/cp_test.go 中 Download 分支。 +""" +import os + +from conftest import MOCK_GLOBALS, make_cos_error + + +class TestDownloadSingle: + + def test_success(self, mock_client, patch_init_client, tmp_path): + """Convey: Download single file success""" + mod = patch_init_client("download_object", mock_client) + mock_client.head_object.return_value = {"Content-Length": "100"} + local = tmp_path / "dst.txt" + + mod.download_object( + {"bucket": "b-1250000000", "cos_key": "k", "local_path": str(local)}, + MOCK_GLOBALS, + ) + + mock_client.download_file.assert_called_once() + kwargs = mock_client.download_file.call_args.kwargs + assert kwargs["Bucket"] == "b-1250000000" + assert kwargs["Key"] == "k" + assert kwargs["DestFilePath"] == str(local) + assert "progress_callback" in kwargs + + def test_with_version_and_rate_limiting(self, mock_client, patch_init_client, tmp_path): + mod = patch_init_client("download_object", mock_client) + local = tmp_path / "dst.txt" + + mod.download_object( + {"bucket": "b", "cos_key": "k", "local_path": str(local), + "version_id": "v-1", "rate_limiting": 5}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.download_file.call_args.kwargs + assert kwargs["VersionId"] == "v-1" + assert kwargs["TrafficLimit"] == str(5 * 1024 * 1024 * 8) + + def test_creates_parent_dir(self, mock_client, patch_init_client, tmp_path): + """本地目标目录不存在时自动创建""" + mod = patch_init_client("download_object", mock_client) + local = tmp_path / "new_dir" / "sub" / "dst.txt" + + mod.download_object( + {"bucket": "b", "cos_key": "k", "local_path": str(local)}, + MOCK_GLOBALS, + ) + + assert os.path.isdir(os.path.dirname(str(local))) + + def test_retry_then_success(self, mock_client, patch_init_client, tmp_path): + mod = patch_init_client("download_object", mock_client) + mock_client.download_file.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "GET", 500), + None, + ] + + mod.download_object( + {"bucket": "b", "cos_key": "k", + "local_path": str(tmp_path / "dst.txt"), "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.download_file.call_count == 2 + + def test_sdk_error(self, mock_client, patch_init_client, tmp_path, capsys): + """Convey: Download error —— 最终失败被上层 except 捕获""" + mod = patch_init_client("download_object", mock_client) + mock_client.download_file.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "GET", 404, + ) + + mod.download_object( + {"bucket": "b", "cos_key": "k", + "local_path": str(tmp_path / "dst.txt"), "retry": 0}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out + + +class TestDownloadRecursive: + + def test_recursive_empty(self, mock_client, patch_init_client, tmp_path): + """Convey: Download directory success (no objects)""" + mod = patch_init_client("download_object", mock_client) + + mod.download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "recursive": True}, + MOCK_GLOBALS, + ) + + mock_client.download_file.assert_not_called() + + def test_recursive_success(self, mock_client, patch_init_client, tmp_path): + """Convey: Download directory success with files""" + mod = patch_init_client("download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "10", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/sub/b.log", "Size": "20", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/empty_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "recursive": True}, + MOCK_GLOBALS, + ) + + # 两个文件都下载 + assert mock_client.download_file.call_count == 2 + # 空目录在本地被创建 + assert os.path.isdir(str(tmp_path / "empty_dir")) + + def test_recursive_include_filter(self, mock_client, patch_init_client, tmp_path): + mod = patch_init_client("download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "recursive": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + assert mock_client.download_file.call_count == 1 + assert mock_client.download_file.call_args.kwargs["Key"] == "pre/a.txt" diff --git a/tccli/plugins/cos/tests/test_extra_branches.py b/tccli/plugins/cos/tests/test_extra_branches.py new file mode 100644 index 0000000000..9f6be121c2 --- /dev/null +++ b/tccli/plugins/cos/tests/test_extra_branches.py @@ -0,0 +1,482 @@ +# -*- coding: utf-8 -*- +""" +进一步补齐覆盖率：copy / download / delete_object / upload 的 recursive 分支、 +空目录过滤、重试错误记录、顶层 CosServiceError、progress bar tick 打印等。 +""" +import time + +from conftest import MOCK_GLOBALS, make_cos_error +from tccli.plugins.cos import utils as _utils + + +# ============================================================= +# copy_object 补充分支 +# ============================================================= + +class TestCopyObjectMoreBranches: + + def test_single_storage_class_only(self, mock_client, patch_init_client): + """Convey: 单文件 copy + 指定 storage_class（只进 storage_class 分支，不进 metadata 分支）""" + mod = patch_init_client("copy_object", mock_client) + + mod.copy_object( + {"bucket": "b", "cos_key": "a", "dest_key": "d", + "storage_class": "ARCHIVE"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["StorageClass"] == "ARCHIVE" + assert "Metadata" not in kwargs + + def test_recursive_pagination(self, mock_client, patch_init_client): + """Convey: recursive 分页（IsTruncated=true → false）""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "true", "NextMarker": "pre/a.txt"}, + {"Contents": [{"Key": "pre/b.txt", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + ] + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", "recursive": True}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_count == 2 + + def test_recursive_empty_dir_excluded(self, mock_client, patch_init_client): + """Convey: recursive 空目录被 include/exclude 过滤掉""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/excluded_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", "recursive": True, + "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # a.txt 被复制；excluded_dir/ 被 include 过滤，put_object 不被调用 + assert mock_client.copy.call_count == 1 + mock_client.put_object.assert_not_called() + + def test_recursive_retry_then_success(self, mock_client, patch_init_client): + """Convey: recursive copy 重试 —— 子文件首次失败、第二次成功""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.copy.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "PUT", 500), + {}, + ] + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True, "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_count == 2 + + def test_recursive_retry_exhausted_logs_err(self, mock_client, patch_init_client): + """Convey: recursive copy 重试耗尽 —— 子文件失败记录 err，整体不抛异常""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.copy.side_effect = make_cos_error( + "InternalError", "mock", "r1", "PUT", 500, + ) + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True, "retry": 1}, + MOCK_GLOBALS, + ) + + # retry=1 -> 共 2 次尝试 + assert mock_client.copy.call_count == 2 + + def test_recursive_empty_dir_put_fails(self, mock_client, patch_init_client): + """Convey: recursive 空目录 put_object 失败 —— 记录 err 不中断""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/sub/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.put_object.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "PUT", 403, + ) + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", "recursive": True}, + MOCK_GLOBALS, + ) + + mock_client.put_object.assert_called_once() + + def test_top_level_cos_error(self, mock_client, patch_init_client, capsys): + """Convey: recursive 顶层 list_objects 抛 CosServiceError""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "GET", 403, + ) + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", "recursive": True}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out + + +# ============================================================= +# download_object 补充分支 +# ============================================================= + +class TestDownloadObjectMoreBranches: + + def test_recursive_include_filters_file_and_dir(self, mock_client, patch_init_client, tmp_path): + """Convey: include 同时过滤普通文件和 COS 上的空目录标记""" + mod = patch_init_client("download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/excluded_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "recursive": True, + "include": "*.txt"}, + MOCK_GLOBALS, + ) + + assert mock_client.download_file.call_count == 1 + # excluded_dir 被过滤 → 不创建本地目录 + assert not (tmp_path / "excluded_dir").exists() + + def test_recursive_pagination(self, mock_client, patch_init_client, tmp_path): + """Convey: recursive 分页""" + mod = patch_init_client("download_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "true", "NextMarker": "pre/a.txt"}, + {"Contents": [{"Key": "pre/b.txt", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + ] + + mod.download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "recursive": True}, + MOCK_GLOBALS, + ) + + assert mock_client.download_file.call_count == 2 + + def test_recursive_retry_exhausted_logs_err(self, mock_client, patch_init_client, tmp_path): + """Convey: recursive 单文件重试耗尽 → 记录 err，整体继续""" + mod = patch_init_client("download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.download_file.side_effect = make_cos_error( + "InternalError", "mock", "r1", "GET", 500, + ) + + mod.download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "recursive": True, "retry": 1}, + MOCK_GLOBALS, + ) + + # retry=1 -> 共 2 次尝试 + assert mock_client.download_file.call_count == 2 + + def test_single_head_fails_file_size_zero(self, mock_client, patch_init_client, tmp_path): + """Convey: 单文件下载 head_object 失败 —— file_size 回退为 0，下载仍成功""" + mod = patch_init_client("download_object", mock_client) + mock_client.head_object.side_effect = RuntimeError("boom") + + mod.download_object( + {"bucket": "b", "cos_key": "k", + "local_path": str(tmp_path / "dst.txt")}, + MOCK_GLOBALS, + ) + + mock_client.download_file.assert_called_once() + + def test_top_level_generic_exception(self, mock_client, patch_init_client, tmp_path, capsys): + """Convey: 顶层非 CosServiceError 异常 —— Error: 打印""" + mod = patch_init_client("download_object", mock_client) + mock_client.list_objects.side_effect = RuntimeError("boom") + + mod.download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "recursive": True}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out + + +# ============================================================= +# delete_object 补充分支 +# ============================================================= + +class TestDeleteObjectMoreBranches: + + def test_recursive_dir_excluded_by_include(self, mock_client, patch_init_client): + """Convey: 目录对象被 include 过滤 → skip""" + mod = patch_init_client("delete_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/excluded_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", + "recursive": True, "force": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # 文件 a.txt 通过 delete_objects；目录 excluded_dir 被过滤 → delete_object 不调用 + mock_client.delete_object.assert_not_called() + + def test_recursive_dir_delete_retry(self, mock_client, patch_init_client): + """Convey: 目录逐个 delete 的重试后成功""" + mod = patch_init_client("delete_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/sub/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + mock_client.delete_object.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "DELETE", 500), + {}, + ] + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", + "recursive": True, "force": True, "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.delete_object.call_count == 2 + + def test_recursive_dir_delete_retry_exhausted(self, mock_client, patch_init_client): + """Convey: 目录 delete 重试耗尽 → 记录 err，不抛异常""" + mod = patch_init_client("delete_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/sub/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + mock_client.delete_object.side_effect = make_cos_error( + "InternalError", "mock", "r1", "DELETE", 500, + ) + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", + "recursive": True, "force": True, "retry": 1}, + MOCK_GLOBALS, + ) + + # retry=1 -> 共 2 次尝试 + assert mock_client.delete_object.call_count == 2 + + def test_recursive_confirm_eof(self, mock_client, patch_init_client, monkeypatch, capsys): + """Convey: input 抛 EOFError —— 取消删除""" + mod = patch_init_client("delete_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + + def _raise_eof(*_a, **_k): + raise EOFError() + + monkeypatch.setattr("builtins.input", _raise_eof) + + mod.delete_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True, "force": False}, + MOCK_GLOBALS, + ) + + mock_client.delete_objects.assert_not_called() + assert "已取消删除" in capsys.readouterr().out + + +# ============================================================= +# upload_object 补充分支 +# ============================================================= + +class TestUploadObjectMoreBranches: + + def test_recursive_empty_dir_put_fails(self, mock_client, patch_init_client, tmp_path): + """Convey: recursive 空目录 put_object 失败 —— 记录 err""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "empty").mkdir() + mock_client.put_object.side_effect = make_cos_error( + "AccessDenied", "denied", "r1", "PUT", 403, + ) + + mod.upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path) + "/", "recursive": True}, + MOCK_GLOBALS, + ) + + mock_client.put_object.assert_called() + + def test_recursive_include_filter_skips_file(self, mock_client, patch_init_client, tmp_path): + """Convey: recursive include 过滤跳过文件""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + (tmp_path / "b.log").write_text("b") + + mod.upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path) + "/", + "recursive": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + assert mock_client.upload_file.call_count == 1 + assert mock_client.upload_file.call_args.kwargs["Key"] == "pre/a.txt" + + def test_recursive_empty_dir_match_filter(self, mock_client, patch_init_client, tmp_path): + """Convey: recursive 空目录被 exclude 过滤跳过""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "keep_dir").mkdir() + (tmp_path / "excluded_dir").mkdir() + + mod.upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path) + "/", + "recursive": True, "exclude": "excluded_dir"}, + MOCK_GLOBALS, + ) + + # keep_dir 对应 put_object 被调用，excluded_dir 不被调用 + keys = [c.kwargs["Key"] for c in mock_client.put_object.call_args_list] + assert any("keep_dir" in k for k in keys) + assert not any("excluded_dir" in k for k in keys) + + def test_recursive_retry_exhausted(self, mock_client, patch_init_client, tmp_path): + """Convey: recursive 重试耗尽 —— 记录 err，整体不抛异常""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + mock_client.upload_file.side_effect = make_cos_error( + "InternalError", "mock", "r1", "PUT", 500, + ) + + mod.upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path) + "/", + "recursive": True, "retry": 1}, + MOCK_GLOBALS, + ) + + # retry=1 -> 2 次尝试 + assert mock_client.upload_file.call_count == 2 + + def test_cos_prefix_without_trailing_slash(self, mock_client, patch_init_client, tmp_path): + """Convey: cos_prefix 不以 / 结尾时的 key 拼接分支""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + + mod.upload_object( + {"bucket": "b", "cos_key": "prefix", # 不以 / 结尾 + "local_path": str(tmp_path) + "/", + "recursive": True}, + MOCK_GLOBALS, + ) + + # 因为 local_path 以 / 结尾，所以不保留顶层目录名；cos_key "prefix" 内部会按 + # 分支 "cos_prefix + '/' + rel_path" 拼接 + keys = [c.kwargs["Key"] for c in mock_client.upload_file.call_args_list] + assert keys == ["prefix/a.txt"] + + +# ============================================================= +# utils.TransferProgressMonitor —— 进度条 tick 打印 +# ============================================================= + +class TestProgressBarTick: + + def test_progress_bar_triggers_tick_print(self, tmp_path): + """走一下 _progress_loop 让 _print_progress_bar 实际被触发打印（不只默认构造）。""" + m = _utils.TransferProgressMonitor("upload") + # 缩短 tick 间隔，确保 _print_progress_bar 能被调用到打印分支 + m._tick_duration = 0.01 + m.set_scan_info(1, 100) + m.start() + # 模拟进度前进 + cb, fid = m.create_progress_callback(100) + cb(50, 100) + # 稍微等待 tick + time.sleep(0.05) + cb(100, 100) + m.update_ok(100, fid) + m.stop() + + def test_progress_bar_scan_not_finished(self, tmp_path): + """scan_end=False 时（未调用 set_scan_info）会走另一个分支""" + m = _utils.TransferProgressMonitor("copy") + m._tick_duration = 0.01 + m.start() + m.update_ok(10) # 不设置 scan_info + time.sleep(0.05) + m.stop() + + def test_finish_bar_with_total_size_but_err(self): + """_print_finish_bar：有 total_size 且存在 err_num → 走 percent 分支""" + m = _utils.TransferProgressMonitor("download") + m.set_scan_info(2, 200) + m.update_ok(100) + m.update_err(path="/x", reason="fail") + m._print_finish_bar() # 直接调用确保覆盖分支 diff --git a/tccli/plugins/cos/tests/test_final_branches.py b/tccli/plugins/cos/tests/test_final_branches.py new file mode 100644 index 0000000000..43f5227986 --- /dev/null +++ b/tccli/plugins/cos/tests/test_final_branches.py @@ -0,0 +1,532 @@ +# -*- coding: utf-8 -*- +""" +最终覆盖率补齐 —— 针对剩余 89-94% 覆盖率的命令文件： +move_object / sync_copy_object / sync_download_object / sync_upload_object / +delete_bucket / lsparts / abort_multipart / copy_object（剩余分支）等。 +""" +import os + +from conftest import MOCK_GLOBALS, make_cos_error + + +# ============================================================= +# move_object —— single 分支补全 +# ============================================================= + +class TestMoveObjectMoreBranches: + + def test_single_head_fails_file_size_zero(self, mock_client, patch_init_client): + """Convey: 单文件 move 时 head_object 失败 → file_size=0，move 继续""" + mod = patch_init_client("move_object", mock_client) + mock_client.head_object.side_effect = RuntimeError("boom") + + mod.move_object( + {"bucket": "b", "cos_key": "a", "dest_key": "b"}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_called_once() + mock_client.delete_object.assert_called_once() + + def test_single_storage_class_passed(self, mock_client, patch_init_client): + """Convey: 单文件 move 透传 storage_class""" + mod = patch_init_client("move_object", mock_client) + + mod.move_object( + {"bucket": "b", "cos_key": "a", "dest_key": "b", + "storage_class": "ARCHIVE"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["StorageClass"] == "ARCHIVE" + + def test_recursive_empty_dir_excluded(self, mock_client, patch_init_client): + """Convey: recursive 空目录被 include/exclude 过滤""" + mod = patch_init_client("move_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/excluded_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.move_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # 空目录被跳过 → put_object 不被调用 + mock_client.put_object.assert_not_called() + + def test_recursive_retry_exhausted_logs_err(self, mock_client, patch_init_client): + """Convey: recursive 子文件 move 重试耗尽 → 记录 err""" + mod = patch_init_client("move_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.copy.side_effect = make_cos_error( + "InternalError", "mock", "r1", "PUT", 500, + ) + + mod.move_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True, "retry": 1}, + MOCK_GLOBALS, + ) + + # retry=1 → 2 次尝试；copy 全失败 → delete_object 不被调用 + assert mock_client.copy.call_count == 2 + + +# ============================================================= +# sync_copy_object 剩余分支 +# ============================================================= + +class TestSyncCopyObjectMoreBranches: + + def test_dir_excluded_by_filter(self, mock_client, patch_init_client): + """Convey: 源端空目录被 include/exclude 过滤跳过""" + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = [ + # 源端 + {"Contents": [ + {"Key": "p/keep_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "p/excluded_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "p/b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], "IsTruncated": "false"}, + # 目标端 + {"Contents": [], "IsTruncated": "false"}, + ] + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/", + "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # include=*.txt → 只有 a.txt 被复制； + # excluded_dir 和 b.log 都被 include 过滤 + assert mock_client.copy.call_count == 1 + assert mock_client.copy.call_args.kwargs["Key"] == "d/a.txt" + + def test_retry_exhausted_logs_err(self, mock_client, patch_init_client): + """Convey: copy 失败 —— 记录 err，顶层不抛异常""" + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + {"Contents": [], "IsTruncated": "false"}, + ] + mock_client.copy.side_effect = make_cos_error( + "InternalError", "mock", "r1", "PUT", 500, + ) + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/", "retry": 0}, + MOCK_GLOBALS, + ) + + # retry=0 → 1 次尝试 + assert mock_client.copy.call_count == 1 + + def test_delete_extra_dir(self, mock_client, patch_init_client): + """Convey: --delete 删除目标端多余的目录对象""" + mod = patch_init_client("sync_copy_object", mock_client) + + src_listing = { + "Contents": [{"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + dest_listing_no_dir = { + "Contents": [ + {"Key": "d/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + # list_all_objects_with_dirs(目标-for delete) 包含多余目录 + dest_with_dirs = { + "Contents": [ + {"Key": "d/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "d/orphan_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + mock_client.list_objects.side_effect = [ + src_listing, dest_listing_no_dir, dest_with_dirs, + ] + # 源目标 CRC 不同 → 仍会复制 a.txt + mock_client.head_object.return_value = {"x-cos-hash-crc64ecma": "0"} + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/", "delete": True}, + MOCK_GLOBALS, + ) + + delete_keys = [c.kwargs["Key"] for c in mock_client.delete_object.call_args_list] + assert "d/orphan_dir/" in delete_keys + + +# ============================================================= +# sync_download_object 剩余分支 +# ============================================================= + +class TestSyncDownloadObjectMoreBranches: + + def test_local_path_auto_created(self, mock_client, patch_init_client, tmp_path): + """Convey: local_path 不存在时自动 makedirs""" + mod = patch_init_client("sync_download_object", mock_client) + target = tmp_path / "new_root" / "subdir" # 不存在 + assert not target.exists() + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(target)}, + MOCK_GLOBALS, + ) + + # 目录被创建 + assert target.is_dir() + + def test_ensure_dir_creates_parent(self, mock_client, patch_init_client, tmp_path): + """Convey: _ensure_dir 分支 —— 下载文件的父目录不存在时自动创建""" + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/sub1/sub2/deep.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + # sub1/sub2 会在 _ensure_dir 中被创建 + assert (tmp_path / "sub1" / "sub2").is_dir() + + def test_retry_exhausted_logs_err(self, mock_client, patch_init_client, tmp_path): + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.download_file.side_effect = make_cos_error( + "InternalError", "mock", "r1", "GET", 500, + ) + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "retry": 0}, + MOCK_GLOBALS, + ) + + assert mock_client.download_file.call_count == 1 + + +# ============================================================= +# sync_upload_object 剩余分支（39, 59, 63, 79-80, 136-137, 157） +# ============================================================= + +class TestSyncUploadObjectMoreBranches: + + def test_local_path_not_dir_returns_early(self, mock_client, patch_init_client, capsys): + """Convey: local_path 是文件而非目录 → 立即 return（已在 test_sync_objects.py 覆盖）""" + # 此处仅走顶层 retry=None 分支：设置 retry=None 走默认 3 + mod = patch_init_client("sync_upload_object", mock_client) + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": "/tmp/tccli-cos-sync-upload-not-exist", + "retry": None}, + MOCK_GLOBALS, + ) + + assert "本地路径不是目录" in capsys.readouterr().out + + def test_local_dir_empty_dir_key_exists_skip(self, mock_client, patch_init_client, tmp_path): + """Convey: 根目录恰为空（rel_dir == "."）—— 走 cos_prefix 尾带 / 的分支""" + mod = patch_init_client("sync_upload_object", mock_client) + # tmp_path 本身是个空目录（rel_dir == "."） + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + # 会尝试创建 pre/ 这个空目录标记（因为根目录是空的） + mock_client.put_object.assert_called() + kwargs = mock_client.put_object.call_args.kwargs + assert kwargs["Key"] == "pre/" + + def test_file_excluded_by_filter(self, mock_client, patch_init_client, tmp_path): + """Convey: 本地文件被 include/exclude 过滤跳过""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + (tmp_path / "b.log").write_text("b") + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "include": "*.txt"}, + MOCK_GLOBALS, + ) + + keys = [c.kwargs["Key"] for c in mock_client.upload_file.call_args_list] + assert keys == ["pre/a.txt"] + + def test_retry_exhausted_logs_err(self, mock_client, patch_init_client, tmp_path): + """Convey: sync_upload 单文件重试耗尽 → 记录 err""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + mock_client.upload_file.side_effect = make_cos_error( + "InternalError", "mock", "r1", "PUT", 500, + ) + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path), "retry": 1}, + MOCK_GLOBALS, + ) + + # retry=1 → 2 次尝试 + assert mock_client.upload_file.call_count == 2 + + +# ============================================================= +# delete_bucket 剩余分支（59, 80, 87, 99-100, 123, 127, 139） +# ============================================================= + +class TestDeleteBucketMoreBranches: + + def test_force_versions_single_version_dict(self, mock_client, patch_init_client): + """Convey: list_objects_versions 返回单个 Version dict（非 list），需要 wrap 成 list""" + mod = patch_init_client("delete_bucket", mock_client) + + mock_client.list_objects.return_value = {"Contents": [], "IsTruncated": "false"} + mock_client.list_objects_versions.return_value = { + "Version": {"Key": "a", "VersionId": "v1"}, # 单对象，非 list + "DeleteMarker": {"Key": "b", "VersionId": "v2"}, # 单对象，非 list + "IsTruncated": "false", + } + mock_client.list_multipart_uploads.return_value = { + "Upload": {"Key": "u1", "UploadId": "up1"}, # 单对象，非 list + "IsTruncated": "false", + } + + mod.delete_bucket({"bucket": "b", "force": True}, MOCK_GLOBALS) + + # 版本 + 删除标记 → 1 次 delete_objects + mock_client.delete_objects.assert_called_once() + # 分片 abort 1 次 + mock_client.abort_multipart_upload.assert_called_once() + + +# ============================================================= +# abort_multipart / lsparts 剩余分支 +# ============================================================= + +class TestAbortMultipartMoreBranches: + + def test_uploads_as_dict(self, mock_client, patch_init_client): + """Convey: list_multipart_uploads 返回单个 Upload dict""" + mod = patch_init_client("abort_multipart", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": {"Key": "a", "UploadId": "u1", "Initiated": "t"}, + "IsTruncated": "false", + } + + mod.abort_multipart({"bucket": "b"}, MOCK_GLOBALS) + + mock_client.abort_multipart_upload.assert_called_once() + + +class TestLspartsMoreBranches: + + def test_uploads_as_dict(self, mock_client, patch_init_client, capsys): + """Convey: list_multipart_uploads 返回单个 Upload dict""" + mod = patch_init_client("lsparts_object", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": {"Key": "a", "UploadId": "u1", "Initiated": "t"}, + "IsTruncated": "false", + } + + mod.lsparts_object({"bucket": "b"}, MOCK_GLOBALS) + + assert "u1" in capsys.readouterr().out + + +# ============================================================= +# copy_object 剩余分支（30, 58-59, 138, 155, 178, 180-181） +# ============================================================= + +class TestCopyObjectRemaining: + + def test_single_dest_bucket_defaults_when_none(self, mock_client, patch_init_client): + """Convey: dest_bucket = None 时，回落为 src bucket（覆盖 or 分支）""" + mod = patch_init_client("copy_object", mock_client) + + # dest_bucket 显式传 None，走 `or bucket` 分支 + mod.copy_object( + {"bucket": "b-125", "cos_key": "a", "dest_bucket": None, "dest_key": "d"}, + MOCK_GLOBALS, + ) + + assert mock_client.copy.call_args.kwargs["Bucket"] == "b-125" + + def test_single_head_fails_file_size_zero(self, mock_client, patch_init_client): + """Convey: 单文件 copy 时 head_object 失败 → file_size=0""" + mod = patch_init_client("copy_object", mock_client) + mock_client.head_object.side_effect = RuntimeError("boom") + + mod.copy_object( + {"bucket": "b", "cos_key": "a", "dest_key": "d"}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_called_once() + + def test_recursive_storage_class_and_metadata_combined(self, mock_client, patch_init_client): + """Convey: recursive + storage_class + metadata （走到 MetadataDirective 分支）""" + mod = patch_init_client("copy_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + + mod.copy_object( + {"bucket": "src", "cos_key": "pre/", + "dest_bucket": "dst", "dest_key": "d/", + "recursive": True, + "storage_class": "ARCHIVE", "meta": "a=1"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.copy.call_args.kwargs + assert kwargs["StorageClass"] == "ARCHIVE" + assert kwargs["Metadata"] == {"x-cos-meta-a": "1"} + assert kwargs["MetadataDirective"] == "Replaced" + + +# ============================================================= +# head_object / list_object / list_buckets / du / restore / acl 剩余一点点分支 +# ============================================================= + +class TestHeadMoreBranches: + + def test_restore_header(self, mock_client, patch_init_client, capsys): + """Convey: head_object 返回 x-cos-restore 时输出""" + mod = patch_init_client("head_object", mock_client) + mock_client.head_object.return_value = { + "Content-Length": "10", + "x-cos-restore": 'ongoing-request="true"', + } + + mod.head_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "Restore:" in capsys.readouterr().out + + +class TestTaggingMoreBranches: + + def test_put_returns_early_on_invalid_format(self, mock_client, patch_init_client, capsys): + """put_object_tagging：tags 只含 `invalid-without-equal` —— 打印 Error 并 return（tagging_object.py 第 25 行）""" + mod = patch_init_client("tagging_object", mock_client) + + mod.put_object_tagging( + {"bucket": "b", "cos_key": "k", "tags": "only-one"}, + MOCK_GLOBALS, + ) + + # 标签格式错误 → 立即 return，不调用 put_object_tagging + mock_client.put_object_tagging.assert_not_called() + assert "Error:" in capsys.readouterr().out + + +class TestRestoreMoreBranches: + + def test_recursive_no_match(self, mock_client, patch_init_client, capsys): + """Convey: 递归只有非归档对象 —— 全部 skip，统计数为 0""" + mod = patch_init_client("restore_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a", "Size": "1", "StorageClass": "STANDARD"}, + ], + "IsTruncated": "false", + } + + mod.restore_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True}, + MOCK_GLOBALS, + ) + + mock_client.restore_object.assert_not_called() + out = capsys.readouterr().out + assert "跳过 1" in out + + +class TestDuMoreBranches: + + def test_pagination_next_marker(self, mock_client, patch_init_client, capsys): + """分页 NextMarker 分支""" + mod = patch_init_client("du_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "a", "Size": "10", "StorageClass": "STANDARD"}], + "IsTruncated": "true", "NextMarker": "a"}, + {"Contents": [], "IsTruncated": "false"}, + ] + + mod.du_object({"bucket": "b"}, MOCK_GLOBALS) + + assert mock_client.list_objects.call_count == 2 + + +class TestAclMoreBranches: + + def test_get_bucket_acl_grant_is_single_dict(self, mock_client, patch_init_client, capsys): + """Convey: Grant 为单个 dict（非 list）""" + mod = patch_init_client("acl_object", mock_client) + mock_client.get_bucket_acl.return_value = { + "Owner": {"ID": "o", "DisplayName": "d"}, + "AccessControlList": {"Grant": { + "Grantee": {"ID": "u1", "type": "CanonicalUser"}, + "Permission": "READ", + }}, + } + + mod.get_bucket_acl({"bucket": "b"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "u1" in out and "READ" in out diff --git a/tccli/plugins/cos/tests/test_head_object.py b/tccli/plugins/cos/tests/test_head_object.py new file mode 100644 index 0000000000..75ffd7e1e2 --- /dev/null +++ b/tccli/plugins/cos/tests/test_head_object.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" +head_object 单测 —— 对齐 coscli cmd/stat_test.go 的打桩风格。 +""" +from conftest import MOCK_GLOBALS, cos_module, make_cos_error + + +class TestHeadObject: + + def test_success(self, mock_client, patch_init_client, capsys): + """Convey: stat success""" + mod = patch_init_client("head_object", mock_client) + mock_client.head_object.return_value = { + "Content-Length": "2048", + "Content-Type": "application/octet-stream", + "ETag": '"etag-xyz"', + "Last-Modified": "Mon, 07 Apr 2026 06:00:00 GMT", + "x-cos-storage-class": "STANDARD_IA", + "x-cos-hash-crc64ecma": "1234567890", + "x-cos-version-id": "v-001", + "x-cos-meta-author": "alice", + } + + mod.head_object( + {"bucket": "test-bucket-1250000000", "cos_key": "path/to/file.txt"}, + MOCK_GLOBALS, + ) + + mock_client.head_object.assert_called_once() + kwargs = mock_client.head_object.call_args.kwargs + assert kwargs["Bucket"] == "test-bucket-1250000000" + assert kwargs["Key"] == "path/to/file.txt" + assert "VersionId" not in kwargs + + out = capsys.readouterr().out + assert "Content-Length" in out + assert "2048" in out + assert "STANDARD_IA" in out + assert "CRC64" in out + assert "1234567890" in out + assert "v-001" in out + assert "x-cos-meta-author" in out + assert "alice" in out + + def test_with_version_id(self, mock_client, patch_init_client): + """Convey: version-id 参数透传""" + mod = patch_init_client("head_object", mock_client) + + mod.head_object( + {"bucket": "b-1250000000", "cos_key": "k", "version_id": "v-abc"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.head_object.call_args.kwargs + assert kwargs["VersionId"] == "v-abc" + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + """Convey: stat NoSuchKey error —— 命令吞掉异常并打印 Error""" + mod = patch_init_client("head_object", mock_client) + mock_client.head_object.side_effect = make_cos_error( + code="NoSuchKey", msg="Object not found", req_id="req-123", + method="GET", status_code=404, + ) + + mod.head_object( + {"bucket": "b-1250000000", "cos_key": "not-exist.txt"}, + MOCK_GLOBALS, + ) + + out = capsys.readouterr().out + assert "Error:" in out + assert "NoSuchKey" in out + assert "Code:" in out + assert "RequestId" in out + assert "req-123" in out + + def test_init_client_error(self, monkeypatch): + """Convey: CreateClient error —— init_cos_client 异常被允许抛出到上层框架处理。""" + mod = cos_module("head_object") + def _raise(_pg): + raise Exception("mock CreateClient error") + monkeypatch.setattr(mod, "init_cos_client", _raise) + + # head_object 未显式捕获 init 异常，允许其抛出（等价于 coscli 的 error 返回） + try: + mod.head_object({"bucket": "b-1250000000", "cos_key": "k"}, MOCK_GLOBALS) + assert False, "expect exception" + except Exception as e: + assert "mock CreateClient error" in str(e) \ No newline at end of file diff --git a/tccli/plugins/cos/tests/test_list_object.py b/tccli/plugins/cos/tests/test_list_object.py new file mode 100644 index 0000000000..4c110da7b8 --- /dev/null +++ b/tccli/plugins/cos/tests/test_list_object.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +""" +list_object 单测 —— 对齐 coscli cmd/ls_test.go。 +""" +from conftest import MOCK_GLOBALS, make_cos_error + + +class TestListObject: + + def test_success_non_recursive(self, mock_client, patch_init_client, capsys): + """Convey: list objects success（非递归，带 CommonPrefixes 目录）""" + mod = patch_init_client("list_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "a.txt", "Size": "100", + "LastModified": "Mon, 07 Apr 2026 06:00:00 GMT", + "StorageClass": "STANDARD", "ETag": '"e1"'}, + {"Key": "b.log", "Size": "200", + "LastModified": "Mon, 07 Apr 2026 06:00:01 GMT", + "StorageClass": "STANDARD_IA", "ETag": '"e2"'}, + ], + "CommonPrefixes": [{"Prefix": "sub/"}], + "IsTruncated": "false", + } + + mod.list_object({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + kwargs = mock_client.list_objects.call_args.kwargs + # 非递归且未指定 delimiter 时默认使用 / + assert kwargs["Delimiter"] == "/" + assert kwargs["Bucket"] == "b-1250000000" + + out = capsys.readouterr().out + assert "DIR" in out and "sub/" in out + assert "a.txt" in out and "b.log" in out + assert "共 2 个对象" in out + + def test_recursive_disables_delimiter(self, mock_client, patch_init_client): + """Convey: recursive 模式下 delimiter 被强制置空""" + mod = patch_init_client("list_object", mock_client) + + mod.list_object( + {"bucket": "b-1250000000", "prefix": "pre/", "recursive": True}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.list_objects.call_args.kwargs + assert kwargs["Delimiter"] == "" + assert kwargs["Prefix"] == "pre/" + + def test_include_exclude_filter(self, mock_client, patch_init_client, capsys): + """Convey: include/exclude 过滤生效（纯逻辑 match_filters 不 mock）""" + mod = patch_init_client("list_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.list_object( + {"bucket": "b-1250000000", "include": "*.txt", "recursive": True}, + MOCK_GLOBALS, + ) + + out = capsys.readouterr().out + assert "a.txt" in out + assert "b.log" not in out + assert "共 1 个对象" in out + + def test_pagination_recursive(self, mock_client, patch_init_client): + """Convey: 分页在 recursive 模式下自动翻页直到 IsTruncated=false""" + mod = patch_init_client("list_object", mock_client) + mock_client.list_objects.side_effect = [ + {"Contents": [{"Key": "p1.txt", "Size": "1", "LastModified": "", + "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "true", "NextMarker": "p1.txt"}, + {"Contents": [{"Key": "p2.txt", "Size": "1", "LastModified": "", + "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + ] + + mod.list_object( + {"bucket": "b-1250000000", "recursive": True}, + MOCK_GLOBALS, + ) + + assert mock_client.list_objects.call_count == 2 + # 第二次请求的 Marker 应来自上一页的 NextMarker + assert mock_client.list_objects.call_args_list[1].kwargs["Marker"] == "p1.txt" + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + """Convey: list objects error""" + mod = patch_init_client("list_object", mock_client) + mock_client.list_objects.side_effect = make_cos_error( + "AccessDenied", "access denied", "req-1", "GET", 403, + ) + + mod.list_object({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "Error:" in out + assert "AccessDenied" in out + assert "req-1" in out diff --git a/tccli/plugins/cos/tests/test_simple_ops.py b/tccli/plugins/cos/tests/test_simple_ops.py new file mode 100644 index 0000000000..d6ed67a8d4 --- /dev/null +++ b/tccli/plugins/cos/tests/test_simple_ops.py @@ -0,0 +1,567 @@ +# -*- coding: utf-8 -*- +""" +signurl / acl / tagging / du / cat / hash / lsparts / abort / restore 单测 +—— 对齐 coscli cmd/signurl_test.go / object_acl_test.go / bucket_acl_test.go + / object_tagging_test.go / du_test.go / cat_test.go / hash_test.go + / lsparts_test.go / abort_test.go / restore_test.go。 + +这些命令都是 "简单查询/操作" 类：直接调用 SDK 后同步输出，无并发。 +""" +from conftest import MOCK_GLOBALS, make_cos_error + + +# ========== signurl ========== + +class TestSignurl: + + def test_get_default(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("signurl_object", mock_client) + mock_client.get_presigned_download_url.return_value = "https://signed-get" + + mod.signurl_object( + {"bucket": "b-1250000000", "cos_key": "k.txt"}, + MOCK_GLOBALS, + ) + + mock_client.get_presigned_download_url.assert_called_once() + kwargs = mock_client.get_presigned_download_url.call_args.kwargs + assert kwargs["Bucket"] == "b-1250000000" + assert kwargs["Key"] == "k.txt" + assert kwargs["Expired"] == 3600 # 默认 + assert "https://signed-get" in capsys.readouterr().out + + def test_put_method(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("signurl_object", mock_client) + mock_client.get_presigned_url.return_value = "https://signed-put" + + mod.signurl_object( + {"bucket": "b-1250000000", "cos_key": "k", + "method": "put", "expired": 60}, + MOCK_GLOBALS, + ) + + mock_client.get_presigned_url.assert_called_once() + kwargs = mock_client.get_presigned_url.call_args.kwargs + assert kwargs["Method"] == "PUT" + assert kwargs["Expired"] == 60 + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("signurl_object", mock_client) + mock_client.get_presigned_download_url.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "GET", 403, + ) + + mod.signurl_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "Error:" in capsys.readouterr().out + + +# ========== bucket_acl / object_acl ========== + +class TestAclObject: + + def _acl_body(self): + return { + "Owner": {"ID": "owner-id", "DisplayName": "owner"}, + "AccessControlList": {"Grant": [ + {"Grantee": {"ID": "u1", "type": "CanonicalUser"}, + "Permission": "FULL_CONTROL"}, + ]}, + } + + def test_get_bucket_acl_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + mock_client.get_bucket_acl.return_value = self._acl_body() + + mod.get_bucket_acl({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "存储桶 ACL" in out and "owner-id" in out and "u1" in out + + def test_get_bucket_acl_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + mock_client.get_bucket_acl.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "GET", 403, + ) + mod.get_bucket_acl({"bucket": "b-1250000000"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + def test_put_bucket_acl_with_grants(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + + mod.put_bucket_acl( + {"bucket": "b-1250000000", "acl": "public-read", + "grant_read": 'id="1001"', "grant_full_control": 'id="1002"'}, + MOCK_GLOBALS, + ) + + mock_client.put_bucket_acl.assert_called_once() + kwargs = mock_client.put_bucket_acl.call_args.kwargs + assert kwargs["ACL"] == "public-read" + assert kwargs["GrantRead"] == 'id="1001"' + assert kwargs["GrantFullControl"] == 'id="1002"' + assert "存储桶 ACL 设置成功" in capsys.readouterr().out + + def test_put_bucket_acl_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + mock_client.put_bucket_acl.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "PUT", 403, + ) + mod.put_bucket_acl({"bucket": "b-1250000000"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + def test_get_object_acl_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + mock_client.get_object_acl.return_value = self._acl_body() + + mod.get_object_acl({"bucket": "b-1250000000", "cos_key": "k"}, MOCK_GLOBALS) + + assert "对象 ACL" in capsys.readouterr().out + + def test_get_object_acl_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + mock_client.get_object_acl.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "GET", 404, + ) + mod.get_object_acl({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + def test_put_object_acl(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + + mod.put_object_acl( + {"bucket": "b-1250000000", "cos_key": "k", "acl": "private"}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.put_object_acl.call_args.kwargs + assert kwargs["Key"] == "k" + assert kwargs["ACL"] == "private" + assert "对象 ACL 设置成功" in capsys.readouterr().out + + def test_put_object_acl_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("acl_object", mock_client) + mock_client.put_object_acl.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "PUT", 403, + ) + mod.put_object_acl({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + +# ========== tagging 三元组 ========== + +class TestObjectTagging: + + def test_get_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + mock_client.get_object_tagging.return_value = { + "TagSet": {"Tag": [ + {"Key": "env", "Value": "prod"}, + {"Key": "team", "Value": "ops"}, + ]} + } + + mod.get_object_tagging({"bucket": "b-1250000000", "cos_key": "k"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "env = prod" in out + assert "team = ops" in out + + def test_get_empty(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + mock_client.get_object_tagging.return_value = {"TagSet": {"Tag": []}} + + mod.get_object_tagging({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "(无标签)" in capsys.readouterr().out + + def test_get_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + mock_client.get_object_tagging.side_effect = make_cos_error( + "NoSuchTagSet", "no tag set", "req-1", "GET", 404, + ) + mod.get_object_tagging({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + def test_put_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + + mod.put_object_tagging( + {"bucket": "b-1250000000", "cos_key": "k", "tags": "env=prod,team=ops"}, + MOCK_GLOBALS, + ) + + mock_client.put_object_tagging.assert_called_once() + kwargs = mock_client.put_object_tagging.call_args.kwargs + assert kwargs["Tagging"] == { + "TagSet": {"Tag": [ + {"Key": "env", "Value": "prod"}, + {"Key": "team", "Value": "ops"}, + ]} + } + assert "对象标签设置成功" in capsys.readouterr().out + + def test_put_invalid_format(self, mock_client, patch_init_client, capsys): + """Convey: invalid tag 格式""" + mod = patch_init_client("tagging_object", mock_client) + + mod.put_object_tagging( + {"bucket": "b", "cos_key": "k", "tags": "invalid-without-equal"}, + MOCK_GLOBALS, + ) + + mock_client.put_object_tagging.assert_not_called() + assert "Error" in capsys.readouterr().out + + def test_put_empty_tags(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + + mod.put_object_tagging({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + mock_client.put_object_tagging.assert_not_called() + assert "Error" in capsys.readouterr().out + + def test_put_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + mock_client.put_object_tagging.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "PUT", 403, + ) + mod.put_object_tagging( + {"bucket": "b", "cos_key": "k", "tags": "a=b"}, MOCK_GLOBALS, + ) + assert "Error:" in capsys.readouterr().out + + def test_delete_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + + mod.delete_object_tagging({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + mock_client.delete_object_tagging.assert_called_once() + assert "对象标签删除成功" in capsys.readouterr().out + + def test_delete_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("tagging_object", mock_client) + mock_client.delete_object_tagging.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "DELETE", 403, + ) + mod.delete_object_tagging({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + +# ========== du ========== + +class TestDuObject: + + def test_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("du_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "a.txt", "Size": "100", "StorageClass": "STANDARD"}, + {"Key": "b.txt", "Size": "200", "StorageClass": "STANDARD_IA"}, + {"Key": "c.txt", "Size": "300", "StorageClass": "STANDARD"}, + {"Key": "dir/", "Size": "0", "StorageClass": "STANDARD"}, + ], + "IsTruncated": "false", + } + + mod.du_object({"bucket": "b-1250000000", "prefix": "p/"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "总文件数: 3" in out + assert "总文件夹数: 1" in out + # 100+200+300=600 字节 + assert "600 字节" in out + assert "STANDARD" in out + assert "STANDARD_IA" in out + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("du_object", mock_client) + mock_client.list_objects.side_effect = make_cos_error( + "NoSuchBucket", "no such", "req-1", "GET", 404, + ) + mod.du_object({"bucket": "b"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + +# ========== cat ========== + +class TestCatObject: + + def test_success_text(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("cat_object", mock_client) + mock_client.head_object.return_value = {"Content-Length": "5"} + + body_stream = mock_client.get_object.return_value["Body"] + body_stream.get_raw_stream.return_value.read.return_value = b"hello" + + mod.cat_object({"bucket": "b-1250000000", "cos_key": "k"}, MOCK_GLOBALS) + + assert "hello" in capsys.readouterr().out + # 小文件不带 Range + assert "Range" not in mock_client.get_object.call_args.kwargs + + def test_big_file_truncated(self, mock_client, patch_init_client, capsys): + """文件超过 max_size 时自动加 Range""" + mod = patch_init_client("cat_object", mock_client) + mock_client.head_object.return_value = { + "Content-Length": str(20 * 1024 * 1024), # 20 MB + } + + mod.cat_object( + {"bucket": "b", "cos_key": "k", "max_size": 10}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.get_object.call_args.kwargs + assert kwargs["Range"].startswith("bytes=0-") + out = capsys.readouterr().out + assert "仅显示前 10MB" in out + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("cat_object", mock_client) + mock_client.head_object.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "GET", 404, + ) + mod.cat_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + +# ========== hash ========== + +class TestHashObject: + + def test_local_md5(self, mock_client, patch_init_client, capsys, tmp_path): + mod = patch_init_client("hash_object", mock_client) + f = tmp_path / "a.txt" + f.write_text("hello") + # "hello" 的 MD5 = 5d41402abc4b2a76b9719d911017c592 + + mod.hash_object({"local_path": str(f), "hash_type": "md5"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "MD5" in out + assert "5d41402abc4b2a76b9719d911017c592" in out + + def test_local_sha256(self, patch_init_client, mock_client, capsys, tmp_path): + mod = patch_init_client("hash_object", mock_client) + f = tmp_path / "a.txt" + f.write_bytes(b"") # 空文件 + + mod.hash_object({"local_path": str(f), "hash_type": "sha256"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + # 空文件 sha256 + assert "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" in out + + def test_local_path_not_exist(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("hash_object", mock_client) + + mod.hash_object( + {"local_path": "/tmp/tccli-cos-test-nonexistent.txt"}, + MOCK_GLOBALS, + ) + + assert "本地文件不存在" in capsys.readouterr().out + + def test_cos_object_hash(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("hash_object", mock_client) + mock_client.head_object.return_value = { + "ETag": '"md5-abc"', + "x-cos-hash-crc64ecma": "123456", + "Content-Length": "100", + } + + mod.hash_object( + {"bucket": "b-1250000000", "cos_key": "k"}, + MOCK_GLOBALS, + ) + + out = capsys.readouterr().out + assert "md5-abc" in out + assert "123456" in out + + def test_cos_hash_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("hash_object", mock_client) + mock_client.head_object.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "GET", 404, + ) + mod.hash_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + def test_nothing_to_hash(self, mock_client, patch_init_client, capsys): + """没有指定 local_path 也没指定 bucket+cos_key""" + mod = patch_init_client("hash_object", mock_client) + + mod.hash_object({}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "Error" in out + + +# ========== lsparts ========== + +class TestLsparts: + + def test_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("lsparts_object", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": [ + {"Key": "big.bin", "UploadId": "upid-1", + "Initiated": "Mon, 07 Apr 2026 06:00:00 GMT"}, + {"Key": "big2.bin", "UploadId": "upid-2", + "Initiated": "Mon, 07 Apr 2026 06:00:01 GMT"}, + ], + "IsTruncated": "false", + } + + mod.lsparts_object({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "upid-1" in out and "upid-2" in out + assert "共 2 个未完成的分片上传" in out + + def test_empty(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("lsparts_object", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": [], "IsTruncated": "false", + } + mod.lsparts_object({"bucket": "b"}, MOCK_GLOBALS) + assert "没有未完成的分片上传" in capsys.readouterr().out + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("lsparts_object", mock_client) + mock_client.list_multipart_uploads.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "GET", 403, + ) + mod.lsparts_object({"bucket": "b"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + +# ========== abort ========== + +class TestAbortMultipart: + + def test_abort_with_upload_id(self, mock_client, patch_init_client, capsys): + """指定 upload_id + cos_key""" + mod = patch_init_client("abort_multipart", mock_client) + + mod.abort_multipart( + {"bucket": "b-1250000000", "cos_key": "k", "upload_id": "upid-1"}, + MOCK_GLOBALS, + ) + + mock_client.abort_multipart_upload.assert_called_once() + kwargs = mock_client.abort_multipart_upload.call_args.kwargs + assert kwargs["UploadId"] == "upid-1" + assert kwargs["Key"] == "k" + assert "已取消分片上传" in capsys.readouterr().out + + def test_abort_upload_id_missing_cos_key(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("abort_multipart", mock_client) + + mod.abort_multipart( + {"bucket": "b", "upload_id": "upid-1"}, MOCK_GLOBALS, + ) + + mock_client.abort_multipart_upload.assert_not_called() + assert "必须同时指定 cos_key" in capsys.readouterr().out + + def test_abort_all(self, mock_client, patch_init_client, capsys): + """无 upload_id —— 列出并全部清理""" + mod = patch_init_client("abort_multipart", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": [ + {"Key": "a", "UploadId": "up1", "Initiated": "t1"}, + {"Key": "b", "UploadId": "up2", "Initiated": "t2"}, + ], + "IsTruncated": "false", + } + + mod.abort_multipart({"bucket": "b-1250000000"}, MOCK_GLOBALS) + + assert mock_client.abort_multipart_upload.call_count == 2 + out = capsys.readouterr().out + assert "共清理 2 个" in out + + def test_abort_none(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("abort_multipart", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": [], "IsTruncated": "false", + } + + mod.abort_multipart({"bucket": "b"}, MOCK_GLOBALS) + + mock_client.abort_multipart_upload.assert_not_called() + assert "没有未完成的分片上传" in capsys.readouterr().out + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("abort_multipart", mock_client) + mock_client.list_multipart_uploads.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "GET", 403, + ) + mod.abort_multipart({"bucket": "b"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + +# ========== restore ========== + +class TestRestoreObject: + + def test_single_success(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("restore_object", mock_client) + + mod.restore_object( + {"bucket": "b-1250000000", "cos_key": "k", + "days": 5, "tier": "Expedited"}, + MOCK_GLOBALS, + ) + + mock_client.restore_object.assert_called_once() + kwargs = mock_client.restore_object.call_args.kwargs + assert kwargs["RestoreRequest"]["Days"] == 5 + assert kwargs["RestoreRequest"]["CASJobParameters"]["Tier"] == "Expedited" + out = capsys.readouterr().out + assert "恢复请求已提交" in out + assert "恢复天数: 5" in out + assert "Expedited" in out + + def test_single_already_in_progress(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("restore_object", mock_client) + mock_client.restore_object.side_effect = make_cos_error( + "RestoreAlreadyInProgress", "already in progress", "req-1", "POST", 409, + ) + + mod.restore_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + + assert "恢复进行中" in capsys.readouterr().out + + def test_single_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("restore_object", mock_client) + mock_client.restore_object.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "POST", 403, + ) + mod.restore_object({"bucket": "b", "cos_key": "k"}, MOCK_GLOBALS) + assert "Error:" in capsys.readouterr().out + + def test_recursive_only_archive(self, mock_client, patch_init_client, capsys): + """递归恢复只处理 ARCHIVE / DEEP_ARCHIVE""" + mod = patch_init_client("restore_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", "StorageClass": "STANDARD"}, + {"Key": "pre/b.txt", "Size": "2", "StorageClass": "ARCHIVE"}, + {"Key": "pre/c.txt", "Size": "3", "StorageClass": "DEEP_ARCHIVE"}, + {"Key": "pre/dir/", "Size": "0", "StorageClass": "STANDARD"}, + ], + "IsTruncated": "false", + } + + mod.restore_object( + {"bucket": "b", "cos_key": "pre/", "recursive": True}, + MOCK_GLOBALS, + ) + + # 只有 b.txt / c.txt 调用了 restore_object + assert mock_client.restore_object.call_count == 2 + out = capsys.readouterr().out + assert "提交 2" in out diff --git a/tccli/plugins/cos/tests/test_sync_objects.py b/tccli/plugins/cos/tests/test_sync_objects.py new file mode 100644 index 0000000000..3c4acbfcba --- /dev/null +++ b/tccli/plugins/cos/tests/test_sync_objects.py @@ -0,0 +1,392 @@ +# -*- coding: utf-8 -*- +""" +sync_upload / sync_download / sync_copy 单测 +—— 对齐 coscli cmd/sync_test.go 的 Upload / Download / CosCopy 分支。 + +重点覆盖跳过逻辑（对齐 coscli sync）： + 1) 默认：CRC64（x-cos-hash-crc64ecma）相等则跳过 + 2) --ignore-existing：目标存在即跳过 + 3) --update：Last-Modified 时间比较 + +以及 --delete（删除目标端多余）。 +""" +import os + +from tccli.plugins.cos.utils import calculate_local_crc64 + +from conftest import MOCK_GLOBALS, make_cos_error + + +# ======================================================== +# sync_upload +# ======================================================== + +class TestSyncUpload: + + def _setup_empty_cos(self, mock_client): + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + + def test_local_path_not_directory(self, mock_client, patch_init_client, capsys): + """本地路径必须是目录""" + mod = patch_init_client("sync_upload_object", mock_client) + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": "/tmp/tccli-cos-sync-upload-not-exist"}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_not_called() + assert "本地路径不是目录" in capsys.readouterr().out + + def test_sync_empty_target_uploads_all(self, mock_client, patch_init_client, tmp_path): + """COS 端为空 —— 全部上传""" + mod = patch_init_client("sync_upload_object", mock_client) + self._setup_empty_cos(mock_client) + (tmp_path / "a.txt").write_text("a") + (tmp_path / "b.txt").write_text("bb") + + mod.sync_upload_object( + {"bucket": "b-1250000000", "cos_key": "pre/", + "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + assert mock_client.upload_file.call_count == 2 + + def test_skip_by_crc64(self, mock_client, patch_init_client, tmp_path): + """Convey: 默认跳过规则 —— 本地 CRC64 与 COS x-cos-hash-crc64ecma 相等则跳过""" + mod = patch_init_client("sync_upload_object", mock_client) + local = tmp_path / "a.txt" + local.write_text("hello") + + local_crc = calculate_local_crc64(str(local)) + # 必须有 CRC64 环境（crcmod 可用），否则跳过逻辑会 fallback 为"不跳过" + assert local_crc is not None, "单测环境缺少 crcmod，请先安装" + + # list_objects 列出对应 key + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "5", + "LastModified": "Mon, 07 Apr 2026 06:00:00 GMT", + "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + # should_skip_sync_upload 内部会 head_object + mock_client.head_object.return_value = { + "Content-Length": "5", + "x-cos-hash-crc64ecma": local_crc, + "Last-Modified": "Mon, 07 Apr 2026 06:00:00 GMT", + } + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_not_called() + + def test_skip_by_ignore_existing(self, mock_client, patch_init_client, tmp_path): + """Convey: --ignore-existing 目标存在即跳过""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "a.txt").write_text("x") + + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.head_object.return_value = {"Content-Length": "1"} + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "ignore_existing": True}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_not_called() + + def test_skip_by_update(self, mock_client, patch_init_client, tmp_path): + """Convey: --update 目标比本地新则跳过""" + mod = patch_init_client("sync_upload_object", mock_client) + local = tmp_path / "a.txt" + local.write_text("x") + # 把本地 mtime 设为一个很早的时间戳（2020 年） + old_ts = 1577836800.0 # 2020-01-01 00:00:00 UTC + os.utime(str(local), (old_ts, old_ts)) + + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "Sun, 07 Apr 2030 06:00:00 GMT", + "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.head_object.return_value = { + "Content-Length": "1", + "Last-Modified": "Sun, 07 Apr 2030 06:00:00 GMT", # 远比本地新 + } + + mod.sync_upload_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "update": True}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_not_called() + + def test_delete_extra(self, mock_client, patch_init_client, tmp_path): + """Convey: --delete 删除 COS 上多余的对象""" + mod = patch_init_client("sync_upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + + # 第一次 list_objects（list_all_objects，不含目录）返回 orphan + a.txt + # 第二次 list_objects（list_all_objects_with_dirs，含目录）同样返回这两个 + listing = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/orphan.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + mock_client.list_objects.return_value = listing + # 目标存在 + CRC 不同 → 不跳过，仍需上传 + mock_client.head_object.return_value = { + "Content-Length": "1", "x-cos-hash-crc64ecma": "0", + } + + mod.sync_upload_object( + {"bucket": "b-1250000000", "cos_key": "pre/", + "local_path": str(tmp_path), "delete": True}, + MOCK_GLOBALS, + ) + + # orphan.log 应被删除 + delete_keys = [c.kwargs["Key"] for c in mock_client.delete_object.call_args_list] + assert "pre/orphan.log" in delete_keys + + +# ======================================================== +# sync_download +# ======================================================== + +class TestSyncDownload: + + def test_sync_download_all(self, mock_client, patch_init_client, tmp_path): + """COS 有文件，本地为空 —— 全部下载""" + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "Mon, 07 Apr 2026 06:00:00 GMT", + "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.log", "Size": "2", + "LastModified": "Mon, 07 Apr 2026 06:00:01 GMT", + "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + assert mock_client.download_file.call_count == 2 + + def test_skip_by_ignore_existing(self, mock_client, patch_init_client, tmp_path): + """--ignore-existing：本地存在即跳过""" + mod = patch_init_client("sync_download_object", mock_client) + (tmp_path / "a.txt").write_text("x") # 本地已存在 + + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "1", + "LastModified": "Mon, 07 Apr 2026 06:00:00 GMT", + "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "ignore_existing": True}, + MOCK_GLOBALS, + ) + + mock_client.download_file.assert_not_called() + + def test_skip_by_crc64(self, mock_client, patch_init_client, tmp_path): + """默认：本地 CRC64 == COS x-cos-hash-crc64ecma → 跳过""" + mod = patch_init_client("sync_download_object", mock_client) + local = tmp_path / "a.txt" + local.write_text("hello") + local_crc = calculate_local_crc64(str(local)) + assert local_crc is not None + + mock_client.list_objects.return_value = { + "Contents": [{"Key": "pre/a.txt", "Size": "5", + "LastModified": "Mon, 07 Apr 2026 06:00:00 GMT", + "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false", + } + mock_client.head_object.return_value = { + "x-cos-hash-crc64ecma": local_crc, + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + mock_client.download_file.assert_not_called() + + def test_delete_extra_local(self, mock_client, patch_init_client, tmp_path): + """--delete：删除本地多余文件""" + mod = patch_init_client("sync_download_object", mock_client) + # 本地 orphan 存在但 COS 没有 + (tmp_path / "orphan.txt").write_text("x") + + mock_client.list_objects.return_value = { + "Contents": [], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "delete": True}, + MOCK_GLOBALS, + ) + + assert not os.path.exists(tmp_path / "orphan.txt") + + def test_sdk_error(self, mock_client, patch_init_client, tmp_path, capsys): + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "GET", 403, + ) + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out + + +# ======================================================== +# sync_copy +# ======================================================== + +class TestSyncCopy: + + def test_empty_src(self, mock_client, patch_init_client): + """源端为空 —— 什么也不做""" + mod = patch_init_client("sync_copy_object", mock_client) + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/"}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_not_called() + + def test_copy_all(self, mock_client, patch_init_client): + """源端有文件 → 目标端全复制""" + mod = patch_init_client("sync_copy_object", mock_client) + # list_all_objects_with_dirs（源） + list_all_objects（目标，空） + # 两次调用返回不同数据，用 side_effect 模拟 + mock_client.list_objects.side_effect = [ + # 源：含一个文件 + 一个目录 + {"Contents": [ + {"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "p/sub/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], "IsTruncated": "false"}, + # 目标：空 + {"Contents": [], "IsTruncated": "false"}, + ] + + mod.sync_copy_object( + {"bucket": "src-125", "cos_key": "p/", + "dest_bucket": "dst-125", "dest_key": "d/"}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_called_once() + copy_kwargs = mock_client.copy.call_args.kwargs + assert copy_kwargs["Bucket"] == "dst-125" + assert copy_kwargs["Key"] == "d/a.txt" + # 空目录在目标端 put_object 创建 + mock_client.put_object.assert_called() + + def test_skip_by_ignore_existing(self, mock_client, patch_init_client): + """目标存在 + --ignore-existing → 跳过""" + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = [ + # 源 + {"Contents": [{"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + # 目标（已有 d/a.txt） + {"Contents": [{"Key": "d/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"}, + ] + # should_skip_sync_copy 会 head_object 目标 + mock_client.head_object.return_value = {"Content-Length": "1"} + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/", + "ignore_existing": True}, + MOCK_GLOBALS, + ) + + mock_client.copy.assert_not_called() + + def test_delete_extra(self, mock_client, patch_init_client): + """--delete：删除目标端多余对象""" + mod = patch_init_client("sync_copy_object", mock_client) + + # list_all_objects_with_dirs(源) / list_all_objects(目标) / list_all_objects_with_dirs(目标 - for delete) + src = {"Contents": [ + {"Key": "p/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"} + dest = {"Contents": [ + {"Key": "d/orphan.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "d/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}], + "IsTruncated": "false"} + mock_client.list_objects.side_effect = [src, dest, dest] + # 目标 a.txt 与源 CRC 对不上 → 仍需复制；但 orphan 会被删 + mock_client.head_object.return_value = { + "Content-Length": "1", "x-cos-hash-crc64ecma": "0", + } + + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/", "delete": True}, + MOCK_GLOBALS, + ) + + delete_keys = [c.kwargs["Key"] for c in mock_client.delete_object.call_args_list] + assert "d/orphan.log" in delete_keys + + def test_sdk_error(self, mock_client, patch_init_client, capsys): + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = make_cos_error( + "AccessDenied", "denied", "req-1", "GET", 403, + ) + + mod.sync_copy_object( + {"bucket": "b", "cos_key": "p/", + "dest_bucket": "d", "dest_key": "d/"}, + MOCK_GLOBALS, + ) + + assert "Error:" in capsys.readouterr().out diff --git a/tccli/plugins/cos/tests/test_tail_branches.py b/tccli/plugins/cos/tests/test_tail_branches.py new file mode 100644 index 0000000000..32f73ff37f --- /dev/null +++ b/tccli/plugins/cos/tests/test_tail_branches.py @@ -0,0 +1,327 @@ +# -*- coding: utf-8 -*- +""" +收尾覆盖 —— 拉到 ≥98% 所需的最后几个分支。 +""" +import os + +from conftest import MOCK_GLOBALS, make_cos_error + + +# ============================================================= +# upload_object 的空 cos_prefix 分支 +# ============================================================= + +class TestUploadObjectEmptyCosPrefix: + + def test_recursive_empty_cos_prefix_file(self, mock_client, patch_init_client, tmp_path): + """Convey: cos_key 为空字符串 + recursive —— 普通文件 key 直接为 rel_path""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + + mod.upload_object( + {"bucket": "b", "cos_key": "", + "local_path": str(tmp_path) + "/", + "recursive": True}, + MOCK_GLOBALS, + ) + + keys = [c.kwargs["Key"] for c in mock_client.upload_file.call_args_list] + assert keys == ["a.txt"] + + def test_recursive_empty_cos_prefix_empty_dir(self, mock_client, patch_init_client, tmp_path): + """Convey: cos_key 为空字符串 + recursive —— 空目录 key = rel_dir + '/'""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "empty_dir").mkdir() + + mod.upload_object( + {"bucket": "b", "cos_key": "", + "local_path": str(tmp_path) + "/", + "recursive": True}, + MOCK_GLOBALS, + ) + + keys = [c.kwargs["Key"] for c in mock_client.put_object.call_args_list] + assert keys == ["empty_dir/"] + + +# ============================================================= +# sync_download_object 的 include/exclude 过滤 + 创建空目录 +# ============================================================= + +class TestSyncDownloadFilterAndCreate: + + def test_include_filter_for_file(self, mock_client, patch_init_client, tmp_path): + """Convey: COS 普通文件被 include 过滤掉""" + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/a.txt", "Size": "1", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + {"Key": "pre/b.log", "Size": "2", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path), + "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # 只 a.txt 被下载 + assert mock_client.download_file.call_count == 1 + + def test_creates_cos_empty_dir_locally(self, mock_client, patch_init_client, tmp_path): + """Convey: COS 上有 / 结尾空目录 + 本地不存在 → 本地创建""" + mod = patch_init_client("sync_download_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/empty_dir/", "Size": "0", + "LastModified": "", "StorageClass": "STANDARD", "ETag": '"e"'}, + ], + "IsTruncated": "false", + } + + mod.sync_download_object( + {"bucket": "b", "cos_key": "pre/", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + # 空目录被创建到本地 + assert (tmp_path / "empty_dir").is_dir() + + +# ============================================================= +# sync_copy_object 剩余：ignore_existing 为 True 但 dest 不存在（dest_head=None） +# ============================================================= + +class TestSyncCopyTopLevelExceptionOther: + + def test_generic_exception_not_captured_by_cos_error(self, mock_client, patch_init_client): + """Convey: 顶层非 CosServiceError 异常会让函数直接抛出（顶层只捕获 CosServiceError）""" + mod = patch_init_client("sync_copy_object", mock_client) + mock_client.list_objects.side_effect = RuntimeError("boom") + + # 不捕获 RuntimeError，函数会直接抛异常 + try: + mod.sync_copy_object( + {"bucket": "src", "cos_key": "p/", + "dest_bucket": "dst", "dest_key": "d/"}, + MOCK_GLOBALS, + ) + assert False, "expected exception" + except RuntimeError: + pass + + +# ============================================================= +# delete_bucket 剩余：多页 objects_versions 分页 +# ============================================================= + +class TestDeleteBucketVersionPagination: + + def test_version_pagination(self, mock_client, patch_init_client): + """Convey: list_objects_versions 分页（IsTruncated=true → false）""" + mod = patch_init_client("delete_bucket", mock_client) + + mock_client.list_objects.return_value = {"Contents": [], "IsTruncated": "false"} + mock_client.list_objects_versions.side_effect = [ + {"Version": [{"Key": "a", "VersionId": "v1"}], + "DeleteMarker": [], "IsTruncated": "true", + "NextKeyMarker": "a", "NextVersionIdMarker": "v1"}, + {"Version": [{"Key": "b", "VersionId": "v2"}], + "DeleteMarker": [], "IsTruncated": "false"}, + ] + mock_client.list_multipart_uploads.return_value = { + "Upload": [], "IsTruncated": "false", + } + + mod.delete_bucket({"bucket": "b", "force": True}, MOCK_GLOBALS) + + # 两次 delete_objects（两页版本） + assert mock_client.delete_objects.call_count == 2 + + +# ============================================================= +# lsparts 剩余：empty upload item 被跳过 +# ============================================================= + +class TestLspartsEmptyItem: + + def test_empty_upload_item_skipped(self, mock_client, patch_init_client, capsys): + """Convey: Upload 列表含空 item —— 代码用 `if not upload: continue` 防御，被跳过""" + mod = patch_init_client("lsparts_object", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": [ + None, + {"Key": "a", "UploadId": "u1", "Initiated": "t1"}, + ], + "IsTruncated": "false", + } + + mod.lsparts_object({"bucket": "b"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "u1" in out + # 只展示 1 条 + assert "共 1 个" in out + + +# ============================================================= +# list_buckets 单桶 dict 分支 +# ============================================================= + +class TestListBucketsSingleDict: + + def test_single_bucket_as_dict(self, mock_client, patch_init_client, capsys): + """Convey: Bucket 为单 dict（非 list）""" + mod = patch_init_client("list_buckets", mock_client) + mock_client.list_buckets.return_value = { + "Buckets": {"Bucket": { + "Name": "only-125", "Location": "ap-guangzhou", + "CreationDate": "2026-04-01T00:00:00Z", + }}, + } + + mod.list_buckets({}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "only-125" in out + assert "共 1 个存储桶" in out + + +# ============================================================= +# list_object non-recursive pagination break 之后的统计行 +# ============================================================= + +class TestListObjectTotalCount: + + def test_total_count_zero_no_output(self, mock_client, patch_init_client, capsys): + """Convey: 非 recursive 且无 Contents —— 不输出统计行""" + mod = patch_init_client("list_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [], "IsTruncated": "false", + } + + mod.list_object({"bucket": "b"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "共 " not in out # 不输出统计行（total_count 为 0 且非 recursive） + + +# ============================================================= +# du_object：子目录标记（以 / 结尾、Size=0）被单独计数 +# ============================================================= + +class TestDuObjectDirMarker: + + def test_dir_marker_counted(self, mock_client, patch_init_client, capsys): + """Convey: / 结尾的目录对象被统计为文件夹数（不计入总大小）""" + mod = patch_init_client("du_object", mock_client) + mock_client.list_objects.return_value = { + "Contents": [ + {"Key": "pre/dir/", "Size": "0", "StorageClass": "STANDARD"}, + {"Key": "pre/dir/a.txt", "Size": "100", "StorageClass": "STANDARD"}, + ], + "IsTruncated": "false", + } + + mod.du_object({"bucket": "b", "prefix": "pre/"}, MOCK_GLOBALS) + + out = capsys.readouterr().out + assert "总文件数: 1" in out + assert "总文件夹数: 1" in out + + +# ============================================================= +# abort_multipart 剩余：upload 中出现空条目被 continue +# ============================================================= + +class TestAbortMultipartEmptyItem: + + def test_empty_upload_item_continue(self, mock_client, patch_init_client): + """Convey: Upload 中含空条目 → continue，不调用 abort""" + mod = patch_init_client("abort_multipart", mock_client) + mock_client.list_multipart_uploads.return_value = { + "Upload": [None, {"Key": "a", "UploadId": "u1", "Initiated": "t"}], + "IsTruncated": "false", + } + + mod.abort_multipart({"bucket": "b"}, MOCK_GLOBALS) + + # 只 abort 非空条目 + assert mock_client.abort_multipart_upload.call_count == 1 + + +# ============================================================= +# acl_object：Grant list 中 grantee 含 ID 和 type 的两种分支 +# ============================================================= + +class TestAclTypeBranches: + + def test_put_object_acl_with_grant_read(self, mock_client, patch_init_client): + """Convey: put_object_acl 传 grant_read（显式走该分支）""" + mod = patch_init_client("acl_object", mock_client) + + mod.put_object_acl( + {"bucket": "b", "cos_key": "k", + "grant_read": 'id="u1"', + "grant_full_control": 'id="u2"'}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.put_object_acl.call_args.kwargs + assert kwargs["GrantRead"] == 'id="u1"' + assert kwargs["GrantFullControl"] == 'id="u2"' + + +# ============================================================= +# copy_object 最后几行：dest_region 默认值 +# ============================================================= + +class TestCopyDestRegionDefault: + + def test_dest_region_falls_back_to_src(self, mock_client, patch_init_client): + """Convey: dest_region=None → 走 `or region` 回落到 src region""" + mod = patch_init_client("copy_object", mock_client) + + mod.copy_object( + {"bucket": "b", "cos_key": "a", "dest_key": "d", + "dest_region": None}, + MOCK_GLOBALS, + ) + + # CopySource 的 Region 来自 src 的 ap-guangzhou + source = mock_client.copy.call_args.kwargs["CopySource"] + assert source["Region"] == "ap-guangzhou" + + +# ============================================================= +# hash_object 剩余 17 行：crcmod 未安装时返回错误信息 +# ============================================================= + +class TestHashObjectNoCrcmod: + + def test_crc64_without_crcmod(self, mock_client, patch_init_client, monkeypatch, capsys, tmp_path): + """Convey: 本地 crc64 但 crcmod 未安装 —— 打印错误提示""" + mod = patch_init_client("hash_object", mock_client) + f = tmp_path / "a.txt" + f.write_text("x") + + import builtins + real_import = builtins.__import__ + + def fake_import(name, *a, **kw): + if name == "crcmod": + raise ImportError("no crcmod") + return real_import(name, *a, **kw) + + monkeypatch.setattr(builtins, "__import__", fake_import) + + mod.hash_object( + {"local_path": str(f), "hash_type": "crc64"}, + MOCK_GLOBALS, + ) + + assert "crcmod" in capsys.readouterr().out diff --git a/tccli/plugins/cos/tests/test_upload_object.py b/tccli/plugins/cos/tests/test_upload_object.py new file mode 100644 index 0000000000..e334e49e70 --- /dev/null +++ b/tccli/plugins/cos/tests/test_upload_object.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +""" +upload_object 单测 —— 对齐 coscli cmd/cp_test.go 中 Upload 分支。 +""" +import os + +from conftest import MOCK_GLOBALS, make_cos_error + + +class TestUploadSingle: + + def test_success(self, mock_client, patch_init_client, tmp_path): + """Convey: Upload single file success""" + mod = patch_init_client("upload_object", mock_client) + f = tmp_path / "src.txt" + f.write_text("hello") + + mod.upload_object( + {"bucket": "b-1250000000", "cos_key": "dest.txt", "local_path": str(f)}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_called_once() + kwargs = mock_client.upload_file.call_args.kwargs + assert kwargs["Bucket"] == "b-1250000000" + assert kwargs["Key"] == "dest.txt" + assert kwargs["LocalFilePath"] == str(f) + assert kwargs["PartSize"] == 20 + assert kwargs["MAXThread"] == 5 + assert "progress_callback" in kwargs + + def test_storage_class_and_meta(self, mock_client, patch_init_client, tmp_path): + """storage_class / meta / content_type / rate_limiting 透传""" + mod = patch_init_client("upload_object", mock_client) + f = tmp_path / "src.txt" + f.write_text("hello") + + mod.upload_object( + {"bucket": "b", "cos_key": "k", "local_path": str(f), + "storage_class": "STANDARD_IA", "content_type": "text/plain", + "meta": "author=alice#version=1.0", + "rate_limiting": 10}, + MOCK_GLOBALS, + ) + + kwargs = mock_client.upload_file.call_args.kwargs + assert kwargs["StorageClass"] == "STANDARD_IA" + assert kwargs["ContentType"] == "text/plain" + assert kwargs["Metadata"] == { + "x-cos-meta-author": "alice", "x-cos-meta-version": "1.0", + } + # 10 MB/s = 10*1024*1024*8 bit/s + assert kwargs["TrafficLimit"] == str(10 * 1024 * 1024 * 8) + + def test_local_path_not_exist(self, mock_client, patch_init_client, capsys): + """Convey: Upload no local file —— 打印错误并直接 return""" + mod = patch_init_client("upload_object", mock_client) + + mod.upload_object( + {"bucket": "b", "cos_key": "k", + "local_path": "/tmp/tccli-cos-nonexistent-upload-src"}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_not_called() + assert "本地文件不存在" in capsys.readouterr().out + + def test_path_is_directory_without_recursive(self, mock_client, patch_init_client, capsys, tmp_path): + """本地是目录但未指定 recursive —— 报错""" + mod = patch_init_client("upload_object", mock_client) + + mod.upload_object( + {"bucket": "b", "cos_key": "k", "local_path": str(tmp_path)}, + MOCK_GLOBALS, + ) + + mock_client.upload_file.assert_not_called() + assert "指定路径不是文件" in capsys.readouterr().out + + def test_retry_then_success(self, mock_client, patch_init_client, tmp_path): + """Convey: err-retry-num —— 第一次失败、第二次成功""" + mod = patch_init_client("upload_object", mock_client) + f = tmp_path / "src.txt" + f.write_text("hello") + mock_client.upload_file.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "PUT", 500), + None, + ] + + mod.upload_object( + {"bucket": "b", "cos_key": "k", "local_path": str(f), "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.upload_file.call_count == 2 + + def test_retry_exhausted_raises(self, mock_client, patch_init_client, tmp_path, capsys): + """重试耗尽：_upload_single 会 raise，上层 upload_object 的 except 捕获并打印 Error""" + mod = patch_init_client("upload_object", mock_client) + f = tmp_path / "src.txt" + f.write_text("hello") + mock_client.upload_file.side_effect = make_cos_error( + "InternalError", "mock", "r1", "PUT", 500, + ) + + mod.upload_object( + {"bucket": "b", "cos_key": "k", "local_path": str(f), "retry": 1}, + MOCK_GLOBALS, + ) + + # retry=1 -> 共 2 次尝试 + assert mock_client.upload_file.call_count == 2 + assert "Error:" in capsys.readouterr().out + + +class TestUploadRecursive: + + def test_recursive_success(self, mock_client, patch_init_client, tmp_path): + """Convey: Upload directory success —— 两个文件都被上传""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "a.txt").write_text("aaa") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "b.log").write_text("bbb") + + mod.upload_object( + {"bucket": "b", "cos_key": "pre", + "local_path": str(tmp_path), "recursive": True}, + MOCK_GLOBALS, + ) + + # 2 个文件都被上传 + assert mock_client.upload_file.call_count == 2 + # cos_key 会根据目录名自动拼接：pre//... + keys = {c.kwargs["Key"] for c in mock_client.upload_file.call_args_list} + # tmp_path 最后一级目录名会被保留（local_path 不以 / 结尾） + dir_name = os.path.basename(str(tmp_path).rstrip(os.sep)) + assert ("pre/%s/a.txt" % dir_name) in keys + assert ("pre/%s/sub/b.log" % dir_name) in keys + + def test_recursive_include_exclude(self, mock_client, patch_init_client, tmp_path): + """include/exclude 过滤在 recursive 模式下生效""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + (tmp_path / "b.log").write_text("b") + + mod.upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path) + "/", + "recursive": True, "include": "*.txt"}, + MOCK_GLOBALS, + ) + + # 只有 a.txt 被上传 + assert mock_client.upload_file.call_count == 1 + assert mock_client.upload_file.call_args.kwargs["Key"] == "pre/a.txt" + + def test_recursive_empty_dir_creates_marker(self, mock_client, patch_init_client, tmp_path): + """空目录在 COS 上创建 / 结尾的空对象标记""" + mod = patch_init_client("upload_object", mock_client) + # 创建一个完全空的目录 + empty = tmp_path / "empty_dir" + empty.mkdir() + + mod.upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path) + "/", + "recursive": True}, + MOCK_GLOBALS, + ) + + # 没有文件可上传 + mock_client.upload_file.assert_not_called() + # 空目录被创建为 put_object（Key 以 / 结尾） + mock_client.put_object.assert_called() + keys = [c.kwargs["Key"] for c in mock_client.put_object.call_args_list] + assert any(k.endswith("empty_dir/") for k in keys) + + def test_recursive_file_retry(self, mock_client, patch_init_client, tmp_path): + """recursive 模式下单个文件重试逻辑（失败不中断整体）""" + mod = patch_init_client("upload_object", mock_client) + (tmp_path / "a.txt").write_text("a") + mock_client.upload_file.side_effect = [ + make_cos_error("InternalError", "mock", "r1", "PUT", 500), + None, + ] + + mod.upload_object( + {"bucket": "b", "cos_key": "pre/", + "local_path": str(tmp_path) + "/", + "recursive": True, "retry": 3}, + MOCK_GLOBALS, + ) + + assert mock_client.upload_file.call_count == 2 + + +class TestUploadErrorPaths: + + def test_init_client_fails_caught(self, monkeypatch, capsys): + """init_cos_client 抛异常 —— 命令允许异常抛出， + 但此处我们验证上层 try/except 吞掉运行期错误。""" + from conftest import cos_module + mod = cos_module("upload_object") + def _raise(_pg): + raise Exception("boom") + monkeypatch.setattr(mod, "init_cos_client", _raise) + + try: + mod.upload_object({"bucket": "b", "cos_key": "k", "local_path": "/x"}, + MOCK_GLOBALS) + # 若抛出到这里以上，下一行不会执行；若被吞则 OK + except Exception as e: + assert "boom" in str(e) diff --git a/tccli/plugins/cos/tests/test_utils.py b/tccli/plugins/cos/tests/test_utils.py new file mode 100644 index 0000000000..c61d234224 --- /dev/null +++ b/tccli/plugins/cos/tests/test_utils.py @@ -0,0 +1,696 @@ +# -*- coding: utf-8 -*- +""" +tccli.plugins.cos.utils 工具函数单测。 + +覆盖目标： +- parse_global_arg：凭据优先级（命令行 > 环境变量 > 配置文件）+ 校验失败 +- init_cos_client：成功构造 CosS3Client（mock qcloud_cos 类） +- format_size / match_filters / parse_meta / build_cos_key：全部分支 +- calculate_local_crc64 / parse_http_time / get_object_head +- should_skip_sync_upload / should_skip_sync_download / should_skip_sync_copy +- list_all_objects / list_all_objects_with_dirs / list_local_files +- TransferProgressMonitor：start / stop / update_ok / update_skip / update_err / callback / log_file +""" +import os +import json + +import pytest +from unittest.mock import MagicMock, patch + +from tccli.plugins.cos import utils as _utils + +from conftest import make_cos_error + + +# ============================================================= +# _load_json_file +# ============================================================= + +class TestLoadJsonFile: + + def test_ok(self, tmp_path): + f = tmp_path / "a.json" + f.write_text('{"k": 1}') + assert _utils._load_json_file(str(f)) == {"k": 1} + + def test_missing_file_returns_empty(self, tmp_path): + assert _utils._load_json_file(str(tmp_path / "not-exist.json")) == {} + + def test_invalid_json_returns_empty(self, tmp_path): + f = tmp_path / "a.json" + f.write_text("not-json") + assert _utils._load_json_file(str(f)) == {} + + +# ============================================================= +# parse_global_arg 凭据优先级 +# ============================================================= + +class TestParseGlobalArg: + + def _isolate_env(self, monkeypatch): + """隔离掉所有腾讯云相关环境变量，避免宿主机污染测试。""" + for k in ("TENCENTCLOUD_SECRET_ID", "TENCENTCLOUD_SECRET_KEY", + "TENCENTCLOUD_TOKEN", "TENCENTCLOUD_REGION", "TCCLI_PROFILE"): + monkeypatch.delenv(k, raising=False) + + def _isolate_home(self, monkeypatch, tmp_path): + """把 HOME 指到 tmp_path，确保 ~/.tccli/*.configure/credential 读不到真实文件。""" + monkeypatch.setenv("HOME", str(tmp_path)) + + def test_cli_arg_highest_priority(self, monkeypatch, tmp_path): + """命令行参数优先于环境变量与配置文件""" + self._isolate_env(monkeypatch) + self._isolate_home(monkeypatch, tmp_path) + monkeypatch.setenv("TENCENTCLOUD_SECRET_ID", "env-sid") + monkeypatch.setenv("TENCENTCLOUD_SECRET_KEY", "env-sk") + + pg = {"secretId": "cli-sid", "secretKey": "cli-sk", + "token": None, "region": None, "endpoint": None, "profile": None} + out = _utils.parse_global_arg(pg) + assert out["secretId"] == "cli-sid" + assert out["secretKey"] == "cli-sk" + assert out["profile"] == "default" + + def test_env_var_fallback(self, monkeypatch, tmp_path): + """未指定命令行参数时，使用环境变量""" + self._isolate_env(monkeypatch) + self._isolate_home(monkeypatch, tmp_path) + monkeypatch.setenv("TENCENTCLOUD_SECRET_ID", "env-sid") + monkeypatch.setenv("TENCENTCLOUD_SECRET_KEY", "env-sk") + monkeypatch.setenv("TENCENTCLOUD_TOKEN", "env-tk") + monkeypatch.setenv("TENCENTCLOUD_REGION", "ap-beijing") + + pg = {"secretId": None, "secretKey": None, + "token": None, "region": None, "endpoint": None, "profile": None} + out = _utils.parse_global_arg(pg) + assert out["secretId"] == "env-sid" + assert out["secretKey"] == "env-sk" + assert out["token"] == "env-tk" + assert out["region"] == "ap-beijing" + + def test_credential_file_fallback(self, monkeypatch, tmp_path): + """命令行 + 环境变量都没给时，读 ~/.tccli/.credential""" + self._isolate_env(monkeypatch) + self._isolate_home(monkeypatch, tmp_path) + tccli_dir = tmp_path / ".tccli" + tccli_dir.mkdir() + (tccli_dir / "default.credential").write_text(json.dumps({ + "secretId": "cred-sid", + "secretKey": "cred-sk", + "token": "cred-tk", + })) + (tccli_dir / "default.configure").write_text(json.dumps({ + "_sys_param": {"region": "ap-chengdu"}, + })) + + pg = {"secretId": None, "secretKey": None, + "token": None, "region": None, "endpoint": None, "profile": None} + out = _utils.parse_global_arg(pg) + assert out["secretId"] == "cred-sid" + assert out["secretKey"] == "cred-sk" + assert out["token"] == "cred-tk" + assert out["region"] == "ap-chengdu" + assert out["endpoint"] is None + + def test_profile_env_override(self, monkeypatch, tmp_path): + """TCCLI_PROFILE 环境变量能影响读取的配置文件""" + self._isolate_env(monkeypatch) + self._isolate_home(monkeypatch, tmp_path) + monkeypatch.setenv("TCCLI_PROFILE", "prod") + tccli_dir = tmp_path / ".tccli" + tccli_dir.mkdir() + (tccli_dir / "prod.credential").write_text(json.dumps({ + "secretId": "prod-sid", "secretKey": "prod-sk", + })) + + pg = {"secretId": None, "secretKey": None, + "token": None, "region": None, "endpoint": None, "profile": None} + out = _utils.parse_global_arg(pg) + assert out["profile"] == "prod" + assert out["secretId"] == "prod-sid" + + def test_missing_secret_id_raises(self, monkeypatch, tmp_path): + self._isolate_env(monkeypatch) + self._isolate_home(monkeypatch, tmp_path) + + with pytest.raises(Exception) as exc: + _utils.parse_global_arg({"secretId": None, "secretKey": None, + "token": None, "region": None, + "endpoint": None, "profile": None}) + assert "secretId 未配置" in str(exc.value) + + def test_missing_secret_key_raises(self, monkeypatch, tmp_path): + self._isolate_env(monkeypatch) + self._isolate_home(monkeypatch, tmp_path) + + with pytest.raises(Exception) as exc: + _utils.parse_global_arg({"secretId": "x", "secretKey": None, + "token": None, "region": None, + "endpoint": None, "profile": None}) + assert "secretKey 未配置" in str(exc.value) + + +# ============================================================= +# init_cos_client +# ============================================================= + +class TestInitCosClient: + + def test_success(self, monkeypatch): + """mock qcloud_cos.CosConfig + CosS3Client，验证 init_cos_client 的装配逻辑""" + fake_config_cls = MagicMock(return_value="fake-config-obj") + fake_client_cls = MagicMock(return_value="fake-client-obj") + monkeypatch.setattr("qcloud_cos.CosConfig", fake_config_cls) + monkeypatch.setattr("qcloud_cos.CosS3Client", fake_client_cls) + + pg = {"secretId": "sid", "secretKey": "sk", "token": "tk", + "region": None, "endpoint": None, "profile": "default"} + client, region = _utils.init_cos_client(pg) + + assert client == "fake-client-obj" + # region 为空时默认 ap-guangzhou + assert region == "ap-guangzhou" + fake_config_cls.assert_called_once_with( + Region="ap-guangzhou", SecretId="sid", SecretKey="sk", + Token="tk", Endpoint=None, + ) + fake_client_cls.assert_called_once_with("fake-config-obj") + + +# ============================================================= +# format_size 各阈值 +# ============================================================= + +class TestFormatSize: + @pytest.mark.parametrize("size,expected_substr", [ + (0, "0 B"), + (1023, "1023 B"), + (1024, "1.00 KB"), + (1024 * 1024, "1.00 MB"), + (1024 * 1024 * 1024, "1.00 GB"), + (1024 * 1024 * 1024 * 1024, "1.00 TB"), + ]) + def test_boundaries(self, size, expected_substr): + assert _utils.format_size(size) == expected_substr + + +# ============================================================= +# match_filters +# ============================================================= + +class TestMatchFilters: + @pytest.mark.parametrize("name,inc,exc,expected", [ + ("a.txt", "", "", True), + ("a.txt", "*.txt", "", True), + ("a.log", "*.txt", "", False), + ("a.txt", "", "*.log", True), + ("a.log", "", "*.log", False), + ("a.txt", "*.txt", "*.log", True), # include 过，且 exclude 不匹配 + ("a.log", "*.txt", "*.log", False), # include 不过 + ]) + def test_all_cases(self, name, inc, exc, expected): + assert _utils.match_filters(name, inc, exc) is expected + + +# ============================================================= +# parse_meta +# ============================================================= + +class TestParseMeta: + + def test_normal(self): + assert _utils.parse_meta("a=1#b=2") == { + "x-cos-meta-a": "1", "x-cos-meta-b": "2", + } + + def test_strip_spaces(self): + assert _utils.parse_meta(" a = 1 # b = 2 ") == { + "x-cos-meta-a": "1", "x-cos-meta-b": "2", + } + + def test_empty_and_invalid(self): + assert _utils.parse_meta("") == {} + # 不含 "="，会被忽略 + assert _utils.parse_meta("invalid#a=1") == {"x-cos-meta-a": "1"} + + +# ============================================================= +# build_cos_key +# ============================================================= + +class TestBuildCosKey: + @pytest.mark.parametrize("prefix,rel,expected", [ + ("", "dir/file.txt", "dir/file.txt"), + ("backup", "dir/file.txt", "backup/dir/file.txt"), + ("backup/", "dir/file.txt", "backup/dir/file.txt"), + ]) + def test_cases(self, prefix, rel, expected): + assert _utils.build_cos_key(prefix, rel) == expected + + +# ============================================================= +# calculate_local_crc64 +# ============================================================= + +class TestCalcLocalCrc64: + + def test_normal_file(self, tmp_path): + f = tmp_path / "a.txt" + f.write_text("hello") + crc = _utils.calculate_local_crc64(str(f)) + assert crc is not None + # 稳定值断言 + assert _utils.calculate_local_crc64(str(f)) == crc + + def test_missing_file_returns_none(self, tmp_path): + assert _utils.calculate_local_crc64(str(tmp_path / "nonexistent.bin")) is None + + def test_crcmod_not_installed(self, monkeypatch, tmp_path): + """模拟 crcmod 未安装 —— 导入失败时返回 None""" + import builtins + real_import = builtins.__import__ + + def fake_import(name, *a, **kw): + if name == "crcmod": + raise ImportError("no crcmod") + return real_import(name, *a, **kw) + + monkeypatch.setattr(builtins, "__import__", fake_import) + f = tmp_path / "a.txt" + f.write_text("x") + assert _utils.calculate_local_crc64(str(f)) is None + + +# ============================================================= +# get_object_head +# ============================================================= + +class TestGetObjectHead: + + def test_ok(self): + cli = MagicMock() + cli.head_object.return_value = {"Content-Length": "1"} + assert _utils.get_object_head(cli, "b", "k") == {"Content-Length": "1"} + + def test_cos_service_error_returns_none(self): + cli = MagicMock() + cli.head_object.side_effect = make_cos_error( + "NoSuchKey", "no key", "req-1", "HEAD", 404, + ) + assert _utils.get_object_head(cli, "b", "k") is None + + def test_generic_exception_returns_none(self): + cli = MagicMock() + cli.head_object.side_effect = RuntimeError("boom") + assert _utils.get_object_head(cli, "b", "k") is None + + +# ============================================================= +# parse_http_time +# ============================================================= + +class TestParseHttpTime: + + def test_empty(self): + assert _utils.parse_http_time("") is None + assert _utils.parse_http_time(None) is None + + def test_rfc1123(self): + ts = _utils.parse_http_time("Mon, 02 Jan 2006 15:04:05 GMT") + # 2006-01-02T15:04:05Z = 1136214245 + assert ts == 1136214245.0 + + def test_rfc3339(self): + ts = _utils.parse_http_time("2006-01-02T15:04:05Z") + assert ts == 1136214245.0 + + def test_invalid(self): + assert _utils.parse_http_time("not a time") is None + + +# ============================================================= +# should_skip_sync_upload / download / copy 全分支 +# ============================================================= + +class TestShouldSkipSyncUpload: + + def test_target_missing_no_skip(self): + cli = MagicMock() + cli.head_object.side_effect = make_cos_error( + "NoSuchKey", "no key", "r", "HEAD", 404, + ) + assert _utils.should_skip_sync_upload(cli, "b", "k", "/tmp/x", 0) is False + + def test_ignore_existing(self): + cli = MagicMock() + cli.head_object.return_value = {"Content-Length": "1"} + assert _utils.should_skip_sync_upload( + cli, "b", "k", "/tmp/x", 0, ignore_existing=True + ) is True + + def test_update_remote_newer(self): + cli = MagicMock() + cli.head_object.return_value = { + "Last-Modified": "Sun, 07 Apr 2030 06:00:00 GMT", + } + # local mtime = 2020-01-01 + assert _utils.should_skip_sync_upload( + cli, "b", "k", "/tmp/x", 1577836800.0, update=True, + ) is True + + def test_update_remote_older(self): + cli = MagicMock() + cli.head_object.return_value = { + "Last-Modified": "Sun, 07 Apr 2000 06:00:00 GMT", + } + assert _utils.should_skip_sync_upload( + cli, "b", "k", "/tmp/x", 1577836800.0, update=True, + ) is False + + def test_default_crc_match(self, tmp_path): + cli = MagicMock() + local = tmp_path / "a.txt" + local.write_text("hello") + crc = _utils.calculate_local_crc64(str(local)) + assert crc is not None + cli.head_object.return_value = {"x-cos-hash-crc64ecma": crc} + assert _utils.should_skip_sync_upload( + cli, "b", "k", str(local), 0, + ) is True + + def test_default_crc_mismatch(self, tmp_path): + cli = MagicMock() + local = tmp_path / "a.txt" + local.write_text("hello") + cli.head_object.return_value = {"x-cos-hash-crc64ecma": "999"} + assert _utils.should_skip_sync_upload( + cli, "b", "k", str(local), 0, + ) is False + + def test_default_remote_crc_empty(self, tmp_path): + cli = MagicMock() + local = tmp_path / "a.txt" + local.write_text("hello") + cli.head_object.return_value = {"x-cos-hash-crc64ecma": ""} + assert _utils.should_skip_sync_upload( + cli, "b", "k", str(local), 0, + ) is False + + +class TestShouldSkipSyncDownload: + + def test_local_missing_no_skip(self, tmp_path): + cli = MagicMock() + assert _utils.should_skip_sync_download( + cli, "b", "k", {"LastModified": ""}, + str(tmp_path / "missing.bin"), + ) is False + + def test_ignore_existing(self, tmp_path): + cli = MagicMock() + f = tmp_path / "a.txt" + f.write_text("x") + assert _utils.should_skip_sync_download( + cli, "b", "k", {"LastModified": ""}, str(f), ignore_existing=True, + ) is True + + def test_update_local_newer(self, tmp_path): + cli = MagicMock() + f = tmp_path / "a.txt" + f.write_text("x") + # 给个很新的本地 mtime + future = 4102444800.0 # 2100-01-01 + os.utime(str(f), (future, future)) + assert _utils.should_skip_sync_download( + cli, "b", "k", + {"LastModified": "Sun, 07 Apr 2000 06:00:00 GMT"}, + str(f), update=True, + ) is True + + def test_update_remote_newer(self, tmp_path): + cli = MagicMock() + f = tmp_path / "a.txt" + f.write_text("x") + old = 946684800.0 # 2000-01-01 + os.utime(str(f), (old, old)) + assert _utils.should_skip_sync_download( + cli, "b", "k", + {"LastModified": "Sun, 07 Apr 2030 06:00:00 GMT"}, + str(f), update=True, + ) is False + + def test_default_crc_match(self, tmp_path): + cli = MagicMock() + f = tmp_path / "a.txt" + f.write_text("hello") + crc = _utils.calculate_local_crc64(str(f)) + cli.head_object.return_value = {"x-cos-hash-crc64ecma": crc} + assert _utils.should_skip_sync_download( + cli, "b", "k", {}, str(f), + ) is True + + def test_default_remote_head_missing(self, tmp_path): + cli = MagicMock() + f = tmp_path / "a.txt" + f.write_text("hello") + # head_object 抛错 → get_object_head 返回 None → 不跳过 + cli.head_object.side_effect = make_cos_error( + "NoSuchKey", "no", "r", "HEAD", 404, + ) + assert _utils.should_skip_sync_download( + cli, "b", "k", {}, str(f), + ) is False + + def test_default_remote_crc_empty(self, tmp_path): + cli = MagicMock() + f = tmp_path / "a.txt" + f.write_text("hello") + cli.head_object.return_value = {"x-cos-hash-crc64ecma": ""} + assert _utils.should_skip_sync_download( + cli, "b", "k", {}, str(f), + ) is False + + +class TestShouldSkipSyncCopy: + + def test_dest_missing_no_skip(self): + cli = MagicMock() + cli.head_object.side_effect = make_cos_error( + "NoSuchKey", "no", "r", "HEAD", 404, + ) + assert _utils.should_skip_sync_copy(cli, "sb", "sk", "db", "dk") is False + + def test_ignore_existing(self): + cli = MagicMock() + cli.head_object.return_value = {"Content-Length": "1"} + assert _utils.should_skip_sync_copy( + cli, "sb", "sk", "db", "dk", ignore_existing=True, + ) is True + + def test_update_dest_newer(self): + cli = MagicMock() + # 先 dest_head，再 src_head + cli.head_object.side_effect = [ + {"Last-Modified": "Sun, 07 Apr 2030 06:00:00 GMT"}, # dest 新 + {"Last-Modified": "Sun, 07 Apr 2000 06:00:00 GMT"}, # src 旧 + ] + assert _utils.should_skip_sync_copy( + cli, "sb", "sk", "db", "dk", update=True, + ) is True + + def test_update_src_newer(self): + cli = MagicMock() + cli.head_object.side_effect = [ + {"Last-Modified": "Sun, 07 Apr 2000 06:00:00 GMT"}, # dest 旧 + {"Last-Modified": "Sun, 07 Apr 2030 06:00:00 GMT"}, # src 新 + ] + assert _utils.should_skip_sync_copy( + cli, "sb", "sk", "db", "dk", update=True, + ) is False + + def test_default_crc_match(self): + cli = MagicMock() + cli.head_object.side_effect = [ + {"x-cos-hash-crc64ecma": "12345"}, # dest + {"x-cos-hash-crc64ecma": "12345"}, # src + ] + assert _utils.should_skip_sync_copy( + cli, "sb", "sk", "db", "dk", + ) is True + + def test_default_crc_missing_on_either_side(self): + cli = MagicMock() + cli.head_object.side_effect = [ + {"x-cos-hash-crc64ecma": ""}, + {"x-cos-hash-crc64ecma": "12345"}, + ] + assert _utils.should_skip_sync_copy( + cli, "sb", "sk", "db", "dk", + ) is False + + def test_src_missing(self): + cli = MagicMock() + cli.head_object.side_effect = [ + {"x-cos-hash-crc64ecma": "1"}, # dest ok + make_cos_error("NoSuchKey", "no", "r", "HEAD", 404), # src missing + ] + assert _utils.should_skip_sync_copy( + cli, "sb", "sk", "db", "dk", + ) is False + + +# ============================================================= +# list_all_objects / list_all_objects_with_dirs / list_local_files +# ============================================================= + +class TestListAllObjects: + + def test_pagination_and_skip_dir_markers(self): + cli = MagicMock() + cli.list_objects.side_effect = [ + # 第一页：含目录标记 + 普通文件 + 截断 + {"Contents": [ + {"Key": "pre/dir/", "Size": "0"}, # 目录标记会被跳过 + {"Key": "pre/a.txt", "Size": "10", + "ETag": '"1"', "LastModified": "t1", + "StorageClass": "STANDARD"}, + ], "IsTruncated": "true", "NextMarker": "pre/a.txt"}, + # 第二页 + {"Contents": [ + {"Key": "pre/b.txt", "Size": "20", + "ETag": '"2"', "LastModified": "t2", + "StorageClass": "STANDARD_IA"}, + ], "IsTruncated": "false"}, + ] + + result = _utils.list_all_objects(cli, "b", "pre/") + assert set(result.keys()) == {"pre/a.txt", "pre/b.txt"} + assert result["pre/a.txt"]["Size"] == 10 + assert result["pre/b.txt"]["StorageClass"] == "STANDARD_IA" + assert cli.list_objects.call_count == 2 + + def test_with_dirs_keeps_markers(self): + cli = MagicMock() + cli.list_objects.return_value = { + "Contents": [ + {"Key": "pre/dir/", "Size": "0", + "ETag": '"0"', "LastModified": "t", + "StorageClass": "STANDARD"}, + {"Key": "pre/a.txt", "Size": "5", + "ETag": '"1"', "LastModified": "t", + "StorageClass": "STANDARD"}, + ], + "IsTruncated": "false", + } + result = _utils.list_all_objects_with_dirs(cli, "b", "pre/") + assert set(result.keys()) == {"pre/dir/", "pre/a.txt"} + assert result["pre/dir/"]["IsDir"] is True + assert result["pre/a.txt"]["IsDir"] is False + + +class TestListLocalFiles: + + def test_recursive_and_mtime(self, tmp_path): + (tmp_path / "a.txt").write_text("aaa") + sub = tmp_path / "sub" + sub.mkdir() + (sub / "b.log").write_text("bb") + + result = _utils.list_local_files(str(tmp_path)) + assert set(result.keys()) == {"a.txt", "sub/b.log"} + assert result["a.txt"]["Size"] == 3 + assert "FullPath" in result["a.txt"] + assert "MTime" in result["a.txt"] + + def test_empty_dir(self, tmp_path): + assert _utils.list_local_files(str(tmp_path)) == {} + + +# ============================================================= +# TransferProgressMonitor +# ============================================================= + +class TestTransferProgressMonitor: + + def test_happy_path_with_callback_and_log(self, tmp_path): + """完整走一遍：start → set_scan_info → ok + skip + err → stop → 写 log""" + m = _utils.TransferProgressMonitor("upload") + m.set_scan_info(3, 300) + m.start() + + # 成功（通过 callback 模拟进度更新） + cb, fid = m.create_progress_callback(100) + cb(50, 100) # 进行中 + cb(100, 100) # 完成 + m.update_ok(100, fid) + + # 跳过 + m.update_skip(100) + + # 失败 + fail_cb, fail_fid = m.create_progress_callback(100) + m.update_err( + fail_fid, + src_path="/local/a.txt", + dest_path="cos://b/a.txt", + reason="mock", + request_id="req-1", + ) + + log_path = tmp_path / "logs" / "fail.log" + m.stop(log_file=str(log_path)) + + # 失败日志被写出（目录会自动创建） + assert log_path.exists() + body = log_path.read_text(encoding="utf-8") + assert "Source : /local/a.txt" in body + assert "Dest : cos://b/a.txt" in body + assert "Reason : mock" in body + assert "RequestId : req-1" in body + + def test_update_ok_without_callback(self): + """不使用 progress_callback 时，update_ok 直接累加 transfer_size""" + m = _utils.TransferProgressMonitor("copy") + m.set_scan_info(1, 10) + m.update_ok(10) # 无 file_id + assert m.transfer_size == 10 + assert m.ok_num == 1 + + def test_update_err_legacy_path_kw(self): + """update_err 通过旧的 path= 兼容参数工作""" + m = _utils.TransferProgressMonitor("download") + m.update_err(path="/a", reason="x") + assert m.err_num == 1 + assert m._fail_records[0]["src_path"] == "/a" + + def test_write_log_file_err_path(self, monkeypatch, tmp_path): + """写日志失败（open 异常）时，应打印错误到 stderr，但不抛异常""" + m = _utils.TransferProgressMonitor("upload") + m.update_err(path="/a", reason="x") + + real_open = open + + def fake_open(path, *a, **kw): + if str(path).endswith("fail.log"): + raise IOError("disk full") + return real_open(path, *a, **kw) + + monkeypatch.setattr("builtins.open", fake_open) + + # 不应抛异常 + m._write_log_file(str(tmp_path / "fail.log")) + + def test_start_stop_without_callbacks(self): + """空 monitor 也能正常 start/stop""" + m = _utils.TransferProgressMonitor("delete") + m.set_scan_info(0, 0) + m.start() + m.stop() # log_file 为空 → 不写文件 + + def test_stop_empty_log_file_no_write(self, tmp_path): + """stop(log_file="") 时即使有失败记录也不写文件""" + m = _utils.TransferProgressMonitor("upload") + m.update_err(path="/a", reason="x") + m.stop(log_file="") + assert not (tmp_path / "any.log").exists() diff --git a/tccli/plugins/cos/upload_object.py b/tccli/plugins/cos/upload_object.py new file mode 100644 index 0000000000..d920f8095c --- /dev/null +++ b/tccli/plugins/cos/upload_object.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- +""" +upload 操作：上传本地文件到 COS +对齐 coscli cp (本地->COS) 命令 +- thread_num: 单文件分块上传并发线程数（传给 SDK 的 MAXThread） +- routines: 文件间并发数（同时上传的文件数） +""" +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from qcloud_cos import CosServiceError +from .utils import init_cos_client, match_filters, parse_meta, TransferProgressMonitor + + +def upload_object(args, parsed_globals): + """上传本地文件到 COS""" + client, region = init_cos_client(parsed_globals) + + bucket = args["bucket"] + local_path = args["local_path"] + cos_key = args["cos_key"] + storage_class = args.get("storage_class", "") or "" + content_type = args.get("content_type", "") or "" + meta = args.get("meta", "") or "" + recursive = args.get("recursive", False) + include = args.get("include", "") or "" + exclude = args.get("exclude", "") or "" + thread_num = args.get("thread_num", 5) or 5 + routines = args.get("routines", 3) or 3 + part_size = args.get("part_size", 20) or 20 + rate_limiting = args.get("rate_limiting", 0) or 0 + retry = args.get("retry", 3) + if retry is None: + retry = 3 + retry = int(retry) + log_file = args.get("log_file", "") or "" + + # 解析自定义元数据 + metadata = parse_meta(meta) + + try: + if recursive and os.path.isdir(local_path): + # 对齐 coscli cp 行为： + # - local_path 以 / 结尾（如 /tmp/dir/）：不保留目录名，直接映射内容 + # - local_path 不以 / 结尾（如 /tmp/dir）：保留目录名，映射为 cos_key/dir/ + if not local_path.endswith(os.sep) and not local_path.endswith("/"): + dir_name = os.path.basename(local_path.rstrip(os.sep)) + if cos_key: + cos_key = cos_key.rstrip("/") + "/" + dir_name + "/" + else: + cos_key = dir_name + "/" + _upload_directory(client, bucket, local_path, cos_key, include, exclude, + storage_class, content_type, metadata, thread_num, routines, + part_size, rate_limiting, retry, log_file) + else: + if not os.path.exists(local_path): + print("Error: 本地文件不存在: %s" % local_path) + return + if not os.path.isfile(local_path): + print("Error: 指定路径不是文件: %s（如需上传目录请使用 --recursive true）" % local_path) + return + + _upload_single(client, bucket, local_path, cos_key, + storage_class, content_type, metadata, thread_num, part_size, rate_limiting, retry, log_file) + + except CosServiceError as e: + print("Error: %s (Code: %s, RequestId: %s)" % ( + e.get_error_msg(), e.get_error_code(), e.get_request_id())) + except Exception as e: + print("Error: %s" % str(e)) + + +def _build_upload_kwargs(bucket, local_path, cos_key, storage_class, content_type, + metadata, thread_num, part_size, rate_limiting): + """构造 upload_file 的参数""" + kwargs = { + "Bucket": bucket, + "LocalFilePath": local_path, + "Key": cos_key, + "PartSize": part_size, + "MAXThread": thread_num, + } + if storage_class: + kwargs["StorageClass"] = storage_class + if content_type: + kwargs["ContentType"] = content_type + if metadata: + kwargs["Metadata"] = metadata + if rate_limiting: + kwargs["TrafficLimit"] = str(int(rate_limiting) * 1024 * 1024 * 8) + return kwargs + + +def _upload_single(client, bucket, local_path, cos_key, + storage_class, content_type, metadata, thread_num, part_size, rate_limiting, retry=3, log_file=""): + """上传单个文件（带进度监控）""" + monitor = TransferProgressMonitor("upload") + file_size = os.path.getsize(local_path) + monitor.set_scan_info(1, file_size) + monitor.start() + + progress_cb, file_id = monitor.create_progress_callback(file_size) + last_err = None + for attempt in range(max(1, retry + 1)): + try: + kwargs = _build_upload_kwargs(bucket, local_path, cos_key, storage_class, content_type, + metadata, thread_num, part_size, rate_limiting) + kwargs["progress_callback"] = progress_cb + client.upload_file(**kwargs) + monitor.update_ok(file_size, file_id) + last_err = None + break + except CosServiceError as e: + last_err = e + if attempt < retry: + # 重置该文件的进度，准备重试 + progress_cb, file_id = monitor.create_progress_callback(file_size) + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(file_id, + src_path=local_path, + dest_path="cos://%s/%s" % (bucket, cos_key), + reason=err_reason, + request_id=last_err.get_request_id()) + monitor.stop(log_file=log_file) + if last_err is not None: + raise last_err + + +def _upload_directory(client, bucket, local_dir, cos_prefix, include, exclude, + storage_class, content_type, metadata, thread_num, routines, + part_size, rate_limiting, retry=3, log_file=""): + """递归上传目录 + - thread_num: 单文件分块并发（传给 SDK MAXThread） + - routines: 文件间并发（同时上传的文件数） + """ + monitor = TransferProgressMonitor("upload") + monitor.start() + + # 先收集所有待上传的文件任务，同时统计总大小 + tasks = [] + empty_dir_keys = [] # 空目录对应的 COS key（以 / 结尾的空对象） + total_size = 0 + skip_count = 0 + for root, dirs, files in os.walk(local_dir): + # 检测空目录：当前目录下没有文件，且没有子目录（或子目录也都是空的） + # 简单判断：当前 root 下既无文件也无子目录，则为空目录 + if not files and not dirs: + rel_dir = os.path.relpath(root, local_dir).replace(os.sep, "/") + if rel_dir == ".": + # 根目录本身为空 + dir_key = cos_prefix.rstrip("/") + "/" if cos_prefix else "" + else: + # include/exclude 过滤目录 + if not match_filters(rel_dir, include, exclude): + skip_count += 1 + continue + if cos_prefix: + if cos_prefix.endswith("/"): + dir_key = cos_prefix + rel_dir + "/" + else: + dir_key = cos_prefix + "/" + rel_dir + "/" + else: + dir_key = rel_dir + "/" + if dir_key: + empty_dir_keys.append(dir_key) + + for filename in files: + full_path = os.path.join(root, filename) + rel_path = os.path.relpath(full_path, local_dir).replace(os.sep, "/") + + # include/exclude 过滤 + if not match_filters(rel_path, include, exclude): + skip_count += 1 + continue + + # 构造 COS key + if cos_prefix: + if cos_prefix.endswith("/"): + key = cos_prefix + rel_path + else: + key = cos_prefix + "/" + rel_path + else: + key = rel_path + + file_size = os.path.getsize(full_path) + total_size += file_size + tasks.append((full_path, key, file_size)) + + # 设置扫描结果（文件数 + 空目录数 + 跳过数） + monitor.set_scan_info(len(tasks) + len(empty_dir_keys) + skip_count, total_size) + for _ in range(skip_count): + monitor.update_skip(0) + + def _do_upload(full_path, key, file_size): + """单个文件上传任务（含重试）""" + last_err = None + progress_cb, file_id = monitor.create_progress_callback(file_size) + for attempt in range(max(1, retry + 1)): + try: + kwargs = _build_upload_kwargs(bucket, full_path, key, storage_class, content_type, + metadata, thread_num, part_size, rate_limiting) + kwargs["progress_callback"] = progress_cb + client.upload_file(**kwargs) + monitor.update_ok(file_size, file_id) + last_err = None + break + except CosServiceError as e: + last_err = e + if attempt < retry: + # 重置该文件的进度，准备重试 + progress_cb, file_id = monitor.create_progress_callback(file_size) + if last_err is not None: + err_reason = "%s (Code: %s)" % (last_err.get_error_msg(), last_err.get_error_code()) + monitor.update_err(file_id, + src_path=full_path, + dest_path="cos://%s/%s" % (bucket, key), + reason=err_reason, + request_id=last_err.get_request_id()) + + # 使用线程池并发上传多个文件，routines 控制文件间并发 + if tasks: + max_workers = min(routines, len(tasks)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for full_path, key, file_size in tasks: + futures.append(executor.submit(_do_upload, full_path, key, file_size)) + for future in as_completed(futures): + future.result() + + # 在 COS 上创建空目录标记（以 / 结尾的空对象） + for dir_key in empty_dir_keys: + try: + client.put_object(Bucket=bucket, Key=dir_key, Body=b"") + monitor.update_ok(0) + except CosServiceError as e: + monitor.update_err(src_path=dir_key, + reason="创建空目录失败: %s (Code: %s)" % (e.get_error_msg(), e.get_error_code()), + request_id=e.get_request_id()) + + monitor.stop(log_file=log_file) \ No newline at end of file diff --git a/tccli/plugins/cos/utils.py b/tccli/plugins/cos/utils.py new file mode 100644 index 0000000000..e9bd3beeb4 --- /dev/null +++ b/tccli/plugins/cos/utils.py @@ -0,0 +1,689 @@ +# -*- coding: utf-8 -*- +""" +COS 插件工具模块 +提供凭据解析、文件过滤、格式化等通用功能 +""" +import os +import json +import fnmatch + + +def _load_json_file(filepath): + """加载 JSON 配置文件""" + try: + with open(filepath, "r") as f: + return json.load(f) + except (IOError, json.JSONDecodeError): + return {} + + +def parse_global_arg(parsed_globals): + """ + 从 TCCLI 配置文件和环境变量中加载凭据信息，填充到 parsed_globals 中。 + 对齐标准 TCCLI 服务（如 CVM）的 parse_global_arg 逻辑。 + """ + g_param = parsed_globals + + # 确定 profile + profile = g_param.get("profile") or os.environ.get("TCCLI_PROFILE", "default") + g_param["profile"] = profile + + # 加载配置文件 + configure_path = os.path.join(os.path.expanduser("~"), ".tccli") + conf_path = os.path.join(configure_path, profile + ".configure") + cred_path = os.path.join(configure_path, profile + ".credential") + + conf = _load_json_file(conf_path) + cred = _load_json_file(cred_path) + + # 从环境变量加载凭据（优先级高于配置文件） + env_secret_id = os.environ.get("TENCENTCLOUD_SECRET_ID") + env_secret_key = os.environ.get("TENCENTCLOUD_SECRET_KEY") + env_token = os.environ.get("TENCENTCLOUD_TOKEN") + env_region = os.environ.get("TENCENTCLOUD_REGION") + + # 填充 secretId + if g_param.get("secretId") is None: + if env_secret_id: + g_param["secretId"] = env_secret_id + elif "secretId" in cred: + g_param["secretId"] = cred["secretId"] + + # 填充 secretKey + if g_param.get("secretKey") is None: + if env_secret_key: + g_param["secretKey"] = env_secret_key + elif "secretKey" in cred: + g_param["secretKey"] = cred["secretKey"] + + # 填充 token + if g_param.get("token") is None: + if env_token: + g_param["token"] = env_token + elif "token" in cred: + g_param["token"] = cred["token"] + else: + g_param["token"] = None + + # 填充 region + if g_param.get("region") is None: + if env_region: + g_param["region"] = env_region + elif isinstance(conf.get("_sys_param"), dict) and "region" in conf["_sys_param"]: + g_param["region"] = conf["_sys_param"]["region"] + + # 填充 endpoint + if g_param.get("endpoint") is None: + g_param["endpoint"] = None + + # 校验必要参数 + if not g_param.get("secretId"): + raise Exception( + "secretId 未配置。请通过以下方式之一配置：\n" + " 1. tccli configure (交互式配置)\n" + " 2. 设置环境变量 TENCENTCLOUD_SECRET_ID\n" + " 3. 命令行参数 --secretId YOUR_SECRET_ID" + ) + if not g_param.get("secretKey"): + raise Exception( + "secretKey 未配置。请通过以下方式之一配置：\n" + " 1. tccli configure (交互式配置)\n" + " 2. 设置环境变量 TENCENTCLOUD_SECRET_KEY\n" + " 3. 命令行参数 --secretKey YOUR_SECRET_KEY" + ) + + return g_param + + +def init_cos_client(parsed_globals): + """ + 标准 COS 客户端初始化。 + 返回 (client, region) 元组。 + """ + from qcloud_cos import CosConfig + from qcloud_cos import CosS3Client + + parsed_globals = parse_global_arg(parsed_globals) + secret_id = parsed_globals["secretId"] + secret_key = parsed_globals["secretKey"] + token = parsed_globals["token"] + region = parsed_globals["region"] or "ap-guangzhou" + endpoint = parsed_globals["endpoint"] + + config = CosConfig( + Region=region, + SecretId=secret_id, + SecretKey=secret_key, + Token=token, + Endpoint=endpoint, + ) + client = CosS3Client(config) + return client, region + + +def format_size(size_bytes): + """格式化文件大小为人类可读的字符串""" + if size_bytes < 1024: + return "%d B" % size_bytes + elif size_bytes < 1024 * 1024: + return "%.2f KB" % (size_bytes / 1024.0) + elif size_bytes < 1024 * 1024 * 1024: + return "%.2f MB" % (size_bytes / (1024.0 * 1024)) + elif size_bytes < 1024 * 1024 * 1024 * 1024: + return "%.2f GB" % (size_bytes / (1024.0 * 1024 * 1024)) + else: + return "%.2f TB" % (size_bytes / (1024.0 * 1024 * 1024 * 1024)) + + +def match_filters(name, include, exclude): + """ + 根据 include/exclude 模式过滤文件名。 + 返回 True 表示文件应被处理，False 表示应跳过。 + """ + if include and not fnmatch.fnmatch(name, include): + return False + if exclude and fnmatch.fnmatch(name, exclude): + return False + return True + + +def parse_meta(meta_str): + """ + 解析自定义元数据字符串。 + 格式: key1=value1#key2=value2 + 返回 dict，key 自动加上 x-cos-meta- 前缀。 + """ + metadata = {} + if meta_str: + for pair in meta_str.split("#"): + pair = pair.strip() + if "=" in pair: + k, v = pair.split("=", 1) + metadata["x-cos-meta-" + k.strip()] = v.strip() + return metadata + + +def build_cos_key(prefix, rel_path): + """ + 根据前缀和相对路径构造 COS 对象键。 + """ + if not prefix: + return rel_path + if prefix.endswith("/"): + return prefix + rel_path + return prefix + "/" + rel_path + + +def list_all_objects(client, bucket, prefix=""): + """列出存储桶中指定前缀下的所有对象（跳过目录标记）""" + objects = {} + marker = "" + while True: + response = client.list_objects( + Bucket=bucket, + Prefix=prefix, + Marker=marker, + MaxKeys=1000, + ) + if "Contents" in response: + for content in response["Contents"]: + key = content["Key"] + if key.endswith("/"): + continue + objects[key] = { + "Size": int(content.get("Size", 0)), + "ETag": content.get("ETag", ""), + "LastModified": content.get("LastModified", ""), + "StorageClass": content.get("StorageClass", "STANDARD"), + } + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + else: + break + return objects + + +def list_all_objects_with_dirs(client, bucket, prefix=""): + """列出存储桶中指定前缀下的所有对象（包含 / 结尾的目录标记）""" + objects = {} + marker = "" + while True: + response = client.list_objects( + Bucket=bucket, + Prefix=prefix, + Marker=marker, + MaxKeys=1000, + ) + if "Contents" in response: + for content in response["Contents"]: + key = content["Key"] + objects[key] = { + "Size": int(content.get("Size", 0)), + "ETag": content.get("ETag", ""), + "LastModified": content.get("LastModified", ""), + "StorageClass": content.get("StorageClass", "STANDARD"), + "IsDir": key.endswith("/"), + } + if response.get("IsTruncated") == "true": + marker = response.get("NextMarker", "") + else: + break + return objects + + +def list_local_files(local_dir): + """递归列出本地目录下的所有文件""" + files = {} + for root, dirs, filenames in os.walk(local_dir): + for filename in filenames: + full_path = os.path.join(root, filename) + rel_path = os.path.relpath(full_path, local_dir) + rel_path = rel_path.replace(os.sep, "/") + files[rel_path] = { + "Size": os.path.getsize(full_path), + "FullPath": full_path, + "MTime": os.path.getmtime(full_path), + } + return files + + +def calculate_local_crc64(file_path): + """ + 计算本地文件的 CRC64 (ECMA-182 多项式)，对齐 COS 返回的 x-cos-hash-crc64ecma 头。 + 参数：polynomial=0x142F0E1EBA9EA3693, initCrc=0, xorOut=0xFFFFFFFFFFFFFFFF, rev=True + （该参数组合经实测与 COS 服务端 CRC64 结果精确匹配；与 Go 标准库 hash/crc64 搭配 + crc64.MakeTable(crc64.ECMA) 并以 ^uint64(0) 作为 xorOut 的行为一致。） + 返回字符串形式的无符号十进制数；失败返回 None（例如未安装 crcmod 时）。 + """ + try: + import crcmod + except ImportError: + return None + try: + crc64_fn = crcmod.mkCrcFun( + 0x142F0E1EBA9EA3693, + initCrc=0, + xorOut=0xFFFFFFFFFFFFFFFF, + rev=True, + ) + crc = 0 + with open(file_path, "rb") as f: + while True: + data = f.read(65536) + if not data: + break + crc = crc64_fn(data, crc) + return str(crc) + except (IOError, OSError): + return None + + +def get_object_head(client, bucket, cos_key): + """ + 获取 COS 对象的 HEAD 响应。返回 dict（可直接取 'x-cos-hash-crc64ecma'/'Last-Modified' 等头）。 + 对象不存在或任何异常时返回 None。用于 sync 跳过判断。 + """ + from qcloud_cos import CosServiceError + try: + return client.head_object(Bucket=bucket, Key=cos_key) + except CosServiceError: + return None + except Exception: + return None + + +def parse_http_time(time_str): + """ + 解析 HTTP 时间字符串（RFC1123/RFC3339），返回 Unix 时间戳（float）；失败返回 None。 + 用于 sync --update 模式对比 Last-Modified 时间。 + """ + if not time_str: + return None + import calendar + from email.utils import parsedate_tz, mktime_tz + # 优先按 RFC1123/RFC822（如 "Mon, 02 Jan 2006 15:04:05 GMT"） + parsed = parsedate_tz(time_str) + if parsed is not None: + try: + return float(mktime_tz(parsed)) + except (TypeError, ValueError, OverflowError): + pass + # 回退按 RFC3339（如 "2006-01-02T15:04:05Z"） + try: + import datetime + s = time_str.rstrip("Z") + dt = datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S") + return float(calendar.timegm(dt.timetuple())) + except ValueError: + return None + + +def should_skip_sync_upload(client, bucket, cos_key, local_full_path, local_mtime, + ignore_existing=False, update=False): + """ + 判断同步上传时是否跳过该文件，对齐 coscli 的 skipUpload。 + 跳过规则（优先级从高到低）： + 1. --ignore-existing：目标存在即跳过 + 2. --update：目标存在且 Last-Modified >= 本地 mtime 则跳过 + 3. 默认（CRC64）：对比本地 CRC64 与 COS HEAD 的 x-cos-hash-crc64ecma；相等则跳过 + 返回 True 表示跳过，False 表示需要上传。 + 目标不存在、无法获取 CRC64 或比较不相等时均返回 False。 + """ + head = get_object_head(client, bucket, cos_key) + if head is None: + return False # 目标不存在或异常，不跳过 + if ignore_existing: + return True + if update: + last_modified = head.get("Last-Modified") or head.get("last-modified", "") + remote_ts = parse_http_time(last_modified) + if remote_ts is not None and remote_ts >= local_mtime: + return True + return False + # 默认：CRC64 比较 + cos_crc = head.get("x-cos-hash-crc64ecma", "") + if not cos_crc: + return False + local_crc = calculate_local_crc64(local_full_path) + if local_crc is None: + return False + return cos_crc == local_crc + + +def should_skip_sync_download(client, bucket, cos_key, cos_head_info, local_full_path, + ignore_existing=False, update=False): + """ + 判断同步下载时是否跳过该文件，对齐 coscli 的 skipDownload。 + - cos_head_info: 从 list_objects 得到的对象信息 dict（含 'Size' / 'LastModified' 等） + 跳过规则（优先级从高到低）： + 1. --ignore-existing：本地存在即跳过 + 2. --update：本地 mtime >= COS LastModified 则跳过 + 3. 默认（CRC64）：对比本地 CRC64 与 COS HEAD 的 x-cos-hash-crc64ecma；相等则跳过 + 返回 True 表示跳过，False 表示需要下载。 + """ + if not os.path.exists(local_full_path): + return False + if ignore_existing: + return True + if update: + local_mtime = os.path.getmtime(local_full_path) + remote_ts = parse_http_time(cos_head_info.get("LastModified", "")) + if remote_ts is not None and local_mtime >= remote_ts: + return True + return False + # 默认：CRC64 比较 + local_crc = calculate_local_crc64(local_full_path) + if local_crc is None: + return False + head = get_object_head(client, bucket, cos_key) + if head is None: + return False + cos_crc = head.get("x-cos-hash-crc64ecma", "") + if not cos_crc: + return False + return cos_crc == local_crc + + +def should_skip_sync_copy(client, src_bucket, src_key, dest_bucket, dest_key, + ignore_existing=False, update=False): + """ + 判断同步复制时是否跳过该文件，对齐 coscli 的 skipCopy。 + 跳过规则（优先级从高到低）： + 1. --ignore-existing：目标存在即跳过 + 2. --update：目标 Last-Modified >= 源 Last-Modified 则跳过 + 3. 默认（CRC64）：对比源 CRC64 与目标 CRC64；相等则跳过 + 返回 True 表示跳过，False 表示需要复制。 + """ + dest_head = get_object_head(client, dest_bucket, dest_key) + if dest_head is None: + return False + if ignore_existing: + return True + src_head = get_object_head(client, src_bucket, src_key) + if src_head is None: + return False + if update: + src_ts = parse_http_time(src_head.get("Last-Modified", "")) + dest_ts = parse_http_time(dest_head.get("Last-Modified", "")) + if src_ts is not None and dest_ts is not None and dest_ts >= src_ts: + return True + return False + # 默认：CRC64 比较 + src_crc = src_head.get("x-cos-hash-crc64ecma", "") + dest_crc = dest_head.get("x-cos-hash-crc64ecma", "") + if not src_crc or not dest_crc: + return False + return src_crc == dest_crc + + +# ============================================================ +# 进度监控模块 - 对齐 COSCLI 的 FileProcessMonitor +# ============================================================ +import sys +import time +import threading as _threading + + +class TransferProgressMonitor(object): + """ + 文件传输进度监控器，对齐 COSCLI 的 FileProcessMonitor。 + 支持实时显示：总数/已处理数/成功/跳过/失败/进度百分比/速度。 + 支持通过 SDK progress_callback 实现分片级别的实时进度更新。 + """ + + def __init__(self, op_type="upload"): + self.op_type = op_type # upload / download / copy / move + self._lock = _threading.Lock() + # 扫描统计 + self.total_num = 0 + self.total_size = 0 + self.scan_end = False + # 处理统计 + self.ok_num = 0 + self.skip_num = 0 + self.err_num = 0 + self.deal_size = 0 # 已处理的总大小（含跳过） + self.transfer_size = 0 # 实际传输的大小（通过 progress_callback 实时更新） + self.skip_size = 0 + # 每个文件的已传输字节数追踪（用于 progress_callback） + self._file_progress = {} # file_id -> consumed_bytes + self._file_id_counter = 0 + # 失败记录列表：每项为 dict {"path": ..., "reason": ...} + self._fail_records = [] + # 速度计算 + self._start_time = time.time() + self._last_snap_time = time.time() + self._last_snap_size = 0 + self._tick_duration = 0.5 # 刷新间隔（秒） + self._finished = False + # 上一次输出的可见字符长度（用于清除残留） + self._last_bar_len = 0 + # 进度条线程 + self._progress_thread = None + self._stop_event = _threading.Event() + + def set_scan_info(self, total_num, total_size): + """设置扫描结果（文件总数和总大小）""" + with self._lock: + self.total_num = total_num + self.total_size = total_size + self.scan_end = True + + def update_ok(self, size, file_id=None): + """更新成功计数 + 如果使用了 progress_callback（file_id 不为 None），则不再累加 transfer_size， + 因为已经通过 _update_file_progress 实时更新了。 + """ + with self._lock: + self.ok_num += 1 + if file_id is not None: + # 使用了 progress_callback，确保该文件的进度被标记为完成 + already = self._file_progress.pop(file_id, 0) + # 修正：确保 transfer_size 精确等于文件大小 + delta = size - already + if delta > 0: + self.transfer_size += delta + self.deal_size += size + else: + # 没有使用 progress_callback，直接累加 + self.deal_size += size + self.transfer_size += size + + def update_skip(self, size): + """更新跳过计数""" + with self._lock: + self.skip_num += 1 + self.deal_size += size + self.skip_size += size + + def update_err(self, file_id=None, path=None, reason=None, + src_path=None, dest_path=None, request_id=None): + """更新失败计数，可选记录失败路径和原因 + - path: 兼容旧接口，作为 src_path 使用（若 src_path 未指定） + - src_path: 源路径（本地文件路径或 COS key） + - dest_path: 目标路径（本地文件路径或 COS key） + - request_id: SDK 返回的 RequestId + - reason: 失败原因（SDK 错误信息） + """ + import datetime + with self._lock: + self.err_num += 1 + if file_id is not None: + self._file_progress.pop(file_id, None) + record_src = src_path or path or "" + record_dest = dest_path or "" + if record_src or reason: + self._fail_records.append({ + "time": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "src_path": record_src, + "dest_path": record_dest, + "reason": reason or "", + "request_id": request_id or "", + }) + + def create_progress_callback(self, file_size): + """创建一个可以传给 COS SDK 的 progress_callback 函数。 + SDK 会在每个分片上传/下载完成后调用 callback(consumed_bytes, total_bytes)。 + 返回 (callback_func, file_id) 元组。 + """ + with self._lock: + self._file_id_counter += 1 + file_id = self._file_id_counter + self._file_progress[file_id] = 0 + + def _callback(consumed_bytes, total_bytes): + with self._lock: + old_consumed = self._file_progress.get(file_id, 0) + delta = consumed_bytes - old_consumed + if delta > 0: + self.transfer_size += delta + self._file_progress[file_id] = consumed_bytes + + return _callback, file_id + + def start(self): + """启动进度条刷新线程""" + self._start_time = time.time() + self._last_snap_time = time.time() + self._stop_event.clear() + self._progress_thread = _threading.Thread(target=self._progress_loop, daemon=True) + self._progress_thread.start() + + def stop(self, log_file=None): + """停止进度条并输出最终结果，如果指定 log_file 则写入失败日志""" + self._stop_event.set() + if self._progress_thread: + self._progress_thread.join(timeout=2) + self._print_finish_bar() + if log_file and self._fail_records: + self._write_log_file(log_file) + + def _write_log_file(self, log_file): + """将失败记录写入日志文件（结构化格式，每条记录含时间/源路径/目标路径/错误信息/RequestId）""" + import datetime + try: + log_dir = os.path.dirname(log_file) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + elapsed = time.time() - self._start_time + with open(log_file, "w", encoding="utf-8") as f: + f.write("# %s 失败日志\n" % self.op_type) + f.write("# 生成时间: %s\n" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + f.write("# 执行耗时: %.1fs\n" % elapsed) + f.write("# 失败总数: %d\n" % len(self._fail_records)) + f.write("#\n") + for i, record in enumerate(self._fail_records, 1): + f.write("[%d]\n" % i) + f.write(" Time : %s\n" % record.get("time", "")) + f.write(" Source : %s\n" % record.get("src_path", "")) + if record.get("dest_path"): + f.write(" Dest : %s\n" % record["dest_path"]) + f.write(" Reason : %s\n" % record.get("reason", "")) + if record.get("request_id"): + f.write(" RequestId : %s\n" % record["request_id"]) + f.write("\n") + sys.stderr.write("失败日志已写入: %s\n" % log_file) + sys.stderr.flush() + except Exception as e: + sys.stderr.write("写入失败日志出错: %s\n" % str(e)) + sys.stderr.flush() + + def _progress_loop(self): + """进度条刷新循环""" + while not self._stop_event.is_set(): + self._print_progress_bar() + self._stop_event.wait(self._tick_duration) + + def _print_progress_bar(self): + """打印实时进度条（覆盖当前行）""" + with self._lock: + now = time.time() + duration = now - self._last_snap_time + if duration < self._tick_duration: + return + + increment_size = self.transfer_size - self._last_snap_size + speed = increment_size / duration if duration > 0 else 0 + self._last_snap_time = now + self._last_snap_size = self.transfer_size + + deal_num = self.ok_num + self.skip_num + self.err_num + # 已传输 + 已跳过 = 总进度字节数 + progress_size = self.transfer_size + self.skip_size + + if self.scan_end and self.total_num > 0: + # 扫描完成，显示百分比 + if self.total_size > 0: + percent = min(float(progress_size) * 100.0 / float(self.total_size), 99.9) + else: + percent = min(float(deal_num) * 100.0 / float(self.total_num), 99.9) + bar = "Total num: %d, size: %s. Processed num: %d(%d ok, %d skip, %d err), " \ + "OK size: %s, Progress: %.1f%%, Speed: %s/s" % ( + self.total_num, format_size(self.total_size), + deal_num, self.ok_num, self.skip_num, self.err_num, + format_size(progress_size), + percent, format_size(int(speed))) + else: + # 扫描中 + scan_num = max(self.total_num, deal_num) + bar = "Scanned num: %d. Processed num: %d(%d ok, %d skip, %d err), " \ + "OK size: %s, Speed: %s/s" % ( + scan_num, + deal_num, self.ok_num, self.skip_num, self.err_num, + format_size(progress_size), + format_size(int(speed))) + + # 回到行首，写入新内容，用空格覆盖上一次多出的部分 + padding = max(0, self._last_bar_len - len(bar)) + sys.stderr.write("\r" + bar + " " * padding) + sys.stderr.flush() + self._last_bar_len = len(bar) + + def _print_finish_bar(self): + """打印最终完成信息（显示100%）""" + with self._lock: + elapsed = time.time() - self._start_time + avg_speed = self.transfer_size / elapsed if elapsed > 0 else 0 + deal_num = self.ok_num + self.skip_num + self.err_num + total_done_size = self.transfer_size + self.skip_size + + if self.err_num == 0: + status = "Succeed" + else: + status = "FinishWithError" + + if self.scan_end: + bar = "%s: Total num: %d, size: %s. OK num: %d" % ( + status, self.total_num, format_size(self.total_size), self.ok_num) + else: + bar = "%s: Scanned num: %d. OK num: %d" % ( + status, max(self.total_num, deal_num), self.ok_num) + + detail_parts = [] + if self.skip_num > 0: + detail_parts.append("skip %d" % self.skip_num) + if self.err_num > 0: + detail_parts.append("err %d" % self.err_num) + if detail_parts: + bar += "(%s)" % ", ".join(detail_parts) + + if self.skip_size > 0: + bar += ", Skip size: %s" % format_size(self.skip_size) + bar += ", OK size: %s" % format_size(self.transfer_size) + + # 显示最终进度 100%（如果没有错误） + if self.err_num == 0 and self.total_num > 0: + bar += ", Progress: 100.0%" + elif self.total_size > 0: + percent = float(total_done_size) * 100.0 / float(self.total_size) + bar += ", Progress: %.1f%%" % percent + + # 回到行首，写入最终结果，用空格覆盖上一次多出的部分 + padding = max(0, self._last_bar_len - len(bar)) + sys.stderr.write("\r" + bar + " " * padding + "\n") + sys.stderr.flush() + + # 输出平均速度和总耗时 + if elapsed > 0: + sys.stderr.write("AvgSpeed: %s/s, Elapsed: %.1fs\n" % (format_size(int(avg_speed)), elapsed)) + sys.stderr.flush() \ No newline at end of file