AWS S3 存储桶复制及权限同步

1、存储桶复制

分为2种： SCR , CCR 

SCR和CCR的操作文档可以参考AWS 官方文档，这里就不重复了：

复制对象 - Amazon Simple Storage Service

使用 S3 分批复制以复制现有对象 - Amazon Simple Storage Service

授予 Amazon S3 分批操作的权限 - Amazon Simple Storage Service

SCR可以同步对象的权限，不需要额外的权限同步操作。

CCR无法同步除所有者之外的权限，需要进行其他权限的同步，需要通过写批量同步权限的脚本完成同步操作

下面是同步公开READ的权限脚本示例，供参考：

#!/usr/bin/python3
# -*- coding: utf-8 -*-

# Copyright WUZL. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

"""
Purpose
Show how to use AWS SDK for Python (Boto3) with Amazon Simple Storage Service
(Amazon S3) to perform basic object acl operations, Synchronize the public read permissions of the source and target buckets. 
"""

import json
import logging

# 在操作系统里需要先安全AWS boto3 SDK包 # pip3 install boto3
import boto3
from botocore.exceptions import ClientError

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# 建立一个filehandler来把日志记录在文件里，级别为debug以上
fh = logging.FileHandler("boto3_s3_object_acl_modi.log")
fh.setLevel(logging.DEBUG)
# 建立一个streamhandler来把日志打在CMD窗口上，级别为error以上
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
# 设置日志格式
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(lineno)s %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
ch.setFormatter(formatter)
fh.setFormatter(formatter)
#将相应的handler添加在logger对象中
logger.addHandler(ch)
logger.addHandler(fh)
# 开始打日志
# logger.debug("debug message")
# logger.info("info message")
# logger.warn("warn message")
# logger.error("error message")
# logger.critical("critical message")

# snippet-start:[python.example_code.s3.helper.ObjectWrapper]

# args 变量值根据实际情况自己定义
s_region_name='eu-west-2'
s_aws_access_key_id='xxx'
s_aws_secret_access_key='xxx'
s_bucket='xxx'

t_region_name='us-west-2'
t_awt_accest_key_id=''
t_awt_secret_accest_key='xxx'
target_bucket='xxx'


class ObjectWrapper:
    """Encapsulates S3 object actions."""
    def __init__(self, s3_object):
        """
        :param s3_object: A Boto3 Object resource. This is a high-level resource in Boto3
                          that wraps object actions in a class-like structure.
        """
        self.object = s3_object
        self.key = self.object.key
# snippet-end:[python.example_code.s3.helper.ObjectWrapper]

# snippet-start:[python.example_code.s3.GetObject]
    def get(self):
        """
        Gets the object.

        :return: The object data in bytes.
        """
        try:
            body = self.object.get()['Body'].read()
            logger.info(
                "Got object '%s' from bucket '%s'.",
                self.object.key, self.object.bucket_name)
        except ClientError:
            logger.exception(
                "Couldn't get object '%s' from bucket '%s'.",
                self.object.key, self.object.bucket_name)
            raise
        else:
            return body
# snippet-end:[python.example_code.s3.GetObject]

# snippet-start:[python.example_code.s3.ListObjects]
    @staticmethod
    def list(bucket, prefix=None):
        """
        Lists the objects in a bucket, optionally filtered by a prefix.

        :param bucket: The bucket to query. This is a Boto3 Bucket resource.
        :param prefix: When specified, only objects that start with this prefix are listed.
        :return: The list of objects.
        """
        try:
            if not prefix:
                objects = list(bucket.objects.all())
            else:
                objects = list(bucket.objects.filter(Prefix=prefix))
            # logger.info("Got objects %s from bucket '%s'", [o.key for o in objects], bucket.name)
            logger.info("Got objects from bucket '%s'", bucket.name)
        except ClientError:
            logger.exception("Couldn't get objects for bucket '%s'.", bucket.name)
            raise
        else:
            return objects
# snippet-end:[python.example_code.s3.ListObjects]

# snippet-start:[python.example_code.s3.ListObjectsKeys]
    @staticmethod
    def list_all_keys(bucket, prefix=None):
        """
        Lists the ListObjectsKeys in a bucket, optionally filtered by a prefix.

        :param bucket: The bucket to query. This is a Boto3 Bucket resource.
        :param prefix: When specified, only objects that start with this prefix are listed.
        :return: The list of objects.
        """
        try:
            if not prefix:
                objects = list(bucket.objects.all())
            else:
                objects = list(bucket.objects.filter(Prefix=prefix))
            all_keys = [o.key for o in objects]
            # logger.info("Got objects %s from bucket '%s'", [o.key for o in objects], bucket.name)
            logger.info("Got objects list from bucket '%s'", bucket.name)
        except ClientError:
            logger.exception("Couldn't get objects for bucket '%s'.", bucket.name)
            raise
        else:
            return all_keys
# snippet-end:[python.example_code.s3.ListObjectsKeys]

# snippet-start:[python.example_code.s3.GetObjectAcl]
    def get_acl(self):
        """
        Gets the ACL of the object.

        :return: The ACL of the object.
        """
        try:
            acl = self.object.Acl()
            # logger.info("Got ACL for object %s owned by %s.", self.object.key, acl.owner['DisplayName'])
        except ClientError:
            logger.exception("Couldn't get ACL for object %s.", self.object.key)
            raise
        else:
            return acl
# snippet-end:[python.example_code.s3.GetObjectAcl]

# snippet-start:[python.example_code.s3.PutObjectAcl]
    def put_acl(self, uri):
        """
        Applies an ACL to the object that grants read access to an AWS user identified
        by email address.

        :param email: The email address of the user to grant access.
        """
        try:
            acl = self.object.Acl()
            # Putting an ACL overwrites the existing ACL, so append new grants
            # if you want to preserve existing grants.
            grants = acl.grants if acl.grants else []
            grants.append({'Grantee': {'Type': 'Group', 'URI': uri}, 'Permission': 'READ'})
            acl.put(
                AccessControlPolicy={
                    'Grants': grants,
                    'Owner': acl.owner
                }
            )
            # logger.info("Granted read access to %s.", uri)
        except ClientError:
            logger.exception("Couldn't add ACL to object '%s'.", self.object.key)
            raise
# snippet-end:[python.example_code.s3.PutObjectAcl]


# snippet-start:[python.example_code.s3.Scenario_ObjectManagement]
def usage_demo():
    # print('-'*88)
    # print("Welcome to the Amazon S3 object acl modi demo!")
    # print('-'*88)

    # logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
    # LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
    # logging.basicConfig(filename='boto3_s3_object_acl_modi.log', level=logging.DEBUG, format=LOG_FORMAT)
    
    # s3_client = boto3.client('s3', region_name=s_region_name, aws_access_key_id=s_aws_access_key_id, aws_secret_access_key=s_aws_secret_access_key)
    # response = s3_client.list_buckets()
    # print('Existing buckets:')
    # for bucket in response['Buckets']:
    #     print(f'  {bucket["Name"]}')

    s3_resource = boto3.resource('s3', region_name=s_region_name, aws_access_key_id=s_aws_access_key_id, aws_secret_access_key=s_aws_secret_access_key)
    bucket = s3_resource.Bucket(s_bucket)
    # print(dir(bucket))

    t_s3_resource = boto3.resource('s3', region_name=t_region_name, aws_access_key_id=t_awt_accest_key_id, aws_secret_access_key=t_awt_secret_accest_key)
    t_bucket = t_s3_resource.Bucket(target_bucket)
    # print(dir(t_bucket))
    # t_objects = ObjectWrapper.list(t_bucket)
    # print(t_objects)

    # prepare objects keys for modi
    objects = ObjectWrapper.list(bucket)
    all_keys = ObjectWrapper.list_all_keys(bucket)
    # print(objects)
    try:
        keys=[]
        len_all_keys = len(all_keys)
        logger.info("len_all_keys: %s", len_all_keys)
        for object_summary in objects:
            len_all_keys = len_all_keys - 1
            logger.info("left_keys: %s", len_all_keys)
            key=str(object_summary.key)
            # logger.info("object_key: '%s'", key)
            # print(key+':')
            object_acl = object_summary.Acl()
            # print(object_acl)
            # print(object_acl.grants)
            # logger.info("object_grants: '%s'", str(object_acl.grants))
            for grant in object_acl.grants:
                if 'READ' == grant['Permission']:
                    # print('very good!') 
                    keys.append(key)
                    break
    except ClientError as error:
        print(error)
        
    # print(keys)
    logger.info("keys list len: %s", len(keys))
    logger.info("source keys: %s", keys)
    
    logger.info("Modi target bucket object grants:")
    
    # prepare target objects keys for modi
    t_objects = ObjectWrapper.list(t_bucket)
    # print(t_objects)
    # exit()
    t_all_keys = ObjectWrapper.list_all_keys(t_bucket)
    logger.info("t_all_keys list len: %s", len(t_all_keys))
    try:
        modi_keys=[]
        t_keys=[]
        tmp_keys = []
        for tmp_key in keys:
            tmp_keys.append(tmp_key)
        len_left_t_keys = len(keys)
        logger.info("len_left_t_keys: %s", len_left_t_keys)
        for key in keys:
            # logger.info("len of keys: %s, keys: %s", len(keys), keys)
            len_left_t_keys = len_left_t_keys - 1
            logger.info("len_left_t_keys: %s", len_left_t_keys)
            if key in t_all_keys:
                t_key=key
                object_summary = t_s3_resource.ObjectSummary(target_bucket,t_key)                
                # logger.info("t_object_key: '%s'", t_key)
                # print(key+':')
                object_acl = object_summary.Acl()
                # print(object_acl)
                # print(object_acl.grants)
                # logger.info("object_grants: '%s'", str(object_acl.grants))
                # t_keys.append(t_key)
                # logger.info("len of t_keys: %s, t_keys: %s", len(t_keys), t_keys)
                for grant in object_acl.grants:
                    # logger.info("grant: %s", grant)
                    # if 'READ' == grant['Permission']:
                    if grant['Permission'] == 'READ':
                        # logger.info("object %s have permission READ", t_key)
                        tmp_keys.remove(t_key)
                        break
                    
            # logger.info("len of tmp_keys: %s, keys: %s", len(tmp_keys), tmp_keys)
            modi_keys=tmp_keys
        logger.info("len of modi_keys: %s ,modi_keys: '%s'", len(modi_keys), str(modi_keys))
    except ClientError as error:
        print(error)
    
    len_left_modi_keys = len(modi_keys)
    for key in modi_keys:
        len_left_modi_keys = len_left_modi_keys - 1
        logger.info("len_left_modi_keys: %s", len_left_modi_keys)
        object_key = key
        # print(object_key)
        obj_wrapper = ObjectWrapper(t_bucket.Object(object_key))
        # print(t_bucket.Object(object_key))
        object_acl = t_bucket.Object(object_key).Acl()
        # print(object_acl)
        # print(object_acl.grants)
        try:
            obj_wrapper.put_acl(uri='http://acs.amazonaws.com/groups/global/AllUsers')
            acl = obj_wrapper.get_acl()
            # logger.info("Put ACL grants on object '%s': '%s'", str(obj_wrapper.key), str(json.dumps(acl.grants)))
            logger.info("Put ACL grants on object '%s'", str(obj_wrapper.key))
        except ClientError as error:
            if error.response['Error']['Code'] == 'UnresolvableGrantByEmailAddress':
                print('*'*88)
                print("This demo couldn't apply the ACL to the object because the email\n"
                    "address specified as the grantee is for a test user who does not\n"
                    "exist. For this request to succeed, you must replace the grantee\n"
                    "email with one for an existing AWS user.")
                print('*' * 88)
            else:
                raise


# snippet-end:[python.example_code.s3.Scenario_ObjectManagement]


if __name__ == '__main__':
    usage_demo()

代码参考：

S3 — Boto3 Docs 1.26.26 documentation

aws-doc-sdk-examples/object_wrapper.py at main · awsdocs/aws-doc-sdk-examples · GitHub文章来源地址https://www.toymoban.com/news/detail-538994.html

到了这里，关于AWS S3 存储桶复制及权限同步的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！