AWS S3 存储桶复制及权限同步

1、存储桶复制
分为2种： SCR , CCR 
SCR和CCR的操作文档可以参考AWS 官方文档，这里就不重复了：
复制对象 - Amazon Simple Storage Service
使用 S3 分批复制以复制现有对象 - Amazon Simple Storage Service
授予 Amazon S3 分批操作的权限 - Amazon Simple Storage Service

SCR可以同步对象的权限，不必要额外的权限同步操作。
CCR无法同步除所有者之外的权限，必要举行其他权限的同步，必要通过写批量同步权限的脚本完成同步操作
下面是同步公开READ的权限脚本示例，供参考：

1. #!/usr/bin/python3
2. # -*- coding: utf-8 -*-
4. # Copyright WUZL. or its affiliates. All Rights Reserved.
5. # SPDX-License-Identifier: Apache-2.0
7. """
8. Purpose
9. Show how to use AWS SDK for Python (Boto3) with Amazon Simple Storage Service
10. (Amazon S3) to perform basic object acl operations, Synchronize the public read permissions of the source and target buckets.
11. """
13. import json
14. import logging
16. # 在操作系统里需要先安全AWS boto3 SDK包 # pip3 install boto3
17. import boto3
18. from botocore.exceptions import ClientError
20. logger = logging.getLogger(__name__)
21. logger.setLevel(logging.DEBUG)
22. # 建立一个filehandler来把日志记录在文件里，级别为debug以上
23. fh = logging.FileHandler("boto3_s3_object_acl_modi.log")
24. fh.setLevel(logging.DEBUG)
25. # 建立一个streamhandler来把日志打在CMD窗口上，级别为error以上
26. ch = logging.StreamHandler()
27. ch.setLevel(logging.ERROR)
28. # 设置日志格式
29. formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(lineno)s %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
30. ch.setFormatter(formatter)
31. fh.setFormatter(formatter)
32. #将相应的handler添加在logger对象中
33. logger.addHandler(ch)
34. logger.addHandler(fh)
35. # 开始打日志
36. # logger.debug("debug message")
37. # logger.info("info message")
38. # logger.warn("warn message")
39. # logger.error("error message")
40. # logger.critical("critical message")
42. # snippet-start:[python.example_code.s3.helper.ObjectWrapper]
44. # args 变量值根据实际情况自己定义
45. s_region_name='eu-west-2'
46. s_aws_access_key_id='xxx'
47. s_aws_secret_access_key='xxx'
48. s_bucket='xxx'
50. t_region_name='us-west-2'
51. t_awt_accest_key_id=''
52. t_awt_secret_accest_key='xxx'
53. target_bucket='xxx'
56. class ObjectWrapper:
57.     """Encapsulates S3 object actions."""
58.     def __init__(self, s3_object):
59.         """
60.         :param s3_object: A Boto3 Object resource. This is a high-level resource in Boto3
61.                           that wraps object actions in a class-like structure.
62.         """
63.         self.object = s3_object
64.         self.key = self.object.key
65. # snippet-end:[python.example_code.s3.helper.ObjectWrapper]
67. # snippet-start:[python.example_code.s3.GetObject]
68.     def get(self):
69.         """
70.         Gets the object.
72.         :return: The object data in bytes.
73.         """
74.         try:
75.             body = self.object.get()['Body'].read()
76.             logger.info(
77.                 "Got object '%s' from bucket '%s'.",
78.                 self.object.key, self.object.bucket_name)
79.         except ClientError:
80.             logger.exception(
81.                 "Couldn't get object '%s' from bucket '%s'.",
82.                 self.object.key, self.object.bucket_name)
83.             raise
84.         else:
85.             return body
86. # snippet-end:[python.example_code.s3.GetObject]
88. # snippet-start:[python.example_code.s3.ListObjects]
89.     @staticmethod
90.     def list(bucket, prefix=None):
91.         """
92.         Lists the objects in a bucket, optionally filtered by a prefix.
94.         :param bucket: The bucket to query. This is a Boto3 Bucket resource.
95.         :param prefix: When specified, only objects that start with this prefix are listed.
96.         :return: The list of objects.
97.         """
98.         try:
99.             if not prefix:
100.                 objects = list(bucket.objects.all())
101.             else:
102.                 objects = list(bucket.objects.filter(Prefix=prefix))
103.             # logger.info("Got objects %s from bucket '%s'", [o.key for o in objects], bucket.name)
104.             logger.info("Got objects from bucket '%s'", bucket.name)
105.         except ClientError:
106.             logger.exception("Couldn't get objects for bucket '%s'.", bucket.name)
107.             raise
108.         else:
109.             return objects
110. # snippet-end:[python.example_code.s3.ListObjects]
112. # snippet-start:[python.example_code.s3.ListObjectsKeys]
113.     @staticmethod
114.     def list_all_keys(bucket, prefix=None):
115.         """
116.         Lists the ListObjectsKeys in a bucket, optionally filtered by a prefix.
118.         :param bucket: The bucket to query. This is a Boto3 Bucket resource.
119.         :param prefix: When specified, only objects that start with this prefix are listed.
120.         :return: The list of objects.
121.         """
122.         try:
123.             if not prefix:
124.                 objects = list(bucket.objects.all())
125.             else:
126.                 objects = list(bucket.objects.filter(Prefix=prefix))
127.             all_keys = [o.key for o in objects]
128.             # logger.info("Got objects %s from bucket '%s'", [o.key for o in objects], bucket.name)
129.             logger.info("Got objects list from bucket '%s'", bucket.name)
130.         except ClientError:
131.             logger.exception("Couldn't get objects for bucket '%s'.", bucket.name)
132.             raise
133.         else:
134.             return all_keys
135. # snippet-end:[python.example_code.s3.ListObjectsKeys]
137. # snippet-start:[python.example_code.s3.GetObjectAcl]
138.     def get_acl(self):
139.         """
140.         Gets the ACL of the object.
142.         :return: The ACL of the object.
143.         """
144.         try:
145.             acl = self.object.Acl()
146.             # logger.info("Got ACL for object %s owned by %s.", self.object.key, acl.owner['DisplayName'])
147.         except ClientError:
148.             logger.exception("Couldn't get ACL for object %s.", self.object.key)
149.             raise
150.         else:
151.             return acl
152. # snippet-end:[python.example_code.s3.GetObjectAcl]
154. # snippet-start:[python.example_code.s3.PutObjectAcl]
155.     def put_acl(self, uri):
156.         """
157.         Applies an ACL to the object that grants read access to an AWS user identified
158.         by email address.
160.         :param email: The email address of the user to grant access.
161.         """
162.         try:
163.             acl = self.object.Acl()
164.             # Putting an ACL overwrites the existing ACL, so append new grants
165.             # if you want to preserve existing grants.
166.             grants = acl.grants if acl.grants else []
167.             grants.append({'Grantee': {'Type': 'Group', 'URI': uri}, 'Permission': 'READ'})
168.             acl.put(
169.                 AccessControlPolicy={
170.                     'Grants': grants,
171.                     'Owner': acl.owner
172.                 }
173.             )
174.             # logger.info("Granted read access to %s.", uri)
175.         except ClientError:
176.             logger.exception("Couldn't add ACL to object '%s'.", self.object.key)
177.             raise
178. # snippet-end:[python.example_code.s3.PutObjectAcl]
181. # snippet-start:[python.example_code.s3.Scenario_ObjectManagement]
182. def usage_demo():
183.     # print('-'*88)
184.     # print("Welcome to the Amazon S3 object acl modi demo!")
185.     # print('-'*88)
187.     # logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
188.     # LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
189.     # logging.basicConfig(filename='boto3_s3_object_acl_modi.log', level=logging.DEBUG, format=LOG_FORMAT)
190.    
191.     # s3_client = boto3.client('s3', region_name=s_region_name, aws_access_key_id=s_aws_access_key_id, aws_secret_access_key=s_aws_secret_access_key)
192.     # response = s3_client.list_buckets()
193.     # print('Existing buckets:')
194.     # for bucket in response['Buckets']:
195.     #     print(f'  {bucket["Name"]}')
197.     s3_resource = boto3.resource('s3', region_name=s_region_name, aws_access_key_id=s_aws_access_key_id, aws_secret_access_key=s_aws_secret_access_key)
198.     bucket = s3_resource.Bucket(s_bucket)
199.     # print(dir(bucket))
201.     t_s3_resource = boto3.resource('s3', region_name=t_region_name, aws_access_key_id=t_awt_accest_key_id, aws_secret_access_key=t_awt_secret_accest_key)
202.     t_bucket = t_s3_resource.Bucket(target_bucket)
203.     # print(dir(t_bucket))
204.     # t_objects = ObjectWrapper.list(t_bucket)
205.     # print(t_objects)
207.     # prepare objects keys for modi
208.     objects = ObjectWrapper.list(bucket)
209.     all_keys = ObjectWrapper.list_all_keys(bucket)
210.     # print(objects)
211.     try:
212.         keys=[]
213.         len_all_keys = len(all_keys)
214.         logger.info("len_all_keys: %s", len_all_keys)
215.         for object_summary in objects:
216.             len_all_keys = len_all_keys - 1
217.             logger.info("left_keys: %s", len_all_keys)
218.             key=str(object_summary.key)
219.             # logger.info("object_key: '%s'", key)
220.             # print(key+':')
221.             object_acl = object_summary.Acl()
222.             # print(object_acl)
223.             # print(object_acl.grants)
224.             # logger.info("object_grants: '%s'", str(object_acl.grants))
225.             for grant in object_acl.grants:
226.                 if 'READ' == grant['Permission']:
227.                     # print('very good!')
228.                     keys.append(key)
229.                     break
230.     except ClientError as error:
231.         print(error)
232.         
233.     # print(keys)
234.     logger.info("keys list len: %s", len(keys))
235.     logger.info("source keys: %s", keys)
236.    
237.     logger.info("Modi target bucket object grants:")
238.    
239.     # prepare target objects keys for modi
240.     t_objects = ObjectWrapper.list(t_bucket)
241.     # print(t_objects)
242.     # exit()
243.     t_all_keys = ObjectWrapper.list_all_keys(t_bucket)
244.     logger.info("t_all_keys list len: %s", len(t_all_keys))
245.     try:
246.         modi_keys=[]
247.         t_keys=[]
248.         tmp_keys = []
249.         for tmp_key in keys:
250.             tmp_keys.append(tmp_key)
251.         len_left_t_keys = len(keys)
252.         logger.info("len_left_t_keys: %s", len_left_t_keys)
253.         for key in keys:
254.             # logger.info("len of keys: %s, keys: %s", len(keys), keys)
255.             len_left_t_keys = len_left_t_keys - 1
256.             logger.info("len_left_t_keys: %s", len_left_t_keys)
257.             if key in t_all_keys:
258.                 t_key=key
259.                 object_summary = t_s3_resource.ObjectSummary(target_bucket,t_key)               
260.                 # logger.info("t_object_key: '%s'", t_key)
261.                 # print(key+':')
262.                 object_acl = object_summary.Acl()
263.                 # print(object_acl)
264.                 # print(object_acl.grants)
265.                 # logger.info("object_grants: '%s'", str(object_acl.grants))
266.                 # t_keys.append(t_key)
267.                 # logger.info("len of t_keys: %s, t_keys: %s", len(t_keys), t_keys)
268.                 for grant in object_acl.grants:
269.                     # logger.info("grant: %s", grant)
270.                     # if 'READ' == grant['Permission']:
271.                     if grant['Permission'] == 'READ':
272.                         # logger.info("object %s have permission READ", t_key)
273.                         tmp_keys.remove(t_key)
274.                         break
275.                     
276.             # logger.info("len of tmp_keys: %s, keys: %s", len(tmp_keys), tmp_keys)
277.             modi_keys=tmp_keys
278.         logger.info("len of modi_keys: %s ,modi_keys: '%s'", len(modi_keys), str(modi_keys))
279.     except ClientError as error:
280.         print(error)
281.    
282.     len_left_modi_keys = len(modi_keys)
283.     for key in modi_keys:
284.         len_left_modi_keys = len_left_modi_keys - 1
285.         logger.info("len_left_modi_keys: %s", len_left_modi_keys)
286.         object_key = key
287.         # print(object_key)
288.         obj_wrapper = ObjectWrapper(t_bucket.Object(object_key))
289.         # print(t_bucket.Object(object_key))
290.         object_acl = t_bucket.Object(object_key).Acl()
291.         # print(object_acl)
292.         # print(object_acl.grants)
293.         try:
294.             obj_wrapper.put_acl(uri='http://acs.amazonaws.com/groups/global/AllUsers')
295.             acl = obj_wrapper.get_acl()
296.             # logger.info("Put ACL grants on object '%s': '%s'", str(obj_wrapper.key), str(json.dumps(acl.grants)))
297.             logger.info("Put ACL grants on object '%s'", str(obj_wrapper.key))
298.         except ClientError as error:
299.             if error.response['Error']['Code'] == 'UnresolvableGrantByEmailAddress':
300.                 print('*'*88)
301.                 print("This demo couldn't apply the ACL to the object because the email\n"
302.                     "address specified as the grantee is for a test user who does not\n"
303.                     "exist. For this request to succeed, you must replace the grantee\n"
304.                     "email with one for an existing AWS user.")
305.                 print('*' * 88)
306.             else:
307.                 raise
310. # snippet-end:[python.example_code.s3.Scenario_ObjectManagement]
313. if __name__ == '__main__':
314.     usage_demo()

复制代码

代码参考：
S3 — Boto3 Docs 1.26.26 documentation
aws-doc-sdk-examples/object_wrapper.py at main · awsdocs/aws-doc-sdk-examples · GitHub

免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。