Added list_objects_v2(), download_all_objects(), and download_object() to s3storage.py

lib_afc_s3storage/s3storage.py

@@ -96,6 +96,19 @@ class S3Storage:
         return r['Contents']
 
 
+    def list_objects_v2( self, bucket=None, prefix=None ):
+        """
+        list_objects
+        """
+
+        if self.client is None or bucket is None:
+            return None
+
+        r = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix)
+
+        return r['Contents']
+
+
     def upload_file( self, filepathname=None, bucket=None, key=None ):
         """
         upload_file
@@ -206,3 +219,72 @@ class S3Storage:
             self.client.delete_object(Bucket=bucket, Key=obj['Key'])
 
         return
+
+
+    def download_all_objects( self, bucket=None, prefix=None, localdir=None):
+        """
+        download_all_objects
+        """
+
+        if bucket == None:
+            raise AFS3Error("S3Storage.download_all_objects(): missing required bucket name")
+
+        if prefix == None:
+            raise AFS3Error("S3Storage.download_all_objects(): missing required key prefix string")
+
+        if localdir == None:
+            raise AFS3Error("S3Storage.download_all_objects(): missing required localdir name")
+
+        objects = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix)
+        # keys in 'objects' are: ['ResponseMetadata', 'IsTruncated', 'Contents', 'Name', 'Prefix', 'MaxKeys', 'EncodingType', 'KeyCount']
+
+        for obj in objects['Contents']:
+            # keys in 'obj' are: ['Key', 'LastModified', 'ETag', 'Size', 'StorageClass']
+
+            lessPrefix = obj['Key'].split(prefix)[-1]
+
+            fullPathname = f"{localdir}/{lessPrefix}"
+            dirList = fullPathname.split('/')
+            basefilename = dirList.pop(-1)
+            dirList.remove('')
+            mkDirPath = "/".join(dirList)
+            fullPathname = f"{mkDirPath}/{basefilename}"
+
+            if not os.path.exists(mkDirPath):
+                #print(f"making dir {mkDirPath}")
+                os.makedirs(mkDirPath)
+            #print(f"downloading {fullPathname}")
+            self.client.download_file(Bucket=bucket, Key=obj['Key'], Filename=fullPathname)
+
+        return
+
+
+    def download_object( self, bucket=None, key=None, filename=None ):
+        """
+        download_object
+        """
+
+        if bucket == None:
+            raise AFS3Error("S3Storage.download_object(): missing required bucket name")
+
+        if key == None:
+            raise AFS3Error("S3Storage.download_object(): missing required key")
+
+        if filename == None:
+            raise AFS3Error("S3Storage.download_object(): missing required filename")
+
+        response = self.client.download_file(Bucket=bucket, Key=key, Filename=filename)
+
+        # response looks like this:
+        # response = {'ResponseMetadata':
+        #              {'RequestId': 'tx00000ed21e851e5b2e313-00656c222c-32c0e88-default', 'HostId': '', 'HTTPStatusCode': 200,
+        #               'HTTPHeaders': {'date': 'Sun, 03 Dec 2023 06:37:32 GMT',
+        #                               'content-length': '0',
+        #                               'connection': 'keep-alive',
+        #                               'etag': '"5d6ef8943a9f853a82247c9e87152c1f"',
+        #                               'accept-ranges': 'bytes',
+        #                               'x-amz-request-id': 'tx00000ed21e851e5b2e313-00656c222c-32c0e88-default'},
+        #               'RetryAttempts': 0},
+        #             'ETag': '"5d6ef8943a9f853a82247c9e87152c1f"'}
+
+        return response