Added put_csv_data() for pushing csv data into a file key in S3

This commit is contained in:
Rick Ross 2023-12-05 13:06:58 -08:00
parent 8274917ee6
commit 15a12cf9f6
1 changed files with 23 additions and 0 deletions

View File

@ -6,6 +6,8 @@ import os
import logging import logging
import boto3 import boto3
import csv
import smart_open
from lib_afc_s3storage.afs3exception import AFS3Error from lib_afc_s3storage.afs3exception import AFS3Error
@ -170,6 +172,27 @@ class S3Storage:
return response return response
def put_csv_data( self, bucket=None, key=None, rowdata=None ):
if bucket == None:
raise AFS3Error("S3Storage.put_csv_data(): missing required bucket name")
if key == None:
raise AFS3Error("S3Storage.put_csv_data(): missing required key")
# should validate 'rowdata' is a list of dicts
if rowdata == None:
raise AFS3Error("S3Storage.put_csv_data(): missing required rowdata")
s3Path = f"s3://{bucket}/{key}"
with smart_open.open(s3Path, 'w', transport_params={'client': self.client}) as ofh:
writer = csv.DictWriter(ofh, fieldnames=rowdata[0].keys())
writer.writeheader()
for row in rowdata:
writer.writerow(row)
return
def delete_all_objects( self, bucket=None ): def delete_all_objects( self, bucket=None ):
""" """
delete_all_objects delete_all_objects