Added put_csv_data() for pushing csv data into a file key in S3
This commit is contained in:
parent
8274917ee6
commit
15a12cf9f6
|
|
@ -6,6 +6,8 @@ import os
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import boto3
|
import boto3
|
||||||
|
import csv
|
||||||
|
import smart_open
|
||||||
|
|
||||||
from lib_afc_s3storage.afs3exception import AFS3Error
|
from lib_afc_s3storage.afs3exception import AFS3Error
|
||||||
|
|
||||||
|
|
@ -170,6 +172,27 @@ class S3Storage:
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def put_csv_data( self, bucket=None, key=None, rowdata=None ):
|
||||||
|
if bucket == None:
|
||||||
|
raise AFS3Error("S3Storage.put_csv_data(): missing required bucket name")
|
||||||
|
|
||||||
|
if key == None:
|
||||||
|
raise AFS3Error("S3Storage.put_csv_data(): missing required key")
|
||||||
|
|
||||||
|
# should validate 'rowdata' is a list of dicts
|
||||||
|
if rowdata == None:
|
||||||
|
raise AFS3Error("S3Storage.put_csv_data(): missing required rowdata")
|
||||||
|
|
||||||
|
s3Path = f"s3://{bucket}/{key}"
|
||||||
|
with smart_open.open(s3Path, 'w', transport_params={'client': self.client}) as ofh:
|
||||||
|
writer = csv.DictWriter(ofh, fieldnames=rowdata[0].keys())
|
||||||
|
writer.writeheader()
|
||||||
|
for row in rowdata:
|
||||||
|
writer.writerow(row)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
def delete_all_objects( self, bucket=None ):
|
def delete_all_objects( self, bucket=None ):
|
||||||
"""
|
"""
|
||||||
delete_all_objects
|
delete_all_objects
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue