alliance-forge-libraries
/
lib-af-mosaic
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added raise error when trying to load file from S3 and renamed error column from 'reason' to 'afc_error'

This commit is contained in:
Rick Ross 2023-12-08 12:32:19 -08:00
parent d011b93c85
commit 2f8d06dff4
1 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lib_afc_mosaic/universe_map_file.py
View File

@ -205,9 +205,13 @@ class AFCMUniverseMapFile:
amplify_rows = [] amplify_rows = []
removed_rows = [] removed_rows = []
voterset_filename_s3_key = os.path.join(self.metadata.get("s3_key", ""), voterset_filename) try:
file_data_stream = self.voterset_s3_connection.get_object(self.voterset_s3_bucket, voterset_filename_s3_key = os.path.join(self.metadata.get("s3_key", ""), voterset_filename)
voterset_filename_s3_key) file_data_stream = self.voterset_s3_connection.get_object(self.voterset_s3_bucket,
voterset_filename_s3_key)
except Exception as ex:
raise Exception(f"universe_map_file.process_file: Failed to get {self.voterset_s3_bucket}/{voterset_filename_s3_key}: {ex}") from ex
rows = read_voterdata_csv_stream(file_data_stream) rows = read_voterdata_csv_stream(file_data_stream)
logging.debug("rows = %s", len(rows)) logging.debug("rows = %s", len(rows))
@ -236,7 +240,7 @@ class AFCMUniverseMapFile:
try: try:
parsed_number = phonenumbers.parse(cell_phone_num,'US') parsed_number = phonenumbers.parse(cell_phone_num,'US')
except Exception: # pylint: disable=broad-exception-caught except Exception: # pylint: disable=broad-exception-caught
r['reason'] = f"bad row #{i} in {voterset_filename} reason='could not parse phone number'" r['afc_error'] = f"bad row #{i} in {voterset_filename}; reason='could not parse phone number'"
removed_rows.append(r) removed_rows.append(r)
if "could not parse phone number" in self.stats["removed_details"]: if "could not parse phone number" in self.stats["removed_details"]:
self.stats["removed_details"]["could not parse phone number"] += 1 self.stats["removed_details"]["could not parse phone number"] += 1
@ -248,7 +252,7 @@ class AFCMUniverseMapFile:
if phonenumbers.is_possible_number(parsed_number): if phonenumbers.is_possible_number(parsed_number):
possible_number = parsed_number possible_number = parsed_number
else: else:
r['reason'] = f"bad row #{i} in {voterset_filename} reason='invalid phone number'" r['afc_error'] = f"bad row #{i} in {voterset_filename}; reason='invalid phone number'"
removed_rows.append(r) removed_rows.append(r)
if "invalid phone number" in self.stats["removed_details"]: if "invalid phone number" in self.stats["removed_details"]:
self.stats["removed_details"]["invalid phone number"] += 1 self.stats["removed_details"]["invalid phone number"] += 1
@ -261,7 +265,7 @@ class AFCMUniverseMapFile:
formatted_number = phonenumbers.format_number(possible_number, formatted_number = phonenumbers.format_number(possible_number,
phonenumbers.PhoneNumberFormat.E164) phonenumbers.PhoneNumberFormat.E164)
else: else:
r['reason'] = f"bad row #{i} in {voterset_filename} reason='invalid phone number'" r['afc_error'] = f"bad row #{i} in {voterset_filename}; reason='invalid phone number'"
removed_rows.append(r) removed_rows.append(r)
if "invalid phone number" in self.stats["removed_details"]: if "invalid phone number" in self.stats["removed_details"]:
self.stats["removed_details"]["invalid phone number"] += 1 self.stats["removed_details"]["invalid phone number"] += 1
@ -271,7 +275,7 @@ class AFCMUniverseMapFile:
# look up carrier errors and optouts # look up carrier errors and optouts
if formatted_number in self.error_numbers: if formatted_number in self.error_numbers:
r['reason'] = f"bad row #{i} in {voterset_filename} reason='{self.error_numbers[formatted_number]}'" r['afc_error'] = f"bad row #{i} in {voterset_filename}; reason='{self.error_numbers[formatted_number]}'"
removed_rows.append(r) removed_rows.append(r)
if self.error_numbers[formatted_number] in self.stats["removed_details"]: if self.error_numbers[formatted_number] in self.stats["removed_details"]:
self.stats["removed_details"][self.error_numbers[formatted_number]] += 1 self.stats["removed_details"][self.error_numbers[formatted_number]] += 1
@ -281,7 +285,7 @@ class AFCMUniverseMapFile:
# last check, make sure we haven't added this number already # last check, make sure we haven't added this number already
if formatted_number in self.running_phone_numbers: if formatted_number in self.running_phone_numbers:
r['reason'] = f"bad row #{i} in {voterset_filename} reason='duplicate phone number'" r['afc_error'] = f"bad row #{i} in {voterset_filename}; reason='duplicate phone number'"
removed_rows.append(r) removed_rows.append(r)
if "duplicate phone number" in self.stats["removed_details"]: if "duplicate phone number" in self.stats["removed_details"]:
self.stats["removed_details"]["duplicate phone number"] += 1 self.stats["removed_details"]["duplicate phone number"] += 1