From ea4cdd2362df7ed842426a56eeaa993c07b73a28 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Tue, 7 Apr 2026 07:21:33 +0000 Subject: [PATCH 1/3] Fixed Code for 3 files --- scripts/un/energy/download.py | 4 ++-- scripts/un/energy/process.py | 3 +++ scripts/un/energy/process_test.py | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index 4542b4068c..b3dab19d4c 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -90,7 +90,6 @@ def download_energy_dataset( str(supported_datasets)) return output_files # Download data in batches of years as the download has a limit of 100k rows. - years_list = list(range(start_year, years_per_batch + 1)) years_list = [str(y) for y in range(start_year, end_year + 1)] batch_years = [ years_list[i:i + years_per_batch] @@ -117,7 +116,8 @@ def download_energy_dataset( if download_successful: logging.info(f"Download of '{download_url}' completed.") for f in os.listdir(output): - output_files.append(os.path.join(output, f)) + if f.endswith('.csv'): + output_files.append(os.path.join(output, f)) else: logging.fatal(f"Download or processing of '{download_url}' failed") return output_files diff --git a/scripts/un/energy/process.py b/scripts/un/energy/process.py index eedbe872d8..5791f7644d 100644 --- a/scripts/un/energy/process.py +++ b/scripts/un/energy/process.py @@ -545,6 +545,9 @@ def process(in_paths: list, with open(mcf_file_path, 'w+', newline='') as f_out_mcf: # Process each CSV input file, one row at a time. for in_file in in_paths: + if not in_file.endswith('.csv'): + logging.info(f'Skipping non-CSV file: {in_file}') + continue logging.info(f'Processing data file: {in_file}') with open(in_file) as csvfile: counters['input_files'] += 1 diff --git a/scripts/un/energy/process_test.py b/scripts/un/energy/process_test.py index 776e45ccbf..bb7fb81c77 100644 --- a/scripts/un/energy/process_test.py +++ b/scripts/un/energy/process_test.py @@ -67,5 +67,4 @@ def test_un_energy_process(self): if __name__ == '__main__': - app.run() unittest.main() From 91d22a72b80b57ddae12b3a9d23be9f72ec9430d Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Wed, 8 Apr 2026 10:59:08 +0000 Subject: [PATCH 2/3] NameError Resolved --- scripts/un/energy/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index b3dab19d4c..1ef6e37f13 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -83,6 +83,7 @@ def download_energy_dataset( Returns: A list of output files downloaded. """ + output_files = [] supported_datasets = get_all_energy_source_codes() if energy_dataset not in supported_datasets: logging.info( @@ -95,7 +96,6 @@ def download_energy_dataset( years_list[i:i + years_per_batch] for i in range(0, len(years_list), years_per_batch) ] - output_files = [] for year_batch in batch_years: start_year = year_batch[0] end_year = year_batch[-1] From 932aafcddccb55c9e2102e570377b3b7a3982ac2 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Thu, 11 Jun 2026 12:36:23 +0000 Subject: [PATCH 3/3] fema: Increase retry count and delay to handle transient 503 errors --- statvar_imports/fema/flood_insurance_claims/fema_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statvar_imports/fema/flood_insurance_claims/fema_download.py b/statvar_imports/fema/flood_insurance_claims/fema_download.py index 5cd76ff730..93ed3171c4 100644 --- a/statvar_imports/fema/flood_insurance_claims/fema_download.py +++ b/statvar_imports/fema/flood_insurance_claims/fema_download.py @@ -127,8 +127,8 @@ def download_data(api_url: str, temp_dir: str): download_success = download_file(url=csv_url, output_folder=temp_dir, unzip=False, - tries=3, - delay=5, + tries=10, + delay=10, backoff=2) if not download_success or not os.path.exists(util_output_path):