fixed pipeline for UNDATA

statvar_imports/undata/UNData_metadata.csv

@@ -16,7 +16,6 @@ mapped_columns,9
 debug,0
 #input_rows,5
 #log_level,2
-existing_statvar_mcf,""""""
 #resolve_places,0
 word_delimiter,""""""
 filter_data_min_value,2

statvar_imports/undata/UNData_places_resolved_csv.csv

@@ -2688,6 +2688,7 @@ Jaleeb Al-Shuykh Kuwait,
 Salmiya Kuwait,
 BISHKEK Kyrgyzstan,wikidataId/Q9361
 Jalal-Abad Kyrgyzstan,wikidataId/Q487689
+Manas (Jalal-Abad) Kyrgyzstan,wikidataId/Q487689
 Osh Kyrgyzstan,wikidataId/Q47282
 VIENTIANE Lao People's Democratic Republic,wikidataId/Q9326
 Daugavpils Latvia,wikidataId/Q80021
@@ -4092,14 +4093,18 @@ Tashauz Turkmenistan,wikidataId/Q487672
 GRAND TURK Turks and Caicos Islands,wikidataId/Q34205
 FUNAFUTI Tuvalu,wikidataId/Q34126
 Gulu Uganda,wikidataId/Q581379
+Gulu City Uganda,wikidataId/Q581379
 Hoima Uganda,wikidataId/Q1624192
+Hoima City Uganda,wikidataId/Q1624192
 KAMPALA Uganda,wikidataId/Q3894
 Kasese Uganda,wikidataId/Q1232016
 Kira Uganda,wikidataId/Q966819
 Lugazi Uganda,wikidataId/Q6699738
 Makindye Ssabagabo Uganda,
-Masaka Uganda,wikidataId/Q1032233
+Masaka Uganda,wikidataId/Q956659
+Masaka City Uganda,wikidataId/Q956659
 Mbarara Uganda,wikidataId/Q731039
+Mbarara City Uganda,wikidataId/Q731039
 Mukono Uganda,wikidataId/Q179198
 Nansana Uganda,wikidataId/Q965956
 Alchevsk Ukraine,wikidataId/Q161144

statvar_imports/undata/config.ini

@@ -1,2 +1,2 @@
 [DEFAULT]
-UN_ZIP_URL = https://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=tableCode:240&DataMartId=POP&Format=csv&c=2,3,6,8,10,12,14,16,17,18&s=_countryEnglishNameOrderBy:asc,refYear:desc,areaCode:asc
+UN_ZIP_URL = https://data.un.org/legacy/Handlers/DownloadHandler.ashx?DataFilter=tableCode:240&DataMartId=POP&Format=csv&c=2,3,6,8,10,12,14,16,17,18&s=_countryEnglishNameOrderBy:asc,refYear:desc,areaCode:asc

statvar_imports/undata/download.py

@@ -17,6 +17,7 @@
 '''
 
 import os
+import io
 import requests
 import zipfile
 from pathlib import Path
@@ -40,38 +41,58 @@
 # Read the config file
 def read_config_file():
     config = configparser.ConfigParser()
-    config.read(_FLAGS.config_path)
+    config_path = _FLAGS.config_path
+    if not os.path.isabs(config_path):
+        config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), config_path)
+    config.read(config_path)
     return config
 
-# Retry function for handling request failures
-@retry(tries=3, delay=5, backoff=2)
-def retry_method(url, headers=None):
+HEADERS = {
+    'User-Agent': (
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+        '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+    ),
+    'Accept': 'application/zip,application/octet-stream,*/*',
+}
+
+# Retry function for handling request failures and invalid zip files
+@retry(exceptions=(requests.exceptions.RequestException, zipfile.BadZipFile),
+       tries=4, delay=5, backoff=2)
+def retry_method(url, headers=HEADERS):
     response = requests.get(url, headers=headers, timeout=60)
     response.raise_for_status()
+    if not zipfile.is_zipfile(io.BytesIO(response.content)):
+        snippet = response.content[:300]
+        logging.warning(
+            f"Downloaded content from {url} is not a valid ZIP file! HTTP {response.status_code}, "
+            f"Content-Type: {response.headers.get('content-type')}, "
+            f"Length: {len(response.content)} bytes. Snippet: {snippet}")
+        raise zipfile.BadZipFile(f"Downloaded content from {url} is not a valid ZIP file.")
     return response
 
 # Function to download and save the ZIP file
 def download_file():
-    logging.info("Starting download...")
     config = read_config_file()
     UN_ZIP_URL = config["DEFAULT"]["UN_ZIP_URL"]
+    logging.info(f"Starting download from {UN_ZIP_URL}...")
     zip_path = os.path.join(OUTPUT_DIR, "UNdata_Export.zip")
 
     try:
         response = retry_method(UN_ZIP_URL)
         with open(zip_path, "wb") as f:
             f.write(response.content)
 
-        logging.info(f"Downloaded file saved to {zip_path}")
+        logging.info(f"Downloaded file from {UN_ZIP_URL} saved to {zip_path}")
         return zip_path
 
     except requests.exceptions.RequestException as e:
-        logging.fatal(f"Failed to download file: {e}")
-        return None
+        logging.fatal(f"Failed to download file from {UN_ZIP_URL} after retries: {e}")
+
+    except zipfile.BadZipFile as e:
+        logging.fatal(f"Failed to obtain a valid ZIP file from {UN_ZIP_URL} after retries: {e}")
 
     except Exception as e:
-        logging.fatal(f"An unexpected error occurred during file download: {e}")
-        return None
+        logging.fatal(f"An unexpected error occurred during file download from {UN_ZIP_URL}: {e}")
 
 # Function to extract and process the CSV file from ZIP
 def extract_and_process(zip_path):
@@ -84,7 +105,6 @@ def extract_and_process(zip_path):
 
             if not csv_files:
                 logging.fatal("No CSV files found in the ZIP archive!")
-                return
 
             for csv_file in csv_files:
                 logging.info(f"Processing extracted file: {csv_file}")
@@ -95,7 +115,8 @@ def extract_and_process(zip_path):
                     logging.info(f"Read {len(lines)} lines from {csv_file}")
 
     except zipfile.BadZipFile:
-        logging.fatal("Invalid ZIP file format!")
+        file_size = os.path.getsize(zip_path) if os.path.exists(zip_path) else 0
+        logging.fatal(f"Invalid ZIP file format! File size: {file_size} bytes")
 
     except Exception as e:
         logging.fatal(f"An unexpected error occurred during ZIP extraction: {e}")
@@ -106,8 +127,7 @@ def main(argv):
 
     # Run the download and extraction process
     zip_path = download_file()
-    if zip_path:
-        extract_and_process(zip_path)
+    extract_and_process(zip_path)
 
 if __name__ == '__main__':
     app.run(main)