# -*- coding: utf-8 -*- # --------------------------------------------------------------------------------- # Name: gpm_all.py # Purpose: This script will update the GPM mosaic dataset by removing all # rasters older than a specified time period. New images will be # downloaded and loaded into existing mosaic datasets. # The script was originally provided by Esri but has been modified # based on lessons learned and an effort to simplify the code. # # Author: Esri (v1), jshute (v2) # # Created: Revised on 08/24/2018 # Copyright: (c) NASA 2018 # # --------------------------------------------------------------------------------- # Import modules # --------------- import os import sys import socket import datetime import json import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning import arcpy now = datetime.datetime.now() # Script parameters # ------------------ root_directory = "C:\\att\\project\\arcshare\\public\\disaster_response\\nrt_products\\gpm\\" # product_file_prefix, output_directory, mosaic_dataset_name, product_query_parameter, product_name product_list = [ ["global_landslide_nowcast_3hr", root_directory + "rasters_landslide\\", "landslide", "global_landslide_nowcast_3hr", "Landslide 3hr"], ["gpm_7d", root_directory + "rasters_gpm_nrt_7_day\\", "gpm_nrt_7_day", "precip_7d", "7 day precip"], ["gpm_3hr", root_directory + "rasters_gpm_nrt_3_hour\\", "gpm_nrt_3_hour", "precip_3hr", "3 hour precip"], ["gpm_3hr_1d", root_directory + "rasters_gpm_nrt_1_day\\", "gpm_nrt_1_day", "precip_3hr_1d", "1 day precip 3hr"], ["gpm_30mn", root_directory + "rasters_gpm_nrt_30_minute\\", "gpm_nrt_30_minute", "precip_30mn", "30 min precip"] ] url = "https://pmmpublisher.pps.eosdis.nasa.gov/opensearch" age_of_imagery = 7 # Determines retention time for imagery query_result_limit = "10000" # Determines amount of results for query mosaic_dataset_fields = ["Name", "Date", "Product"] # Import script class files # -------------------------- sys.path.insert(0, root_directory + "classes\\") import UtilitiesGeneral import UtilitiesRaster # Disable url request warnings # ----------------------------- requests.packages.urllib3.disable_warnings(InsecureRequestWarning) # Global logger settings # ----------------------- logger_timestamp = now.strftime("%Y%m%d_%H%M_%S") log_directory = root_directory + "logs" + os.sep log_file = log_directory + "gpm_log_" + logger_timestamp + "_" + socket.gethostname() + ".log" # Main body of the script # ------------------------------------------------------------------------------- # Defines the entry point into the script def main(): # Set up the logger # ------------------ logger = UtilitiesGeneral.configure_logging(log_file) logger.info("Logger configuration complete") logger.info("log_directory ==> " + log_directory) logger.info("log_file location ==> " + log_file) logger.info("Script start time ==> " + now.strftime("%I:%M:%S %p")) # Verify class file connectivity is working # ------------------------------------------ utilities_general_test = "Checking connection to UtilitiesGeneral file ... " logger.info("File check ==> " + utilities_general_test) utilities_general_result = UtilitiesGeneral.connection_test(utilities_general_test) logger.info("Result ==> " + utilities_general_result) utilities_raster_test = "Checking connection to UtilitiesRaster file ... " logger.info("File check ==> " + utilities_raster_test) utilities_raster_result = UtilitiesRaster.connection_test(utilities_raster_test) logger.info("Result ==> " + utilities_raster_result) # Check script parameters # ------------------------ logger.info("Listing script parameters ... ") logger.info("Script start time ==> " + now.strftime("%Y%m%d_%H%M_%S")) logger.info("root_directory ==> " + root_directory) logger.info("product_list ==> " + str(product_list)) logger.info("url ==> " + url) logger.info("age_of_imagery ==> " + str(age_of_imagery)) logger.info("query_result_limit ==> " + query_result_limit) logger.info("mosaic_dataset_fields ==> " + str(mosaic_dataset_fields)) # Configure dates # ---------------- logger.info("Configuring dates...") # End time is always today # ------------------------- end_time = now.strftime("%Y-%m-%d") logger.info("end_time parameter ==> " + str(end_time)) start_time = (now - datetime.timedelta(days=age_of_imagery)).strftime("%Y-%m-%d") logger.info("start_time parameter ==> " + str(start_time)) # Remove date today minus the age_of_imagery parameter # ----------------------------------------------------- remove_date = (now - datetime.timedelta(days=age_of_imagery)).strftime("%Y%m%d") logger.info("remove_date parameter ==> " + str(remove_date)) remove_date_query = str(now - datetime.timedelta(days=age_of_imagery)).split(".")[0] logger.info("remove_date_query parameter ==> " + remove_date_query) logger.info("done") # Begin product loop # ------------------- logger.info("Beginning product loop...") for product in product_list: logger.info("================================================================================") logger.info("Beginning processing for " + product[3]) logger.info("================================================================================") # Set product_type variables # --------------------------- product_file_prefix = product[0] output_directory = product[1] mosaic_dataset_name = product[2] product_query_parameter = product[3] product_name = product[4] logger.info("product_file_prefix ==> " + product_file_prefix) logger.info("output_directory ==> " + output_directory) logger.info("mosaic_dataset_name ==> " + mosaic_dataset_name) logger.info("product_query_parameter ==> " + product_query_parameter) logger.info("product_name ==> " + product_name) mosaic_dataset_full_path = root_directory + "data_" + mosaic_dataset_name + ".gdb\\" + mosaic_dataset_name logger.info("mosaic_dataset_full_path ==> " + mosaic_dataset_full_path) # Find out if directory is empty. If it is empty, set initial run to true # ------------------------------------------------------------------------ file_list = os.listdir(output_directory) if len(file_list) > 0: initial_run = False logger.info("Files already exist in the directory. Adding new files only (initial_run = {})" .format(str(initial_run))) else: initial_run = True logger.info("No files in the directory. Adding all files (initial_run = {})".format(str(initial_run))) # Set full URL for query # ----------------------- full_url = url + "?q={}&startTime={}&endTime={}&limit={}".format(product_query_parameter, start_time, end_time, query_result_limit) logger.info("full_url ==> " + full_url) # Making URL request # ------------------- logger.info("Making URL request...") response = requests.get(full_url, verify=False) logger.info("URL response ==> " + str(response)) # Process URL response # --------------------- logger.info("Processing URL response...") json_response = json.loads(response.text) # logger.info("json_response ==> " + str(json_response)) # <====== Spams the log file item_list = json_response["items"] # logger.info("item_list ==> " + str(item_list)) # <====== Spams the log file # Parse image files from json response and process # ------------------------------------------------- logger.info("Parsing image files from json response and processing...") for item in item_list: action_list = item["action"] # logger.info("action_list ==> " + str(action_list)) # <====== Spams the log file for action in action_list: for using in action["using"]: if using["mediaType"] == "image/tiff": # Set file name for download and full file path # ---------------------------------------------- # logger.info("Setting file name for download and full file path") # <====== Spams the log file f_name = os.path.basename(using["url"]) out_tif = output_directory + f_name # logger.info("File: {} | Full path: {} ".format(f_name, out_tif)) # <====== Spams the log file # Check to see if file already exists in the directory # ----------------------------------------------------- if not os.path.isfile(out_tif): # Download the file # ------------------ logger.info("Downloading and writing file to output directory ==> " + out_tif) response = requests.get(using["url"], verify=False, stream=True) # Write file to output directory # ------------------------------- with open(out_tif, "wb") as f: f.write(response.content) logger.info("done") if not initial_run: # Add image file to the mosaic dataset # ------------------------------------- logger.info("Adding image file to the mosaic dataset ==> " + mosaic_dataset_full_path) try: arcpy.AddRastersToMosaicDataset_management(mosaic_dataset_full_path, "Raster Dataset", out_tif, build_pyramids=True, duplicate_items_action="EXCLUDE_DUPLICATES") logger.info("Image file added to the mosaic dataset") except Exception as e: logger.error("Unable to add " + out_tif + " image file to the mosaic dataset - " + str(e)) # Add all files to mosaic if this is the initial run # --------------------------------------------------- if initial_run: # Add all image files to the mosaic dataset # ------------------------------------------ logger.info("Adding all image files to the mosaic dataset ==> " + mosaic_dataset_full_path) try: arcpy.AddRastersToMosaicDataset_management(mosaic_dataset_full_path, "Raster Dataset", output_directory, build_pyramids=True, duplicate_items_action="EXCLUDE_DUPLICATES") logger.info("All image files from the directory added to the mosaic dataset") except Exception as e: logger.error("Unable to add " + output_directory + " image files to the mosaic dataset - " + str(e)) logger.info("Image file parsing and processing done ({})".format(product_name)) # Update attributes in mosaic dataset table # ------------------------------------------ logger.info("Updating attributes in the mosaic dataset table for " + product_name) with arcpy.da.UpdateCursor(mosaic_dataset_full_path, mosaic_dataset_fields) as cursor: for row in cursor: image_name_object = row[0].split(".") image_date = image_name_object[1][0:4] + "/" + image_name_object[1][4:6] + "/" + \ image_name_object[1][6:8] if len(image_name_object) > 2: image_time = image_name_object[2][0:2] + ":" + image_name_object[2][2:4] + ":" + \ image_name_object[2][4:6] image_date = image_date + " " + image_time row[1] = image_date row[2] = product_name cursor.updateRow(row) # logger.info("Mosaic dataset row updated: Image name({}), Image date({}), Product({})" # .format(row[0], image_date, row[2])) # <====== Spams the log file del cursor logger.info("done") # Remove directory images older than the age_of_imagery parameter # ---------------------------------------------------------------- logger.info("Removing directory images older than " + str(age_of_imagery) + " days for " + product_name) for root, dirs, files in os.walk(output_directory): for directory_file in files: file_date = directory_file.split(".")[1] if file_date < remove_date: logger.warning("Removing file " + root + directory_file) arcpy.Delete_management(root + directory_file) # os.remove(root + directory_file) # logger.info("Current file does not need removal ==> " + root + file) # <====== Spams the log file logger.info("done") # Set the where clause for mosaic dataset deletions # -------------------------------------------------- where_clause = "Product = '" + product_name + "' AND (Date < date '" + remove_date_query + "' OR Date IS NULL)" logger.info("where clause for mosaic dataset deletions ==> " + where_clause) # Remove mosaic dataset images older than the age_of_imagery parameter # --------------------------------------------------------------------- logger.info("Removing mosaic dataset images older than " + str(age_of_imagery) + " days for " + product_name) try: arcpy.RemoveRastersFromMosaicDataset_management(mosaic_dataset_full_path, where_clause) logger.info("done") except Exception as e: logger.error("Unable to remove mosaic dataset images with the following where clause for " + product_name + ": " + str(where_clause) + " - " + str(e)) # Repair GPM mosaic dataset paths # -------------------------------- logger.info("Repairing mosaic dataset paths for " + product_name) UtilitiesRaster.repair_mosaic_paths(logger, mosaic_dataset_full_path, output_directory) # Set mosaic dataset NoData values for GPM only # ---------------------------------------------- if "gpm" in product_file_prefix: logger.info("Setting GPM mosaic dataset NoData values") try: arcpy.DefineMosaicDatasetNoData_management(mosaic_dataset_full_path, "1", "BAND_1 '0 29999'", "", "", "NO_COMPOSITE_NODATA") logger.info("done") except Exception as e: logger.error("Unable to set GPM mosaic dataset NoData values - " + str(e)) logger.info("Product loop done for {}".format(product_name)) logger.info("Script complete") # Script start # ------------- if __name__ == "__main__": sys.exit(main())