diff --git a/README.md b/README.md index 579a304..c230818 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ ![753 Data Sync logo](https://git.nickhepler.cloud/nick/753-Data-Sync/raw/branch/master/logo.png) # 753 Data Sync -![Gitea Release](https://img.shields.io/gitea/v/release/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud) -![Gitea Open Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud) +![Gitea Release](https://img.shields.io/gitea/v/release/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&style=for-the-badge&logo=Python) +![Gitea Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&labels=enhancement&style=for-the-badge&logo=Gitea&label=Enhancements) +![Gitea Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&labels=bug&style=for-the-badge&logo=Gitea&label=Defects) This script fetches enforcement data from an external API, truncates a specified feature layer in ArcGIS, and adds the fetched data as features to the layer. The script performs the following tasks: diff --git a/app.py b/app.py index 69e176e..0a8e844 100644 --- a/app.py +++ b/app.py @@ -4,9 +4,11 @@ import sys import os import json from datetime import datetime +from datetime import timedelta import argparse import urllib.parse from dotenv import load_dotenv +import time # Load environment variables from .env file load_dotenv("753DataSync.env") @@ -259,6 +261,8 @@ def add_features(token, hostname, instance, fs, layer, aggregated_data, secure=T def main(): """Main entry point for the script.""" + start_time = time.time() + try: logger.info("Starting script execution.") @@ -305,11 +309,11 @@ def main(): timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json" - # Save individual page data + # Save individual page data if in DEBUG mode if log_level == 'DEBUG': save_json(data, page_filename) - # Check if the number of records is less than the results_per_page, indicating last page + # Stop if last page if len(data) < results_per_page: logger.info("No more data to fetch, stopping pagination.") break @@ -320,20 +324,22 @@ def main(): break # Prepare aggregated data - aggregated_data = all_data # Just use the collected features directly + aggregated_data = all_data - # Save aggregated data to a single JSON file + # Save aggregated data aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json" logger.info(f"Saving aggregated data to {aggregated_filename}.") save_json(aggregated_data, aggregated_filename) # Add the features to the feature layer response = add_features(token, hostname, instance, fs, layer, aggregated_data) + except Exception as e: logger.error(f"An unexpected error occurred: {e}", exc_info=True) return finally: - logger.info("Script execution completed.") + elapsed_time = timedelta(seconds=time.time() - start_time) + logger.info(f"Script execution completed in {str(elapsed_time)}.") if __name__ == "__main__": main() \ No newline at end of file