Compare commits

...

2 Commits

Author SHA1 Message Date
Nick Heppler
0ebb27729d Update shields.io badges. 2025-04-15 17:15:19 -04:00
Nick Heppler
b886a4636c log total script execution time in HH:MM:SS format. 2025-04-15 17:00:12 -04:00
2 changed files with 14 additions and 7 deletions

View File

@ -1,7 +1,8 @@
![753 Data Sync logo](https://git.nickhepler.cloud/nick/753-Data-Sync/raw/branch/master/logo.png) ![753 Data Sync logo](https://git.nickhepler.cloud/nick/753-Data-Sync/raw/branch/master/logo.png)
# 753 Data Sync # 753 Data Sync
![Gitea Release](https://img.shields.io/gitea/v/release/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud) ![Gitea Release](https://img.shields.io/gitea/v/release/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&style=for-the-badge&logo=Python)
![Gitea Open Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud) ![Gitea Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&labels=enhancement&style=for-the-badge&logo=Gitea&label=Enhancements)
![Gitea Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&labels=bug&style=for-the-badge&logo=Gitea&label=Defects)
This script fetches enforcement data from an external API, truncates a specified feature layer in ArcGIS, and adds the fetched data as features to the layer. The script performs the following tasks: This script fetches enforcement data from an external API, truncates a specified feature layer in ArcGIS, and adds the fetched data as features to the layer. The script performs the following tasks:

16
app.py
View File

@ -4,9 +4,11 @@ import sys
import os import os
import json import json
from datetime import datetime from datetime import datetime
from datetime import timedelta
import argparse import argparse
import urllib.parse import urllib.parse
from dotenv import load_dotenv from dotenv import load_dotenv
import time
# Load environment variables from .env file # Load environment variables from .env file
load_dotenv("753DataSync.env") load_dotenv("753DataSync.env")
@ -259,6 +261,8 @@ def add_features(token, hostname, instance, fs, layer, aggregated_data, secure=T
def main(): def main():
"""Main entry point for the script.""" """Main entry point for the script."""
start_time = time.time()
try: try:
logger.info("Starting script execution.") logger.info("Starting script execution.")
@ -305,11 +309,11 @@ def main():
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json" page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json"
# Save individual page data # Save individual page data if in DEBUG mode
if log_level == 'DEBUG': if log_level == 'DEBUG':
save_json(data, page_filename) save_json(data, page_filename)
# Check if the number of records is less than the results_per_page, indicating last page # Stop if last page
if len(data) < results_per_page: if len(data) < results_per_page:
logger.info("No more data to fetch, stopping pagination.") logger.info("No more data to fetch, stopping pagination.")
break break
@ -320,20 +324,22 @@ def main():
break break
# Prepare aggregated data # Prepare aggregated data
aggregated_data = all_data # Just use the collected features directly aggregated_data = all_data
# Save aggregated data to a single JSON file # Save aggregated data
aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json" aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json"
logger.info(f"Saving aggregated data to {aggregated_filename}.") logger.info(f"Saving aggregated data to {aggregated_filename}.")
save_json(aggregated_data, aggregated_filename) save_json(aggregated_data, aggregated_filename)
# Add the features to the feature layer # Add the features to the feature layer
response = add_features(token, hostname, instance, fs, layer, aggregated_data) response = add_features(token, hostname, instance, fs, layer, aggregated_data)
except Exception as e: except Exception as e:
logger.error(f"An unexpected error occurred: {e}", exc_info=True) logger.error(f"An unexpected error occurred: {e}", exc_info=True)
return return
finally: finally:
logger.info("Script execution completed.") elapsed_time = timedelta(seconds=time.time() - start_time)
logger.info(f"Script execution completed in {str(elapsed_time)}.")
if __name__ == "__main__": if __name__ == "__main__":
main() main()