nick/753-Data-Sync
1
0
Fork 0
Code Issues 4 Pull Requests Releases 2 Activity

Compare commits

base: nick:0ebb27729dc17d3c2f29758064dd256fe19f6427
Branches Tags
nick:master
nick:feature/logfile-purge-support
nick:enhancement/log-execution-time
nick:v2025.04
nick:v2025.03
..
compare: nick:3765c78de0637a0c90a17ec433dbbec88df15adf
Branches Tags
nick:master
nick:feature/logfile-purge-support
nick:enhancement/log-execution-time
nick:v2025.04
nick:v2025.03

No commits in common. "0ebb27729dc17d3c2f29758064dd256fe19f6427" and "3765c78de0637a0c90a17ec433dbbec88df15adf" have entirely different histories.
0ebb27729d ... 3765c78de0

2 changed files with 7 additions and 14 deletions
Show Stats Expand all files Collapse all files

5
README.md
View File

@ -1,8 +1,7 @@
![753 Data Sync logo](https://git.nickhepler.cloud/nick/753-Data-Sync/raw/branch/master/logo.png) ![753 Data Sync logo](https://git.nickhepler.cloud/nick/753-Data-Sync/raw/branch/master/logo.png)
# 753 Data Sync # 753 Data Sync
![Gitea Release](https://img.shields.io/gitea/v/release/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&style=for-the-badge&logo=Python) ![Gitea Release](https://img.shields.io/gitea/v/release/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud)
![Gitea Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&labels=enhancement&style=for-the-badge&logo=Gitea&label=Enhancements) ![Gitea Open Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud)
![Gitea Issues](https://img.shields.io/gitea/issues/open/nick/753-Data-Sync?gitea_url=https%3A%2F%2Fgit.nickhepler.cloud%2F&labels=bug&style=for-the-badge&logo=Gitea&label=Defects)
This script fetches enforcement data from an external API, truncates a specified feature layer in ArcGIS, and adds the fetched data as features to the layer. The script performs the following tasks: This script fetches enforcement data from an external API, truncates a specified feature layer in ArcGIS, and adds the fetched data as features to the layer. The script performs the following tasks:

16
app.py
View File

@ -4,11 +4,9 @@ import sys
import os import os
import json import json
from datetime import datetime from datetime import datetime
from datetime import timedelta
import argparse import argparse
import urllib.parse import urllib.parse
from dotenv import load_dotenv from dotenv import load_dotenv
import time
# Load environment variables from .env file # Load environment variables from .env file
load_dotenv("753DataSync.env") load_dotenv("753DataSync.env")
@ -261,8 +259,6 @@ def add_features(token, hostname, instance, fs, layer, aggregated_data, secure=T
def main(): def main():
"""Main entry point for the script.""" """Main entry point for the script."""
start_time = time.time()
try: try:
logger.info("Starting script execution.") logger.info("Starting script execution.")
@ -309,11 +305,11 @@ def main():
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json" page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json"
# Save individual page data if in DEBUG mode # Save individual page data
if log_level == 'DEBUG': if log_level == 'DEBUG':
save_json(data, page_filename) save_json(data, page_filename)
# Stop if last page # Check if the number of records is less than the results_per_page, indicating last page
if len(data) < results_per_page: if len(data) < results_per_page:
logger.info("No more data to fetch, stopping pagination.") logger.info("No more data to fetch, stopping pagination.")
break break
@ -324,22 +320,20 @@ def main():
break break
# Prepare aggregated data # Prepare aggregated data
aggregated_data = all_data aggregated_data = all_data # Just use the collected features directly
# Save aggregated data # Save aggregated data to a single JSON file
aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json" aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json"
logger.info(f"Saving aggregated data to {aggregated_filename}.") logger.info(f"Saving aggregated data to {aggregated_filename}.")
save_json(aggregated_data, aggregated_filename) save_json(aggregated_data, aggregated_filename)
# Add the features to the feature layer # Add the features to the feature layer
response = add_features(token, hostname, instance, fs, layer, aggregated_data) response = add_features(token, hostname, instance, fs, layer, aggregated_data)
except Exception as e: except Exception as e:
logger.error(f"An unexpected error occurred: {e}", exc_info=True) logger.error(f"An unexpected error occurred: {e}", exc_info=True)
return return
finally: finally:
elapsed_time = timedelta(seconds=time.time() - start_time) logger.info("Script execution completed.")
logger.info(f"Script execution completed in {str(elapsed_time)}.")
if __name__ == "__main__": if __name__ == "__main__":
main() main()