nick/753-Data-Sync
1
0
Fork 0
Code Issues 4 Pull Requests Releases 2 Activity

log total script execution time in HH:MM:SS format.

Browse Source
This commit is contained in:
Nick Heppler 2025-04-15 17:00:12 -04:00
parent 3765c78de0
commit b886a4636c
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
app.py
View File

@ -4,9 +4,11 @@ import sys
import os import os
import json import json
from datetime import datetime from datetime import datetime
from datetime import timedelta
import argparse import argparse
import urllib.parse import urllib.parse
from dotenv import load_dotenv from dotenv import load_dotenv
import time
# Load environment variables from .env file # Load environment variables from .env file
load_dotenv("753DataSync.env") load_dotenv("753DataSync.env")
@ -259,6 +261,8 @@ def add_features(token, hostname, instance, fs, layer, aggregated_data, secure=T
def main(): def main():
"""Main entry point for the script.""" """Main entry point for the script."""
start_time = time.time()
try: try:
logger.info("Starting script execution.") logger.info("Starting script execution.")
@ -305,11 +309,11 @@ def main():
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json" page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json"
# Save individual page data # Save individual page data if in DEBUG mode
if log_level == 'DEBUG': if log_level == 'DEBUG':
save_json(data, page_filename) save_json(data, page_filename)
# Check if the number of records is less than the results_per_page, indicating last page # Stop if last page
if len(data) < results_per_page: if len(data) < results_per_page:
logger.info("No more data to fetch, stopping pagination.") logger.info("No more data to fetch, stopping pagination.")
break break
@ -320,20 +324,22 @@ def main():
break break
# Prepare aggregated data # Prepare aggregated data
aggregated_data = all_data # Just use the collected features directly aggregated_data = all_data
# Save aggregated data to a single JSON file # Save aggregated data
aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json" aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json"
logger.info(f"Saving aggregated data to {aggregated_filename}.") logger.info(f"Saving aggregated data to {aggregated_filename}.")
save_json(aggregated_data, aggregated_filename) save_json(aggregated_data, aggregated_filename)
# Add the features to the feature layer # Add the features to the feature layer
response = add_features(token, hostname, instance, fs, layer, aggregated_data) response = add_features(token, hostname, instance, fs, layer, aggregated_data)
except Exception as e: except Exception as e:
logger.error(f"An unexpected error occurred: {e}", exc_info=True) logger.error(f"An unexpected error occurred: {e}", exc_info=True)
return return
finally: finally:
logger.info("Script execution completed.") elapsed_time = timedelta(seconds=time.time() - start_time)
logger.info(f"Script execution completed in {str(elapsed_time)}.")
if __name__ == "__main__": if __name__ == "__main__":
main() main()