diff --git a/app.py b/app.py index 69e176e..0a8e844 100644 --- a/app.py +++ b/app.py @@ -4,9 +4,11 @@ import sys import os import json from datetime import datetime +from datetime import timedelta import argparse import urllib.parse from dotenv import load_dotenv +import time # Load environment variables from .env file load_dotenv("753DataSync.env") @@ -259,6 +261,8 @@ def add_features(token, hostname, instance, fs, layer, aggregated_data, secure=T def main(): """Main entry point for the script.""" + start_time = time.time() + try: logger.info("Starting script execution.") @@ -305,11 +309,11 @@ def main(): timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") page_filename = f"data/enforcement_page_{page_number}_results_{results_per_page}_{timestamp}.json" - # Save individual page data + # Save individual page data if in DEBUG mode if log_level == 'DEBUG': save_json(data, page_filename) - # Check if the number of records is less than the results_per_page, indicating last page + # Stop if last page if len(data) < results_per_page: logger.info("No more data to fetch, stopping pagination.") break @@ -320,20 +324,22 @@ def main(): break # Prepare aggregated data - aggregated_data = all_data # Just use the collected features directly + aggregated_data = all_data - # Save aggregated data to a single JSON file + # Save aggregated data aggregated_filename = f"data/aggregated_enforcement_results_{timestamp}.json" logger.info(f"Saving aggregated data to {aggregated_filename}.") save_json(aggregated_data, aggregated_filename) # Add the features to the feature layer response = add_features(token, hostname, instance, fs, layer, aggregated_data) + except Exception as e: logger.error(f"An unexpected error occurred: {e}", exc_info=True) return finally: - logger.info("Script execution completed.") + elapsed_time = timedelta(seconds=time.time() - start_time) + logger.info(f"Script execution completed in {str(elapsed_time)}.") if __name__ == "__main__": main() \ No newline at end of file