diff --git a/backend/app/data/__init__.py b/backend/app/data/__init__.py index c997d2a..afcfdfc 100644 --- a/backend/app/data/__init__.py +++ b/backend/app/data/__init__.py @@ -7,6 +7,7 @@ from .utils import (get_category_directory, load_yaml_file, validate, test_regex_pattern, test_format_conditions, check_delete_constraints, filename_to_display) from ..db import add_format_to_renames, remove_format_from_renames, is_format_in_renames +from .cache import data_cache logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -16,43 +17,19 @@ bp = Blueprint('data', __name__) @bp.route('/', methods=['GET']) def retrieve_all(category): try: - directory = get_category_directory(category) - files = [f for f in os.listdir(directory) if f.endswith('.yml')] - logger.debug(f"Found {len(files)} files in {category}") - - if not files: - return jsonify([]), 200 - - result = [] - errors = 0 - for file_name in files: - file_path = os.path.join(directory, file_name) - try: - content = load_yaml_file(file_path) - # Add metadata for custom formats - if category == 'custom_format': - content['metadata'] = { - 'includeInRename': - is_format_in_renames(content['name']) + # Use cache instead of reading from disk + items = data_cache.get_all(category) + + # Add metadata for custom formats + if category == 'custom_format': + for item in items: + if 'content' in item and 'name' in item['content']: + item['content']['metadata'] = { + 'includeInRename': is_format_in_renames(item['content']['name']) } - result.append({ - "file_name": - file_name, - "content": - content, - "modified_date": - get_file_modified_date(file_path) - }) - except yaml.YAMLError: - errors += 1 - result.append({ - "file_name": file_name, - "error": "Failed to parse YAML" - }) - - logger.info( - f"Processed {len(files)} {category} files ({errors} errors)") - return jsonify(result), 200 + + logger.info(f"Retrieved {len(items)} {category} items from cache") + return jsonify(items), 200 except ValueError as ve: logger.error(ve) @@ -127,6 +104,10 @@ def handle_item(category, name): # Then delete the file os.remove(file_path) + + # Update cache + data_cache.remove_item(category, file_name) + return jsonify( {"message": f"Successfully deleted {file_name}"}), 200 except OSError as e: diff --git a/backend/app/data/cache.py b/backend/app/data/cache.py new file mode 100644 index 0000000..9fa43f3 --- /dev/null +++ b/backend/app/data/cache.py @@ -0,0 +1,113 @@ +import os +import yaml +import logging +from typing import Dict, List, Any, Optional +from datetime import datetime +import threading +from .utils import get_category_directory, get_file_modified_date, filename_to_display + +logger = logging.getLogger(__name__) + +class DataCache: + """In-memory cache for YAML data""" + + def __init__(self): + self._cache = { + 'regex_pattern': {}, + 'custom_format': {}, + 'profile': {} + } + self._lock = threading.RLock() + self._initialized = False + + def initialize(self): + """Load all data into memory on startup""" + with self._lock: + if self._initialized: + return + + logger.info("Initializing data cache...") + for category in self._cache.keys(): + self._load_category(category) + + self._initialized = True + logger.info("Data cache initialized successfully") + + def _load_category(self, category: str): + """Load all items from a category into cache""" + try: + directory = get_category_directory(category) + items = {} + + for filename in os.listdir(directory): + if not filename.endswith('.yml'): + continue + + file_path = os.path.join(directory, filename) + try: + with open(file_path, 'r') as f: + content = yaml.safe_load(f) + if content: + # Store with metadata + items[filename] = { + 'file_name': filename, + 'modified_date': get_file_modified_date(file_path), + 'content': content + } + except Exception as e: + logger.error(f"Error loading {file_path}: {e}") + + self._cache[category] = items + logger.info(f"Loaded {len(items)} items for category {category}") + + except Exception as e: + logger.error(f"Error loading category {category}: {e}") + + def get_all(self, category: str) -> List[Dict[str, Any]]: + """Get all items from a category""" + with self._lock: + if not self._initialized: + self.initialize() + + return list(self._cache.get(category, {}).values()) + + def get_item(self, category: str, name: str) -> Optional[Dict[str, Any]]: + """Get a specific item""" + with self._lock: + if not self._initialized: + self.initialize() + + # Convert name to filename + filename = f"{name.replace('[', '(').replace(']', ')')}.yml" + return self._cache.get(category, {}).get(filename) + + def update_item(self, category: str, filename: str, content: Dict[str, Any]): + """Update an item in cache""" + with self._lock: + if category in self._cache: + file_path = os.path.join(get_category_directory(category), filename) + self._cache[category][filename] = { + 'file_name': filename, + 'modified_date': get_file_modified_date(file_path), + 'content': content + } + logger.debug(f"Updated cache for {category}/{filename}") + + def remove_item(self, category: str, filename: str): + """Remove an item from cache""" + with self._lock: + if category in self._cache and filename in self._cache[category]: + del self._cache[category][filename] + logger.debug(f"Removed from cache: {category}/{filename}") + + def rename_item(self, category: str, old_filename: str, new_filename: str): + """Rename an item in cache""" + with self._lock: + if category in self._cache and old_filename in self._cache[category]: + item = self._cache[category].pop(old_filename) + item['file_name'] = new_filename + self._cache[category][new_filename] = item + logger.debug(f"Renamed in cache: {category}/{old_filename} -> {new_filename}") + +# Global cache instance +data_cache = DataCache() \ No newline at end of file diff --git a/backend/app/data/utils.py b/backend/app/data/utils.py index ccd2769..902cfc9 100644 --- a/backend/app/data/utils.py +++ b/backend/app/data/utils.py @@ -154,6 +154,11 @@ def save_yaml_file(file_path: str, with open(safe_file_path, 'w') as f: yaml.safe_dump(ordered_data, f, sort_keys=False) + + # Update cache + from .cache import data_cache + filename = os.path.basename(safe_file_path) + data_cache.update_item(category, filename, ordered_data) def update_yaml_file(file_path: str, data: Dict[str, Any], @@ -218,6 +223,12 @@ def update_yaml_file(file_path: str, data: Dict[str, Any], os.rename(file_path, new_file_path) # Stage the new file repo.index.add([rel_new_path]) + + # Update cache for rename + from .cache import data_cache + old_filename = os.path.basename(file_path) + new_filename = os.path.basename(new_file_path) + data_cache.rename_item(category, old_filename, new_filename) except git.GitCommandError as e: logger.error(f"Git operation failed: {e}") diff --git a/backend/app/main.py b/backend/app/main.py index efcb54c..bc7f138 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -18,6 +18,7 @@ from .logs import bp as logs_bp from .media_management import media_management_bp from .middleware import init_middleware from .init import setup_logging, init_app_config, init_git_user +from .data.cache import data_cache def create_app(): @@ -48,6 +49,10 @@ def create_app(): # Initialize Git user configuration logger.info("Initializing Git user") success, message = init_git_user() + + # Initialize data cache + logger.info("Initializing data cache") + data_cache.initialize() if not success: logger.warning(f"Git user initialization issue: {message}") else: