feat(cache): implement in-memory caching for YAML data to improve performance

backend/app/data/__init__.py

@@ -7,6 +7,7 @@ from .utils import (get_category_directory, load_yaml_file, validate,
                     test_regex_pattern, test_format_conditions,
                     check_delete_constraints, filename_to_display)
 from ..db import add_format_to_renames, remove_format_from_renames, is_format_in_renames
+from .cache import data_cache
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -16,43 +17,19 @@ bp = Blueprint('data', __name__)
 @bp.route('/<string:category>', methods=['GET'])
 def retrieve_all(category):
     try:
-        directory = get_category_directory(category)
-        files = [f for f in os.listdir(directory) if f.endswith('.yml')]
-        logger.debug(f"Found {len(files)} files in {category}")
-
-        if not files:
-            return jsonify([]), 200
-
-        result = []
-        errors = 0
-        for file_name in files:
-            file_path = os.path.join(directory, file_name)
-            try:
-                content = load_yaml_file(file_path)
-                # Add metadata for custom formats
-                if category == 'custom_format':
-                    content['metadata'] = {
-                        'includeInRename':
-                        is_format_in_renames(content['name'])
+        # Use cache instead of reading from disk
+        items = data_cache.get_all(category)
+        
+        # Add metadata for custom formats
+        if category == 'custom_format':
+            for item in items:
+                if 'content' in item and 'name' in item['content']:
+                    item['content']['metadata'] = {
+                        'includeInRename': is_format_in_renames(item['content']['name'])
                     }
-                result.append({
-                    "file_name":
-                    file_name,
-                    "content":
-                    content,
-                    "modified_date":
-                    get_file_modified_date(file_path)
-                })
-            except yaml.YAMLError:
-                errors += 1
-                result.append({
-                    "file_name": file_name,
-                    "error": "Failed to parse YAML"
-                })
-
-        logger.info(
-            f"Processed {len(files)} {category} files ({errors} errors)")
-        return jsonify(result), 200
+        
+        logger.info(f"Retrieved {len(items)} {category} items from cache")
+        return jsonify(items), 200
 
     except ValueError as ve:
         logger.error(ve)
@@ -127,6 +104,10 @@ def handle_item(category, name):
 
                 # Then delete the file
                 os.remove(file_path)
+                
+                # Update cache
+                data_cache.remove_item(category, file_name)
+                
                 return jsonify(
                     {"message": f"Successfully deleted {file_name}"}), 200
             except OSError as e:

backend/app/data/cache.py

@@ -0,0 +1,113 @@
+import os
+import yaml
+import logging
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+import threading
+from .utils import get_category_directory, get_file_modified_date, filename_to_display
+
+logger = logging.getLogger(__name__)
+
+class DataCache:
+    """In-memory cache for YAML data"""
+    
+    def __init__(self):
+        self._cache = {
+            'regex_pattern': {},
+            'custom_format': {},
+            'profile': {}
+        }
+        self._lock = threading.RLock()
+        self._initialized = False
+    
+    def initialize(self):
+        """Load all data into memory on startup"""
+        with self._lock:
+            if self._initialized:
+                return
+            
+            logger.info("Initializing data cache...")
+            for category in self._cache.keys():
+                self._load_category(category)
+            
+            self._initialized = True
+            logger.info("Data cache initialized successfully")
+    
+    def _load_category(self, category: str):
+        """Load all items from a category into cache"""
+        try:
+            directory = get_category_directory(category)
+            items = {}
+            
+            for filename in os.listdir(directory):
+                if not filename.endswith('.yml'):
+                    continue
+                
+                file_path = os.path.join(directory, filename)
+                try:
+                    with open(file_path, 'r') as f:
+                        content = yaml.safe_load(f)
+                        if content:
+                            # Store with metadata
+                            items[filename] = {
+                                'file_name': filename,
+                                'modified_date': get_file_modified_date(file_path),
+                                'content': content
+                            }
+                except Exception as e:
+                    logger.error(f"Error loading {file_path}: {e}")
+            
+            self._cache[category] = items
+            logger.info(f"Loaded {len(items)} items for category {category}")
+            
+        except Exception as e:
+            logger.error(f"Error loading category {category}: {e}")
+    
+    def get_all(self, category: str) -> List[Dict[str, Any]]:
+        """Get all items from a category"""
+        with self._lock:
+            if not self._initialized:
+                self.initialize()
+            
+            return list(self._cache.get(category, {}).values())
+    
+    def get_item(self, category: str, name: str) -> Optional[Dict[str, Any]]:
+        """Get a specific item"""
+        with self._lock:
+            if not self._initialized:
+                self.initialize()
+            
+            # Convert name to filename
+            filename = f"{name.replace('[', '(').replace(']', ')')}.yml"
+            return self._cache.get(category, {}).get(filename)
+    
+    def update_item(self, category: str, filename: str, content: Dict[str, Any]):
+        """Update an item in cache"""
+        with self._lock:
+            if category in self._cache:
+                file_path = os.path.join(get_category_directory(category), filename)
+                self._cache[category][filename] = {
+                    'file_name': filename,
+                    'modified_date': get_file_modified_date(file_path),
+                    'content': content
+                }
+                logger.debug(f"Updated cache for {category}/{filename}")
+    
+    def remove_item(self, category: str, filename: str):
+        """Remove an item from cache"""
+        with self._lock:
+            if category in self._cache and filename in self._cache[category]:
+                del self._cache[category][filename]
+                logger.debug(f"Removed from cache: {category}/{filename}")
+    
+    def rename_item(self, category: str, old_filename: str, new_filename: str):
+        """Rename an item in cache"""
+        with self._lock:
+            if category in self._cache and old_filename in self._cache[category]:
+                item = self._cache[category].pop(old_filename)
+                item['file_name'] = new_filename
+                self._cache[category][new_filename] = item
+                logger.debug(f"Renamed in cache: {category}/{old_filename} -> {new_filename}")
+
+# Global cache instance
+data_cache = DataCache()

backend/app/data/utils.py

@@ -154,6 +154,11 @@ def save_yaml_file(file_path: str,
 
     with open(safe_file_path, 'w') as f:
         yaml.safe_dump(ordered_data, f, sort_keys=False)
+    
+    # Update cache
+    from .cache import data_cache
+    filename = os.path.basename(safe_file_path)
+    data_cache.update_item(category, filename, ordered_data)
 
 
 def update_yaml_file(file_path: str, data: Dict[str, Any],
@@ -218,6 +223,12 @@ def update_yaml_file(file_path: str, data: Dict[str, Any],
                     os.rename(file_path, new_file_path)
                     # Stage the new file
                     repo.index.add([rel_new_path])
+                
+                # Update cache for rename
+                from .cache import data_cache
+                old_filename = os.path.basename(file_path)
+                new_filename = os.path.basename(new_file_path)
+                data_cache.rename_item(category, old_filename, new_filename)
 
             except git.GitCommandError as e:
                 logger.error(f"Git operation failed: {e}")

backend/app/main.py

@@ -18,6 +18,7 @@ from .logs import bp as logs_bp
 from .media_management import media_management_bp
 from .middleware import init_middleware
 from .init import setup_logging, init_app_config, init_git_user
+from .data.cache import data_cache
 
 
 def create_app():
@@ -48,6 +49,10 @@ def create_app():
     # Initialize Git user configuration
     logger.info("Initializing Git user")
     success, message = init_git_user()
+    
+    # Initialize data cache
+    logger.info("Initializing data cache")
+    data_cache.initialize()
     if not success:
         logger.warning(f"Git user initialization issue: {message}")
     else: