From ef86fa251f8979917514af6a778bff72eb8bfc40 Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Wed, 27 Aug 2025 01:34:10 +0930 Subject: [PATCH 1/9] feat(regex): add .NET regex validation via PowerShell and integrate into frontend --- Dockerfile | 19 ++++- backend/Dockerfile | 14 ++++ backend/app/data/__init__.py | 26 +++++++ backend/app/data/utils.py | 64 ++++++++++++++++ backend/scripts/validate.ps1 | 73 +++++++++++++++++++ frontend/src/api/data.js | 12 ++- .../src/components/regex/RegexGeneralTab.jsx | 58 ++++++++++++--- frontend/src/hooks/useRegexModal.js | 13 ++++ 8 files changed, 266 insertions(+), 13 deletions(-) create mode 100755 backend/scripts/validate.ps1 diff --git a/Dockerfile b/Dockerfile index 3edf8c5..6bb99f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,28 @@ # Dockerfile FROM python:3.9-slim WORKDIR /app -# Install git and gosu for user switching -RUN apt-get update && apt-get install -y git gosu && rm -rf /var/lib/apt/lists/* +# Install git, gosu, and PowerShell Core +RUN apt-get update && apt-get install -y \ + git \ + gosu \ + wget \ + ca-certificates \ + libicu-dev \ + && wget -O /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.0/powershell-7.4.0-linux-x64.tar.gz \ + && mkdir -p /opt/microsoft/powershell/7 \ + && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 \ + && chmod +x /opt/microsoft/powershell/7/pwsh \ + && ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh \ + && rm /tmp/powershell.tar.gz \ + && rm -rf /var/lib/apt/lists/* # Copy pre-built files from dist directory COPY dist/backend/app ./app +COPY dist/backend/scripts ./app/scripts COPY dist/static ./app/static COPY dist/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt +# Ensure scripts are executable +RUN chmod +x /app/scripts/*.ps1 || true # Copy and setup entrypoint script COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/backend/Dockerfile b/backend/Dockerfile index 6e6b528..453bc4f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,7 +1,21 @@ FROM python:3.9 WORKDIR /app +# Install PowerShell Core +RUN apt-get update && apt-get install -y \ + wget \ + ca-certificates \ + libicu-dev \ + && wget -O /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.0/powershell-7.4.0-linux-x64.tar.gz \ + && mkdir -p /opt/microsoft/powershell/7 \ + && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 \ + && chmod +x /opt/microsoft/powershell/7/pwsh \ + && ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh \ + && rm /tmp/powershell.tar.gz \ + && rm -rf /var/lib/apt/lists/* COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . +# Ensure scripts are executable +RUN chmod +x /app/scripts/*.ps1 || true # Use gunicorn with 10-minute timeout CMD ["python", "-m", "app.main"] \ No newline at end of file diff --git a/backend/app/data/__init__.py b/backend/app/data/__init__.py index 0267b9d..ab6bbdb 100644 --- a/backend/app/data/__init__.py +++ b/backend/app/data/__init__.py @@ -226,6 +226,32 @@ def handle_item(category, name): return jsonify({"error": "An unexpected error occurred"}), 500 +@bp.route('/regex/verify', methods=['POST']) +def verify_regex(): + """Verify a regex pattern using .NET regex engine via PowerShell""" + try: + data = request.get_json() + if not data: + return jsonify({"error": "No JSON data provided"}), 400 + + pattern = data.get('pattern') + if not pattern: + return jsonify({"error": "Pattern is required"}), 400 + + from .utils import verify_dotnet_regex + + success, message = verify_dotnet_regex(pattern) + + if success: + return jsonify({"valid": True, "message": "Pattern is valid"}), 200 + else: + return jsonify({"valid": False, "error": message}), 200 + + except Exception as e: + logger.exception("Error verifying regex pattern") + return jsonify({"valid": False, "error": str(e)}), 500 + + @bp.route('//test', methods=['POST']) def run_tests(category): logger.info(f"Received test request for category: {category}") diff --git a/backend/app/data/utils.py b/backend/app/data/utils.py index 41d53b8..fefc40c 100644 --- a/backend/app/data/utils.py +++ b/backend/app/data/utils.py @@ -7,6 +7,8 @@ from typing import Dict, List, Any, Tuple, Union import git import regex import logging +import subprocess +import json from ..db.queries.arr import update_arr_config_on_rename, update_arr_config_on_delete logger = logging.getLogger(__name__) @@ -360,6 +362,68 @@ def check_delete_constraints(category: str, name: str) -> Tuple[bool, str]: return False, f"Error checking references: {str(e)}" +def verify_dotnet_regex(pattern: str) -> Tuple[bool, str]: + """ + Verify a regex pattern using .NET regex engine via PowerShell. + Returns (success, message) tuple. + """ + try: + # Get the path to the validate.ps1 script + # In Docker, the structure is /app/app/data/utils.py and script is at /app/scripts/validate.ps1 + script_path = os.path.join('/app', 'scripts', 'validate.ps1') + if not os.path.exists(script_path): + # Fallback for local development + script_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts', 'validate.ps1') + + # Run PowerShell script, passing pattern via stdin to avoid shell escaping issues + result = subprocess.run( + ['pwsh', '-File', script_path], + input=pattern, + capture_output=True, + text=True, + timeout=5 + ) + + if result.returncode != 0 and not result.stdout: + logger.error(f"PowerShell script failed: {result.stderr}") + return False, "Failed to validate pattern" + + # Log the raw output for debugging + logger.debug(f"PowerShell output: {result.stdout}") + + # Parse JSON output + try: + output = json.loads(result.stdout.strip()) + except json.JSONDecodeError: + # Try to find JSON in the output + lines = result.stdout.strip().split('\n') + for line in reversed(lines): + if line.strip(): + try: + output = json.loads(line) + break + except json.JSONDecodeError: + continue + else: + logger.error(f"No valid JSON found in output: {result.stdout}") + return False, "Failed to parse validation result" + + if output.get('valid'): + return True, output.get('message', 'Pattern is valid') + else: + return False, output.get('error', 'Invalid pattern') + + except subprocess.TimeoutExpired: + logger.error("Pattern validation timed out") + return False, "Pattern validation timed out" + except FileNotFoundError: + logger.error("PowerShell (pwsh) not found") + return False, "PowerShell is not available" + except Exception as e: + logger.error(f"Error validating pattern: {e}") + return False, f"Validation error: {str(e)}" + + def update_references(category: str, old_name: str, new_name: str) -> List[str]: """ diff --git a/backend/scripts/validate.ps1 b/backend/scripts/validate.ps1 new file mode 100755 index 0000000..07e9dc9 --- /dev/null +++ b/backend/scripts/validate.ps1 @@ -0,0 +1,73 @@ +#!/usr/bin/env pwsh +# Validate a .NET regex pattern + +param( + [Parameter(Mandatory=$false)] + [string]$Pattern +) + +# Set output encoding to UTF-8 +[Console]::OutputEncoding = [System.Text.Encoding]::UTF8 +$ErrorActionPreference = "Stop" + +# Read pattern from stdin if not provided as parameter +if (-not $Pattern) { + $Pattern = [System.Console]::In.ReadToEnd() +} + +# Ensure we have a pattern +if ([string]::IsNullOrWhiteSpace($Pattern)) { + $result = @{ + valid = $false + error = "No pattern provided" + } + Write-Output (ConvertTo-Json $result -Compress) + exit 0 +} + +try { + # Attempt to create a .NET Regex object with the pattern + # Using IgnoreCase option as per requirement + $regex = [System.Text.RegularExpressions.Regex]::new($Pattern, [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) + + # If we get here, the pattern is valid + $result = @{ + valid = $true + message = "Pattern is valid .NET regex" + } + + Write-Output (ConvertTo-Json $result -Compress) + exit 0 +} +catch { + # Pattern is invalid, extract the meaningful part of the error message + $errorMessage = $_.Exception.Message + + # Try to extract just the useful part of .NET regex errors + if ($errorMessage -match "Invalid pattern '.*?' at offset (\d+)\. (.+)") { + $errorMessage = "At position $($matches[1]): $($matches[2])" + } + elseif ($errorMessage -match 'parsing ".*?" - (.+)') { + $errorMessage = $matches[1] + } + elseif ($errorMessage -match 'Exception calling .* with .* argument\(s\): "(.+)"') { + $innerError = $matches[1] + if ($innerError -match "Invalid pattern '.*?' at offset (\d+)\. (.+)") { + $errorMessage = "At position $($matches[1]): $($matches[2])" + } + else { + $errorMessage = $innerError + } + } + + # Remove any trailing quotes or periods followed by quotes + $errorMessage = $errorMessage -replace '\."$', '.' -replace '"$', '' + + $result = @{ + valid = $false + error = $errorMessage + } + + Write-Output (ConvertTo-Json $result -Compress) + exit 0 +} \ No newline at end of file diff --git a/frontend/src/api/data.js b/frontend/src/api/data.js index 526f670..1d9dfa5 100644 --- a/frontend/src/api/data.js +++ b/frontend/src/api/data.js @@ -301,5 +301,15 @@ export const RegexPatterns = { update: (name, data, newName) => updateItem('regex_pattern', name, data, newName), delete: name => deleteItem('regex_pattern', name), - runTests: createSpecialEndpoint('regex_pattern', 'test') + runTests: createSpecialEndpoint('regex_pattern', 'test'), + verify: async pattern => { + try { + const response = await axios.post(`${BASE_URL}/regex/verify`, { + pattern + }); + return response.data; + } catch (error) { + throw handleError(error, 'verify regex pattern'); + } + } }; diff --git a/frontend/src/components/regex/RegexGeneralTab.jsx b/frontend/src/components/regex/RegexGeneralTab.jsx index b18db9f..06ab2a9 100644 --- a/frontend/src/components/regex/RegexGeneralTab.jsx +++ b/frontend/src/components/regex/RegexGeneralTab.jsx @@ -2,7 +2,9 @@ import React, {useState} from 'react'; import PropTypes from 'prop-types'; import MarkdownEditor from '@ui/MarkdownEditor'; import AddButton from '@ui/DataBar/AddButton'; -import {InfoIcon} from 'lucide-react'; +import {Regex, Loader} from 'lucide-react'; +import {RegexPatterns} from '@api/data'; +import Alert from '@ui/Alert'; const RegexGeneralTab = ({ name, @@ -18,6 +20,7 @@ const RegexGeneralTab = ({ patternError }) => { const [newTag, setNewTag] = useState(''); + const [validating, setValidating] = useState(false); const handleAddTag = () => { if (newTag.trim() && !tags.includes(newTag.trim())) { @@ -33,6 +36,30 @@ const RegexGeneralTab = ({ } }; + const handleValidatePattern = async () => { + if (!pattern?.trim()) { + Alert.warning('Please enter a pattern to validate'); + return; + } + + setValidating(true); + + try { + const result = await RegexPatterns.verify(pattern); + + if (result.valid) { + Alert.success('Pattern is valid .NET regex'); + } else { + Alert.error(result.error || 'Invalid pattern'); + } + } catch (error) { + console.error('Validation error:', error); + Alert.error('Failed to validate pattern'); + } finally { + setValidating(false); + } + }; + return (
{error && ( @@ -89,17 +116,28 @@ const RegexGeneralTab = ({
- -
- - Case insensitive PCRE2 +
+ +

+ Enter your regular expression pattern (case-insensitive .NET) +

+
-

- Enter your regular expression pattern -

{patternError && (

diff --git a/frontend/src/hooks/useRegexModal.js b/frontend/src/hooks/useRegexModal.js index 2a730db..4ddb9c7 100644 --- a/frontend/src/hooks/useRegexModal.js +++ b/frontend/src/hooks/useRegexModal.js @@ -65,6 +65,19 @@ export const useRegexModal = (initialPattern, onSave) => { return; } + // Validate pattern with .NET regex engine + try { + const validationResult = await RegexPatterns.verify(patternValue); + if (!validationResult.valid) { + Alert.error(`Invalid regex pattern: ${validationResult.error || 'Pattern validation failed'}`); + return; + } + } catch (error) { + console.error('Pattern validation error:', error); + Alert.error('Failed to validate pattern. Please check the pattern and try again.'); + return; + } + try { const data = { name, From 77f996f8c53a89488bbc2a9f39fe888b2b19228d Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Wed, 27 Aug 2025 02:38:01 +0930 Subject: [PATCH 2/9] feat(regex): implement .NET regex testing via PowerShell and enhance UI components --- backend/app/data/__init__.py | 16 +- backend/app/data/utils.py | 119 ++++++------- backend/scripts/test.ps1 | 107 ++++++++++++ .../src/components/regex/AddUnitTestModal.jsx | 6 +- frontend/src/components/regex/RegexModal.jsx | 29 ++-- .../src/components/regex/RegexTestingTab.jsx | 156 +++++++++--------- frontend/src/components/regex/UnitTest.jsx | 8 +- frontend/src/hooks/useRegexModal.js | 18 +- frontend/src/hooks/useRegexTesting.js | 8 +- 9 files changed, 281 insertions(+), 186 deletions(-) create mode 100755 backend/scripts/test.ps1 diff --git a/backend/app/data/__init__.py b/backend/app/data/__init__.py index ab6bbdb..c997d2a 100644 --- a/backend/app/data/__init__.py +++ b/backend/app/data/__init__.py @@ -259,25 +259,29 @@ def run_tests(category): try: data = request.get_json() if not data: - logger.warning("Rejected test request - no JSON data provided") + logger.warning("Test request rejected: no JSON data") return jsonify({"error": "No JSON data provided"}), 400 tests = data.get('tests', []) if not tests: - logger.warning("Rejected test request - no test cases provided") + logger.warning("Test request rejected: no tests provided") return jsonify({"error": "At least one test case is required"}), 400 if category == 'regex_pattern': pattern = data.get('pattern') - logger.info(f"Processing regex test request - Pattern: {pattern}") if not pattern: - logger.warning("Rejected test request - missing pattern") + logger.warning("Test request rejected: missing pattern") return jsonify({"error": "Pattern is required"}), 400 success, message, updated_tests = test_regex_pattern( pattern, tests) + + if success and updated_tests: + passed = sum(1 for t in updated_tests if t.get('passes')) + total = len(updated_tests) + logger.info(f"Tests completed: {passed}/{total} passed") elif category == 'custom_format': conditions = data.get('conditions', []) @@ -300,10 +304,8 @@ def run_tests(category): return jsonify( {"error": "Testing not supported for this category"}), 400 - logger.info(f"Test execution completed - Success: {success}") - if not success: - logger.warning(f"Test execution failed - {message}") + logger.error(f"Test execution failed: {message}") return jsonify({"success": False, "message": message}), 400 return jsonify({"success": True, "tests": updated_tests}), 200 diff --git a/backend/app/data/utils.py b/backend/app/data/utils.py index fefc40c..ccd2769 100644 --- a/backend/app/data/utils.py +++ b/backend/app/data/utils.py @@ -542,76 +542,67 @@ def test_regex_pattern( pattern: str, tests: List[Dict[str, Any]]) -> Tuple[bool, str, List[Dict[str, Any]]]: """ - Test a regex pattern against a list of test cases using PCRE2 compatible engine. + Test a regex pattern against a list of test cases using .NET regex engine via PowerShell. Returns match information along with test results. """ - logger.info(f"Starting regex pattern test - Pattern: {pattern}") try: - try: - compiled_pattern = regex.compile(pattern, - regex.V1 | regex.IGNORECASE) - logger.info( - "Pattern compiled successfully with PCRE2 compatibility") - except regex.error as e: - logger.warning(f"Invalid regex pattern: {str(e)}") - return False, f"Invalid regex pattern: {str(e)}", tests - - current_time = datetime.now().isoformat() - logger.info(f"Processing {len(tests)} test cases") - - for test in tests: - test_id = test.get('id', 'unknown') - test_input = test.get('input', '') - expected = test.get('expected', False) - - try: - match = compiled_pattern.search(test_input) - matches = bool(match) - - # Update test result with basic fields - test['passes'] = matches == expected - test['lastRun'] = current_time - - # Add match information - if match: - test['matchedContent'] = match.group(0) - test['matchSpan'] = { - 'start': match.start(), - 'end': match.end() - } - # Get all capture groups if they exist - test['matchedGroups'] = [g for g in match.groups() - ] if match.groups() else [] - else: - test['matchedContent'] = None - test['matchSpan'] = None - test['matchedGroups'] = [] - - logger.info( - f"Test {test_id} {'passed' if test['passes'] else 'failed'} - Match: {matches}, Expected: {expected}" - ) - - except Exception as e: - logger.error(f"Error running test {test_id}: {str(e)}") - test['passes'] = False - test['lastRun'] = current_time - test['matchedContent'] = None - test['matchSpan'] = None - test['matchedGroups'] = [] - - # Log overall results - passed_tests = sum(1 for test in tests if test.get('passes', False)) - logger.info( - f"Test execution complete - {passed_tests}/{len(tests)} tests passed" + # Get the path to the test.ps1 script + script_path = os.path.join('/app', 'scripts', 'test.ps1') + if not os.path.exists(script_path): + # Fallback for local development + script_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts', 'test.ps1') + + # Prepare the input data + input_data = { + 'pattern': pattern, + 'tests': tests + } + + # Run PowerShell script + result = subprocess.run( + ['pwsh', '-File', script_path], + input=json.dumps(input_data), + capture_output=True, + text=True, + timeout=10 ) - - return True, "", tests - + + if result.returncode != 0 and not result.stdout: + logger.error(f"PowerShell script failed: {result.stderr}") + return False, "Failed to run tests", tests + + # Parse JSON output + try: + output = json.loads(result.stdout.strip()) + except json.JSONDecodeError: + # Try to find JSON in the output + lines = result.stdout.strip().split('\n') + for line in reversed(lines): + if line.strip(): + try: + output = json.loads(line) + break + except json.JSONDecodeError: + continue + else: + logger.error(f"No valid JSON found in output: {result.stdout}") + return False, "Failed to parse test results", tests + + if output.get('success'): + return True, "Tests completed successfully", output.get('tests', tests) + else: + return False, output.get('message', 'Tests failed'), tests + + except subprocess.TimeoutExpired: + logger.error("Test execution timed out") + return False, "Test execution timed out", tests + except FileNotFoundError: + logger.error("PowerShell (pwsh) not found") + return False, "PowerShell is not available", tests except Exception as e: - logger.error(f"Unexpected error in test_regex_pattern: {str(e)}", - exc_info=True) - return False, str(e), tests + logger.error(f"Error running tests: {e}") + return False, f"Test error: {str(e)}", tests def test_format_conditions(conditions: List[Dict], diff --git a/backend/scripts/test.ps1 b/backend/scripts/test.ps1 new file mode 100755 index 0000000..eab6743 --- /dev/null +++ b/backend/scripts/test.ps1 @@ -0,0 +1,107 @@ +#!/usr/bin/env pwsh +# Run regex tests against a pattern + +# Set output encoding to UTF-8 +[Console]::OutputEncoding = [System.Text.Encoding]::UTF8 +$ErrorActionPreference = "Stop" + +# Read from stdin +$inputText = $input +if (-not $inputText) { + $inputText = [System.Console]::In.ReadToEnd() +} + +if (-not $inputText) { + Write-Output (ConvertTo-Json @{ + success = $false + message = "No input provided" + } -Compress) + exit 0 +} + +try { + $data = $inputText | ConvertFrom-Json + $Pattern = $data.pattern + $tests = $data.tests +} +catch { + Write-Output (ConvertTo-Json @{ + success = $false + message = "Failed to parse input JSON: $_" + } -Compress) + exit 0 +} + +# Ensure we have required inputs +if ([string]::IsNullOrWhiteSpace($Pattern)) { + Write-Output (ConvertTo-Json @{ + success = $false + message = "No pattern provided" + } -Compress) + exit 0 +} + +if (-not $tests -or $tests.Count -eq 0) { + Write-Output (ConvertTo-Json @{ + success = $false + message = "No tests provided" + } -Compress) + exit 0 +} + +try { + + # Create the regex object with case-insensitive option + $regex = [System.Text.RegularExpressions.Regex]::new($Pattern, [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) + + # Process each test + $results = @() + + foreach ($test in $tests) { + $match = $regex.Match($test.input) + $passes = ($match.Success -eq $test.expected) + + $result = @{ + id = $test.id + input = $test.input + expected = $test.expected + passes = $passes + } + + if ($match.Success) { + # Include match details for highlighting (using original format) + $result.matchedContent = $match.Value + $result.matchSpan = @{ + start = $match.Index + end = $match.Index + $match.Length + } + + # Include capture groups if any + $groups = @() + for ($i = 1; $i -lt $match.Groups.Count; $i++) { + if ($match.Groups[$i].Success) { + $groups += $match.Groups[$i].Value + } + } + $result.matchedGroups = $groups + } + else { + $result.matchedContent = $null + $result.matchSpan = $null + $result.matchedGroups = @() + } + + $results += $result + } + + Write-Output (ConvertTo-Json @{ + success = $true + tests = $results + } -Compress -Depth 10) +} +catch { + Write-Output (ConvertTo-Json @{ + success = $false + message = $_.Exception.Message + } -Compress) +} \ No newline at end of file diff --git a/frontend/src/components/regex/AddUnitTestModal.jsx b/frontend/src/components/regex/AddUnitTestModal.jsx index 4efe4cd..7456675 100644 --- a/frontend/src/components/regex/AddUnitTestModal.jsx +++ b/frontend/src/components/regex/AddUnitTestModal.jsx @@ -23,15 +23,13 @@ const AddUnitTestModal = ({isOpen, onClose, onAdd, tests, editTest = null}) => { const handleSubmit = () => { const getNextTestId = testArray => { if (!testArray || testArray.length === 0) return 1; - return Math.max(...testArray.map(test => test.id)) + 1; + return Math.max(...testArray.map(test => test.id || 0)) + 1; }; const testData = { id: editTest ? editTest.id : getNextTestId(tests), input, - expected: shouldMatch, - passes: false, - lastRun: null + expected: shouldMatch }; onAdd(testData); diff --git a/frontend/src/components/regex/RegexModal.jsx b/frontend/src/components/regex/RegexModal.jsx index d07359a..2275018 100644 --- a/frontend/src/components/regex/RegexModal.jsx +++ b/frontend/src/components/regex/RegexModal.jsx @@ -6,7 +6,7 @@ import RegexTestingTab from './RegexTestingTab'; import {useRegexModal} from '@hooks/useRegexModal'; import {RegexPatterns} from '@api/data'; import Alert from '@ui/Alert'; -import {Loader, Play} from 'lucide-react'; +import {Loader, Play, Save, Trash2, Check} from 'lucide-react'; const RegexModal = ({ pattern: initialPattern, @@ -84,12 +84,13 @@ const RegexModal = ({ {initialPattern && !isCloning && ( )}

@@ -97,20 +98,20 @@ const RegexModal = ({ )}
diff --git a/frontend/src/components/regex/RegexTestingTab.jsx b/frontend/src/components/regex/RegexTestingTab.jsx index 8b64dc4..cf26c5a 100644 --- a/frontend/src/components/regex/RegexTestingTab.jsx +++ b/frontend/src/components/regex/RegexTestingTab.jsx @@ -13,52 +13,47 @@ const RegexTestingTab = ({ }) => { const [isModalOpen, setIsModalOpen] = useState(false); const [editingTest, setEditingTest] = useState(null); + const [testResults, setTestResults] = useState({}); + + // Wrapped run tests function that stores results + const handleRunTests = useCallback(async (testPattern, testData) => { + const results = await onRunTests(testPattern, testData); + if (results && Array.isArray(results)) { + // Store results by test ID + const resultsMap = {}; + results.forEach(result => { + resultsMap[result.id] = result; + }); + setTestResults(resultsMap); + } + return results; + }, [onRunTests]); useEffect(() => { - const needsAutoRun = - tests?.length > 0 && - pattern && - tests.some(test => test.passes !== undefined && !test.matchSpan); - - if (needsAutoRun && !isRunningTests) { - onRunTests(pattern, tests); + // Run tests when pattern or tests change + if (tests?.length > 0 && pattern && !isRunningTests) { + handleRunTests(pattern, tests); } - }, []); + }, [pattern]); // Only re-run when pattern changes const handleAddOrUpdateTest = useCallback( testData => { let updatedTests; if (editingTest) { updatedTests = tests.map(test => - test.id === testData.id - ? { - ...testData, - passes: false, - lastRun: null, - matchedContent: null, - matchSpan: null, - matchedGroups: [] - } - : test + test.id === testData.id ? testData : test ); } else { - updatedTests = [ - ...tests, - { - ...testData, - passes: false, - lastRun: null, - matchedContent: null, - matchSpan: null, - matchedGroups: [] - } - ]; + updatedTests = [...tests, testData]; } onTestsChange(updatedTests); - onRunTests(pattern, updatedTests); + // Run tests automatically after adding/updating + if (pattern) { + handleRunTests(pattern, updatedTests); + } setEditingTest(null); }, - [tests, onTestsChange, onRunTests, pattern, editingTest] + [tests, onTestsChange, handleRunTests, pattern, editingTest] ); const handleEditTest = useCallback(test => { @@ -80,72 +75,81 @@ const RegexTestingTab = ({ }, []); const totalTests = tests?.length || 0; - const passedTests = tests?.filter(test => test.passes)?.length || 0; + const passedTests = tests?.filter(test => { + const result = testResults[test.id]; + return result?.passes; + })?.length || 0; return (
- {/* Header with Progress Bar */} -
+ {/* Header */} +
-

+

Unit Tests

-
-
-
-
- - {totalTests > 0 - ? `${passedTests}/${totalTests} tests passing` - : 'No tests added yet'} - -
+ {totalTests > 0 && ( +

+ {passedTests} of {totalTests} tests passing + {totalTests > 0 && ` (${Math.round((passedTests / totalTests) * 100)}%)`} +

+ )}
{tests?.length > 0 && ( )}
+ + {/* Progress Bar */} + {totalTests > 0 && ( +
+
+
+
+
+ )} {/* Test List */}
{tests?.length > 0 ? (
- {tests.map(test => ( - handleDeleteTest(test.id)} - onEdit={() => handleEditTest(test)} - /> - ))} + {tests.map(test => { + // Merge saved test with runtime results + const testWithResults = { + ...test, + ...testResults[test.id] + }; + return ( + handleDeleteTest(test.id)} + onEdit={() => handleEditTest(test)} + /> + ); + })}
) : (
@@ -173,15 +177,7 @@ RegexTestingTab.propTypes = { PropTypes.shape({ id: PropTypes.number.isRequired, input: PropTypes.string.isRequired, - expected: PropTypes.bool.isRequired, - passes: PropTypes.bool.isRequired, - lastRun: PropTypes.string, - matchedContent: PropTypes.string, - matchedGroups: PropTypes.arrayOf(PropTypes.string), - matchSpan: PropTypes.shape({ - start: PropTypes.number, - end: PropTypes.number - }) + expected: PropTypes.bool.isRequired }) ), onTestsChange: PropTypes.func.isRequired, diff --git a/frontend/src/components/regex/UnitTest.jsx b/frontend/src/components/regex/UnitTest.jsx index 77bc381..ee3eb10 100644 --- a/frontend/src/components/regex/UnitTest.jsx +++ b/frontend/src/components/regex/UnitTest.jsx @@ -68,11 +68,7 @@ const UnitTest = ({test, pattern, onDelete, onEdit}) => { : 'Should Not Match'}
-
- - Last run: {test.lastRun} - -
+
-
@@ -112,7 +107,6 @@ UnitTest.propTypes = { input: PropTypes.string.isRequired, expected: PropTypes.bool.isRequired, passes: PropTypes.bool.isRequired, - lastRun: PropTypes.string, matchedContent: PropTypes.string, matchedGroups: PropTypes.arrayOf(PropTypes.string), matchSpan: PropTypes.shape({ diff --git a/frontend/src/hooks/useRegexModal.js b/frontend/src/hooks/useRegexModal.js index 4ddb9c7..2a63778 100644 --- a/frontend/src/hooks/useRegexModal.js +++ b/frontend/src/hooks/useRegexModal.js @@ -79,12 +79,19 @@ export const useRegexModal = (initialPattern, onSave) => { } try { + // Clean tests to only include saved data + const cleanTests = tests.map((test, index) => ({ + id: test.id || index + 1, + input: test.input, + expected: test.expected + })); + const data = { name, pattern: patternValue, description, tags, - tests + tests: cleanTests }; if (initialPattern && !isCloning) { @@ -111,15 +118,16 @@ export const useRegexModal = (initialPattern, onSave) => { const handleRunTests = useCallback( async (pattern, tests) => { try { - const updatedTests = await runTests(pattern, tests); - if (updatedTests) { - setTests(updatedTests); - } + const testResults = await runTests(pattern, tests); + // We don't update the tests state with results + // Results are only used for display, not saved + return testResults; } catch (error) { console.error('Error running tests:', error); Alert.error( error.message || 'Failed to run tests. Please try again.' ); + return null; } }, [runTests] diff --git a/frontend/src/hooks/useRegexTesting.js b/frontend/src/hooks/useRegexTesting.js index 2011a2a..96fedd5 100644 --- a/frontend/src/hooks/useRegexTesting.js +++ b/frontend/src/hooks/useRegexTesting.js @@ -34,14 +34,12 @@ export const useRegexTesting = onUpdateTests => { } ); - // Update tests through the callback - if (onUpdateTests) { - onUpdateTests(result.tests); - } + // Return the test results (with match information) + // Don't save these results, just return them for display return result.tests; } else { Alert.error(result.message || 'Failed to run tests'); - return tests; + return null; } } catch (error) { console.error('Error running tests:', error); From 666f98c68b962e0935233277ee9f1e04d1241b46 Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Wed, 27 Aug 2025 03:23:24 +0930 Subject: [PATCH 3/9] feat(card): add visibility handling and loading placeholders for FormatCard and RegexCard components to improve performance --- frontend/src/components/format/FormatCard.jsx | 35 ++++++++++++++++-- frontend/src/components/regex/RegexCard.jsx | 36 +++++++++++++++++-- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/format/FormatCard.jsx b/frontend/src/components/format/FormatCard.jsx index 506cb43..a0860a3 100644 --- a/frontend/src/components/format/FormatCard.jsx +++ b/frontend/src/components/format/FormatCard.jsx @@ -1,4 +1,4 @@ -import React, {useState} from 'react'; +import React, {useState, useEffect, useRef} from 'react'; import PropTypes from 'prop-types'; import {Copy, Check, FlaskConical, FileText, ListFilter} from 'lucide-react'; import Tooltip from '@ui/Tooltip'; @@ -14,6 +14,8 @@ function FormatCard({ willBeSelected, onSelect }) { + const [isVisible, setIsVisible] = useState(false); + const cardRef = useRef(null); const [showDescription, setShowDescription] = useState(() => { const saved = localStorage.getItem(`format-view-${format.file_name}`); return saved !== null ? JSON.parse(saved) : true; @@ -64,8 +66,27 @@ function FormatCard({ } }; + useEffect(() => { + const observer = new IntersectionObserver( + ([entry]) => { + setIsVisible(entry.isIntersecting); + }, + { + threshold: 0, + rootMargin: '100px' // Keep cards rendered 100px outside viewport + } + ); + + if (cardRef.current) { + observer.observe(cardRef.current); + } + + return () => observer.disconnect(); + }, []); + return (
-
+ {isVisible ? ( +
{/* Header Section */}
@@ -237,6 +259,15 @@ function FormatCard({ )}
+ ) : ( +
+
+
+
+
+
+
+ )}
); } diff --git a/frontend/src/components/regex/RegexCard.jsx b/frontend/src/components/regex/RegexCard.jsx index 1fde0ac..5f5e434 100644 --- a/frontend/src/components/regex/RegexCard.jsx +++ b/frontend/src/components/regex/RegexCard.jsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, {useState, useEffect, useRef} from 'react'; import PropTypes from 'prop-types'; import {Copy, Check, FlaskConical} from 'lucide-react'; import Tooltip from '@ui/Tooltip'; @@ -15,6 +15,9 @@ const RegexCard = ({ willBeSelected, onSelect }) => { + const [isVisible, setIsVisible] = useState(false); + const cardRef = useRef(null); + const totalTests = pattern.tests?.length || 0; const passedTests = pattern.tests?.filter(t => t.passes)?.length || 0; const passRate = @@ -46,8 +49,27 @@ const RegexCard = ({ return 'text-red-400'; }; + useEffect(() => { + const observer = new IntersectionObserver( + ([entry]) => { + setIsVisible(entry.isIntersecting); + }, + { + threshold: 0, + rootMargin: '100px' // Keep cards rendered 100px outside viewport + } + ); + + if (cardRef.current) { + observer.observe(cardRef.current); + } + + return () => observer.disconnect(); + }, []); + return (
-
+ {isVisible ? ( +
{/* Header Section */}
@@ -183,6 +206,15 @@ const RegexCard = ({ )}
+ ) : ( +
+
+
+
+
+
+
+ )}
); }; From 61854e3d02960ad0a94cb369f4e4a54c03d9bf89 Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Wed, 27 Aug 2025 03:38:07 +0930 Subject: [PATCH 4/9] feat(cache): implement in-memory caching for YAML data to improve performance --- backend/app/data/__init__.py | 53 ++++++---------- backend/app/data/cache.py | 113 +++++++++++++++++++++++++++++++++++ backend/app/data/utils.py | 11 ++++ backend/app/main.py | 5 ++ 4 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 backend/app/data/cache.py diff --git a/backend/app/data/__init__.py b/backend/app/data/__init__.py index c997d2a..afcfdfc 100644 --- a/backend/app/data/__init__.py +++ b/backend/app/data/__init__.py @@ -7,6 +7,7 @@ from .utils import (get_category_directory, load_yaml_file, validate, test_regex_pattern, test_format_conditions, check_delete_constraints, filename_to_display) from ..db import add_format_to_renames, remove_format_from_renames, is_format_in_renames +from .cache import data_cache logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -16,43 +17,19 @@ bp = Blueprint('data', __name__) @bp.route('/', methods=['GET']) def retrieve_all(category): try: - directory = get_category_directory(category) - files = [f for f in os.listdir(directory) if f.endswith('.yml')] - logger.debug(f"Found {len(files)} files in {category}") - - if not files: - return jsonify([]), 200 - - result = [] - errors = 0 - for file_name in files: - file_path = os.path.join(directory, file_name) - try: - content = load_yaml_file(file_path) - # Add metadata for custom formats - if category == 'custom_format': - content['metadata'] = { - 'includeInRename': - is_format_in_renames(content['name']) + # Use cache instead of reading from disk + items = data_cache.get_all(category) + + # Add metadata for custom formats + if category == 'custom_format': + for item in items: + if 'content' in item and 'name' in item['content']: + item['content']['metadata'] = { + 'includeInRename': is_format_in_renames(item['content']['name']) } - result.append({ - "file_name": - file_name, - "content": - content, - "modified_date": - get_file_modified_date(file_path) - }) - except yaml.YAMLError: - errors += 1 - result.append({ - "file_name": file_name, - "error": "Failed to parse YAML" - }) - - logger.info( - f"Processed {len(files)} {category} files ({errors} errors)") - return jsonify(result), 200 + + logger.info(f"Retrieved {len(items)} {category} items from cache") + return jsonify(items), 200 except ValueError as ve: logger.error(ve) @@ -127,6 +104,10 @@ def handle_item(category, name): # Then delete the file os.remove(file_path) + + # Update cache + data_cache.remove_item(category, file_name) + return jsonify( {"message": f"Successfully deleted {file_name}"}), 200 except OSError as e: diff --git a/backend/app/data/cache.py b/backend/app/data/cache.py new file mode 100644 index 0000000..9fa43f3 --- /dev/null +++ b/backend/app/data/cache.py @@ -0,0 +1,113 @@ +import os +import yaml +import logging +from typing import Dict, List, Any, Optional +from datetime import datetime +import threading +from .utils import get_category_directory, get_file_modified_date, filename_to_display + +logger = logging.getLogger(__name__) + +class DataCache: + """In-memory cache for YAML data""" + + def __init__(self): + self._cache = { + 'regex_pattern': {}, + 'custom_format': {}, + 'profile': {} + } + self._lock = threading.RLock() + self._initialized = False + + def initialize(self): + """Load all data into memory on startup""" + with self._lock: + if self._initialized: + return + + logger.info("Initializing data cache...") + for category in self._cache.keys(): + self._load_category(category) + + self._initialized = True + logger.info("Data cache initialized successfully") + + def _load_category(self, category: str): + """Load all items from a category into cache""" + try: + directory = get_category_directory(category) + items = {} + + for filename in os.listdir(directory): + if not filename.endswith('.yml'): + continue + + file_path = os.path.join(directory, filename) + try: + with open(file_path, 'r') as f: + content = yaml.safe_load(f) + if content: + # Store with metadata + items[filename] = { + 'file_name': filename, + 'modified_date': get_file_modified_date(file_path), + 'content': content + } + except Exception as e: + logger.error(f"Error loading {file_path}: {e}") + + self._cache[category] = items + logger.info(f"Loaded {len(items)} items for category {category}") + + except Exception as e: + logger.error(f"Error loading category {category}: {e}") + + def get_all(self, category: str) -> List[Dict[str, Any]]: + """Get all items from a category""" + with self._lock: + if not self._initialized: + self.initialize() + + return list(self._cache.get(category, {}).values()) + + def get_item(self, category: str, name: str) -> Optional[Dict[str, Any]]: + """Get a specific item""" + with self._lock: + if not self._initialized: + self.initialize() + + # Convert name to filename + filename = f"{name.replace('[', '(').replace(']', ')')}.yml" + return self._cache.get(category, {}).get(filename) + + def update_item(self, category: str, filename: str, content: Dict[str, Any]): + """Update an item in cache""" + with self._lock: + if category in self._cache: + file_path = os.path.join(get_category_directory(category), filename) + self._cache[category][filename] = { + 'file_name': filename, + 'modified_date': get_file_modified_date(file_path), + 'content': content + } + logger.debug(f"Updated cache for {category}/{filename}") + + def remove_item(self, category: str, filename: str): + """Remove an item from cache""" + with self._lock: + if category in self._cache and filename in self._cache[category]: + del self._cache[category][filename] + logger.debug(f"Removed from cache: {category}/{filename}") + + def rename_item(self, category: str, old_filename: str, new_filename: str): + """Rename an item in cache""" + with self._lock: + if category in self._cache and old_filename in self._cache[category]: + item = self._cache[category].pop(old_filename) + item['file_name'] = new_filename + self._cache[category][new_filename] = item + logger.debug(f"Renamed in cache: {category}/{old_filename} -> {new_filename}") + +# Global cache instance +data_cache = DataCache() \ No newline at end of file diff --git a/backend/app/data/utils.py b/backend/app/data/utils.py index ccd2769..902cfc9 100644 --- a/backend/app/data/utils.py +++ b/backend/app/data/utils.py @@ -154,6 +154,11 @@ def save_yaml_file(file_path: str, with open(safe_file_path, 'w') as f: yaml.safe_dump(ordered_data, f, sort_keys=False) + + # Update cache + from .cache import data_cache + filename = os.path.basename(safe_file_path) + data_cache.update_item(category, filename, ordered_data) def update_yaml_file(file_path: str, data: Dict[str, Any], @@ -218,6 +223,12 @@ def update_yaml_file(file_path: str, data: Dict[str, Any], os.rename(file_path, new_file_path) # Stage the new file repo.index.add([rel_new_path]) + + # Update cache for rename + from .cache import data_cache + old_filename = os.path.basename(file_path) + new_filename = os.path.basename(new_file_path) + data_cache.rename_item(category, old_filename, new_filename) except git.GitCommandError as e: logger.error(f"Git operation failed: {e}") diff --git a/backend/app/main.py b/backend/app/main.py index efcb54c..bc7f138 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -18,6 +18,7 @@ from .logs import bp as logs_bp from .media_management import media_management_bp from .middleware import init_middleware from .init import setup_logging, init_app_config, init_git_user +from .data.cache import data_cache def create_app(): @@ -48,6 +49,10 @@ def create_app(): # Initialize Git user configuration logger.info("Initializing Git user") success, message = init_git_user() + + # Initialize data cache + logger.info("Initializing data cache") + data_cache.initialize() if not success: logger.warning(f"Git user initialization issue: {message}") else: From 5ee22f72017b483720c0826f80025f8d07a9c357 Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Thu, 28 Aug 2025 01:45:14 +0930 Subject: [PATCH 5/9] fix(cache): reload cache on git operations --- backend/app/git/branches/checkout.py | 6 ++++++ backend/app/git/operations/delete.py | 5 +++++ backend/app/git/operations/merge.py | 5 +++++ backend/app/git/operations/pull.py | 5 +++++ backend/app/git/operations/resolve.py | 5 +++++ backend/app/git/operations/revert.py | 8 ++++++++ backend/app/git/repo/clone.py | 5 +++++ backend/app/git/repo/unlink.py | 6 ++++++ 8 files changed, 45 insertions(+) diff --git a/backend/app/git/branches/checkout.py b/backend/app/git/branches/checkout.py index f688dd7..a814bcf 100644 --- a/backend/app/git/branches/checkout.py +++ b/backend/app/git/branches/checkout.py @@ -44,6 +44,12 @@ def checkout_branch(repo_path, branch_name): return False, f"Branch '{branch_name}' does not exist locally or in any remote." logger.debug(f"Successfully checked out branch: {branch_name}") + + # Reload cache after branch checkout since files may have changed + from ...data.cache import data_cache + logger.info("Reloading data cache after branch checkout") + data_cache.initialize() + return True, { "message": f"Checked out branch: {branch_name}", "current_branch": branch_name diff --git a/backend/app/git/operations/delete.py b/backend/app/git/operations/delete.py index 2713a77..681b63b 100644 --- a/backend/app/git/operations/delete.py +++ b/backend/app/git/operations/delete.py @@ -11,6 +11,11 @@ def delete_file(repo_path, file_path): if os.path.exists(full_file_path): os.remove(full_file_path) + + # Reload cache after file deletion + from ...data.cache import data_cache + data_cache.initialize() + message = f"File {file_path} has been deleted." return True, message else: diff --git a/backend/app/git/operations/merge.py b/backend/app/git/operations/merge.py index be34700..b998fc9 100644 --- a/backend/app/git/operations/merge.py +++ b/backend/app/git/operations/merge.py @@ -60,6 +60,11 @@ def finalize_merge(repo) -> Dict[str, Any]: if status_manager: status_manager.update_remote_status() + # Reload cache for modified data files + from ...data.cache import data_cache + logger.info("Reloading data cache after merge completion") + data_cache.initialize() # This will reload all data + return {'success': True, 'message': 'Merge completed successfully'} except git.GitCommandError as e: logger.error(f"Git command error during commit: {str(e)}") diff --git a/backend/app/git/operations/pull.py b/backend/app/git/operations/pull.py index 61202d0..ad2a638 100644 --- a/backend/app/git/operations/pull.py +++ b/backend/app/git/operations/pull.py @@ -35,6 +35,11 @@ def pull_branch(repo_path, branch_name): if status_manager: status_manager.update_remote_status() + # Reload cache for updated data files + from ...data.cache import data_cache + logger.info("Reloading data cache after pull") + data_cache.initialize() # This will reload all data + # ------------------------------- # *** "On pull" ARR import logic using new importer: # 1) Query all ARR configs that have sync_method="pull" diff --git a/backend/app/git/operations/resolve.py b/backend/app/git/operations/resolve.py index 363d718..c0b08e8 100644 --- a/backend/app/git/operations/resolve.py +++ b/backend/app/git/operations/resolve.py @@ -310,6 +310,11 @@ def resolve_conflicts( logger.debug(f"File status: {item}") logger.debug("=======================================") + # Reload cache after conflict resolution + from ...data.cache import data_cache + logger.info("Reloading data cache after conflict resolution") + data_cache.initialize() + return {'success': True, 'results': results} except Exception as e: diff --git a/backend/app/git/operations/revert.py b/backend/app/git/operations/revert.py index 77a5bc3..220386f 100644 --- a/backend/app/git/operations/revert.py +++ b/backend/app/git/operations/revert.py @@ -51,6 +51,10 @@ def revert_file(repo_path, file_path): repo.git.restore('--staged', "--", file_path) message = f"File {file_path} has been reverted." + # Reload cache after revert + from ...data.cache import data_cache + data_cache.initialize() + return True, message except git.exc.GitCommandError as e: @@ -98,6 +102,10 @@ def revert_all(repo_path): message += f" and {len(untracked_files)} new file(s) have been removed" message += "." + # Reload cache after reverting all + from ...data.cache import data_cache + data_cache.initialize() + return True, message except git.exc.GitCommandError as e: diff --git a/backend/app/git/repo/clone.py b/backend/app/git/repo/clone.py index 801f82a..e85df67 100644 --- a/backend/app/git/repo/clone.py +++ b/backend/app/git/repo/clone.py @@ -116,6 +116,11 @@ def clone_repository(repo_url, repo_path): logger.info("Removing backup directory") shutil.rmtree(backup_dir) + # Reload cache after clone operation + from ...data.cache import data_cache + logger.info("Reloading data cache after clone") + data_cache.initialize() + logger.info("Clone operation completed successfully") return True, "Repository cloned and local files merged successfully" diff --git a/backend/app/git/repo/unlink.py b/backend/app/git/repo/unlink.py index a2a3395..747cd0a 100644 --- a/backend/app/git/repo/unlink.py +++ b/backend/app/git/repo/unlink.py @@ -68,6 +68,12 @@ def unlink_repository(repo_path, remove_files=False): save_settings({'gitRepo': None}) logger.info("Updated settings to remove git information") + # Reload cache if files were removed + if remove_files: + from ...data.cache import data_cache + logger.info("Reloading data cache after removing repository files") + data_cache.initialize() + return True, "Repository successfully unlinked" except Exception as e: logger.error(f"Error unlinking repository: {str(e)}", exc_info=True) From 9e5e5ce523790d685fd04ab9030a6034e29c92be Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Thu, 28 Aug 2025 13:24:43 +0930 Subject: [PATCH 6/9] fix(cache): force reinitialize on git ops --- backend/app/data/cache.py | 14 +++++++++----- backend/app/git/branches/checkout.py | 2 +- backend/app/git/operations/delete.py | 2 +- backend/app/git/operations/merge.py | 2 +- backend/app/git/operations/pull.py | 2 +- backend/app/git/operations/resolve.py | 2 +- backend/app/git/operations/revert.py | 4 ++-- backend/app/git/repo/clone.py | 2 +- backend/app/git/repo/unlink.py | 2 +- 9 files changed, 18 insertions(+), 14 deletions(-) diff --git a/backend/app/data/cache.py b/backend/app/data/cache.py index 9fa43f3..56774b7 100644 --- a/backend/app/data/cache.py +++ b/backend/app/data/cache.py @@ -20,18 +20,22 @@ class DataCache: self._lock = threading.RLock() self._initialized = False - def initialize(self): - """Load all data into memory on startup""" + def initialize(self, force_reload=False): + """Load all data into memory on startup + + Args: + force_reload: If True, force a reload even if already initialized + """ with self._lock: - if self._initialized: + if self._initialized and not force_reload: return - logger.info("Initializing data cache...") + logger.info("Initializing data cache..." if not force_reload else "Reloading data cache...") for category in self._cache.keys(): self._load_category(category) self._initialized = True - logger.info("Data cache initialized successfully") + logger.info("Data cache initialized successfully" if not force_reload else "Data cache reloaded successfully") def _load_category(self, category: str): """Load all items from a category into cache""" diff --git a/backend/app/git/branches/checkout.py b/backend/app/git/branches/checkout.py index a814bcf..6899a61 100644 --- a/backend/app/git/branches/checkout.py +++ b/backend/app/git/branches/checkout.py @@ -48,7 +48,7 @@ def checkout_branch(repo_path, branch_name): # Reload cache after branch checkout since files may have changed from ...data.cache import data_cache logger.info("Reloading data cache after branch checkout") - data_cache.initialize() + data_cache.initialize(force_reload=True) return True, { "message": f"Checked out branch: {branch_name}", diff --git a/backend/app/git/operations/delete.py b/backend/app/git/operations/delete.py index 681b63b..88c89ef 100644 --- a/backend/app/git/operations/delete.py +++ b/backend/app/git/operations/delete.py @@ -14,7 +14,7 @@ def delete_file(repo_path, file_path): # Reload cache after file deletion from ...data.cache import data_cache - data_cache.initialize() + data_cache.initialize(force_reload=True) message = f"File {file_path} has been deleted." return True, message diff --git a/backend/app/git/operations/merge.py b/backend/app/git/operations/merge.py index b998fc9..e7390d7 100644 --- a/backend/app/git/operations/merge.py +++ b/backend/app/git/operations/merge.py @@ -63,7 +63,7 @@ def finalize_merge(repo) -> Dict[str, Any]: # Reload cache for modified data files from ...data.cache import data_cache logger.info("Reloading data cache after merge completion") - data_cache.initialize() # This will reload all data + data_cache.initialize(force_reload=True) # This will reload all data return {'success': True, 'message': 'Merge completed successfully'} except git.GitCommandError as e: diff --git a/backend/app/git/operations/pull.py b/backend/app/git/operations/pull.py index ad2a638..d6f4721 100644 --- a/backend/app/git/operations/pull.py +++ b/backend/app/git/operations/pull.py @@ -38,7 +38,7 @@ def pull_branch(repo_path, branch_name): # Reload cache for updated data files from ...data.cache import data_cache logger.info("Reloading data cache after pull") - data_cache.initialize() # This will reload all data + data_cache.initialize(force_reload=True) # This will reload all data # ------------------------------- # *** "On pull" ARR import logic using new importer: diff --git a/backend/app/git/operations/resolve.py b/backend/app/git/operations/resolve.py index c0b08e8..87bb1f8 100644 --- a/backend/app/git/operations/resolve.py +++ b/backend/app/git/operations/resolve.py @@ -313,7 +313,7 @@ def resolve_conflicts( # Reload cache after conflict resolution from ...data.cache import data_cache logger.info("Reloading data cache after conflict resolution") - data_cache.initialize() + data_cache.initialize(force_reload=True) return {'success': True, 'results': results} diff --git a/backend/app/git/operations/revert.py b/backend/app/git/operations/revert.py index 220386f..4badb52 100644 --- a/backend/app/git/operations/revert.py +++ b/backend/app/git/operations/revert.py @@ -53,7 +53,7 @@ def revert_file(repo_path, file_path): # Reload cache after revert from ...data.cache import data_cache - data_cache.initialize() + data_cache.initialize(force_reload=True) return True, message @@ -104,7 +104,7 @@ def revert_all(repo_path): # Reload cache after reverting all from ...data.cache import data_cache - data_cache.initialize() + data_cache.initialize(force_reload=True) return True, message diff --git a/backend/app/git/repo/clone.py b/backend/app/git/repo/clone.py index e85df67..2c99dd3 100644 --- a/backend/app/git/repo/clone.py +++ b/backend/app/git/repo/clone.py @@ -119,7 +119,7 @@ def clone_repository(repo_url, repo_path): # Reload cache after clone operation from ...data.cache import data_cache logger.info("Reloading data cache after clone") - data_cache.initialize() + data_cache.initialize(force_reload=True) logger.info("Clone operation completed successfully") return True, "Repository cloned and local files merged successfully" diff --git a/backend/app/git/repo/unlink.py b/backend/app/git/repo/unlink.py index 747cd0a..1d54263 100644 --- a/backend/app/git/repo/unlink.py +++ b/backend/app/git/repo/unlink.py @@ -72,7 +72,7 @@ def unlink_repository(repo_path, remove_files=False): if remove_files: from ...data.cache import data_cache logger.info("Reloading data cache after removing repository files") - data_cache.initialize() + data_cache.initialize(force_reload=True) return True, "Repository successfully unlinked" except Exception as e: From baba7ad3c59b89689c2c5620e739e565b779ceb4 Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Wed, 3 Sep 2025 08:43:04 +0930 Subject: [PATCH 7/9] fix(cache): remove regex compiler cache, reload cache after ANY git revert --- backend/app/git/operations/revert.py | 11 ++--------- backend/app/importer/compiler.py | 12 +----------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/backend/app/git/operations/revert.py b/backend/app/git/operations/revert.py index 4badb52..7940238 100644 --- a/backend/app/git/operations/revert.py +++ b/backend/app/git/operations/revert.py @@ -33,14 +33,7 @@ def revert_file(repo_path, file_path): message = f"New file {file_path} has been removed." except FileNotFoundError: message = f"File {file_path} was already removed." - return True, message - - # Check if file is staged for deletion - staged_deletions = repo.index.diff("HEAD", R=True) - is_staged_for_deletion = any(d.a_path == file_path - for d in staged_deletions) - - if is_staged_for_deletion: + elif is_staged_for_deletion: # Restore file staged for deletion repo.git.reset("--", file_path) repo.git.checkout('HEAD', "--", file_path) @@ -51,7 +44,7 @@ def revert_file(repo_path, file_path): repo.git.restore('--staged', "--", file_path) message = f"File {file_path} has been reverted." - # Reload cache after revert + # Reload cache after ANY revert operation from ...data.cache import data_cache data_cache.initialize(force_reload=True) diff --git a/backend/app/importer/compiler.py b/backend/app/importer/compiler.py index 3e9c7f3..9a65f57 100644 --- a/backend/app/importer/compiler.py +++ b/backend/app/importer/compiler.py @@ -9,16 +9,6 @@ from .logger import get_import_logger logger = logging.getLogger(__name__) -# Cache patterns at module level to avoid reloading -_CACHED_PATTERNS = None - -def get_cached_patterns(): - """Get cached regex patterns, loading them once on first access.""" - global _CACHED_PATTERNS - if _CACHED_PATTERNS is None: - _CACHED_PATTERNS = load_regex_patterns() - return _CACHED_PATTERNS - def compile_format_to_api_structure( format_yaml: Dict[str, Any], @@ -35,7 +25,7 @@ def compile_format_to_api_structure( Compiled format ready for API """ target_app = TargetApp.RADARR if arr_type.lower() == 'radarr' else TargetApp.SONARR - patterns = get_cached_patterns() + patterns = load_regex_patterns() compiled = { 'name': format_yaml.get('name', 'Unknown') From 88f11b65aae4023204cb680c89d557c11e3d20ca Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Wed, 3 Sep 2025 09:28:38 +0930 Subject: [PATCH 8/9] fix(revert): add deletion check back --- backend/app/git/operations/revert.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/app/git/operations/revert.py b/backend/app/git/operations/revert.py index 7940238..6d11b69 100644 --- a/backend/app/git/operations/revert.py +++ b/backend/app/git/operations/revert.py @@ -26,6 +26,11 @@ def revert_file(repo_path, file_path): untracked_files = repo.untracked_files is_untracked = any(f == file_path for f in untracked_files) + # Check if file is staged for deletion + staged_deletions = repo.index.diff("HEAD", R=True) + is_staged_for_deletion = any(d.a_path == file_path + for d in staged_deletions) + if is_untracked: # For untracked files, we need to remove them try: From cf67a1c9850114529968ebd3cdc81aa31c58acc0 Mon Sep 17 00:00:00 2001 From: Sam Chau Date: Thu, 4 Sep 2025 00:17:55 +0930 Subject: [PATCH 9/9] perf(cache): add proper file caching at the import instantiation --- backend/app/importer/compiler.py | 8 ++++++-- backend/app/importer/strategies/format.py | 7 ++++++- backend/app/importer/strategies/profile.py | 9 +++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/backend/app/importer/compiler.py b/backend/app/importer/compiler.py index 9a65f57..f64e21e 100644 --- a/backend/app/importer/compiler.py +++ b/backend/app/importer/compiler.py @@ -12,7 +12,8 @@ logger = logging.getLogger(__name__) def compile_format_to_api_structure( format_yaml: Dict[str, Any], - arr_type: str + arr_type: str, + patterns: Dict[str, str] = None ) -> Dict[str, Any]: """ Compile a format from YAML to Arr API structure. @@ -20,12 +21,15 @@ def compile_format_to_api_structure( Args: format_yaml: Format data from YAML file arr_type: 'radarr' or 'sonarr' + patterns: Pre-loaded regex patterns (if None, will load from disk) Returns: Compiled format ready for API """ target_app = TargetApp.RADARR if arr_type.lower() == 'radarr' else TargetApp.SONARR - patterns = load_regex_patterns() + # Only load patterns if not provided + if patterns is None: + patterns = load_regex_patterns() compiled = { 'name': format_yaml.get('name', 'Unknown') diff --git a/backend/app/importer/strategies/format.py b/backend/app/importer/strategies/format.py index 76ad96a..aedcc40 100644 --- a/backend/app/importer/strategies/format.py +++ b/backend/app/importer/strategies/format.py @@ -22,6 +22,11 @@ class FormatStrategy(ImportStrategy): Returns: Dictionary with 'formats' key containing compiled formats """ + from ..utils import load_regex_patterns + + # Load all regex patterns once at the start + patterns = load_regex_patterns() + formats = [] failed = [] import_logger = get_import_logger() @@ -35,7 +40,7 @@ class FormatStrategy(ImportStrategy): format_yaml = load_yaml(f"custom_format/{filename}.yml") # Compile to API structure - compiled = compile_format_to_api_structure(format_yaml, self.arr_type) + compiled = compile_format_to_api_structure(format_yaml, self.arr_type, patterns) # Add unique suffix if needed if self.import_as_unique: diff --git a/backend/app/importer/strategies/profile.py b/backend/app/importer/strategies/profile.py index 6a2dbc8..198bcda 100644 --- a/backend/app/importer/strategies/profile.py +++ b/backend/app/importer/strategies/profile.py @@ -22,6 +22,11 @@ class ProfileStrategy(ImportStrategy): Returns: Dictionary with 'profiles' and 'formats' keys """ + from ..utils import load_regex_patterns + + # Load all regex patterns once at the start + patterns = load_regex_patterns() + profiles = [] all_formats = [] processed_formats: Set[str] = set() @@ -49,7 +54,7 @@ class ProfileStrategy(ImportStrategy): try: format_yaml = load_yaml(f"custom_format/{format_name}.yml") - compiled_format = compile_format_to_api_structure(format_yaml, self.arr_type) + compiled_format = compile_format_to_api_structure(format_yaml, self.arr_type, patterns) if self.import_as_unique: compiled_format['name'] = self.add_unique_suffix(compiled_format['name']) @@ -72,7 +77,7 @@ class ProfileStrategy(ImportStrategy): for lang_format in language_formats: lang_name = lang_format.get('name', 'Language format') - compiled_lang = compile_format_to_api_structure(lang_format, self.arr_type) + compiled_lang = compile_format_to_api_structure(lang_format, self.arr_type, patterns) if self.import_as_unique: compiled_lang['name'] = self.add_unique_suffix(compiled_lang['name'])