From 2dd93e2588712327c4b4108e348e43dccbfc79db Mon Sep 17 00:00:00 2001 From: santiagosayshey Date: Fri, 16 Aug 2024 15:08:15 +0930 Subject: [PATCH] refactor(regex, format): Change the way regexes and formats are saved - regex IDs are now saved in conditions, not names - sanitisation in custom formats --- backend/app.py | 2 +- backend/app/routes/format_routes.py | 32 +++-- backend/app/routes/regex_routes.py | 34 ++--- backend/app/utils/file_operations.py | 110 --------------- backend/app/utils/file_utils.py | 15 +++ backend/app/utils/format_operations.py | 126 ++++++++++++++++++ backend/app/utils/regex_operations.py | 69 ++++++++++ backend/requirements.txt | 2 +- .../components/condition/ConditionCard.jsx | 28 ++-- .../components/condition/ConditionModal.jsx | 76 +++++------ .../src/components/format/FormatModal.jsx | 1 + 11 files changed, 292 insertions(+), 203 deletions(-) delete mode 100644 backend/app/utils/file_operations.py create mode 100644 backend/app/utils/file_utils.py create mode 100644 backend/app/utils/format_operations.py create mode 100644 backend/app/utils/regex_operations.py diff --git a/backend/app.py b/backend/app.py index cb03e86..117111d 100644 --- a/backend/app.py +++ b/backend/app.py @@ -4,7 +4,7 @@ from app.routes import regex_routes, format_routes def create_app(): app = Flask(__name__) - CORS(app, resources={r"/*": {"origins": "http://localhost:3000"}}) + CORS(app, resources={r"/*": {"origins": "*"}}) app.register_blueprint(regex_routes.bp) app.register_blueprint(format_routes.bp) return app diff --git a/backend/app/routes/format_routes.py b/backend/app/routes/format_routes.py index 9862e7d..932eba3 100644 --- a/backend/app/routes/format_routes.py +++ b/backend/app/routes/format_routes.py @@ -1,33 +1,31 @@ -# app/routes/format_routes.py - from flask import Blueprint, request, jsonify -from app.utils.file_operations import FORMAT_DIR, REGEX_DIR, save_to_file, load_all_from_directory, delete_file, load_from_file +from app.utils.format_operations import save_format, load_all_formats, delete_format, load_format -bp = Blueprint('format', __name__, url_prefix='/format') +bp = Blueprint('format', __name__, url_prefix='/format') @bp.route('', methods=['GET', 'POST']) -def handle_items(): +def handle_formats(): if request.method == 'POST': data = request.json - saved_data = save_to_file(FORMAT_DIR, data) + saved_data = save_format(data) return jsonify(saved_data), 201 else: - items = load_all_from_directory(FORMAT_DIR) - return jsonify(items) + formats = load_all_formats() + return jsonify(formats) @bp.route('/', methods=['GET', 'PUT', 'DELETE']) -def handle_item(id): +def handle_format(id): if request.method == 'GET': - item = load_from_file(FORMAT_DIR, id) - if item: - return jsonify(item) - return jsonify({"error": "Item not found"}), 404 + format = load_format(id) + if format: + return jsonify(format) + return jsonify({"error": "Format not found"}), 404 elif request.method == 'PUT': data = request.json data['id'] = id - saved_data = save_to_file(FORMAT_DIR, data) + saved_data = save_format(data) return jsonify(saved_data) elif request.method == 'DELETE': - if delete_file(FORMAT_DIR, id): - return jsonify({"message": f"Item with ID {id} deleted."}), 200 - return jsonify({"error": f"Item with ID {id} not found."}), 404 \ No newline at end of file + if delete_format(id): + return jsonify({"message": f"Format with ID {id} deleted."}), 200 + return jsonify({"error": f"Format with ID {id} not found."}), 404 \ No newline at end of file diff --git a/backend/app/routes/regex_routes.py b/backend/app/routes/regex_routes.py index ad08e75..a6e074c 100644 --- a/backend/app/routes/regex_routes.py +++ b/backend/app/routes/regex_routes.py @@ -1,5 +1,5 @@ from flask import Blueprint, request, jsonify -from app.utils.file_operations import REGEX_DIR, save_to_file, load_all_from_directory, delete_file, load_from_file +from app.utils.regex_operations import save_regex, load_all_regexes, delete_regex, load_regex import json import logging import subprocess @@ -70,34 +70,28 @@ def regex101_proxy(): @bp.route('', methods=['GET', 'POST']) -def handle_items(): +def handle_regexes(): if request.method == 'POST': data = request.json - # Ensure regex101Link is included in the data - if 'regex101Link' not in data: - data['regex101Link'] = '' - saved_data = save_to_file(REGEX_DIR, data) + saved_data = save_regex(data) return jsonify(saved_data), 201 else: - items = load_all_from_directory(REGEX_DIR) - return jsonify(items) + regexes = load_all_regexes() + return jsonify(regexes) @bp.route('/', methods=['GET', 'PUT', 'DELETE']) -def handle_item(id): +def handle_regex(id): if request.method == 'GET': - item = load_from_file(REGEX_DIR, id) - if item: - return jsonify(item) - return jsonify({"error": "Item not found"}), 404 + regex = load_regex(id) + if regex: + return jsonify(regex) + return jsonify({"error": "Regex not found"}), 404 elif request.method == 'PUT': data = request.json data['id'] = id - # Ensure regex101Link is included in the data - if 'regex101Link' not in data: - data['regex101Link'] = '' - saved_data = save_to_file(REGEX_DIR, data) + saved_data = save_regex(data) return jsonify(saved_data) elif request.method == 'DELETE': - if delete_file(REGEX_DIR, id): - return jsonify({"message": f"Item with ID {id} deleted."}), 200 - return jsonify({"error": f"Item with ID {id} not found."}), 404 \ No newline at end of file + if delete_regex(id): + return jsonify({"message": f"Regex with ID {id} deleted."}), 200 + return jsonify({"error": f"Regex with ID {id} not found."}), 404 \ No newline at end of file diff --git a/backend/app/utils/file_operations.py b/backend/app/utils/file_operations.py deleted file mode 100644 index bb2150b..0000000 --- a/backend/app/utils/file_operations.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import yaml -import datetime -from collections import OrderedDict - -REGEX_DIR = 'regex_patterns' -FORMAT_DIR = 'custom_formats' - -os.makedirs(REGEX_DIR, exist_ok=True) -os.makedirs(FORMAT_DIR, exist_ok=True) - -# Custom representer to handle OrderedDict -def represent_ordereddict(dumper, data): - return dumper.represent_dict(data.items()) - -yaml.add_representer(OrderedDict, represent_ordereddict, Dumper=yaml.SafeDumper) - -def get_next_id(directory): - files = [f for f in os.listdir(directory) if f.endswith('.yml')] - if not files: - return 1 - return max(int(f.split('_')[0]) for f in files) + 1 - -def generate_filename(directory, id, name): - sanitized_name = name.replace(' ', '_').lower() - return f"{directory}/{id}_{sanitized_name}.yml" - -def get_current_timestamp(): - return datetime.datetime.now().isoformat() - -def save_to_file(directory, data): - # Ensure ID, Name, Description, and Tags are ordered first - ordered_data = OrderedDict() - ordered_data['id'] = data.get('id', get_next_id(directory)) - ordered_data['name'] = data.get('name', '') - ordered_data['description'] = data.get('description', '') - ordered_data['tags'] = data.get('tags', []) - - # Add the rest of the data - for key, value in data.items(): - if key not in ordered_data: - ordered_data[key] = value - - # Update timestamps - if 'id' in data and data['id'] != 0: - existing_files = [f for f in os.listdir(directory) if f.startswith(f"{data['id']}_") and f.endswith('.yml')] - if existing_files: - existing_filename = os.path.join(directory, existing_files[0]) - existing_data = load_from_file(directory, data['id']) - if existing_data: - ordered_data['date_created'] = existing_data.get('date_created', get_current_timestamp()) - else: - ordered_data['date_created'] = get_current_timestamp() - ordered_data['date_modified'] = get_current_timestamp() - new_filename = generate_filename(directory, ordered_data['id'], ordered_data['name']) - - # Remove the old file if the name has changed - if existing_filename != new_filename: - os.remove(existing_filename) - - with open(new_filename, 'w') as file: - yaml.dump(ordered_data, file, default_flow_style=False, Dumper=yaml.SafeDumper) - return ordered_data - else: - # If existing file not found, treat it as new - ordered_data['id'] = get_next_id(directory) - ordered_data['date_created'] = get_current_timestamp() - ordered_data['date_modified'] = get_current_timestamp() - else: - # Handle new records - ordered_data['id'] = get_next_id(directory) - ordered_data['date_created'] = get_current_timestamp() - ordered_data['date_modified'] = get_current_timestamp() - - new_filename = generate_filename(directory, ordered_data['id'], ordered_data['name']) - with open(new_filename, 'w') as file: - yaml.dump(ordered_data, file, default_flow_style=False, Dumper=yaml.SafeDumper) - - return ordered_data - -def load_from_file(directory, id): - files = [f for f in os.listdir(directory) if f.startswith(f"{id}_") and f.endswith('.yml')] - if files: - filename = os.path.join(directory, files[0]) - with open(filename, 'r') as file: - data = yaml.safe_load(file) - if 'conditions' not in data: - data['conditions'] = [] # Ensure conditions is always a list - if 'regex101Link' not in data: - data['regex101Link'] = '' # Ensure regex101Link is always present - return data - return None - -def load_all_from_directory(directory): - items = [] - for filename in os.listdir(directory): - if filename.endswith('.yml'): - with open(os.path.join(directory, filename), 'r') as file: - data = yaml.safe_load(file) - if 'regex101Link' not in data: - data['regex101Link'] = '' # Ensure regex101Link is always present - items.append(data) - return items - -def delete_file(directory, id): - files = [f for f in os.listdir(directory) if f.startswith(f"{id}_") and f.endswith('.yml')] - if files: - os.remove(os.path.join(directory, files[0])) - return True - return False diff --git a/backend/app/utils/file_utils.py b/backend/app/utils/file_utils.py new file mode 100644 index 0000000..002a11e --- /dev/null +++ b/backend/app/utils/file_utils.py @@ -0,0 +1,15 @@ +import os +import datetime + +def get_next_id(directory): + files = [f for f in os.listdir(directory) if f.endswith('.yml')] + if not files: + return 1 + return max(int(f.split('_')[0]) for f in files) + 1 + +def generate_filename(directory, id, name): + sanitized_name = name.replace(' ', '_').lower() + return os.path.join(directory, f"{id}_{sanitized_name}.yml") + +def get_current_timestamp(): + return datetime.datetime.now().isoformat() \ No newline at end of file diff --git a/backend/app/utils/format_operations.py b/backend/app/utils/format_operations.py new file mode 100644 index 0000000..843a7e6 --- /dev/null +++ b/backend/app/utils/format_operations.py @@ -0,0 +1,126 @@ +import os +import yaml +import re +import logging +from collections import OrderedDict +from .file_utils import get_next_id, generate_filename, get_current_timestamp + +FORMAT_DIR = 'custom_formats' + +# Set up basic logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def represent_ordereddict(dumper, data): + return dumper.represent_mapping('tag:yaml.org,2002:map', data.items()) + +yaml.add_representer(OrderedDict, represent_ordereddict, Dumper=yaml.SafeDumper) + +def sanitize_input(input_str): + # Trim leading/trailing whitespace + sanitized_str = input_str.strip() + + # Replace special characters that could affect YAML formatting + sanitized_str = re.sub(r'[:#\-\*>\|&]', '', sanitized_str) + + # Ensure there are no tabs (which can cause issues in YAML) + sanitized_str = sanitized_str.replace('\t', ' ') + + # Optionally: Collapse multiple spaces into a single space + sanitized_str = re.sub(r'\s+', ' ', sanitized_str) + + return sanitized_str + +def save_format(data): + # Log the received data + logger.info("Received data for saving format: %s", data) + + # Sanitize inputs + name = sanitize_input(data.get('name', '')) + description = sanitize_input(data.get('description', '')) + + # Determine if it's a new format or an edit + format_id = data.get('id', None) + if format_id == 0: # If id is 0, treat it as a new format + format_id = get_next_id(FORMAT_DIR) + logger.info("Assigned new format ID: %d", format_id) + date_created = get_current_timestamp() + else: + existing_data = load_format(format_id) + if existing_data: + date_created = existing_data.get('date_created') + old_filename = generate_filename(FORMAT_DIR, format_id, existing_data['name']) + # Delete the old file + if os.path.exists(old_filename): + os.remove(old_filename) + else: + date_created = get_current_timestamp() + + date_modified = get_current_timestamp() + + # Prepare conditions + conditions = [] + for condition in data.get('conditions', []): + logger.info("Processing condition: %s", condition) + cond_dict = OrderedDict([ + ('type', condition['type']), + ('name', sanitize_input(condition['name'])), + ('negate', condition.get('negate', False)), + ('required', condition.get('required', False)) + ]) + if condition['type'] == 'regex': + cond_dict['regex_id'] = condition['regex_id'] + elif condition['type'] == 'size': + cond_dict['min'] = condition['min'] + cond_dict['max'] = condition['max'] + elif condition['type'] == 'flag': + cond_dict['flag'] = sanitize_input(condition['flag']) + conditions.append(cond_dict) + + # Prepare tags + tags = [sanitize_input(tag) for tag in data.get('tags', [])] + + # Create ordered data dictionary + ordered_data = OrderedDict([ + ('id', format_id), + ('name', name), + ('description', description), + ('date_created', str(date_created)), + ('date_modified', str(date_modified)), + ('conditions', conditions), + ('tags', tags) + ]) + + # Generate new filename based on the updated name + new_filename = generate_filename(FORMAT_DIR, format_id, name) + + # Write the YAML file with the new name + with open(new_filename, 'w') as file: + yaml.dump(ordered_data, file, default_flow_style=False, Dumper=yaml.SafeDumper) + + return ordered_data + +def load_format(id): + files = [f for f in os.listdir(FORMAT_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')] + if files: + filename = os.path.join(FORMAT_DIR, files[0]) + with open(filename, 'r') as file: + data = yaml.safe_load(file) + return data + return None + +def load_all_formats(): + formats = [] + for filename in os.listdir(FORMAT_DIR): + if filename.endswith('.yml'): + with open(os.path.join(FORMAT_DIR, filename), 'r') as file: + data = yaml.safe_load(file) + formats.append(data) + return formats + +def delete_format(id): + files = [f for f in os.listdir(FORMAT_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')] + if files: + os.remove(os.path.join(FORMAT_DIR, files[0])) + return True + return False diff --git a/backend/app/utils/regex_operations.py b/backend/app/utils/regex_operations.py new file mode 100644 index 0000000..b9beefc --- /dev/null +++ b/backend/app/utils/regex_operations.py @@ -0,0 +1,69 @@ +import os +import yaml +from collections import OrderedDict +from .file_utils import get_next_id, generate_filename, get_current_timestamp + +REGEX_DIR = 'regex_patterns' + +def save_regex(data): + ordered_data = OrderedDict() + if 'id' in data and data['id'] != 0: + ordered_data['id'] = data['id'] + else: + ordered_data['id'] = get_next_id(REGEX_DIR) + + ordered_data['name'] = data.get('name', '') + ordered_data['description'] = data.get('description', '') + ordered_data['pattern'] = data.get('pattern', '') + ordered_data['regex101Link'] = data.get('regex101Link', '') + + if ordered_data['id'] != 0: # Existing regex + existing_data = load_regex(ordered_data['id']) + if existing_data: + ordered_data['date_created'] = existing_data.get('date_created', get_current_timestamp()) + else: + ordered_data['date_created'] = get_current_timestamp() + else: # New regex + ordered_data['date_created'] = get_current_timestamp() + + ordered_data['date_modified'] = get_current_timestamp() + ordered_data['tags'] = data.get('tags', []) + + filename = generate_filename(REGEX_DIR, ordered_data['id'], ordered_data['name']) + with open(filename, 'w') as file: + for key, value in ordered_data.items(): + if key in ['description', 'date_created', 'date_modified', 'regex101Link']: + file.write(f"{key}: '{value}'\n") + elif key == 'tags': + file.write('tags:\n') + for tag in value: + file.write(f'- {tag}\n') + else: + file.write(f'{key}: {value}\n') + + return ordered_data + +def load_regex(id): + files = [f for f in os.listdir(REGEX_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')] + if files: + filename = os.path.join(REGEX_DIR, files[0]) + with open(filename, 'r') as file: + data = yaml.safe_load(file) + return data + return None + +def load_all_regexes(): + regexes = [] + for filename in os.listdir(REGEX_DIR): + if filename.endswith('.yml'): + with open(os.path.join(REGEX_DIR, filename), 'r') as file: + data = yaml.safe_load(file) + regexes.append(data) + return regexes + +def delete_regex(id): + files = [f for f in os.listdir(REGEX_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')] + if files: + os.remove(os.path.join(REGEX_DIR, files[0])) + return True + return False \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index e2aa953..09feb47 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,4 +2,4 @@ Flask==2.0.1 Flask-CORS==3.0.10 PyYAML==5.4.1 requests==2.26.0 -Werkzeug==2.0.1 +Werkzeug==2.0.1 \ No newline at end of file diff --git a/frontend/src/components/condition/ConditionCard.jsx b/frontend/src/components/condition/ConditionCard.jsx index dfd652e..6d41dcd 100644 --- a/frontend/src/components/condition/ConditionCard.jsx +++ b/frontend/src/components/condition/ConditionCard.jsx @@ -1,12 +1,6 @@ import PropTypes from 'prop-types'; function ConditionCard({ condition, onEdit }) { - const conditionType = condition.regex_name - ? 'Regex' - : condition.min !== undefined && condition.max !== undefined - ? 'Size' - : 'Flag'; - return (
onEdit(condition)} @@ -15,21 +9,26 @@ function ConditionCard({ condition, onEdit }) {

{condition.name}

- {conditionType} + {condition.type.charAt(0).toUpperCase() + condition.type.slice(1)}
- {conditionType === 'Regex' && ( + {condition.type === 'regex' && ( 
-            {condition.regex_name}
+            Regex ID: {condition.regex_id || condition.id}  {/* Display regex_id */}
)} - {conditionType === 'Size' && ( + {condition.type === 'size' && (

Size: {condition.min || 'Any'} - {condition.max || 'Any'} bytes

)} + {condition.type === 'flag' && ( +

+ Flag: {condition.flag} +

+ )}
{condition.required ? 'Required' : 'Optional'} @@ -46,14 +45,15 @@ function ConditionCard({ condition, onEdit }) { ConditionCard.propTypes = { condition: PropTypes.shape({ - name: PropTypes.string, - regex_name: PropTypes.string, + type: PropTypes.string.isRequired, + name: PropTypes.string.isRequired, + regex_id: PropTypes.number, // Updated to regex_id min: PropTypes.number, max: PropTypes.number, + flag: PropTypes.string, negate: PropTypes.bool, required: PropTypes.bool, - flag: PropTypes.string, // Include flag in prop types - }), + }).isRequired, onEdit: PropTypes.func.isRequired, }; diff --git a/frontend/src/components/condition/ConditionModal.jsx b/frontend/src/components/condition/ConditionModal.jsx index aa63bf5..a6b48ce 100644 --- a/frontend/src/components/condition/ConditionModal.jsx +++ b/frontend/src/components/condition/ConditionModal.jsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef } from 'react'; +import React, { useState, useEffect, useRef } from 'react'; import PropTypes from 'prop-types'; import Modal from '../ui/Modal'; @@ -13,7 +13,7 @@ function ConditionModal({ }) { const [name, setName] = useState(''); const [type, setType] = useState('regex'); - const [regexName, setRegexName] = useState(''); + const [regexId, setRegexId] = useState(null); const [minSize, setMinSize] = useState(''); const [maxSize, setMaxSize] = useState(''); const [flag, setFlag] = useState(''); @@ -27,70 +27,68 @@ function ConditionModal({ initialConditionRef.current = condition; if (condition) { setName(condition.name); - if (condition.regex_name) { - setType('regex'); - setRegexName(condition.regex_name); - } else if (condition.min !== undefined && condition.max !== undefined) { - setType('size'); - setMinSize(condition.min); - setMaxSize(condition.max); - } else if (condition.flag) { - setType('flag'); - setFlag(condition.flag); - } + setType(condition.type); + setRegexId(condition.regex_id || condition.id); // Read regex_id instead of id + setMinSize(condition.min?.toString() || ''); + setMaxSize(condition.max?.toString() || ''); + setFlag(condition.flag || ''); setNegate(condition.negate || false); setRequired(condition.required || false); } else { - setName(''); - setType('regex'); - setRegexName(''); - setMinSize(''); - setMaxSize(''); - setFlag(''); - setNegate(false); - setRequired(false); + resetForm(); } - setError(''); } }, [condition, isOpen]); + const resetForm = () => { + setName(''); + setType('regex'); + setRegexId(null); + setMinSize(''); + setMaxSize(''); + setFlag(''); + setNegate(false); + setRequired(false); + setError(''); + }; + const handleSave = () => { if (!name.trim()) { setError('Condition name is required.'); return; } - - if (type === 'regex' && !regexName) { + + if (type === 'regex' && !regexId) { setError('Please select a regex pattern.'); return; } - + if (type === 'size' && (!minSize || !maxSize)) { setError('Both minimum and maximum sizes are required.'); return; } - + if (type === 'flag' && !flag) { setError('Please select a flag.'); return; } - + const newCondition = { type, name, negate, required, - ...(type === 'regex' ? { id: regexes.find(regex => regex.name === regexName)?.id } : {}), + ...(type === 'regex' ? { regex_id: regexId } : {}), // Save regex_id ...(type === 'size' ? { min: parseInt(minSize), max: parseInt(maxSize) } : {}), ...(type === 'flag' ? { flag } : {}), }; - + onSave(newCondition); onClose(); }; const handleDelete = () => { - if (initialConditionRef.current) { + if (initialConditionRef.current && onDelete) { onDelete(initialConditionRef.current); onClose(); } @@ -100,11 +98,8 @@ function ConditionModal({ {error &&
{error}
}
@@ -139,13 +134,13 @@ function ConditionModal({ Regex Pattern