refactor(regex, format): Change the way regexes and formats are saved - regex IDs are now saved in conditions, not names - sanitisation in custom formats

This commit is contained in:
santiagosayshey
2024-08-16 15:08:15 +09:30
committed by Sam Chau
parent 789c5c6115
commit 2dd93e2588
11 changed files with 292 additions and 203 deletions

View File

@@ -4,7 +4,7 @@ from app.routes import regex_routes, format_routes
def create_app():
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "http://localhost:3000"}})
CORS(app, resources={r"/*": {"origins": "*"}})
app.register_blueprint(regex_routes.bp)
app.register_blueprint(format_routes.bp)
return app

View File

@@ -1,33 +1,31 @@
# app/routes/format_routes.py
from flask import Blueprint, request, jsonify
from app.utils.file_operations import FORMAT_DIR, REGEX_DIR, save_to_file, load_all_from_directory, delete_file, load_from_file
from app.utils.format_operations import save_format, load_all_formats, delete_format, load_format
bp = Blueprint('format', __name__, url_prefix='/format')
bp = Blueprint('format', __name__, url_prefix='/format')
@bp.route('', methods=['GET', 'POST'])
def handle_items():
def handle_formats():
if request.method == 'POST':
data = request.json
saved_data = save_to_file(FORMAT_DIR, data)
saved_data = save_format(data)
return jsonify(saved_data), 201
else:
items = load_all_from_directory(FORMAT_DIR)
return jsonify(items)
formats = load_all_formats()
return jsonify(formats)
@bp.route('/<int:id>', methods=['GET', 'PUT', 'DELETE'])
def handle_item(id):
def handle_format(id):
if request.method == 'GET':
item = load_from_file(FORMAT_DIR, id)
if item:
return jsonify(item)
return jsonify({"error": "Item not found"}), 404
format = load_format(id)
if format:
return jsonify(format)
return jsonify({"error": "Format not found"}), 404
elif request.method == 'PUT':
data = request.json
data['id'] = id
saved_data = save_to_file(FORMAT_DIR, data)
saved_data = save_format(data)
return jsonify(saved_data)
elif request.method == 'DELETE':
if delete_file(FORMAT_DIR, id):
return jsonify({"message": f"Item with ID {id} deleted."}), 200
return jsonify({"error": f"Item with ID {id} not found."}), 404
if delete_format(id):
return jsonify({"message": f"Format with ID {id} deleted."}), 200
return jsonify({"error": f"Format with ID {id} not found."}), 404

View File

@@ -1,5 +1,5 @@
from flask import Blueprint, request, jsonify
from app.utils.file_operations import REGEX_DIR, save_to_file, load_all_from_directory, delete_file, load_from_file
from app.utils.regex_operations import save_regex, load_all_regexes, delete_regex, load_regex
import json
import logging
import subprocess
@@ -70,34 +70,28 @@ def regex101_proxy():
@bp.route('', methods=['GET', 'POST'])
def handle_items():
def handle_regexes():
if request.method == 'POST':
data = request.json
# Ensure regex101Link is included in the data
if 'regex101Link' not in data:
data['regex101Link'] = ''
saved_data = save_to_file(REGEX_DIR, data)
saved_data = save_regex(data)
return jsonify(saved_data), 201
else:
items = load_all_from_directory(REGEX_DIR)
return jsonify(items)
regexes = load_all_regexes()
return jsonify(regexes)
@bp.route('/<int:id>', methods=['GET', 'PUT', 'DELETE'])
def handle_item(id):
def handle_regex(id):
if request.method == 'GET':
item = load_from_file(REGEX_DIR, id)
if item:
return jsonify(item)
return jsonify({"error": "Item not found"}), 404
regex = load_regex(id)
if regex:
return jsonify(regex)
return jsonify({"error": "Regex not found"}), 404
elif request.method == 'PUT':
data = request.json
data['id'] = id
# Ensure regex101Link is included in the data
if 'regex101Link' not in data:
data['regex101Link'] = ''
saved_data = save_to_file(REGEX_DIR, data)
saved_data = save_regex(data)
return jsonify(saved_data)
elif request.method == 'DELETE':
if delete_file(REGEX_DIR, id):
return jsonify({"message": f"Item with ID {id} deleted."}), 200
return jsonify({"error": f"Item with ID {id} not found."}), 404
if delete_regex(id):
return jsonify({"message": f"Regex with ID {id} deleted."}), 200
return jsonify({"error": f"Regex with ID {id} not found."}), 404

View File

@@ -1,110 +0,0 @@
import os
import yaml
import datetime
from collections import OrderedDict
REGEX_DIR = 'regex_patterns'
FORMAT_DIR = 'custom_formats'
os.makedirs(REGEX_DIR, exist_ok=True)
os.makedirs(FORMAT_DIR, exist_ok=True)
# Custom representer to handle OrderedDict
def represent_ordereddict(dumper, data):
return dumper.represent_dict(data.items())
yaml.add_representer(OrderedDict, represent_ordereddict, Dumper=yaml.SafeDumper)
def get_next_id(directory):
files = [f for f in os.listdir(directory) if f.endswith('.yml')]
if not files:
return 1
return max(int(f.split('_')[0]) for f in files) + 1
def generate_filename(directory, id, name):
sanitized_name = name.replace(' ', '_').lower()
return f"{directory}/{id}_{sanitized_name}.yml"
def get_current_timestamp():
return datetime.datetime.now().isoformat()
def save_to_file(directory, data):
# Ensure ID, Name, Description, and Tags are ordered first
ordered_data = OrderedDict()
ordered_data['id'] = data.get('id', get_next_id(directory))
ordered_data['name'] = data.get('name', '')
ordered_data['description'] = data.get('description', '')
ordered_data['tags'] = data.get('tags', [])
# Add the rest of the data
for key, value in data.items():
if key not in ordered_data:
ordered_data[key] = value
# Update timestamps
if 'id' in data and data['id'] != 0:
existing_files = [f for f in os.listdir(directory) if f.startswith(f"{data['id']}_") and f.endswith('.yml')]
if existing_files:
existing_filename = os.path.join(directory, existing_files[0])
existing_data = load_from_file(directory, data['id'])
if existing_data:
ordered_data['date_created'] = existing_data.get('date_created', get_current_timestamp())
else:
ordered_data['date_created'] = get_current_timestamp()
ordered_data['date_modified'] = get_current_timestamp()
new_filename = generate_filename(directory, ordered_data['id'], ordered_data['name'])
# Remove the old file if the name has changed
if existing_filename != new_filename:
os.remove(existing_filename)
with open(new_filename, 'w') as file:
yaml.dump(ordered_data, file, default_flow_style=False, Dumper=yaml.SafeDumper)
return ordered_data
else:
# If existing file not found, treat it as new
ordered_data['id'] = get_next_id(directory)
ordered_data['date_created'] = get_current_timestamp()
ordered_data['date_modified'] = get_current_timestamp()
else:
# Handle new records
ordered_data['id'] = get_next_id(directory)
ordered_data['date_created'] = get_current_timestamp()
ordered_data['date_modified'] = get_current_timestamp()
new_filename = generate_filename(directory, ordered_data['id'], ordered_data['name'])
with open(new_filename, 'w') as file:
yaml.dump(ordered_data, file, default_flow_style=False, Dumper=yaml.SafeDumper)
return ordered_data
def load_from_file(directory, id):
files = [f for f in os.listdir(directory) if f.startswith(f"{id}_") and f.endswith('.yml')]
if files:
filename = os.path.join(directory, files[0])
with open(filename, 'r') as file:
data = yaml.safe_load(file)
if 'conditions' not in data:
data['conditions'] = [] # Ensure conditions is always a list
if 'regex101Link' not in data:
data['regex101Link'] = '' # Ensure regex101Link is always present
return data
return None
def load_all_from_directory(directory):
items = []
for filename in os.listdir(directory):
if filename.endswith('.yml'):
with open(os.path.join(directory, filename), 'r') as file:
data = yaml.safe_load(file)
if 'regex101Link' not in data:
data['regex101Link'] = '' # Ensure regex101Link is always present
items.append(data)
return items
def delete_file(directory, id):
files = [f for f in os.listdir(directory) if f.startswith(f"{id}_") and f.endswith('.yml')]
if files:
os.remove(os.path.join(directory, files[0]))
return True
return False

View File

@@ -0,0 +1,15 @@
import os
import datetime
def get_next_id(directory):
files = [f for f in os.listdir(directory) if f.endswith('.yml')]
if not files:
return 1
return max(int(f.split('_')[0]) for f in files) + 1
def generate_filename(directory, id, name):
sanitized_name = name.replace(' ', '_').lower()
return os.path.join(directory, f"{id}_{sanitized_name}.yml")
def get_current_timestamp():
return datetime.datetime.now().isoformat()

View File

@@ -0,0 +1,126 @@
import os
import yaml
import re
import logging
from collections import OrderedDict
from .file_utils import get_next_id, generate_filename, get_current_timestamp
FORMAT_DIR = 'custom_formats'
# Set up basic logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def represent_ordereddict(dumper, data):
return dumper.represent_mapping('tag:yaml.org,2002:map', data.items())
yaml.add_representer(OrderedDict, represent_ordereddict, Dumper=yaml.SafeDumper)
def sanitize_input(input_str):
# Trim leading/trailing whitespace
sanitized_str = input_str.strip()
# Replace special characters that could affect YAML formatting
sanitized_str = re.sub(r'[:#\-\*>\|&]', '', sanitized_str)
# Ensure there are no tabs (which can cause issues in YAML)
sanitized_str = sanitized_str.replace('\t', ' ')
# Optionally: Collapse multiple spaces into a single space
sanitized_str = re.sub(r'\s+', ' ', sanitized_str)
return sanitized_str
def save_format(data):
# Log the received data
logger.info("Received data for saving format: %s", data)
# Sanitize inputs
name = sanitize_input(data.get('name', ''))
description = sanitize_input(data.get('description', ''))
# Determine if it's a new format or an edit
format_id = data.get('id', None)
if format_id == 0: # If id is 0, treat it as a new format
format_id = get_next_id(FORMAT_DIR)
logger.info("Assigned new format ID: %d", format_id)
date_created = get_current_timestamp()
else:
existing_data = load_format(format_id)
if existing_data:
date_created = existing_data.get('date_created')
old_filename = generate_filename(FORMAT_DIR, format_id, existing_data['name'])
# Delete the old file
if os.path.exists(old_filename):
os.remove(old_filename)
else:
date_created = get_current_timestamp()
date_modified = get_current_timestamp()
# Prepare conditions
conditions = []
for condition in data.get('conditions', []):
logger.info("Processing condition: %s", condition)
cond_dict = OrderedDict([
('type', condition['type']),
('name', sanitize_input(condition['name'])),
('negate', condition.get('negate', False)),
('required', condition.get('required', False))
])
if condition['type'] == 'regex':
cond_dict['regex_id'] = condition['regex_id']
elif condition['type'] == 'size':
cond_dict['min'] = condition['min']
cond_dict['max'] = condition['max']
elif condition['type'] == 'flag':
cond_dict['flag'] = sanitize_input(condition['flag'])
conditions.append(cond_dict)
# Prepare tags
tags = [sanitize_input(tag) for tag in data.get('tags', [])]
# Create ordered data dictionary
ordered_data = OrderedDict([
('id', format_id),
('name', name),
('description', description),
('date_created', str(date_created)),
('date_modified', str(date_modified)),
('conditions', conditions),
('tags', tags)
])
# Generate new filename based on the updated name
new_filename = generate_filename(FORMAT_DIR, format_id, name)
# Write the YAML file with the new name
with open(new_filename, 'w') as file:
yaml.dump(ordered_data, file, default_flow_style=False, Dumper=yaml.SafeDumper)
return ordered_data
def load_format(id):
files = [f for f in os.listdir(FORMAT_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')]
if files:
filename = os.path.join(FORMAT_DIR, files[0])
with open(filename, 'r') as file:
data = yaml.safe_load(file)
return data
return None
def load_all_formats():
formats = []
for filename in os.listdir(FORMAT_DIR):
if filename.endswith('.yml'):
with open(os.path.join(FORMAT_DIR, filename), 'r') as file:
data = yaml.safe_load(file)
formats.append(data)
return formats
def delete_format(id):
files = [f for f in os.listdir(FORMAT_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')]
if files:
os.remove(os.path.join(FORMAT_DIR, files[0]))
return True
return False

View File

@@ -0,0 +1,69 @@
import os
import yaml
from collections import OrderedDict
from .file_utils import get_next_id, generate_filename, get_current_timestamp
REGEX_DIR = 'regex_patterns'
def save_regex(data):
ordered_data = OrderedDict()
if 'id' in data and data['id'] != 0:
ordered_data['id'] = data['id']
else:
ordered_data['id'] = get_next_id(REGEX_DIR)
ordered_data['name'] = data.get('name', '')
ordered_data['description'] = data.get('description', '')
ordered_data['pattern'] = data.get('pattern', '')
ordered_data['regex101Link'] = data.get('regex101Link', '')
if ordered_data['id'] != 0: # Existing regex
existing_data = load_regex(ordered_data['id'])
if existing_data:
ordered_data['date_created'] = existing_data.get('date_created', get_current_timestamp())
else:
ordered_data['date_created'] = get_current_timestamp()
else: # New regex
ordered_data['date_created'] = get_current_timestamp()
ordered_data['date_modified'] = get_current_timestamp()
ordered_data['tags'] = data.get('tags', [])
filename = generate_filename(REGEX_DIR, ordered_data['id'], ordered_data['name'])
with open(filename, 'w') as file:
for key, value in ordered_data.items():
if key in ['description', 'date_created', 'date_modified', 'regex101Link']:
file.write(f"{key}: '{value}'\n")
elif key == 'tags':
file.write('tags:\n')
for tag in value:
file.write(f'- {tag}\n')
else:
file.write(f'{key}: {value}\n')
return ordered_data
def load_regex(id):
files = [f for f in os.listdir(REGEX_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')]
if files:
filename = os.path.join(REGEX_DIR, files[0])
with open(filename, 'r') as file:
data = yaml.safe_load(file)
return data
return None
def load_all_regexes():
regexes = []
for filename in os.listdir(REGEX_DIR):
if filename.endswith('.yml'):
with open(os.path.join(REGEX_DIR, filename), 'r') as file:
data = yaml.safe_load(file)
regexes.append(data)
return regexes
def delete_regex(id):
files = [f for f in os.listdir(REGEX_DIR) if f.startswith(f"{id}_") and f.endswith('.yml')]
if files:
os.remove(os.path.join(REGEX_DIR, files[0]))
return True
return False

View File

@@ -2,4 +2,4 @@ Flask==2.0.1
Flask-CORS==3.0.10
PyYAML==5.4.1
requests==2.26.0
Werkzeug==2.0.1
Werkzeug==2.0.1

View File

@@ -1,12 +1,6 @@
import PropTypes from 'prop-types';
function ConditionCard({ condition, onEdit }) {
const conditionType = condition.regex_name
? 'Regex'
: condition.min !== undefined && condition.max !== undefined
? 'Size'
: 'Flag';
return (
<div
onClick={() => onEdit(condition)}
@@ -15,21 +9,26 @@ function ConditionCard({ condition, onEdit }) {
<div className="flex justify-between items-center mb-2">
<h4 className="font-bold text-md dark:text-gray-200">{condition.name}</h4>
<span className="text-xs text-gray-600 dark:text-gray-400">
{conditionType}
{condition.type.charAt(0).toUpperCase() + condition.type.slice(1)}
</span>
</div>
{conditionType === 'Regex' && (
{condition.type === 'regex' && (
<div className="bg-gray-100 dark:bg-gray-600 rounded p-2 mb-2">
<pre className="text-sm font-mono text-gray-800 dark:text-gray-200 whitespace-pre-wrap">
{condition.regex_name}
Regex ID: {condition.regex_id || condition.id} {/* Display regex_id */}
</pre>
</div>
)}
{conditionType === 'Size' && (
{condition.type === 'size' && (
<p className="text-gray-600 dark:text-gray-400 text-sm mb-2">
Size: {condition.min || 'Any'} - {condition.max || 'Any'} bytes
</p>
)}
{condition.type === 'flag' && (
<p className="text-gray-600 dark:text-gray-400 text-sm mb-2">
Flag: {condition.flag}
</p>
)}
<div className="flex space-x-2 mt-2">
<span className={`text-xs font-semibold inline-block py-1 px-2 rounded ${condition.required ? 'bg-green-500 text-white' : 'bg-blue-500 text-white'}`}>
{condition.required ? 'Required' : 'Optional'}
@@ -46,14 +45,15 @@ function ConditionCard({ condition, onEdit }) {
ConditionCard.propTypes = {
condition: PropTypes.shape({
name: PropTypes.string,
regex_name: PropTypes.string,
type: PropTypes.string.isRequired,
name: PropTypes.string.isRequired,
regex_id: PropTypes.number, // Updated to regex_id
min: PropTypes.number,
max: PropTypes.number,
flag: PropTypes.string,
negate: PropTypes.bool,
required: PropTypes.bool,
flag: PropTypes.string, // Include flag in prop types
}),
}).isRequired,
onEdit: PropTypes.func.isRequired,
};

View File

@@ -1,4 +1,4 @@
import { useState, useEffect, useRef } from 'react';
import React, { useState, useEffect, useRef } from 'react';
import PropTypes from 'prop-types';
import Modal from '../ui/Modal';
@@ -13,7 +13,7 @@ function ConditionModal({
}) {
const [name, setName] = useState('');
const [type, setType] = useState('regex');
const [regexName, setRegexName] = useState('');
const [regexId, setRegexId] = useState(null);
const [minSize, setMinSize] = useState('');
const [maxSize, setMaxSize] = useState('');
const [flag, setFlag] = useState('');
@@ -27,70 +27,68 @@ function ConditionModal({
initialConditionRef.current = condition;
if (condition) {
setName(condition.name);
if (condition.regex_name) {
setType('regex');
setRegexName(condition.regex_name);
} else if (condition.min !== undefined && condition.max !== undefined) {
setType('size');
setMinSize(condition.min);
setMaxSize(condition.max);
} else if (condition.flag) {
setType('flag');
setFlag(condition.flag);
}
setType(condition.type);
setRegexId(condition.regex_id || condition.id); // Read regex_id instead of id
setMinSize(condition.min?.toString() || '');
setMaxSize(condition.max?.toString() || '');
setFlag(condition.flag || '');
setNegate(condition.negate || false);
setRequired(condition.required || false);
} else {
setName('');
setType('regex');
setRegexName('');
setMinSize('');
setMaxSize('');
setFlag('');
setNegate(false);
setRequired(false);
resetForm();
}
setError('');
}
}, [condition, isOpen]);
const resetForm = () => {
setName('');
setType('regex');
setRegexId(null);
setMinSize('');
setMaxSize('');
setFlag('');
setNegate(false);
setRequired(false);
setError('');
};
const handleSave = () => {
if (!name.trim()) {
setError('Condition name is required.');
return;
}
if (type === 'regex' && !regexName) {
if (type === 'regex' && !regexId) {
setError('Please select a regex pattern.');
return;
}
if (type === 'size' && (!minSize || !maxSize)) {
setError('Both minimum and maximum sizes are required.');
return;
}
if (type === 'flag' && !flag) {
setError('Please select a flag.');
return;
}
const newCondition = {
type,
name,
negate,
required,
...(type === 'regex' ? { id: regexes.find(regex => regex.name === regexName)?.id } : {}),
...(type === 'regex' ? { regex_id: regexId } : {}), // Save regex_id
...(type === 'size' ? { min: parseInt(minSize), max: parseInt(maxSize) } : {}),
...(type === 'flag' ? { flag } : {}),
};
onSave(newCondition);
onClose();
};
const handleDelete = () => {
if (initialConditionRef.current) {
if (initialConditionRef.current && onDelete) {
onDelete(initialConditionRef.current);
onClose();
}
@@ -100,11 +98,8 @@ function ConditionModal({
<Modal
isOpen={isOpen}
onClose={onClose}
level={level}
title={initialConditionRef.current ? 'Edit Condition' : 'Add Condition'}
disableCloseOnOutsideClick={true}
disableCloseOnEscape={true}
className="max-w-2xl min-h-72"
level={level}
>
{error && <div className="text-red-500 mb-4">{error}</div>}
<div className="mb-4">
@@ -139,13 +134,13 @@ function ConditionModal({
Regex Pattern
</label>
<select
value={regexName}
onChange={(e) => setRegexName(e.target.value)}
value={regexId || ''}
onChange={(e) => setRegexId(e.target.value ? Number(e.target.value) : null)}
className="w-full p-3 border rounded dark:bg-gray-700 dark:text-gray-200 dark:border-gray-600"
>
<option value="">Select a regex</option>
{regexes.map((regex) => (
<option key={regex.id} value={regex.name}>
<option key={regex.id} value={regex.id}>
{regex.name}
</option>
))}
@@ -242,13 +237,14 @@ function ConditionModal({
ConditionModal.propTypes = {
condition: PropTypes.shape({
type: PropTypes.string,
name: PropTypes.string,
regex_name: PropTypes.string,
regex_id: PropTypes.number, // Updated to regex_id
min: PropTypes.number,
max: PropTypes.number,
flag: PropTypes.string,
negate: PropTypes.bool,
required: PropTypes.bool,
flag: PropTypes.string,
}),
isOpen: PropTypes.bool.isRequired,
onClose: PropTypes.func.isRequired,
@@ -261,4 +257,4 @@ ConditionModal.propTypes = {
level: PropTypes.number,
};
export default ConditionModal;
export default ConditionModal;

View File

@@ -63,6 +63,7 @@ function FormatModal({ format = null, isOpen, onClose, onSave }) {
};
const handleDelete = async () => {
console.log(`Trying to delete format with id: ${initialFormatRef.current.id}`);
if (initialFormatRef.current && initialFormatRef.current.id) {
try {
await deleteFormat(initialFormatRef.current.id);