Files
profilarr/backend/app/compile/profile_compiler.py
santiagosayshey d8f944af11 improvements: whole lotta stuff (#18)
* feat: initialise task scheduler

* feat: add "next run" field to task status

* fix: adjust status route path

* feat: task dashboard + api functions

* fix: return sucess object for alert

* refactor: turn task cards into seperate objects

* fix: change task names

* feat: implement compile and import module
- only working for custom formats for now

* refactor/feature: refactor compilation module
- seperate file for mappings
- seperate format compiler into new file
- add compiler for profiles

* feat: add import logic for quality profiles

* fix: properly resolve cutoff IDs for singular qualities

* fix: remux mappings for sonarr

* fix: retain quality group order
- stop groups first, then singular

* fix: dynamically find next group ID to stop duplicate IDs from occuring

* fix: normalise quality letter case

* feat: add api functions for import functionality

* fix: adjust validation for import.js

* feat: add mass selection tool componnet
- keyboard shortcuts to enter state
- mass delete / import

* feat: add loading indicator on import

* style: improve selected card styling

* fix: append extra custom formats with 0 score

* perf: add git status caching to improve load times

* fix: adjust mass import handling and selection logic
- use content.name rather than content.id

* fix: enhance quality name mapping with alternate names and case-insensitive lookups

* feat: add description truncation to ProfileCard for improved readability
- also remove qualitites

* fix: update upgrade quality selection logic to handle disabling scenarios

* feat: expand language mappings with additional languages and identifiers

* feat: enhance profile conversion logging with language handling

* fix: clarify language setting impact in ProfileLanguagesTab component

* feat: add Sonarr language mappings and update language selection logic

* feat: implement language normalization and enhance logging in format conversion

* feat: enhance logging in format conversion and add language specification handling

* feat: add Afrikaans and Albanian languages to the language constants

* refactor: remove language strictness feature and update language handling in ProfileModal

* feat: add logging setup for improved debugging in mappings module

* feat: enhance language compilation to work with new system

* feat: update language handling in ProfileLanguagesTab to support 'any' behavior

* fix: remove redundant import statement in profile.py

* feat: implement in-memory format import functionality and update profile compiler to utilize it for non english language compilation

* feat: add comparison for tweaks in quality profile changes

* feat: add functions to convert display names to filenames and vice versa

* fix: remove unnecessary filename modification in formats data

* feat: add process_tweaks function to handle profile tweaks and import formats before compilation

* feat: refactor process_tweaks to modularize format import and scoring
- include lossless audio tweak handling

* feat: add support for Dolby Vision no fallback and bleeding edge codecs in process_tweaks

* feat: add support for disabling prereleases based on profile tweaks

* fix: reduce cache TTL from 30 seconds to 1 second for quicker updates

* feat: add resetState function to initialize profile modal state

* feat: enhance save_yaml_file function to optionally use data name for filename

* feat: enhance handle_rename function to account for staged renames

* feat: enhance revert functionality to handle untracked files and staged deletions

* chore: update timezone setting in docker-compose.yml

* feat: increase maximum description length in ProfileCard component

* refactor: generic YAML comparison and change summary functionality

* refactor: incoming changes now uses generic comparison logic

* refactor: add YAML conflict comparison functionality with detailed summary generation for merge conflicts

* refactor: heavily simplified resolve conflict modal to work with new generic conflict parsing

* refactor: implement GitStatusManager for improved repository status handling and sync task updates

* refactor: integrate GitStatusManager to update remote status after pull operations

* refactor: remove isDevMode prop from ChangeRow, ConflictRow, ConflictTable, ChangeTable, RepoContainer, and ActionButtons components

* refactor: migrate settings handling from settings_utils to db module and enhance settings management

* refactor: remove deprecated authentication methods and streamline push operations with SSH access

* refactor: enhance error handling and authentication for Git operations, including PAT support

* refactor: remove old settings_utils

* refactor: add settings prop to StatusContainer and update database initialization for profilarr_pat

* refactor: simplify format name handling in ViewChanges component by removing old API call

* fix: update fetchSettings to handle cases with no git repository and ensure settings is null

* fix: enhance get_git_status to return a valid status object when no git repository is found

* style: various improvements to repo container
- add database stats
- add organisation / profile avatar
- parse organisation / repo name
- improvements to branch button

* feat: implement authentication setup and middleware for secure session management

* fix: auto login after setting up authentication

* feat: enhance authentication setup with GET method and track failed attempts

* feat: add authentication setup and login components with API integration

* feat: redesign SetupPage component with improved layout and user guidance

* feat: enhance LoginPage layout with improved design and user guidance

* refactor: remove unused API functions for regex and format management

* improvements: whole lot more stuff (#17)

* feat: implement configuration management for directory paths and session settings

* feat: implement backup management with API endpoints for backup operations

* feat: add backup import functionality with zip file validation and restoration

* feat: implement backup API with endpoints for listing, creating, downloading, restoring, deleting, and importing backups

* feat: enhance backup listing with file size and last modified time

* feat: add backup management interface with listing, creation, restoration, and deletion functionalities

* fix: refresh backups list after successful deletion

* feat: create BackupCard component for displaying backup details and actions

* fix: status parsing improvements
- now properly shows outgoing changes with / without developer mode

* fix: remove authentication bypass for backup routes during testing

* feat: add logging configuration and ensure log directory creation

* feat: implement application-wide logging configuration and ensure log directory creation

* feat: add logging blueprint with endpoints for retrieving and searching log files

* feat: add git logging configuration to enhance logging capabilities

* refactor: enhance logging details and improve error handling in repository cloning and file processing

* refactor: update tab labels for clarity in settings page

* refactor: remove unused tasks and streamline task scheduler

* refactor: improve repository settings handling and UI updates in RepoContainer

* style: add slight gradient to modal

* feat: enhance footer with GitHub repository info and organization avatar

* refactor: update imports and enhance modal layout for linking Git repository

* refactor: remove documentation and issue links from footer component

* refactor: git settings refactor.
- new git container to contain repo / status
- split repo container into active / empty components
- split status into seperate sections for incoming, outgoing, conflict

* chore: simplify environment configuration by using .env file in docker-compose

* fix: update remote status after commit and push operations

* refactor: restructure application initialization and configure Git user settings on startup

* refactor: improve default Git user configuration handling in initialization

* style: enhance UI styling and structure for Git status display

* style: enhance UI layout and styling for ChangeRow and ChangeTable components

* style: enhance UI layout and styling for CommitMessage component

* style: improve selected row styling

* style: enhance UI styling for ConflictRow and ConflictTable components

* style: update merge process to include remote status update after finalization

* style: update noChangesMessages for improved clarity and engagement

* feat: add animated Logo component and integrate it into Navbar

* feat: enhance selection handling
- add shift selection
- add will be selected state and styling

* feat: enhance mouse tracking for shift selection

* refactor: remove deprecated ActionButtons

* style: unify button colors and update tooltips in IncomingChanges and OutgoingChanges components

* feat: add auto-pull feature with toggle in settings and backend support

* fix: update auto-pull implementation to use integer values

* feat: implement auto-pull functionality in remote status update

* file: improve value formatting for new files

* fix: change logger level to DEBUG and add debug message for profile import attempts

* refactor: remove logging statements and streamline exception handling in format compiler

* feat: enhance format import process with detailed logging and error handling

* feat: enhance profile import process with detailed logging and error handling

* refactor: remove non error logging statements for profile compilation

* feat: enhance logging for memory-based format import with detailed success and error messages

* feat: add logging for language settings and compiled profile data in profile import

* feat: add dedicated logging for importarr with separate log file and configuration

* feat: add logging API with functions to fetch logs, search, and filter by level

* feat: add logs tab to settings page with LogContainer component

* feat: add LogContainer and LogViewer components for enhanced log management

* fix: dynamic vertical height for log viewer

* fix: reduce log file size and increase backup count for improved log management

* fix: enhance error logging with exception type and full traceback for better debugging

* fix: use mapped cutoff name for profile conversion to ensure correct quality mapping

* fix: add validation for git repository existence before syncing

* fix: implement delete constraints check before item deletion to prevent breaking references

* fix: enhance delete constraints check with improved logging and name normalization

* fix: add protection against deletion of required custom formats in delete constraints check

* feat: implement ANSI color parsing in LogViewer for improved log readability

* feat: enhance ViewChanges component with improved key parsing and rendering of changes

* refactor: improve styling and structure of ResolveConflicts component for better readability and usability

* fix: improve error handling and response for arr config saving

* feat: extend arr configuration with additional fields and sync methods

* feat: add DataSelectorModal component for selecting data to sync

* feat: enhance ArrModal and DataSelectorModal with improved layout and data display

* feat: update ArrContainer to use new API import and include additional arrConfig fields

* feat: enhance ArrCard component with sync details and improved layout

* feat: add AddButton component with custom positioning and animations

* feat: replace Add New Card section with AddButton component for improved UX

* feat: reposition AddButton in ArrContainer for better visibility

* feat: replace AddNewCard with AddButton in RegexPage for improved UX

* feat: replace AddNewCard with AddButton in FormatPage for improved UX

* feat: replace AddNewCard with AddButton in ProfilePage for improved UX
2025-02-05 16:09:59 +10:30

328 lines
13 KiB
Python

"""Profile compilation module for converting quality profiles"""
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Any, Callable
import json
import yaml
import logging
from .mappings import TargetApp, ValueResolver
from ..data.utils import load_yaml_file, get_category_directory
from ..importarr.format_memory import import_format_from_memory
logger = logging.getLogger(__name__)
@dataclass
class ConvertedProfile:
"""Data class for converted profile output"""
name: str
items: List[Dict]
format_items: List[Dict]
upgrade_allowed: bool
min_format_score: int
cutoff_format_score: int
min_upgrade_format_score: int
language: Dict
cutoff: Optional[int] = None
class ProfileConverter:
"""Converts quality profiles between different formats"""
def __init__(self,
target_app: TargetApp,
base_url: str = None,
api_key: str = None,
format_importer: Callable = None):
self.target_app = target_app
self.base_url = base_url
self.api_key = api_key
self.format_importer = format_importer
self.quality_mappings = ValueResolver.get_qualities(target_app)
def _convert_group_id(self, group_id: int) -> int:
if group_id < 0:
return 1000 + abs(group_id)
return group_id
def _create_all_qualities(self,
allowed_qualities: List[str]) -> List[Dict]:
qualities = []
for quality_name in allowed_qualities:
if quality_name in self.quality_mappings:
qualities.append({
"quality":
self.quality_mappings[quality_name].copy(),
"items": [],
"allowed":
True
})
return qualities
def _process_language_formats(self, behaviour: str,
language: str) -> List[Dict]:
if not self.base_url or not self.api_key or not self.format_importer:
logger.error("Missing required credentials or format importer")
raise ValueError(
"base_url, api_key, and format_importer are required for language format processing"
)
try:
formats_to_import = []
format_configs = []
base_format_path = f"{get_category_directory('custom_format')}/Not English.yml"
base_format = load_yaml_file(base_format_path)
language_data = ValueResolver.get_language(language,
self.target_app,
for_profile=False)
modified_format = base_format.copy()
modified_format['name'] = f"Not {language_data['name']}"
for condition in modified_format['conditions']:
if condition.get('type') == 'language':
condition['language'] = language
if condition.get('name') == 'Not English':
condition['name'] = f"Not {language_data['name']}"
elif condition.get('name') == 'Includes English':
condition['name'] = f"Includes {language_data['name']}"
formats_to_import.append(modified_format)
if behaviour == 'only':
additional_formats = [
"Not Only English", "Not Only English (Missing)"
]
for format_name in additional_formats:
format_path = f"{get_category_directory('custom_format')}/{format_name}.yml"
format_data = load_yaml_file(format_path)
format_data['name'] = format_data['name'].replace(
'English', language_data['name'])
for c in format_data.get('conditions', []):
if c.get('type') == 'language':
c['language'] = language
if c.get('name') == 'Not English':
c['name'] = f"Not {language_data['name']}"
elif c.get('name') == 'Includes English':
c['name'] = f"Includes {language_data['name']}"
formats_to_import.append(format_data)
arr_type = 'radarr' if self.target_app == TargetApp.RADARR else 'sonarr'
for format_data in formats_to_import:
try:
result = import_format_from_memory(format_data,
self.base_url,
self.api_key, arr_type)
if not result.get('success', False):
logger.error(
f"Format import failed for: {format_data['name']}")
raise Exception(
f"Failed to import format {format_data['name']}")
format_configs.append({
'name': format_data['name'],
'score': -9999
})
except Exception as e:
logger.error(
f"Error importing format {format_data['name']}: {str(e)}"
)
raise
return format_configs
except Exception as e:
logger.error(f"Error processing language formats: {str(e)}")
raise
def convert_quality_group(self, group: Dict) -> Dict:
original_id = group.get("id", 0)
converted_id = self._convert_group_id(original_id)
allowed_qualities = []
for q_item in group.get("qualities", []):
input_name = q_item.get("name", "")
quality_map = {k.lower(): k for k in self.quality_mappings}
if input_name.lower() in quality_map:
allowed_qualities.append(quality_map[input_name.lower()])
converted_group = {
"name": group["name"],
"items": self._create_all_qualities(allowed_qualities),
"allowed": True,
"id": converted_id
}
return converted_group
def convert_profile(self, profile: Dict) -> ConvertedProfile:
language = profile.get('language')
if language != 'any':
language_parts = language.split('_', 1)
behaviour, language = language_parts
try:
language_formats = self._process_language_formats(
behaviour, language)
if 'custom_formats' not in profile:
profile['custom_formats'] = []
profile['custom_formats'].extend(language_formats)
except Exception as e:
logger.error(f"Failed to process language formats: {e}")
selected_language = ValueResolver.get_language('any',
self.target_app,
for_profile=True)
converted_profile = ConvertedProfile(
name=profile["name"],
upgrade_allowed=profile.get("upgradesAllowed", True),
items=[],
format_items=[],
min_format_score=profile.get("minCustomFormatScore", 0),
cutoff_format_score=profile.get("upgradeUntilScore", 0),
min_upgrade_format_score=max(1,
profile.get("minScoreIncrement", 1)),
language=selected_language)
used_qualities = set()
tweaks = profile.get('tweaks', {})
allow_prereleases = tweaks.get('allowPrereleases', False)
for quality_entry in profile.get("qualities", []):
if quality_entry.get("id", 0) < 0:
converted_group = self.convert_quality_group(quality_entry)
if (quality_entry.get("name") == "Prereleases"
and not allow_prereleases):
converted_group["allowed"] = False
if converted_group["items"]:
converted_profile.items.append(converted_group)
for q in quality_entry.get("qualities", []):
used_qualities.add(q.get("name", "").upper())
else:
quality_name = quality_entry.get("name")
mapped_name = ValueResolver.get_quality_name(
quality_name, self.target_app)
if mapped_name in self.quality_mappings:
converted_profile.items.append({
"quality":
self.quality_mappings[mapped_name],
"items": [],
"allowed":
True
})
used_qualities.add(mapped_name.upper())
for quality_name, quality_data in self.quality_mappings.items():
if quality_name.upper() not in used_qualities:
converted_profile.items.append({
"quality": quality_data,
"items": [],
"allowed": False
})
if "upgrade_until" in profile and "id" in profile["upgrade_until"]:
cutoff_id = profile["upgrade_until"]["id"]
cutoff_name = profile["upgrade_until"]["name"]
mapped_cutoff_name = ValueResolver.get_quality_name(
cutoff_name, self.target_app)
if cutoff_id < 0:
converted_profile.cutoff = self._convert_group_id(cutoff_id)
else:
# And use mapped_cutoff_name here instead of cutoff_name
converted_profile.cutoff = self.quality_mappings[
mapped_cutoff_name]["id"]
for cf in profile.get("custom_formats", []):
format_item = {"name": cf["name"], "score": cf["score"]}
converted_profile.format_items.append(format_item)
converted_profile.items.reverse()
return converted_profile
class ProfileProcessor:
"""Main class for processing profile files"""
def __init__(self,
input_dir: Path,
output_dir: Path,
target_app: TargetApp,
base_url: str = None,
api_key: str = None,
format_importer: Callable = None):
self.input_dir = input_dir
self.output_dir = output_dir
self.converter = ProfileConverter(target_app, base_url, api_key,
format_importer)
def _load_profile(self, profile_name: str) -> Optional[Dict]:
profile_path = self.input_dir / f"{profile_name}.yml"
if not profile_path.exists():
return None
with profile_path.open('r') as f:
return yaml.safe_load(f)
def process_profile(
self,
profile_name: str,
return_data: bool = False) -> Optional[ConvertedProfile]:
profile_data = self._load_profile(profile_name)
if not profile_data:
return None
converted = self.converter.convert_profile(profile_data)
if return_data:
return converted
output_data = [{
'name': converted.name,
'upgradeAllowed': converted.upgrade_allowed,
'items': converted.items,
'formatItems': converted.format_items,
'minFormatScore': converted.min_format_score,
'cutoffFormatScore': converted.cutoff_format_score,
'minUpgradeFormatScore': converted.min_upgrade_format_score,
'language': converted.language
}]
if converted.cutoff is not None:
output_data[0]['cutoff'] = converted.cutoff
output_path = self.output_dir / f"{profile_name}.json"
with output_path.open('w') as f:
json.dump(output_data, f, indent=2)
return converted
def compile_quality_profile(profile_data: Dict,
target_app: TargetApp,
base_url: str = None,
api_key: str = None,
format_importer: Callable = None) -> List[Dict]:
converter = ProfileConverter(target_app, base_url, api_key,
format_importer)
converted = converter.convert_profile(profile_data)
output = {
'name': converted.name,
'upgradeAllowed': converted.upgrade_allowed,
'items': converted.items,
'formatItems': converted.format_items,
'minFormatScore': converted.min_format_score,
'cutoffFormatScore': converted.cutoff_format_score,
'minUpgradeFormatScore': converted.min_upgrade_format_score,
'language': converted.language
}
if converted.cutoff is not None:
output['cutoff'] = converted.cutoff
return [output]