diff --git a/docs/todo/scratchpad.md b/docs/todo/scratchpad.md index f5cbce9..f3aa8fc 100644 --- a/docs/todo/scratchpad.md +++ b/docs/todo/scratchpad.md @@ -2,8 +2,6 @@ # Feedback from Seraphys -- cache fetches from github on things that dont change often (images, stats for - example) - rethink job polling architecture - maybe move langauges to general tab or put info directly below it to fill space diff --git a/src/lib/server/db/migrations.ts b/src/lib/server/db/migrations.ts index 153a59a..92d5129 100644 --- a/src/lib/server/db/migrations.ts +++ b/src/lib/server/db/migrations.ts @@ -34,6 +34,7 @@ import { migration as migration029 } from './migrations/029_add_database_id_fore import { migration as migration030 } from './migrations/030_create_general_settings.ts'; import { migration as migration031 } from './migrations/031_remove_search_cooldown.ts'; import { migration as migration032 } from './migrations/032_add_filter_id_to_upgrade_runs.ts'; +import { migration as migration033 } from './migrations/033_create_github_cache.ts'; export interface Migration { version: number; @@ -286,7 +287,8 @@ export function loadMigrations(): Migration[] { migration029, migration030, migration031, - migration032 + migration032, + migration033 ]; // Sort by version number diff --git a/src/lib/server/db/migrations/033_create_github_cache.ts b/src/lib/server/db/migrations/033_create_github_cache.ts new file mode 100644 index 0000000..9a425aa --- /dev/null +++ b/src/lib/server/db/migrations/033_create_github_cache.ts @@ -0,0 +1,25 @@ +import type { Migration } from '../migrations.ts'; + +export const migration: Migration = { + version: 33, + name: 'Create GitHub cache table', + + up: ` + CREATE TABLE IF NOT EXISTS github_cache ( + cache_key TEXT PRIMARY KEY, + cache_type TEXT NOT NULL, + data TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + expires_at DATETIME NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_github_cache_type ON github_cache(cache_type); + CREATE INDEX IF NOT EXISTS idx_github_cache_expires ON github_cache(expires_at); + `, + + down: ` + DROP INDEX IF EXISTS idx_github_cache_expires; + DROP INDEX IF EXISTS idx_github_cache_type; + DROP TABLE IF EXISTS github_cache; + ` +}; diff --git a/src/lib/server/db/queries/githubCache.ts b/src/lib/server/db/queries/githubCache.ts new file mode 100644 index 0000000..7af7d9a --- /dev/null +++ b/src/lib/server/db/queries/githubCache.ts @@ -0,0 +1,97 @@ +import { db } from '../db.ts'; + +/** + * Types for github_cache table + */ +export interface GitHubCache { + cache_key: string; + cache_type: string; + data: string; + created_at: string; + expires_at: string; +} + +export type GitHubCacheType = 'repo_info' | 'avatar' | 'releases'; + +/** + * All queries for github_cache table + */ +export const githubCacheQueries = { + /** + * Get cached data by key (returns null if expired) + */ + get(cacheKey: string): GitHubCache | null { + const result = db.queryFirst( + `SELECT * FROM github_cache + WHERE cache_key = ? AND expires_at > datetime('now')`, + cacheKey + ); + return result ?? null; + }, + + /** + * Get cached data by key even if expired (for stale-while-revalidate) + */ + getStale(cacheKey: string): GitHubCache | null { + const result = db.queryFirst( + 'SELECT * FROM github_cache WHERE cache_key = ?', + cacheKey + ); + return result ?? null; + }, + + /** + * Check if cached data is expired + */ + isExpired(cacheKey: string): boolean { + const result = db.queryFirst<{ expired: number }>( + `SELECT CASE WHEN expires_at <= datetime('now') THEN 1 ELSE 0 END as expired + FROM github_cache WHERE cache_key = ?`, + cacheKey + ); + return result?.expired === 1; + }, + + /** + * Store data in cache with TTL (insert or replace) + */ + set(cacheKey: string, cacheType: GitHubCacheType, data: string, ttlMinutes: number): void { + db.execute( + `INSERT OR REPLACE INTO github_cache (cache_key, cache_type, data, expires_at) + VALUES (?, ?, ?, datetime('now', '+' || ? || ' minutes'))`, + cacheKey, + cacheType, + data, + ttlMinutes + ); + }, + + /** + * Delete a cached entry + */ + delete(cacheKey: string): boolean { + const affected = db.execute('DELETE FROM github_cache WHERE cache_key = ?', cacheKey); + return affected > 0; + }, + + /** + * Delete all expired entries + */ + deleteExpired(): number { + return db.execute("DELETE FROM github_cache WHERE expires_at <= datetime('now')"); + }, + + /** + * Clear all cached entries + */ + clear(): number { + return db.execute('DELETE FROM github_cache'); + }, + + /** + * Invalidate all entries of a specific type + */ + invalidateByType(cacheType: GitHubCacheType): number { + return db.execute('DELETE FROM github_cache WHERE cache_type = ?', cacheType); + } +}; diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 214ca59..cc655a9 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -620,3 +620,21 @@ CREATE TABLE general_settings ( created_at DATETIME DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ); + +-- ============================================================================== +-- TABLE: github_cache +-- Purpose: Cache GitHub API responses (repo info, avatars, releases) to reduce API calls +-- Migration: 033_create_github_cache.ts +-- ============================================================================== + +CREATE TABLE github_cache ( + cache_key TEXT PRIMARY KEY, -- e.g., "repo:owner/repo", "avatar:owner", "releases:owner/repo" + cache_type TEXT NOT NULL, -- "repo_info", "avatar", "releases" + data TEXT NOT NULL, -- JSON response data (or base64 data URL for images) + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + expires_at DATETIME NOT NULL -- TTL-based expiration +); + +-- GitHub cache indexes (Migration: 033_create_github_cache.ts) +CREATE INDEX idx_github_cache_type ON github_cache(cache_type); +CREATE INDEX idx_github_cache_expires ON github_cache(expires_at); diff --git a/src/lib/server/utils/git/repo.ts b/src/lib/server/utils/git/repo.ts index afde794..b1be2ba 100644 --- a/src/lib/server/utils/git/repo.ts +++ b/src/lib/server/utils/git/repo.ts @@ -4,6 +4,7 @@ import { execGit, execGitSafe } from './exec.ts'; import type { RepoInfo } from './types.ts'; +import { getCachedRepoInfo } from '../github/cache.ts'; /** * Validate that a repository URL is accessible and detect if it's private @@ -174,54 +175,11 @@ export async function commit(repoPath: string, message: string): Promise { } /** - * Get repository info from GitHub API + * Get repository info from GitHub API (cached) */ export async function getRepoInfo( repositoryUrl: string, personalAccessToken?: string | null ): Promise { - const githubPattern = /^https:\/\/github\.com\/([\w-]+)\/([\w-]+)\/?$/; - const normalizedUrl = repositoryUrl.replace(/\.git$/, ''); - const match = normalizedUrl.match(githubPattern); - - if (!match) { - return null; - } - - const [, owner, repo] = match; - const apiUrl = `https://api.github.com/repos/${owner}/${repo}`; - - const headers: Record = { - Accept: 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28', - 'User-Agent': 'Profilarr' - }; - - if (personalAccessToken) { - headers['Authorization'] = `Bearer ${personalAccessToken}`; - } - - try { - const response = await globalThis.fetch(apiUrl, { headers }); - - if (!response.ok) { - return null; - } - - const data = await response.json(); - - return { - owner: data.owner.login, - repo: data.name, - description: data.description, - stars: data.stargazers_count, - forks: data.forks_count, - openIssues: data.open_issues_count, - ownerAvatarUrl: data.owner.avatar_url, - ownerType: data.owner.type, - htmlUrl: data.html_url - }; - } catch { - return null; - } + return getCachedRepoInfo(repositoryUrl, personalAccessToken); } diff --git a/src/lib/server/utils/github/cache.ts b/src/lib/server/utils/github/cache.ts new file mode 100644 index 0000000..4f886f9 --- /dev/null +++ b/src/lib/server/utils/github/cache.ts @@ -0,0 +1,288 @@ +/** + * GitHub API caching utilities + */ + +import { githubCacheQueries } from '$db/queries/githubCache.ts'; +import { logger } from '$logger/logger.ts'; +import type { RepoInfo } from '../git/types.ts'; + +/** + * TTL Configuration (in minutes) + */ +const TTL = { + REPO_INFO: 60, // 1 hour - stars/forks don't change often + AVATAR: 1440, // 24 hours - avatars rarely change + RELEASES: 30 // 30 minutes - releases are less frequent +}; + +/** + * GitHub Release type + */ +export interface GitHubRelease { + tag_name: string; + name: string; + published_at: string; + html_url: string; + prerelease: boolean; +} + +/** + * Standard GitHub API headers + */ +function getHeaders(pat?: string | null): Record { + const headers: Record = { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'Profilarr' + }; + + if (pat) { + headers['Authorization'] = `Bearer ${pat}`; + } + + return headers; +} + +/** + * Parse GitHub URL to extract owner and repo + */ +function parseGitHubUrl(repositoryUrl: string): { owner: string; repo: string } | null { + const githubPattern = /^https:\/\/github\.com\/([\w-]+)\/([\w-]+)\/?$/; + const normalizedUrl = repositoryUrl.replace(/\.git$/, ''); + const match = normalizedUrl.match(githubPattern); + + if (!match) { + return null; + } + + return { owner: match[1], repo: match[2] }; +} + +/** + * Get cached repo info or fetch from GitHub API + */ +export async function getCachedRepoInfo( + repositoryUrl: string, + pat?: string | null +): Promise { + const parsed = parseGitHubUrl(repositoryUrl); + if (!parsed) { + return null; + } + + const { owner, repo } = parsed; + const cacheKey = `repo:${owner}/${repo}`; + + // Check cache + const cached = githubCacheQueries.get(cacheKey); + if (cached) { + await logger.debug('GitHub repo info cache hit', { + source: 'GitHubCache', + meta: { cacheKey } + }); + return JSON.parse(cached.data) as RepoInfo; + } + + await logger.debug('GitHub repo info cache miss', { + source: 'GitHubCache', + meta: { cacheKey } + }); + + // Fetch from API + const apiUrl = `https://api.github.com/repos/${owner}/${repo}`; + const headers = getHeaders(pat); + + try { + const response = await globalThis.fetch(apiUrl, { headers }); + + if (!response.ok) { + return null; + } + + const data = await response.json(); + + const repoInfo: RepoInfo = { + owner: data.owner.login, + repo: data.name, + description: data.description, + stars: data.stargazers_count, + forks: data.forks_count, + openIssues: data.open_issues_count, + ownerAvatarUrl: data.owner.avatar_url, + ownerType: data.owner.type, + htmlUrl: data.html_url + }; + + // Cache the result + githubCacheQueries.set(cacheKey, 'repo_info', JSON.stringify(repoInfo), TTL.REPO_INFO); + + return repoInfo; + } catch (err) { + await logger.error('Failed to fetch GitHub repo info', { + source: 'GitHubCache', + meta: { error: String(err), repositoryUrl } + }); + return null; + } +} + +/** + * Fetch avatar from GitHub and cache it + */ +async function fetchAndCacheAvatar(owner: string, cacheKey: string): Promise { + const avatarUrl = `https://github.com/${owner}.png?size=80`; + + try { + const response = await globalThis.fetch(avatarUrl); + + if (!response.ok) { + return null; + } + + const buffer = await response.arrayBuffer(); + const base64 = btoa(String.fromCharCode(...new Uint8Array(buffer))); + const contentType = response.headers.get('content-type') || 'image/png'; + const dataUrl = `data:${contentType};base64,${base64}`; + + // Cache the result + githubCacheQueries.set(cacheKey, 'avatar', dataUrl, TTL.AVATAR); + + return dataUrl; + } catch (err) { + await logger.error('Failed to fetch GitHub avatar', { + source: 'GitHubCache', + meta: { error: String(err), owner } + }); + return null; + } +} + +/** + * Get cached avatar with stale-while-revalidate + * Always returns cached data if available, refreshes in background when stale + * Returns base64 encoded image data + */ +export async function getCachedAvatar(owner: string): Promise { + const cacheKey = `avatar:${owner}`; + + // Check for any cached data (even if expired) + const cached = githubCacheQueries.getStale(cacheKey); + + if (cached) { + // Check if data is expired + const isExpired = githubCacheQueries.isExpired(cacheKey); + + if (isExpired) { + await logger.debug('GitHub avatar cache stale, revalidating in background', { + source: 'GitHubCache', + meta: { owner } + }); + // Trigger background refresh (don't await) + fetchAndCacheAvatar(owner, cacheKey).catch(() => { + // Silently ignore background refresh errors + }); + } else { + await logger.debug('GitHub avatar cache hit', { + source: 'GitHubCache', + meta: { owner } + }); + } + + return cached.data; + } + + await logger.debug('GitHub avatar cache miss', { + source: 'GitHubCache', + meta: { owner } + }); + + // No cached data at all - fetch synchronously + return fetchAndCacheAvatar(owner, cacheKey); +} + +/** + * Get cached releases or fetch from GitHub API + */ +export async function getCachedReleases( + owner: string, + repo: string +): Promise { + const cacheKey = `releases:${owner}/${repo}`; + + // Check cache + const cached = githubCacheQueries.get(cacheKey); + if (cached) { + await logger.debug('GitHub releases cache hit', { + source: 'GitHubCache', + meta: { owner, repo } + }); + return JSON.parse(cached.data) as GitHubRelease[]; + } + + await logger.debug('GitHub releases cache miss', { + source: 'GitHubCache', + meta: { owner, repo } + }); + + // Fetch from API + const apiUrl = `https://api.github.com/repos/${owner}/${repo}/releases`; + const headers = getHeaders(); + + try { + const response = await globalThis.fetch(apiUrl, { headers }); + + if (!response.ok) { + return []; + } + + const data = await response.json(); + const releases: GitHubRelease[] = data.map( + (release: Record) => ({ + tag_name: release.tag_name, + name: release.name, + published_at: release.published_at, + html_url: release.html_url, + prerelease: release.prerelease + }) + ); + + // Cache the result + githubCacheQueries.set(cacheKey, 'releases', JSON.stringify(releases), TTL.RELEASES); + + return releases; + } catch (err) { + await logger.error('Failed to fetch GitHub releases', { + source: 'GitHubCache', + meta: { error: String(err), owner, repo } + }); + return []; + } +} + +/** + * Invalidate all cache entries for a repository + */ +export function invalidateRepo(repositoryUrl: string): void { + const parsed = parseGitHubUrl(repositoryUrl); + if (!parsed) { + return; + } + + const { owner, repo } = parsed; + + // Delete repo info cache + githubCacheQueries.delete(`repo:${owner}/${repo}`); + + // Delete avatar cache + githubCacheQueries.delete(`avatar:${owner}`); + + // Delete releases cache + githubCacheQueries.delete(`releases:${owner}/${repo}`); +} + +/** + * Clean up expired cache entries + */ +export function cleanupExpiredCache(): number { + return githubCacheQueries.deleteExpired(); +} diff --git a/src/routes/api/github/avatar/[owner]/+server.ts b/src/routes/api/github/avatar/[owner]/+server.ts new file mode 100644 index 0000000..aa6f593 --- /dev/null +++ b/src/routes/api/github/avatar/[owner]/+server.ts @@ -0,0 +1,39 @@ +import { error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getCachedAvatar } from '$lib/server/utils/github/cache.ts'; + +export const GET: RequestHandler = async ({ params }) => { + const { owner } = params; + + if (!owner) { + throw error(400, 'Missing owner parameter'); + } + + const dataUrl = await getCachedAvatar(owner); + + if (!dataUrl) { + throw error(404, 'Avatar not found'); + } + + // Parse the data URL to extract content type and base64 data + const match = dataUrl.match(/^data:([^;]+);base64,(.+)$/); + if (!match) { + throw error(500, 'Invalid cached avatar data'); + } + + const [, contentType, base64Data] = match; + + // Convert base64 to binary + const binaryString = atob(base64Data); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + return new Response(bytes, { + headers: { + 'Content-Type': contentType, + 'Cache-Control': 'public, max-age=86400' // 24 hours browser cache + } + }); +}; diff --git a/src/routes/databases/+page.svelte b/src/routes/databases/+page.svelte index 1480300..4539f06 100644 --- a/src/routes/databases/+page.svelte +++ b/src/routes/databases/+page.svelte @@ -49,11 +49,11 @@ // Track loaded images let loadedImages: Set = new Set(); - // Extract GitHub username/org from repository URL + // Extract GitHub username/org from repository URL and use local proxy function getGitHubAvatar(repoUrl: string): string { const match = repoUrl.match(/github\.com\/([^\/]+)\//); if (match) { - return `https://github.com/${match[1]}.png?size=40`; + return `/api/github/avatar/${match[1]}`; } return ''; } diff --git a/src/routes/settings/about/+page.server.ts b/src/routes/settings/about/+page.server.ts index 21e2d9e..c6a20cf 100644 --- a/src/routes/settings/about/+page.server.ts +++ b/src/routes/settings/about/+page.server.ts @@ -1,37 +1,12 @@ import { migrationRunner } from '$db/migrations.ts'; import { config } from '$config'; import packageJson from '../../../../package.json' with { type: 'json' }; - -type GitHubRelease = { - tag_name: string; - name: string; - published_at: string; - html_url: string; - prerelease: boolean; -}; +import { getCachedReleases, type GitHubRelease } from '$lib/server/utils/github/cache.ts'; type VersionStatus = 'up-to-date' | 'out-of-date' | 'dev-build'; async function fetchGitHubReleases(): Promise { - try { - const response = await fetch( - 'https://api.github.com/repos/Dictionarry-Hub/profilarr/releases', - { - headers: { - Accept: 'application/vnd.github+json', - 'User-Agent': 'Profilarr' - } - } - ); - - if (!response.ok) { - return []; - } - - return await response.json(); - } catch { - return []; - } + return getCachedReleases('Dictionarry-Hub', 'profilarr'); } function compareVersions(v1: string, v2: string): number {