diff --git a/catalogue.py b/catalogue.py new file mode 100644 index 0000000..c1ff627 --- /dev/null +++ b/catalogue.py @@ -0,0 +1,1076 @@ +#!/usr/bin/env python3 +""" +catalogue.py — Read-only HTML catalogue generator for o-phone.py backups. + +Does not import, modify, or depend on o-phone.py. +Reads the existing SQLite database and generates a single offline index.html. +""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import sqlite3 +import sys +import webbrowser +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any + + +# ============================================================================ +# CONFIGURATION (must match o-phone.py) +# ============================================================================ + +DATABASE_NAME = ".photo_organiser.db" + +# file_type values produced by o-phone.py classify_file() +TYPE_IMAGE = "image" +TYPE_VIDEO = "video" +TYPE_SCREENSHOT = "screenshot" +TYPE_UNKNOWN = "unknown" + +MONTH_NAMES = [ + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", +] + +# Extensions that browsers generally cannot display as +RAW_EXTENSIONS = { + ".dng", ".raw", ".arw", ".cr2", ".cr3", ".nef", ".nrw", + ".orf", ".rw2", ".raf", ".pef", ".srw", ".x3f", +} + + +# ============================================================================ +# UTILITIES +# ============================================================================ + +def eprint(*args: Any, **kwargs: Any) -> None: + print(*args, file=sys.stderr, **kwargs) + + +def parse_date_taken(value: str) -> datetime | None: + """Parse date_taken stored by o-phone.py (ISO-ish).""" + if not value: + return None + value = value.strip() + # Handle fractional seconds and timezone suffixes loosely + for fmt in ( + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + "%Y:%m:%d %H:%M:%S", + "%Y:%m:%d", + ): + try: + return datetime.strptime(value, fmt) + except ValueError: + continue + # Last resort: take first 19 chars as YYYY-MM-DDTHH:MM:SS + try: + return datetime.strptime(value[:19], "%Y-%m-%dT%H:%M:%S") + except ValueError: + return None + + +def format_display_date(dt: datetime) -> str: + return f"{dt.day} {MONTH_NAMES[dt.month - 1]} {dt.year}" + + +def format_display_day(dt: datetime) -> str: + return f"{dt.day} {MONTH_NAMES[dt.month - 1][:3]}" + + +def is_raw(path: str, original_name: str) -> bool: + ext = Path(path).suffix.lower() or Path(original_name).suffix.lower() + return ext in RAW_EXTENSIONS + + +def open_database_readonly(db_path: Path) -> sqlite3.Connection: + """Open SQLite in read-only mode. Fail if the file does not exist.""" + if not db_path.is_file(): + eprint(f"ERROR: SQLite database not found:\n{db_path}") + sys.exit(1) + + # URI mode forces read-only; also prevents accidental creation + uri = f"file:{db_path.resolve().as_posix()}?mode=ro" + try: + conn = sqlite3.connect(uri, uri=True) + conn.row_factory = sqlite3.Row + # Quick schema sanity check + cols = { + row[1] + for row in conn.execute("PRAGMA table_info(files)").fetchall() + } + required = { + "id", "sha256", "size", "original_name", "stored_name", + "relative_path", "date_taken", "file_type", + "device_make", "device_model", "imported_at", + } + if not required.issubset(cols): + eprint("ERROR: Database schema is not recognised.") + eprint(f"Expected columns including: {sorted(required)}") + eprint(f"Found: {sorted(cols)}") + sys.exit(1) + return conn + except sqlite3.Error as exc: + eprint(f"ERROR: Unable to open database.\n{exc}") + sys.exit(1) + + +def load_records(conn: sqlite3.Connection) -> list[dict[str, Any]]: + rows = conn.execute( + """ + SELECT + id, + sha256, + size, + original_name, + stored_name, + relative_path, + date_taken, + file_type, + device_make, + device_model, + imported_at + FROM files + ORDER BY date_taken DESC, id DESC + """ + ).fetchall() + + records: list[dict[str, Any]] = [] + for row in rows: + dt = parse_date_taken(row["date_taken"]) + records.append({ + "id": row["id"], + "sha256": row["sha256"], + "size": row["size"], + "original_name": row["original_name"], + "stored_name": row["stored_name"], + "relative_path": row["relative_path"], + "date_taken": row["date_taken"], + "file_type": row["file_type"] or TYPE_UNKNOWN, + "device_make": row["device_make"] or "", + "device_model": row["device_model"] or "", + "imported_at": row["imported_at"], + "_dt": dt, + }) + return records + + +# ============================================================================ +# CHECK MODE +# ============================================================================ + +def run_check( + backup_root: Path, + records: list[dict[str, Any]], + verbose: bool, +) -> int: + """ + Report consistency issues. Returns number of problems found. + Does not modify anything. + """ + print("=" * 60) + print("CATALOGUE CHECK") + print("=" * 60) + print(f"Backup: {backup_root}") + print(f"Database: {backup_root / DATABASE_NAME}") + print(f"Records: {len(records):,}") + print() + + missing: list[str] = [] + path_escape: list[str] = [] + present = 0 + hash_counts: dict[str, int] = defaultdict(int) + + for rec in records: + rel = rec["relative_path"] + hash_counts[rec["sha256"]] += 1 + + # Detect path traversal / escape + try: + full = (backup_root / rel).resolve() + if not str(full).startswith(str(backup_root.resolve())): + path_escape.append(rel) + continue + except Exception: + path_escape.append(rel) + continue + + if full.is_file(): + present += 1 + else: + missing.append(rel) + + duplicate_hashes = { + h: c for h, c in hash_counts.items() if c > 1 + } + + # Files on disk not in database (optional, can be slow on huge trees) + db_paths = {rec["relative_path"] for rec in records} + unindexed: list[str] = [] + skip_names = { + DATABASE_NAME, + f"{DATABASE_NAME}-wal", + f"{DATABASE_NAME}-shm", + "index.html", + } + + for path in backup_root.rglob("*"): + if not path.is_file(): + continue + if path.name in skip_names or path.name.startswith("._"): + continue + if path.name == ".DS_Store": + continue + # Ignore anything under a .catalogue folder if present later + try: + rel = str(path.relative_to(backup_root)) + except ValueError: + continue + if rel.startswith(".catalogue/") or rel == "index.html": + continue + if rel not in db_paths: + unindexed.append(rel) + + print(f"Files present: {present:,}") + print(f"Missing files: {len(missing):,}") + print(f"Path escape attempts: {len(path_escape):,}") + print(f"Duplicate SHA-256: {len(duplicate_hashes):,}") + print(f"Unindexed on disk: {len(unindexed):,}") + + problems = ( + len(missing) + + len(path_escape) + + len(duplicate_hashes) + + len(unindexed) + ) + + if missing: + print() + print("MISSING FILES:") + for p in missing[:50]: + print(f" {p}") + if len(missing) > 50: + print(f" ... and {len(missing) - 50} more") + + if path_escape: + print() + print("PATHS THAT ESCAPE BACKUP ROOT:") + for p in path_escape: + print(f" {p}") + + if duplicate_hashes and verbose: + print() + print("DUPLICATE HASHES (first few):") + for i, (h, c) in enumerate(duplicate_hashes.items()): + if i >= 10: + print(f" ... and {len(duplicate_hashes) - 10} more") + break + print(f" {h[:16]}… ×{c}") + + if unindexed: + print() + print("UNINDEXED FILES ON DISK (sample):") + for p in unindexed[:30]: + print(f" {p}") + if len(unindexed) > 30: + print(f" ... and {len(unindexed) - 30} more") + + print() + if problems == 0: + print("RESULT: CONSISTENT") + else: + print("RESULT: PROBLEMS FOUND") + print("=" * 60) + return problems + + +# ============================================================================ +# BUILD CATALOGUE DATA +# ============================================================================ + +def build_items( + backup_root: Path, + records: list[dict[str, Any]], + verbose: bool, +) -> tuple[list[dict[str, Any]], int, int, int]: + """ + Convert DB records into catalogue items. + Returns (items, present_count, missing_count, unknown_count). + Only includes items whose files exist. + """ + items: list[dict[str, Any]] = [] + missing = 0 + unknown = 0 + present = 0 + + for rec in records: + rel = rec["relative_path"] + full = backup_root / rel + + if not full.is_file(): + missing += 1 + if verbose: + eprint(f" missing: {rel}") + continue + + present += 1 + dt = rec["_dt"] + ftype = rec["file_type"] + + if ftype == TYPE_UNKNOWN: + unknown += 1 + + # Determine display type for the UI + if ftype == TYPE_SCREENSHOT: + ui_type = "screenshot" + elif ftype == TYPE_VIDEO: + ui_type = "video" + elif ftype == TYPE_IMAGE: + if is_raw(rel, rec["original_name"]): + ui_type = "raw" + else: + ui_type = "image" + else: + ui_type = "unknown" + + # Relative path suitable for HTML (forward slashes, no leading ./) + web_path = rel.replace("\\", "/") + + device = " ".join( + part for part in (rec["device_make"], rec["device_model"]) if part + ).strip() + + item = { + "id": rec["id"], + "name": rec["stored_name"], + "original": rec["original_name"], + "path": web_path, + "type": ui_type, + "date": dt.strftime("%Y-%m-%d") if dt else "", + "displayDate": format_display_date(dt) if dt else "", + "displayDay": format_display_day(dt) if dt else "", + "time": dt.strftime("%H:%M:%S") if dt and (dt.hour or dt.minute or dt.second) else "", + "year": dt.year if dt else 0, + "month": dt.month if dt else 0, + "monthName": MONTH_NAMES[dt.month - 1] if dt else "", + "day": dt.day if dt else 0, + "device": device, + "size": rec["size"], + # Sort key: newest first + "sort": dt.timestamp() if dt else 0, + } + items.append(item) + + # Newest first overall + items.sort(key=lambda x: (-x["sort"], -x["id"])) + return items, present, missing, unknown + + +# ============================================================================ +# HTML GENERATION +# ============================================================================ + +# Large HTML/JS template. Data is injected via a unique placeholder +# so we never use Python f-strings for the JS body (avoids { } conflicts). + +HTML_TEMPLATE = r''' + + + + +Photo Archive + + + + +
+
Photo Archive
+
+ +
+
+ + + + +
+
+ +
+ + + + + + +''' + + +def generate_html(items: list[dict[str, Any]]) -> str: + """Build final index.html with JSON data safely embedded.""" + # Strip internal sort helper before serialising + clean = [] + for it in items: + clean.append({ + "id": it["id"], + "name": it["name"], + "original": it["original"], + "path": it["path"], + "type": it["type"], + "date": it["date"], + "displayDate": it["displayDate"], + "displayDay": it["displayDay"], + "time": it["time"], + "year": it["year"], + "month": it["month"], + "monthName": it["monthName"], + "day": it["day"], + "device": it["device"], + "size": it["size"], + "sort": it["sort"], + }) + + payload = {"items": clean} + # Ensure safe embedding inside + json_text = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + json_text = json_text.replace("<", "\\u003c").replace(">", "\\u003e") + + html_out = HTML_TEMPLATE.replace("__CATALOGUE_DATA__", json_text) + return html_out + + +# ============================================================================ +# MAIN +# ============================================================================ + +def main() -> None: + parser = argparse.ArgumentParser( + prog="catalogue.py", + description=( + "Generate a read-only offline HTML catalogue from an " + "o-phone.py photo/video backup." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python3 catalogue.py "/Volumes/all/phone-media-backup" + python3 catalogue.py "/Volumes/all/phone-media-backup" --open + python3 catalogue.py "/Volumes/all/phone-media-backup" --check + python3 catalogue.py "/Volumes/all/phone-media-backup" --check --verbose + """.strip(), + ) + + parser.add_argument( + "backup", + type=Path, + nargs="?", + help="Root folder of the existing backup (contains .photo_organiser.db)", + ) + parser.add_argument( + "--check", + action="store_true", + help="Check database/filesystem consistency without generating the catalogue", + ) + parser.add_argument( + "--open", + action="store_true", + help="After generating, open index.html in the default browser", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Show additional diagnostic information", + ) + + args = parser.parse_args() + + if args.backup is None: + parser.print_help() + sys.exit(1) + + backup_root = args.backup.expanduser().resolve() + + if not backup_root.exists(): + eprint(f"ERROR: Backup folder does not exist.\n{backup_root}") + sys.exit(1) + if not backup_root.is_dir(): + eprint(f"ERROR: Not a folder.\n{backup_root}") + sys.exit(1) + + db_path = backup_root / DATABASE_NAME + + print("Reading database...") + conn = open_database_readonly(db_path) + try: + records = load_records(conn) + finally: + conn.close() + + print(f"Found {len(records):,} records.") + + if args.check: + problems = run_check(backup_root, records, args.verbose) + sys.exit(1 if problems else 0) + + # ---- Generate catalogue ---- + print("Checking files...") + items, present, missing, unknown = build_items( + backup_root, records, args.verbose + ) + print(f"{present:,} files present.") + if missing: + print(f"WARNING: {missing:,} files referenced by the database are missing.") + if unknown: + print(f"Note: {unknown:,} items have unknown/unsupported type.") + + print("Generating catalogue...") + html_content = generate_html(items) + + out_path = backup_root / "index.html" + try: + out_path.write_text(html_content, encoding="utf-8") + except OSError as exc: + eprint(f"ERROR: Could not write index.html.\n{exc}") + sys.exit(1) + + print("Wrote:") + print(f" {out_path}") + + if args.open: + try: + # Prefer OS open on macOS; fall back to webbrowser + if sys.platform == "darwin": + os.system(f'open "{out_path}"') # noqa: S605 + else: + webbrowser.open(out_path.as_uri()) + except Exception as exc: + eprint(f"WARNING: Could not open browser: {exc}") + + +if __name__ == "__main__": + main() diff --git a/o-phone.py b/o-phone.py new file mode 100644 index 0000000..45641d4 --- /dev/null +++ b/o-phone.py @@ -0,0 +1,1832 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import os +import random +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path + + +# ============================================================================ +# CONFIGURATION +# ============================================================================ + +DATABASE_NAME = ".photo_organiser.db" + +IMAGE_EXTENSIONS = { + ".jpg", ".jpeg", ".jpe", + ".png", + ".gif", + ".bmp", + ".tif", ".tiff", + ".webp", + ".heic", ".heif", + ".avif", + + # RAW + ".dng", + ".raw", + ".arw", + ".cr2", ".cr3", + ".nef", ".nrw", + ".orf", + ".rw2", + ".raf", + ".pef", + ".srw", + ".x3f", +} + +VIDEO_EXTENSIONS = { + ".mp4", + ".mov", + ".m4v", + ".avi", + ".mkv", + ".wmv", + ".webm", + ".3gp", + ".3g2", + ".mts", + ".m2ts", + ".ts", + ".mpg", + ".mpeg", + ".mxf", +} + +MONTH_NAMES = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +] + + +# ============================================================================ +# DATABASE +# ============================================================================ + +def database_path(destination: Path) -> Path: + return destination / DATABASE_NAME + + +def connect_database(destination: Path) -> sqlite3.Connection: + db_path = database_path(destination) + + connection = sqlite3.connect(db_path) + + # Good settings for a database on an external drive. + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + connection.execute("PRAGMA foreign_keys=ON") + + return connection + + +def create_database(connection: sqlite3.Connection): + connection.execute( + """ + CREATE TABLE IF NOT EXISTS files ( + id INTEGER PRIMARY KEY, + + sha256 TEXT NOT NULL UNIQUE, + + size INTEGER NOT NULL, + + original_name TEXT NOT NULL, + + stored_name TEXT NOT NULL, + + relative_path TEXT NOT NULL UNIQUE, + + date_taken TEXT NOT NULL, + + file_type TEXT NOT NULL, + + device_make TEXT, + + device_model TEXT, + + imported_at TEXT NOT NULL + ) + """ + ) + + connection.execute( + """ + CREATE INDEX IF NOT EXISTS idx_files_sha256 + ON files(sha256) + """ + ) + + connection.execute( + """ + CREATE INDEX IF NOT EXISTS idx_files_date_taken + ON files(date_taken) + """ + ) + + connection.execute( + """ + CREATE INDEX IF NOT EXISTS idx_files_device + ON files(device_make, device_model) + """ + ) + + connection.commit() + + +def hash_exists( + connection: sqlite3.Connection, + file_hash: str, +): + cursor = connection.execute( + """ + SELECT id, relative_path + FROM files + WHERE sha256 = ? + """, + (file_hash,), + ) + + return cursor.fetchone() + + +# ============================================================================ +# FILE HASHING +# ============================================================================ + +def calculate_sha256( + path: Path, + chunk_size: int = 8 * 1024 * 1024, +) -> str: + """ + Calculate SHA-256 without loading the entire file into memory. + """ + + sha256 = hashlib.sha256() + + with path.open("rb") as file: + while True: + chunk = file.read(chunk_size) + + if not chunk: + break + + sha256.update(chunk) + + return sha256.hexdigest() + + +# ============================================================================ +# METADATA +# ============================================================================ + +def parse_metadata_date(value: str): + if not value: + return None + + value = value.strip() + + # Remove fractional seconds. + if "." in value: + value = value.split(".", 1)[0] + + formats = [ + "%Y:%m:%d %H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M", + "%Y:%m:%d %H:%M", + "%Y-%m-%d", + "%Y:%m:%d", + ] + + for fmt in formats: + try: + return datetime.strptime(value, fmt) + except ValueError: + continue + + return None + + +def run_exiftool(path: Path, tag: str): + """ + Get a single metadata tag from ExifTool. + + Returns None if ExifTool isn't installed or the tag isn't available. + """ + + try: + result = subprocess.run( + [ + "exiftool", + "-s3", + f"-{tag}", + str(path), + ], + capture_output=True, + text=True, + timeout=15, + ) + + except ( + FileNotFoundError, + subprocess.SubprocessError, + OSError, + ): + return None + + if result.returncode != 0: + return None + + value = result.stdout.strip() + + return value or None + + +def get_exiftool_date(path: Path): + """ + Try the most useful capture/creation date fields. + """ + + tags = [ + "DateTimeOriginal", + "CreateDate", + "MediaCreateDate", + "CreationDate", + ] + + for tag in tags: + value = run_exiftool(path, tag) + + if value: + parsed = parse_metadata_date(value) + + if parsed: + return parsed + + return None + + +def get_pillow_date(path: Path): + """ + Optional fallback for image EXIF. + + Pillow is not required. + """ + + try: + from PIL import Image + except ImportError: + return None + + try: + with Image.open(path) as image: + + exif = image.getexif() + + if not exif: + return None + + # 36867 = DateTimeOriginal + # 36868 = DateTimeDigitized + # 306 = DateTime + + for tag in (36867, 36868, 306): + + value = exif.get(tag) + + if value: + parsed = parse_metadata_date( + str(value) + ) + + if parsed: + return parsed + + except Exception: + pass + + return None + + +def get_file_date(path: Path) -> datetime: + """ + Date priority: + + 1. ExifTool metadata + 2. Pillow EXIF + 3. Filesystem modification time + """ + + date = get_exiftool_date(path) + + if date: + return date + + if path.suffix.lower() in IMAGE_EXTENSIONS: + + date = get_pillow_date(path) + + if date: + return date + + return datetime.fromtimestamp( + path.stat().st_mtime + ) + + +def get_device_metadata(path: Path): + """ + Return: + + (make, model) + + from ExifTool. + + Examples: + + ("Motorola", "moto g13") + ("Apple", "iPhone 15") + ("GoPro", "HERO12 Black") + + """ + + make = run_exiftool(path, "Make") + model = run_exiftool(path, "Model") + + if make: + make = make.strip() + + if model: + model = model.strip() + + return make, model + + +# ============================================================================ +# FILE CLASSIFICATION +# ============================================================================ + +def classify_file(path: Path) -> str: + + extension = path.suffix.lower() + + # PNG is deliberately treated as screenshot. + if extension == ".png": + return "screenshot" + + if extension in IMAGE_EXTENSIONS: + return "image" + + if extension in VIDEO_EXTENSIONS: + return "video" + + return "unknown" + + +# ============================================================================ +# DESTINATION PATH +# ============================================================================ + +def unknown_folder( + destination: Path, + extension: str, +) -> Path: + + if extension: + + clean_extension = ( + extension + .lower() + .lstrip(".") + ) + + return destination / f"u-{clean_extension}" + + return destination / "u-no-extension" + + +def get_destination_directory( + destination: Path, + source_file: Path, + file_type: str, + date: datetime, +) -> Path: + + if file_type == "screenshot": + + root = destination / "screen shots" + + elif file_type in ("image", "video"): + + root = destination + + else: + + return unknown_folder( + destination, + source_file.suffix, + ) + + year = date.strftime("%Y") + + month = MONTH_NAMES[ + date.month - 1 + ] + + day = date.strftime( + "%d-%m-%Y" + ) + + return ( + root + / year + / month + / day + ) + + +# ============================================================================ +# FILE NAMING +# ============================================================================ + +def random_six_digits() -> str: + return ( + f"{random.SystemRandom().randint(0, 999999):06d}" + ) + + +def create_destination_filename( + directory: Path, + date: datetime, + extension: str, +) -> Path: + + directory.mkdir( + parents=True, + exist_ok=True, + ) + + date_string = date.strftime( + "%d-%m-%Y" + ) + + for _ in range(10000): + + random_number = random_six_digits() + + filename = ( + f"{date_string}-" + f"{random_number}" + f"{extension.lower()}" + ) + + candidate = directory / filename + + if not candidate.exists(): + return candidate + + raise RuntimeError( + "Could not generate a unique filename." + ) + + +# ============================================================================ +# SOURCE FILE DISCOVERY +# ============================================================================ + +def find_source_files(source: Path): + + for path in source.rglob("*"): + + if not path.is_file(): + continue + + # macOS resource fork. + if path.name.startswith("._"): + continue + + # macOS directory metadata. + if path.name == ".DS_Store": + continue + + yield path + + +# ============================================================================ +# DESTINATION SAFETY +# ============================================================================ + +def destination_contains_library_files( + destination: Path, +) -> bool: + + database = database_path(destination) + + for path in destination.rglob("*"): + + if not path.is_file(): + continue + + if path == database: + continue + + if path.name.startswith( + database.name + ): + continue + + if path.name.startswith("._"): + continue + + if path.name == ".DS_Store": + continue + + return True + + return False + + +# ============================================================================ +# SAFE COPY +# ============================================================================ + +def safe_copy_and_verify( + source: Path, + destination: Path, + source_hash: str, +): + """ + Copy to a temporary file. + + Hash the temporary copy. + + Only rename to the final filename after the hash matches. + + This protects against: + - incomplete copies + - disconnected drives + - corrupted transfers + """ + + destination.parent.mkdir( + parents=True, + exist_ok=True, + ) + + temporary_path = None + + try: + + fd, temp_name = tempfile.mkstemp( + prefix=".photo-import-", + suffix=".tmp", + dir=str(destination.parent), + ) + + os.close(fd) + + temporary_path = Path(temp_name) + + shutil.copy2( + source, + temporary_path, + ) + + copied_hash = calculate_sha256( + temporary_path + ) + + if copied_hash != source_hash: + + raise RuntimeError( + "Hash mismatch after copy.\n" + f"Source: {source_hash}\n" + f"Copy: {copied_hash}" + ) + + # Same filesystem = atomic rename. + temporary_path.replace( + destination + ) + + temporary_path = None + + finally: + + if temporary_path is not None: + + try: + temporary_path.unlink() + + except OSError: + pass + + +# ============================================================================ +# IMPORT ONE FILE +# ============================================================================ + +def process_file( + source_file: Path, + source_root: Path, + destination_root: Path, + connection: sqlite3.Connection, + dry_run: bool, + move: bool, +): + relative_source = ( + source_file.relative_to( + source_root + ) + ) + + print() + print( + f"Processing: {relative_source}" + ) + + # ------------------------------------------------------------------------ + # Hash source + # ------------------------------------------------------------------------ + + try: + + source_hash = calculate_sha256( + source_file + ) + + except Exception as exc: + + print( + f" ERROR hashing: {exc}" + ) + + return "error" + + # ------------------------------------------------------------------------ + # Duplicate check + # ------------------------------------------------------------------------ + + existing = hash_exists( + connection, + source_hash, + ) + + if existing: + + _, existing_path = existing + + print(" DUPLICATE") + print( + f" Existing file: {existing_path}" + ) + + return "duplicate" + + # ------------------------------------------------------------------------ + # Metadata + # ------------------------------------------------------------------------ + + try: + + date_taken = get_file_date( + source_file + ) + + except Exception as exc: + + print( + f" ERROR getting date: {exc}" + ) + + return "error" + + file_type = classify_file( + source_file + ) + + device_make, device_model = ( + get_device_metadata( + source_file + ) + ) + + # ------------------------------------------------------------------------ + # Destination + # ------------------------------------------------------------------------ + + destination_directory = ( + get_destination_directory( + destination_root, + source_file, + file_type, + date_taken, + ) + ) + + try: + + destination_file = ( + create_destination_filename( + destination_directory, + date_taken, + source_file.suffix, + ) + ) + + except Exception as exc: + + print( + f" ERROR creating filename: {exc}" + ) + + return "error" + + relative_destination = ( + destination_file.relative_to( + destination_root + ) + ) + + print( + f" Type: {file_type}" + ) + + print( + f" Date: " + f"{date_taken.strftime('%d-%m-%Y')}" + ) + + if device_make or device_model: + + device = " ".join( + value + for value in ( + device_make, + device_model, + ) + if value + ) + + print( + f" Device: {device}" + ) + + else: + + print( + " Device: unknown" + ) + + print( + f" SHA-256: {source_hash}" + ) + + print( + f" Destination: {relative_destination}" + ) + + # ------------------------------------------------------------------------ + # Dry run + # ------------------------------------------------------------------------ + + if dry_run: + + print( + " DRY RUN - nothing copied" + ) + + return "dry-run" + + # ------------------------------------------------------------------------ + # Copy + verify + database transaction + # ------------------------------------------------------------------------ + + try: + + # FIRST: + # Copy and verify the physical file. + safe_copy_and_verify( + source_file, + destination_file, + source_hash, + ) + + # SECOND: + # Only after successful verification do we write to SQLite. + imported_at = datetime.now( + timezone.utc + ).isoformat() + + connection.execute( + "BEGIN" + ) + + try: + + connection.execute( + """ + INSERT INTO files ( + sha256, + size, + original_name, + stored_name, + relative_path, + date_taken, + file_type, + device_make, + device_model, + imported_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + source_hash, + source_file.stat().st_size, + source_file.name, + destination_file.name, + str(relative_destination), + date_taken.isoformat(), + file_type, + device_make, + device_model, + imported_at, + ), + ) + + connection.commit() + + except Exception: + + connection.rollback() + + # The database transaction failed. + # Remove the copied file so filesystem and database + # remain consistent. + try: + + destination_file.unlink() + + except OSError: + pass + + raise + + # -------------------------------------------------------------------- + # Optional source deletion + # -------------------------------------------------------------------- + + if move: + + source_file.unlink() + + print(" IMPORTED") + + return ( + "moved" + if move + else "imported" + ) + + except Exception as exc: + + print( + f" ERROR: {exc}" + ) + + return "error" + + +# ============================================================================ +# NORMAL IMPORT +# ============================================================================ + +def import_files( + source: Path, + destination: Path, + dry_run: bool, + move: bool, +): + destination.mkdir( + parents=True, + exist_ok=True, + ) + + db_path = database_path( + destination + ) + + # ------------------------------------------------------------------------ + # Safety check + # + # If files already exist in the destination but there is no DB, + # refuse to continue. + # + # This prevents a user accidentally importing thousands of duplicates + # into an existing library. + # ------------------------------------------------------------------------ + + if ( + not db_path.exists() + and destination_contains_library_files( + destination + ) + ): + + print( + "ERROR: The destination already contains files," + ) + + print( + "but there is no photo organiser database." + ) + + print() + print( + "Before importing, build the database with:" + ) + + print() + + print( + f'python3 organise_phone.py ' + f'--build-db "{destination}"' + ) + + print() + + print( + "This is a safety feature to prevent accidental" + ) + + print( + "duplicate imports." + ) + + sys.exit(1) + + connection = connect_database( + destination + ) + + try: + + create_database( + connection + ) + + files = list( + find_source_files( + source + ) + ) + + print("=" * 70) + print("PHOTO ORGANISER") + print("=" * 70) + + print( + f"Source: {source}" + ) + + print( + f"Destination: {destination}" + ) + + print( + f"Database: {db_path}" + ) + + print( + f"Operation: " + f"{'MOVE' if move else 'COPY'}" + ) + + print( + f"Mode: " + f"{'DRY RUN' if dry_run else 'LIVE'}" + ) + + print() + + print( + f"Found {len(files)} files." + ) + + print("=" * 70) + + imported = 0 + duplicates = 0 + errors = 0 + dry_runs = 0 + moved = 0 + + for source_file in files: + + result = process_file( + source_file=source_file, + source_root=source, + destination_root=destination, + connection=connection, + dry_run=dry_run, + move=move, + ) + + if result == "imported": + + imported += 1 + + elif result == "moved": + + moved += 1 + + elif result == "duplicate": + + duplicates += 1 + + elif result == "error": + + errors += 1 + + elif result == "dry-run": + + dry_runs += 1 + + print() + print("=" * 70) + print("IMPORT COMPLETE") + print("=" * 70) + + if dry_run: + + print( + f"Would import: {dry_runs}" + ) + + else: + + print( + f"Imported: {imported}" + ) + + if moved: + + print( + f"Moved: {moved}" + ) + + print( + f"Duplicates skipped: {duplicates}" + ) + + print( + f"Errors: {errors}" + ) + + print("=" * 70) + + finally: + + connection.close() + + +# ============================================================================ +# BUILD DATABASE +# ============================================================================ + +def build_database( + destination: Path, +): + """ + Build a database for an existing library. + + This does not move, rename or delete anything. + """ + + db_path = database_path( + destination + ) + + connection = connect_database( + destination + ) + + try: + + create_database( + connection + ) + + files = list( + find_source_files( + destination + ) + ) + + print("=" * 70) + print("BUILDING PHOTO DATABASE") + print("=" * 70) + + print( + f"Library: {destination}" + ) + + print( + f"Database: {db_path}" + ) + + print( + f"Files found: {len(files)}" + ) + + print() + + added = 0 + already_indexed = 0 + duplicates = 0 + errors = 0 + + for path in files: + + # Never index the database itself. + if path == db_path: + continue + + # SQLite WAL / SHM files. + if path.name in ( + f"{DATABASE_NAME}-wal", + f"{DATABASE_NAME}-shm", + ): + continue + + try: + + relative_path = ( + path.relative_to( + destination + ) + ) + + # Check whether this exact path is already in DB. + path_exists = connection.execute( + """ + SELECT id + FROM files + WHERE relative_path = ? + """, + (str(relative_path),), + ).fetchone() + + if path_exists: + + already_indexed += 1 + continue + + print( + f"Hashing: {relative_path}" + ) + + file_hash = ( + calculate_sha256( + path + ) + ) + + existing = hash_exists( + connection, + file_hash, + ) + + if existing: + + print( + " DUPLICATE HASH:" + ) + + print( + f" {existing[1]}" + ) + + duplicates += 1 + continue + + date_taken = ( + get_file_date( + path + ) + ) + + device_make, device_model = ( + get_device_metadata( + path + ) + ) + + connection.execute( + """ + INSERT INTO files ( + sha256, + size, + original_name, + stored_name, + relative_path, + date_taken, + file_type, + device_make, + device_model, + imported_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + file_hash, + path.stat().st_size, + path.name, + path.name, + str(relative_path), + date_taken.isoformat(), + classify_file(path), + device_make, + device_model, + datetime.now( + timezone.utc + ).isoformat(), + ), + ) + + connection.commit() + + added += 1 + + except Exception as exc: + + print( + f" ERROR: {exc}" + ) + + errors += 1 + + print() + print("=" * 70) + print("DATABASE BUILD COMPLETE") + print("=" * 70) + + print( + f"Added: {added}" + ) + + print( + f"Already indexed: {already_indexed}" + ) + + print( + f"Duplicate hashes: {duplicates}" + ) + + print( + f"Errors: {errors}" + ) + + print("=" * 70) + + finally: + + connection.close() + + +# ============================================================================ +# VERIFY DATABASE +# ============================================================================ + +def verify_database( + destination: Path, + deep: bool, +): + """ + Verify database ↔ filesystem consistency. + + Normal mode checks: + - DB files exist + - sizes match + - every library file is indexed + + Deep mode additionally: + - hashes every file + - compares against DB SHA-256 + """ + + db_path = database_path( + destination + ) + + if not db_path.exists(): + + print( + "ERROR: Database does not exist:" + ) + + print( + db_path + ) + + print() + print( + "Build it with:" + ) + + print() + + print( + f'python3 organise_phone.py ' + f'--build-db "{destination}"' + ) + + sys.exit(1) + + connection = connect_database( + destination + ) + + try: + + create_database( + connection + ) + + print("=" * 70) + print("VERIFYING PHOTO LIBRARY") + print("=" * 70) + + print( + f"Library: {destination}" + ) + + print( + f"Database: {db_path}" + ) + + print( + f"Mode: " + f"{'DEEP - hash every file' if deep else 'NORMAL'}" + ) + + print() + + rows = connection.execute( + """ + SELECT + id, + sha256, + size, + relative_path + FROM files + ORDER BY id + """ + ).fetchall() + + database_paths = set() + + missing = 0 + size_mismatches = 0 + hash_mismatches = 0 + errors = 0 + + # -------------------------------------------------------------------- + # Check every database entry + # -------------------------------------------------------------------- + + for ( + row_id, + expected_hash, + expected_size, + relative_path, + ) in rows: + + database_paths.add( + relative_path + ) + + actual_path = ( + destination + / relative_path + ) + + if not actual_path.exists(): + + print( + f"MISSING: {relative_path}" + ) + + missing += 1 + continue + + if not actual_path.is_file(): + + print( + f"NOT A FILE: {relative_path}" + ) + + errors += 1 + continue + + try: + + actual_size = ( + actual_path.stat().st_size + ) + + if actual_size != expected_size: + + print( + f"SIZE MISMATCH: " + f"{relative_path}" + ) + + size_mismatches += 1 + + continue + + if deep: + + print( + f"Hashing: " + f"{relative_path}" + ) + + actual_hash = ( + calculate_sha256( + actual_path + ) + ) + + if ( + actual_hash + != expected_hash + ): + + print( + f"HASH MISMATCH: " + f"{relative_path}" + ) + + hash_mismatches += 1 + + except Exception as exc: + + print( + f"ERROR: " + f"{relative_path}: " + f"{exc}" + ) + + errors += 1 + + # -------------------------------------------------------------------- + # Find files on disk that aren't in DB + # -------------------------------------------------------------------- + + unindexed = [] + + for path in find_source_files( + destination + ): + + if path == db_path: + continue + + if path.name in ( + f"{DATABASE_NAME}-wal", + f"{DATABASE_NAME}-shm", + ): + continue + + try: + + relative_path = str( + path.relative_to( + destination + ) + ) + + except ValueError: + + continue + + if ( + relative_path + not in database_paths + ): + + unindexed.append( + relative_path + ) + + # -------------------------------------------------------------------- + # Results + # -------------------------------------------------------------------- + + print() + print("=" * 70) + print("VERIFICATION COMPLETE") + print("=" * 70) + + print( + f"Database records: {len(rows)}" + ) + + print( + f"Missing files: {missing}" + ) + + print( + f"Size mismatches: {size_mismatches}" + ) + + if deep: + + print( + f"Hash mismatches: {hash_mismatches}" + ) + + print( + f"Unindexed files: {len(unindexed)}" + ) + + print( + f"Other errors: {errors}" + ) + + if unindexed: + + print() + print( + "UNINDEXED FILES:" + ) + + for path in unindexed: + + print( + f" {path}" + ) + + print() + + if ( + missing == 0 + and size_mismatches == 0 + and hash_mismatches == 0 + and len(unindexed) == 0 + and errors == 0 + ): + + print( + "RESULT: LIBRARY IS CONSISTENT" + ) + + else: + + print( + "RESULT: PROBLEMS FOUND" + ) + + print("=" * 70) + + finally: + + connection.close() + + +# ============================================================================ +# MAIN +# ============================================================================ + +def main(): + + parser = argparse.ArgumentParser( + description=( + "Organise photos and videos into a date-based " + "library with SHA-256 duplicate detection." + ) + ) + + parser.add_argument( + "source", + type=Path, + nargs="?", + help="Source folder", + ) + + parser.add_argument( + "destination", + type=Path, + nargs="?", + help="Destination folder / external drive", + ) + + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would happen without copying", + ) + + parser.add_argument( + "--move", + action="store_true", + help="Move files instead of copying them", + ) + + parser.add_argument( + "--build-db", + metavar="DESTINATION", + type=Path, + help="Build database for an existing library", + ) + + parser.add_argument( + "--verify", + metavar="DESTINATION", + type=Path, + help="Verify database against library", + ) + + parser.add_argument( + "--deep", + action="store_true", + help=( + "With --verify, calculate SHA-256 for every " + "file and verify its contents" + ), + ) + + args = parser.parse_args() + + # ------------------------------------------------------------------------ + # BUILD DATABASE + # ------------------------------------------------------------------------ + + if args.build_db: + + destination = ( + args.build_db + .expanduser() + .resolve() + ) + + if not destination.exists(): + + print( + f"ERROR: Folder does not exist:\n" + f"{destination}", + file=sys.stderr, + ) + + sys.exit(1) + + if not destination.is_dir(): + + print( + f"ERROR: Not a folder:\n" + f"{destination}", + file=sys.stderr, + ) + + sys.exit(1) + + build_database( + destination + ) + + return + + # ------------------------------------------------------------------------ + # VERIFY + # ------------------------------------------------------------------------ + + if args.verify: + + destination = ( + args.verify + .expanduser() + .resolve() + ) + + if not destination.exists(): + + print( + f"ERROR: Folder does not exist:\n" + f"{destination}", + file=sys.stderr, + ) + + sys.exit(1) + + if not destination.is_dir(): + + print( + f"ERROR: Not a folder:\n" + f"{destination}", + file=sys.stderr, + ) + + sys.exit(1) + + verify_database( + destination, + args.deep, + ) + + return + + # ------------------------------------------------------------------------ + # NORMAL IMPORT + # ------------------------------------------------------------------------ + + if ( + not args.source + or not args.destination + ): + + parser.print_help() + sys.exit(1) + + source = ( + args.source + .expanduser() + .resolve() + ) + + destination = ( + args.destination + .expanduser() + .resolve() + ) + + if not source.exists(): + + print( + f"ERROR: Source folder does not exist:\n" + f"{source}", + file=sys.stderr, + ) + + sys.exit(1) + + if not source.is_dir(): + + print( + f"ERROR: Source is not a folder:\n" + f"{source}", + file=sys.stderr, + ) + + sys.exit(1) + + # Destination cannot be inside source. + try: + + destination.relative_to( + source + ) + + print( + "ERROR: Destination cannot be " + "inside the source folder.", + file=sys.stderr, + ) + + sys.exit(1) + + except ValueError: + + pass + + import_files( + source=source, + destination=destination, + dry_run=args.dry_run, + move=args.move, + ) + + +if __name__ == "__main__": + main() +