1833 lines
37 KiB
Python
1833 lines
37 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import random
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
|
|
# ============================================================================
|
|
# CONFIGURATION
|
|
# ============================================================================
|
|
|
|
DATABASE_NAME = ".photo_organiser.db"
|
|
|
|
IMAGE_EXTENSIONS = {
|
|
".jpg", ".jpeg", ".jpe",
|
|
".png",
|
|
".gif",
|
|
".bmp",
|
|
".tif", ".tiff",
|
|
".webp",
|
|
".heic", ".heif",
|
|
".avif",
|
|
|
|
# RAW
|
|
".dng",
|
|
".raw",
|
|
".arw",
|
|
".cr2", ".cr3",
|
|
".nef", ".nrw",
|
|
".orf",
|
|
".rw2",
|
|
".raf",
|
|
".pef",
|
|
".srw",
|
|
".x3f",
|
|
}
|
|
|
|
VIDEO_EXTENSIONS = {
|
|
".mp4",
|
|
".mov",
|
|
".m4v",
|
|
".avi",
|
|
".mkv",
|
|
".wmv",
|
|
".webm",
|
|
".3gp",
|
|
".3g2",
|
|
".mts",
|
|
".m2ts",
|
|
".ts",
|
|
".mpg",
|
|
".mpeg",
|
|
".mxf",
|
|
}
|
|
|
|
MONTH_NAMES = [
|
|
"January",
|
|
"February",
|
|
"March",
|
|
"April",
|
|
"May",
|
|
"June",
|
|
"July",
|
|
"August",
|
|
"September",
|
|
"October",
|
|
"November",
|
|
"December",
|
|
]
|
|
|
|
|
|
# ============================================================================
|
|
# DATABASE
|
|
# ============================================================================
|
|
|
|
def database_path(destination: Path) -> Path:
|
|
return destination / DATABASE_NAME
|
|
|
|
|
|
def connect_database(destination: Path) -> sqlite3.Connection:
|
|
db_path = database_path(destination)
|
|
|
|
connection = sqlite3.connect(db_path)
|
|
|
|
# Good settings for a database on an external drive.
|
|
connection.execute("PRAGMA journal_mode=WAL")
|
|
connection.execute("PRAGMA synchronous=FULL")
|
|
connection.execute("PRAGMA foreign_keys=ON")
|
|
|
|
return connection
|
|
|
|
|
|
def create_database(connection: sqlite3.Connection):
|
|
connection.execute(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS files (
|
|
id INTEGER PRIMARY KEY,
|
|
|
|
sha256 TEXT NOT NULL UNIQUE,
|
|
|
|
size INTEGER NOT NULL,
|
|
|
|
original_name TEXT NOT NULL,
|
|
|
|
stored_name TEXT NOT NULL,
|
|
|
|
relative_path TEXT NOT NULL UNIQUE,
|
|
|
|
date_taken TEXT NOT NULL,
|
|
|
|
file_type TEXT NOT NULL,
|
|
|
|
device_make TEXT,
|
|
|
|
device_model TEXT,
|
|
|
|
imported_at TEXT NOT NULL
|
|
)
|
|
"""
|
|
)
|
|
|
|
connection.execute(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_files_sha256
|
|
ON files(sha256)
|
|
"""
|
|
)
|
|
|
|
connection.execute(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_files_date_taken
|
|
ON files(date_taken)
|
|
"""
|
|
)
|
|
|
|
connection.execute(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_files_device
|
|
ON files(device_make, device_model)
|
|
"""
|
|
)
|
|
|
|
connection.commit()
|
|
|
|
|
|
def hash_exists(
|
|
connection: sqlite3.Connection,
|
|
file_hash: str,
|
|
):
|
|
cursor = connection.execute(
|
|
"""
|
|
SELECT id, relative_path
|
|
FROM files
|
|
WHERE sha256 = ?
|
|
""",
|
|
(file_hash,),
|
|
)
|
|
|
|
return cursor.fetchone()
|
|
|
|
|
|
# ============================================================================
|
|
# FILE HASHING
|
|
# ============================================================================
|
|
|
|
def calculate_sha256(
|
|
path: Path,
|
|
chunk_size: int = 8 * 1024 * 1024,
|
|
) -> str:
|
|
"""
|
|
Calculate SHA-256 without loading the entire file into memory.
|
|
"""
|
|
|
|
sha256 = hashlib.sha256()
|
|
|
|
with path.open("rb") as file:
|
|
while True:
|
|
chunk = file.read(chunk_size)
|
|
|
|
if not chunk:
|
|
break
|
|
|
|
sha256.update(chunk)
|
|
|
|
return sha256.hexdigest()
|
|
|
|
|
|
# ============================================================================
|
|
# METADATA
|
|
# ============================================================================
|
|
|
|
def parse_metadata_date(value: str):
|
|
if not value:
|
|
return None
|
|
|
|
value = value.strip()
|
|
|
|
# Remove fractional seconds.
|
|
if "." in value:
|
|
value = value.split(".", 1)[0]
|
|
|
|
formats = [
|
|
"%Y:%m:%d %H:%M:%S",
|
|
"%Y-%m-%d %H:%M:%S",
|
|
"%Y-%m-%dT%H:%M:%S",
|
|
"%Y-%m-%d %H:%M",
|
|
"%Y:%m:%d %H:%M",
|
|
"%Y-%m-%d",
|
|
"%Y:%m:%d",
|
|
]
|
|
|
|
for fmt in formats:
|
|
try:
|
|
return datetime.strptime(value, fmt)
|
|
except ValueError:
|
|
continue
|
|
|
|
return None
|
|
|
|
|
|
def run_exiftool(path: Path, tag: str):
|
|
"""
|
|
Get a single metadata tag from ExifTool.
|
|
|
|
Returns None if ExifTool isn't installed or the tag isn't available.
|
|
"""
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
[
|
|
"exiftool",
|
|
"-s3",
|
|
f"-{tag}",
|
|
str(path),
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
)
|
|
|
|
except (
|
|
FileNotFoundError,
|
|
subprocess.SubprocessError,
|
|
OSError,
|
|
):
|
|
return None
|
|
|
|
if result.returncode != 0:
|
|
return None
|
|
|
|
value = result.stdout.strip()
|
|
|
|
return value or None
|
|
|
|
|
|
def get_exiftool_date(path: Path):
|
|
"""
|
|
Try the most useful capture/creation date fields.
|
|
"""
|
|
|
|
tags = [
|
|
"DateTimeOriginal",
|
|
"CreateDate",
|
|
"MediaCreateDate",
|
|
"CreationDate",
|
|
]
|
|
|
|
for tag in tags:
|
|
value = run_exiftool(path, tag)
|
|
|
|
if value:
|
|
parsed = parse_metadata_date(value)
|
|
|
|
if parsed:
|
|
return parsed
|
|
|
|
return None
|
|
|
|
|
|
def get_pillow_date(path: Path):
|
|
"""
|
|
Optional fallback for image EXIF.
|
|
|
|
Pillow is not required.
|
|
"""
|
|
|
|
try:
|
|
from PIL import Image
|
|
except ImportError:
|
|
return None
|
|
|
|
try:
|
|
with Image.open(path) as image:
|
|
|
|
exif = image.getexif()
|
|
|
|
if not exif:
|
|
return None
|
|
|
|
# 36867 = DateTimeOriginal
|
|
# 36868 = DateTimeDigitized
|
|
# 306 = DateTime
|
|
|
|
for tag in (36867, 36868, 306):
|
|
|
|
value = exif.get(tag)
|
|
|
|
if value:
|
|
parsed = parse_metadata_date(
|
|
str(value)
|
|
)
|
|
|
|
if parsed:
|
|
return parsed
|
|
|
|
except Exception:
|
|
pass
|
|
|
|
return None
|
|
|
|
|
|
def get_file_date(path: Path) -> datetime:
|
|
"""
|
|
Date priority:
|
|
|
|
1. ExifTool metadata
|
|
2. Pillow EXIF
|
|
3. Filesystem modification time
|
|
"""
|
|
|
|
date = get_exiftool_date(path)
|
|
|
|
if date:
|
|
return date
|
|
|
|
if path.suffix.lower() in IMAGE_EXTENSIONS:
|
|
|
|
date = get_pillow_date(path)
|
|
|
|
if date:
|
|
return date
|
|
|
|
return datetime.fromtimestamp(
|
|
path.stat().st_mtime
|
|
)
|
|
|
|
|
|
def get_device_metadata(path: Path):
|
|
"""
|
|
Return:
|
|
|
|
(make, model)
|
|
|
|
from ExifTool.
|
|
|
|
Examples:
|
|
|
|
("Motorola", "moto g13")
|
|
("Apple", "iPhone 15")
|
|
("GoPro", "HERO12 Black")
|
|
|
|
"""
|
|
|
|
make = run_exiftool(path, "Make")
|
|
model = run_exiftool(path, "Model")
|
|
|
|
if make:
|
|
make = make.strip()
|
|
|
|
if model:
|
|
model = model.strip()
|
|
|
|
return make, model
|
|
|
|
|
|
# ============================================================================
|
|
# FILE CLASSIFICATION
|
|
# ============================================================================
|
|
|
|
def classify_file(path: Path) -> str:
|
|
|
|
extension = path.suffix.lower()
|
|
|
|
# PNG is deliberately treated as screenshot.
|
|
if extension == ".png":
|
|
return "screenshot"
|
|
|
|
if extension in IMAGE_EXTENSIONS:
|
|
return "image"
|
|
|
|
if extension in VIDEO_EXTENSIONS:
|
|
return "video"
|
|
|
|
return "unknown"
|
|
|
|
|
|
# ============================================================================
|
|
# DESTINATION PATH
|
|
# ============================================================================
|
|
|
|
def unknown_folder(
|
|
destination: Path,
|
|
extension: str,
|
|
) -> Path:
|
|
|
|
if extension:
|
|
|
|
clean_extension = (
|
|
extension
|
|
.lower()
|
|
.lstrip(".")
|
|
)
|
|
|
|
return destination / f"u-{clean_extension}"
|
|
|
|
return destination / "u-no-extension"
|
|
|
|
|
|
def get_destination_directory(
|
|
destination: Path,
|
|
source_file: Path,
|
|
file_type: str,
|
|
date: datetime,
|
|
) -> Path:
|
|
|
|
if file_type == "screenshot":
|
|
|
|
root = destination / "screen shots"
|
|
|
|
elif file_type in ("image", "video"):
|
|
|
|
root = destination
|
|
|
|
else:
|
|
|
|
return unknown_folder(
|
|
destination,
|
|
source_file.suffix,
|
|
)
|
|
|
|
year = date.strftime("%Y")
|
|
|
|
month = MONTH_NAMES[
|
|
date.month - 1
|
|
]
|
|
|
|
day = date.strftime(
|
|
"%d-%m-%Y"
|
|
)
|
|
|
|
return (
|
|
root
|
|
/ year
|
|
/ month
|
|
/ day
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# FILE NAMING
|
|
# ============================================================================
|
|
|
|
def random_six_digits() -> str:
|
|
return (
|
|
f"{random.SystemRandom().randint(0, 999999):06d}"
|
|
)
|
|
|
|
|
|
def create_destination_filename(
|
|
directory: Path,
|
|
date: datetime,
|
|
extension: str,
|
|
) -> Path:
|
|
|
|
directory.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
date_string = date.strftime(
|
|
"%d-%m-%Y"
|
|
)
|
|
|
|
for _ in range(10000):
|
|
|
|
random_number = random_six_digits()
|
|
|
|
filename = (
|
|
f"{date_string}-"
|
|
f"{random_number}"
|
|
f"{extension.lower()}"
|
|
)
|
|
|
|
candidate = directory / filename
|
|
|
|
if not candidate.exists():
|
|
return candidate
|
|
|
|
raise RuntimeError(
|
|
"Could not generate a unique filename."
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# SOURCE FILE DISCOVERY
|
|
# ============================================================================
|
|
|
|
def find_source_files(source: Path):
|
|
|
|
for path in source.rglob("*"):
|
|
|
|
if not path.is_file():
|
|
continue
|
|
|
|
# macOS resource fork.
|
|
if path.name.startswith("._"):
|
|
continue
|
|
|
|
# macOS directory metadata.
|
|
if path.name == ".DS_Store":
|
|
continue
|
|
|
|
yield path
|
|
|
|
|
|
# ============================================================================
|
|
# DESTINATION SAFETY
|
|
# ============================================================================
|
|
|
|
def destination_contains_library_files(
|
|
destination: Path,
|
|
) -> bool:
|
|
|
|
database = database_path(destination)
|
|
|
|
for path in destination.rglob("*"):
|
|
|
|
if not path.is_file():
|
|
continue
|
|
|
|
if path == database:
|
|
continue
|
|
|
|
if path.name.startswith(
|
|
database.name
|
|
):
|
|
continue
|
|
|
|
if path.name.startswith("._"):
|
|
continue
|
|
|
|
if path.name == ".DS_Store":
|
|
continue
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
# ============================================================================
|
|
# SAFE COPY
|
|
# ============================================================================
|
|
|
|
def safe_copy_and_verify(
|
|
source: Path,
|
|
destination: Path,
|
|
source_hash: str,
|
|
):
|
|
"""
|
|
Copy to a temporary file.
|
|
|
|
Hash the temporary copy.
|
|
|
|
Only rename to the final filename after the hash matches.
|
|
|
|
This protects against:
|
|
- incomplete copies
|
|
- disconnected drives
|
|
- corrupted transfers
|
|
"""
|
|
|
|
destination.parent.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
temporary_path = None
|
|
|
|
try:
|
|
|
|
fd, temp_name = tempfile.mkstemp(
|
|
prefix=".photo-import-",
|
|
suffix=".tmp",
|
|
dir=str(destination.parent),
|
|
)
|
|
|
|
os.close(fd)
|
|
|
|
temporary_path = Path(temp_name)
|
|
|
|
shutil.copy2(
|
|
source,
|
|
temporary_path,
|
|
)
|
|
|
|
copied_hash = calculate_sha256(
|
|
temporary_path
|
|
)
|
|
|
|
if copied_hash != source_hash:
|
|
|
|
raise RuntimeError(
|
|
"Hash mismatch after copy.\n"
|
|
f"Source: {source_hash}\n"
|
|
f"Copy: {copied_hash}"
|
|
)
|
|
|
|
# Same filesystem = atomic rename.
|
|
temporary_path.replace(
|
|
destination
|
|
)
|
|
|
|
temporary_path = None
|
|
|
|
finally:
|
|
|
|
if temporary_path is not None:
|
|
|
|
try:
|
|
temporary_path.unlink()
|
|
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# ============================================================================
|
|
# IMPORT ONE FILE
|
|
# ============================================================================
|
|
|
|
def process_file(
|
|
source_file: Path,
|
|
source_root: Path,
|
|
destination_root: Path,
|
|
connection: sqlite3.Connection,
|
|
dry_run: bool,
|
|
move: bool,
|
|
):
|
|
relative_source = (
|
|
source_file.relative_to(
|
|
source_root
|
|
)
|
|
)
|
|
|
|
print()
|
|
print(
|
|
f"Processing: {relative_source}"
|
|
)
|
|
|
|
# ------------------------------------------------------------------------
|
|
# Hash source
|
|
# ------------------------------------------------------------------------
|
|
|
|
try:
|
|
|
|
source_hash = calculate_sha256(
|
|
source_file
|
|
)
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" ERROR hashing: {exc}"
|
|
)
|
|
|
|
return "error"
|
|
|
|
# ------------------------------------------------------------------------
|
|
# Duplicate check
|
|
# ------------------------------------------------------------------------
|
|
|
|
existing = hash_exists(
|
|
connection,
|
|
source_hash,
|
|
)
|
|
|
|
if existing:
|
|
|
|
_, existing_path = existing
|
|
|
|
print(" DUPLICATE")
|
|
print(
|
|
f" Existing file: {existing_path}"
|
|
)
|
|
|
|
return "duplicate"
|
|
|
|
# ------------------------------------------------------------------------
|
|
# Metadata
|
|
# ------------------------------------------------------------------------
|
|
|
|
try:
|
|
|
|
date_taken = get_file_date(
|
|
source_file
|
|
)
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" ERROR getting date: {exc}"
|
|
)
|
|
|
|
return "error"
|
|
|
|
file_type = classify_file(
|
|
source_file
|
|
)
|
|
|
|
device_make, device_model = (
|
|
get_device_metadata(
|
|
source_file
|
|
)
|
|
)
|
|
|
|
# ------------------------------------------------------------------------
|
|
# Destination
|
|
# ------------------------------------------------------------------------
|
|
|
|
destination_directory = (
|
|
get_destination_directory(
|
|
destination_root,
|
|
source_file,
|
|
file_type,
|
|
date_taken,
|
|
)
|
|
)
|
|
|
|
try:
|
|
|
|
destination_file = (
|
|
create_destination_filename(
|
|
destination_directory,
|
|
date_taken,
|
|
source_file.suffix,
|
|
)
|
|
)
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" ERROR creating filename: {exc}"
|
|
)
|
|
|
|
return "error"
|
|
|
|
relative_destination = (
|
|
destination_file.relative_to(
|
|
destination_root
|
|
)
|
|
)
|
|
|
|
print(
|
|
f" Type: {file_type}"
|
|
)
|
|
|
|
print(
|
|
f" Date: "
|
|
f"{date_taken.strftime('%d-%m-%Y')}"
|
|
)
|
|
|
|
if device_make or device_model:
|
|
|
|
device = " ".join(
|
|
value
|
|
for value in (
|
|
device_make,
|
|
device_model,
|
|
)
|
|
if value
|
|
)
|
|
|
|
print(
|
|
f" Device: {device}"
|
|
)
|
|
|
|
else:
|
|
|
|
print(
|
|
" Device: unknown"
|
|
)
|
|
|
|
print(
|
|
f" SHA-256: {source_hash}"
|
|
)
|
|
|
|
print(
|
|
f" Destination: {relative_destination}"
|
|
)
|
|
|
|
# ------------------------------------------------------------------------
|
|
# Dry run
|
|
# ------------------------------------------------------------------------
|
|
|
|
if dry_run:
|
|
|
|
print(
|
|
" DRY RUN - nothing copied"
|
|
)
|
|
|
|
return "dry-run"
|
|
|
|
# ------------------------------------------------------------------------
|
|
# Copy + verify + database transaction
|
|
# ------------------------------------------------------------------------
|
|
|
|
try:
|
|
|
|
# FIRST:
|
|
# Copy and verify the physical file.
|
|
safe_copy_and_verify(
|
|
source_file,
|
|
destination_file,
|
|
source_hash,
|
|
)
|
|
|
|
# SECOND:
|
|
# Only after successful verification do we write to SQLite.
|
|
imported_at = datetime.now(
|
|
timezone.utc
|
|
).isoformat()
|
|
|
|
connection.execute(
|
|
"BEGIN"
|
|
)
|
|
|
|
try:
|
|
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO files (
|
|
sha256,
|
|
size,
|
|
original_name,
|
|
stored_name,
|
|
relative_path,
|
|
date_taken,
|
|
file_type,
|
|
device_make,
|
|
device_model,
|
|
imported_at
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
source_hash,
|
|
source_file.stat().st_size,
|
|
source_file.name,
|
|
destination_file.name,
|
|
str(relative_destination),
|
|
date_taken.isoformat(),
|
|
file_type,
|
|
device_make,
|
|
device_model,
|
|
imported_at,
|
|
),
|
|
)
|
|
|
|
connection.commit()
|
|
|
|
except Exception:
|
|
|
|
connection.rollback()
|
|
|
|
# The database transaction failed.
|
|
# Remove the copied file so filesystem and database
|
|
# remain consistent.
|
|
try:
|
|
|
|
destination_file.unlink()
|
|
|
|
except OSError:
|
|
pass
|
|
|
|
raise
|
|
|
|
# --------------------------------------------------------------------
|
|
# Optional source deletion
|
|
# --------------------------------------------------------------------
|
|
|
|
if move:
|
|
|
|
source_file.unlink()
|
|
|
|
print(" IMPORTED")
|
|
|
|
return (
|
|
"moved"
|
|
if move
|
|
else "imported"
|
|
)
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" ERROR: {exc}"
|
|
)
|
|
|
|
return "error"
|
|
|
|
|
|
# ============================================================================
|
|
# NORMAL IMPORT
|
|
# ============================================================================
|
|
|
|
def import_files(
|
|
source: Path,
|
|
destination: Path,
|
|
dry_run: bool,
|
|
move: bool,
|
|
):
|
|
destination.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
db_path = database_path(
|
|
destination
|
|
)
|
|
|
|
# ------------------------------------------------------------------------
|
|
# Safety check
|
|
#
|
|
# If files already exist in the destination but there is no DB,
|
|
# refuse to continue.
|
|
#
|
|
# This prevents a user accidentally importing thousands of duplicates
|
|
# into an existing library.
|
|
# ------------------------------------------------------------------------
|
|
|
|
if (
|
|
not db_path.exists()
|
|
and destination_contains_library_files(
|
|
destination
|
|
)
|
|
):
|
|
|
|
print(
|
|
"ERROR: The destination already contains files,"
|
|
)
|
|
|
|
print(
|
|
"but there is no photo organiser database."
|
|
)
|
|
|
|
print()
|
|
print(
|
|
"Before importing, build the database with:"
|
|
)
|
|
|
|
print()
|
|
|
|
print(
|
|
f'python3 organise_phone.py '
|
|
f'--build-db "{destination}"'
|
|
)
|
|
|
|
print()
|
|
|
|
print(
|
|
"This is a safety feature to prevent accidental"
|
|
)
|
|
|
|
print(
|
|
"duplicate imports."
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
connection = connect_database(
|
|
destination
|
|
)
|
|
|
|
try:
|
|
|
|
create_database(
|
|
connection
|
|
)
|
|
|
|
files = list(
|
|
find_source_files(
|
|
source
|
|
)
|
|
)
|
|
|
|
print("=" * 70)
|
|
print("PHOTO ORGANISER")
|
|
print("=" * 70)
|
|
|
|
print(
|
|
f"Source: {source}"
|
|
)
|
|
|
|
print(
|
|
f"Destination: {destination}"
|
|
)
|
|
|
|
print(
|
|
f"Database: {db_path}"
|
|
)
|
|
|
|
print(
|
|
f"Operation: "
|
|
f"{'MOVE' if move else 'COPY'}"
|
|
)
|
|
|
|
print(
|
|
f"Mode: "
|
|
f"{'DRY RUN' if dry_run else 'LIVE'}"
|
|
)
|
|
|
|
print()
|
|
|
|
print(
|
|
f"Found {len(files)} files."
|
|
)
|
|
|
|
print("=" * 70)
|
|
|
|
imported = 0
|
|
duplicates = 0
|
|
errors = 0
|
|
dry_runs = 0
|
|
moved = 0
|
|
|
|
for source_file in files:
|
|
|
|
result = process_file(
|
|
source_file=source_file,
|
|
source_root=source,
|
|
destination_root=destination,
|
|
connection=connection,
|
|
dry_run=dry_run,
|
|
move=move,
|
|
)
|
|
|
|
if result == "imported":
|
|
|
|
imported += 1
|
|
|
|
elif result == "moved":
|
|
|
|
moved += 1
|
|
|
|
elif result == "duplicate":
|
|
|
|
duplicates += 1
|
|
|
|
elif result == "error":
|
|
|
|
errors += 1
|
|
|
|
elif result == "dry-run":
|
|
|
|
dry_runs += 1
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("IMPORT COMPLETE")
|
|
print("=" * 70)
|
|
|
|
if dry_run:
|
|
|
|
print(
|
|
f"Would import: {dry_runs}"
|
|
)
|
|
|
|
else:
|
|
|
|
print(
|
|
f"Imported: {imported}"
|
|
)
|
|
|
|
if moved:
|
|
|
|
print(
|
|
f"Moved: {moved}"
|
|
)
|
|
|
|
print(
|
|
f"Duplicates skipped: {duplicates}"
|
|
)
|
|
|
|
print(
|
|
f"Errors: {errors}"
|
|
)
|
|
|
|
print("=" * 70)
|
|
|
|
finally:
|
|
|
|
connection.close()
|
|
|
|
|
|
# ============================================================================
|
|
# BUILD DATABASE
|
|
# ============================================================================
|
|
|
|
def build_database(
|
|
destination: Path,
|
|
):
|
|
"""
|
|
Build a database for an existing library.
|
|
|
|
This does not move, rename or delete anything.
|
|
"""
|
|
|
|
db_path = database_path(
|
|
destination
|
|
)
|
|
|
|
connection = connect_database(
|
|
destination
|
|
)
|
|
|
|
try:
|
|
|
|
create_database(
|
|
connection
|
|
)
|
|
|
|
files = list(
|
|
find_source_files(
|
|
destination
|
|
)
|
|
)
|
|
|
|
print("=" * 70)
|
|
print("BUILDING PHOTO DATABASE")
|
|
print("=" * 70)
|
|
|
|
print(
|
|
f"Library: {destination}"
|
|
)
|
|
|
|
print(
|
|
f"Database: {db_path}"
|
|
)
|
|
|
|
print(
|
|
f"Files found: {len(files)}"
|
|
)
|
|
|
|
print()
|
|
|
|
added = 0
|
|
already_indexed = 0
|
|
duplicates = 0
|
|
errors = 0
|
|
|
|
for path in files:
|
|
|
|
# Never index the database itself.
|
|
if path == db_path:
|
|
continue
|
|
|
|
# SQLite WAL / SHM files.
|
|
if path.name in (
|
|
f"{DATABASE_NAME}-wal",
|
|
f"{DATABASE_NAME}-shm",
|
|
):
|
|
continue
|
|
|
|
try:
|
|
|
|
relative_path = (
|
|
path.relative_to(
|
|
destination
|
|
)
|
|
)
|
|
|
|
# Check whether this exact path is already in DB.
|
|
path_exists = connection.execute(
|
|
"""
|
|
SELECT id
|
|
FROM files
|
|
WHERE relative_path = ?
|
|
""",
|
|
(str(relative_path),),
|
|
).fetchone()
|
|
|
|
if path_exists:
|
|
|
|
already_indexed += 1
|
|
continue
|
|
|
|
print(
|
|
f"Hashing: {relative_path}"
|
|
)
|
|
|
|
file_hash = (
|
|
calculate_sha256(
|
|
path
|
|
)
|
|
)
|
|
|
|
existing = hash_exists(
|
|
connection,
|
|
file_hash,
|
|
)
|
|
|
|
if existing:
|
|
|
|
print(
|
|
" DUPLICATE HASH:"
|
|
)
|
|
|
|
print(
|
|
f" {existing[1]}"
|
|
)
|
|
|
|
duplicates += 1
|
|
continue
|
|
|
|
date_taken = (
|
|
get_file_date(
|
|
path
|
|
)
|
|
)
|
|
|
|
device_make, device_model = (
|
|
get_device_metadata(
|
|
path
|
|
)
|
|
)
|
|
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO files (
|
|
sha256,
|
|
size,
|
|
original_name,
|
|
stored_name,
|
|
relative_path,
|
|
date_taken,
|
|
file_type,
|
|
device_make,
|
|
device_model,
|
|
imported_at
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
file_hash,
|
|
path.stat().st_size,
|
|
path.name,
|
|
path.name,
|
|
str(relative_path),
|
|
date_taken.isoformat(),
|
|
classify_file(path),
|
|
device_make,
|
|
device_model,
|
|
datetime.now(
|
|
timezone.utc
|
|
).isoformat(),
|
|
),
|
|
)
|
|
|
|
connection.commit()
|
|
|
|
added += 1
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" ERROR: {exc}"
|
|
)
|
|
|
|
errors += 1
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("DATABASE BUILD COMPLETE")
|
|
print("=" * 70)
|
|
|
|
print(
|
|
f"Added: {added}"
|
|
)
|
|
|
|
print(
|
|
f"Already indexed: {already_indexed}"
|
|
)
|
|
|
|
print(
|
|
f"Duplicate hashes: {duplicates}"
|
|
)
|
|
|
|
print(
|
|
f"Errors: {errors}"
|
|
)
|
|
|
|
print("=" * 70)
|
|
|
|
finally:
|
|
|
|
connection.close()
|
|
|
|
|
|
# ============================================================================
|
|
# VERIFY DATABASE
|
|
# ============================================================================
|
|
|
|
def verify_database(
|
|
destination: Path,
|
|
deep: bool,
|
|
):
|
|
"""
|
|
Verify database ↔ filesystem consistency.
|
|
|
|
Normal mode checks:
|
|
- DB files exist
|
|
- sizes match
|
|
- every library file is indexed
|
|
|
|
Deep mode additionally:
|
|
- hashes every file
|
|
- compares against DB SHA-256
|
|
"""
|
|
|
|
db_path = database_path(
|
|
destination
|
|
)
|
|
|
|
if not db_path.exists():
|
|
|
|
print(
|
|
"ERROR: Database does not exist:"
|
|
)
|
|
|
|
print(
|
|
db_path
|
|
)
|
|
|
|
print()
|
|
print(
|
|
"Build it with:"
|
|
)
|
|
|
|
print()
|
|
|
|
print(
|
|
f'python3 organise_phone.py '
|
|
f'--build-db "{destination}"'
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
connection = connect_database(
|
|
destination
|
|
)
|
|
|
|
try:
|
|
|
|
create_database(
|
|
connection
|
|
)
|
|
|
|
print("=" * 70)
|
|
print("VERIFYING PHOTO LIBRARY")
|
|
print("=" * 70)
|
|
|
|
print(
|
|
f"Library: {destination}"
|
|
)
|
|
|
|
print(
|
|
f"Database: {db_path}"
|
|
)
|
|
|
|
print(
|
|
f"Mode: "
|
|
f"{'DEEP - hash every file' if deep else 'NORMAL'}"
|
|
)
|
|
|
|
print()
|
|
|
|
rows = connection.execute(
|
|
"""
|
|
SELECT
|
|
id,
|
|
sha256,
|
|
size,
|
|
relative_path
|
|
FROM files
|
|
ORDER BY id
|
|
"""
|
|
).fetchall()
|
|
|
|
database_paths = set()
|
|
|
|
missing = 0
|
|
size_mismatches = 0
|
|
hash_mismatches = 0
|
|
errors = 0
|
|
|
|
# --------------------------------------------------------------------
|
|
# Check every database entry
|
|
# --------------------------------------------------------------------
|
|
|
|
for (
|
|
row_id,
|
|
expected_hash,
|
|
expected_size,
|
|
relative_path,
|
|
) in rows:
|
|
|
|
database_paths.add(
|
|
relative_path
|
|
)
|
|
|
|
actual_path = (
|
|
destination
|
|
/ relative_path
|
|
)
|
|
|
|
if not actual_path.exists():
|
|
|
|
print(
|
|
f"MISSING: {relative_path}"
|
|
)
|
|
|
|
missing += 1
|
|
continue
|
|
|
|
if not actual_path.is_file():
|
|
|
|
print(
|
|
f"NOT A FILE: {relative_path}"
|
|
)
|
|
|
|
errors += 1
|
|
continue
|
|
|
|
try:
|
|
|
|
actual_size = (
|
|
actual_path.stat().st_size
|
|
)
|
|
|
|
if actual_size != expected_size:
|
|
|
|
print(
|
|
f"SIZE MISMATCH: "
|
|
f"{relative_path}"
|
|
)
|
|
|
|
size_mismatches += 1
|
|
|
|
continue
|
|
|
|
if deep:
|
|
|
|
print(
|
|
f"Hashing: "
|
|
f"{relative_path}"
|
|
)
|
|
|
|
actual_hash = (
|
|
calculate_sha256(
|
|
actual_path
|
|
)
|
|
)
|
|
|
|
if (
|
|
actual_hash
|
|
!= expected_hash
|
|
):
|
|
|
|
print(
|
|
f"HASH MISMATCH: "
|
|
f"{relative_path}"
|
|
)
|
|
|
|
hash_mismatches += 1
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f"ERROR: "
|
|
f"{relative_path}: "
|
|
f"{exc}"
|
|
)
|
|
|
|
errors += 1
|
|
|
|
# --------------------------------------------------------------------
|
|
# Find files on disk that aren't in DB
|
|
# --------------------------------------------------------------------
|
|
|
|
unindexed = []
|
|
|
|
for path in find_source_files(
|
|
destination
|
|
):
|
|
|
|
if path == db_path:
|
|
continue
|
|
|
|
if path.name in (
|
|
f"{DATABASE_NAME}-wal",
|
|
f"{DATABASE_NAME}-shm",
|
|
):
|
|
continue
|
|
|
|
try:
|
|
|
|
relative_path = str(
|
|
path.relative_to(
|
|
destination
|
|
)
|
|
)
|
|
|
|
except ValueError:
|
|
|
|
continue
|
|
|
|
if (
|
|
relative_path
|
|
not in database_paths
|
|
):
|
|
|
|
unindexed.append(
|
|
relative_path
|
|
)
|
|
|
|
# --------------------------------------------------------------------
|
|
# Results
|
|
# --------------------------------------------------------------------
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("VERIFICATION COMPLETE")
|
|
print("=" * 70)
|
|
|
|
print(
|
|
f"Database records: {len(rows)}"
|
|
)
|
|
|
|
print(
|
|
f"Missing files: {missing}"
|
|
)
|
|
|
|
print(
|
|
f"Size mismatches: {size_mismatches}"
|
|
)
|
|
|
|
if deep:
|
|
|
|
print(
|
|
f"Hash mismatches: {hash_mismatches}"
|
|
)
|
|
|
|
print(
|
|
f"Unindexed files: {len(unindexed)}"
|
|
)
|
|
|
|
print(
|
|
f"Other errors: {errors}"
|
|
)
|
|
|
|
if unindexed:
|
|
|
|
print()
|
|
print(
|
|
"UNINDEXED FILES:"
|
|
)
|
|
|
|
for path in unindexed:
|
|
|
|
print(
|
|
f" {path}"
|
|
)
|
|
|
|
print()
|
|
|
|
if (
|
|
missing == 0
|
|
and size_mismatches == 0
|
|
and hash_mismatches == 0
|
|
and len(unindexed) == 0
|
|
and errors == 0
|
|
):
|
|
|
|
print(
|
|
"RESULT: LIBRARY IS CONSISTENT"
|
|
)
|
|
|
|
else:
|
|
|
|
print(
|
|
"RESULT: PROBLEMS FOUND"
|
|
)
|
|
|
|
print("=" * 70)
|
|
|
|
finally:
|
|
|
|
connection.close()
|
|
|
|
|
|
# ============================================================================
|
|
# MAIN
|
|
# ============================================================================
|
|
|
|
def main():
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description=(
|
|
"Organise photos and videos into a date-based "
|
|
"library with SHA-256 duplicate detection."
|
|
)
|
|
)
|
|
|
|
parser.add_argument(
|
|
"source",
|
|
type=Path,
|
|
nargs="?",
|
|
help="Source folder",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"destination",
|
|
type=Path,
|
|
nargs="?",
|
|
help="Destination folder / external drive",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="Show what would happen without copying",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--move",
|
|
action="store_true",
|
|
help="Move files instead of copying them",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--build-db",
|
|
metavar="DESTINATION",
|
|
type=Path,
|
|
help="Build database for an existing library",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--verify",
|
|
metavar="DESTINATION",
|
|
type=Path,
|
|
help="Verify database against library",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--deep",
|
|
action="store_true",
|
|
help=(
|
|
"With --verify, calculate SHA-256 for every "
|
|
"file and verify its contents"
|
|
),
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# ------------------------------------------------------------------------
|
|
# BUILD DATABASE
|
|
# ------------------------------------------------------------------------
|
|
|
|
if args.build_db:
|
|
|
|
destination = (
|
|
args.build_db
|
|
.expanduser()
|
|
.resolve()
|
|
)
|
|
|
|
if not destination.exists():
|
|
|
|
print(
|
|
f"ERROR: Folder does not exist:\n"
|
|
f"{destination}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
if not destination.is_dir():
|
|
|
|
print(
|
|
f"ERROR: Not a folder:\n"
|
|
f"{destination}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
build_database(
|
|
destination
|
|
)
|
|
|
|
return
|
|
|
|
# ------------------------------------------------------------------------
|
|
# VERIFY
|
|
# ------------------------------------------------------------------------
|
|
|
|
if args.verify:
|
|
|
|
destination = (
|
|
args.verify
|
|
.expanduser()
|
|
.resolve()
|
|
)
|
|
|
|
if not destination.exists():
|
|
|
|
print(
|
|
f"ERROR: Folder does not exist:\n"
|
|
f"{destination}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
if not destination.is_dir():
|
|
|
|
print(
|
|
f"ERROR: Not a folder:\n"
|
|
f"{destination}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
verify_database(
|
|
destination,
|
|
args.deep,
|
|
)
|
|
|
|
return
|
|
|
|
# ------------------------------------------------------------------------
|
|
# NORMAL IMPORT
|
|
# ------------------------------------------------------------------------
|
|
|
|
if (
|
|
not args.source
|
|
or not args.destination
|
|
):
|
|
|
|
parser.print_help()
|
|
sys.exit(1)
|
|
|
|
source = (
|
|
args.source
|
|
.expanduser()
|
|
.resolve()
|
|
)
|
|
|
|
destination = (
|
|
args.destination
|
|
.expanduser()
|
|
.resolve()
|
|
)
|
|
|
|
if not source.exists():
|
|
|
|
print(
|
|
f"ERROR: Source folder does not exist:\n"
|
|
f"{source}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
if not source.is_dir():
|
|
|
|
print(
|
|
f"ERROR: Source is not a folder:\n"
|
|
f"{source}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
# Destination cannot be inside source.
|
|
try:
|
|
|
|
destination.relative_to(
|
|
source
|
|
)
|
|
|
|
print(
|
|
"ERROR: Destination cannot be "
|
|
"inside the source folder.",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
except ValueError:
|
|
|
|
pass
|
|
|
|
import_files(
|
|
source=source,
|
|
destination=destination,
|
|
dry_run=args.dry_run,
|
|
move=args.move,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|