cadmus_core/library/
migrations.rs

1//! Runtime migrations for the library subsystem.
2//!
3//! Each migration is registered automatically at startup via the [`crate::migration!`]
4//! macro and tracked in the `_cadmus_migrations` table.
5//!
6//! # Registered migrations
7//!
8//! | Module | Migration ID |
9//! |---|---|
10//! | [`import_legacy_filesystem_data::MIGRATION_ID`] | `v1_import_legacy_filesystem_data` |
11
12use crate::db::types::UnixTimestamp;
13use crate::helpers::Fp;
14use crate::library::db::conversion::{
15    extract_authors, info_to_book_row, reader_info_to_reading_state_row,
16};
17#[cfg(not(feature = "test"))]
18use crate::library::THUMBNAIL_PREVIEWS_DIRNAME;
19use crate::library::{METADATA_FILENAME, READING_STATES_DIRNAME};
20use crate::metadata::Info;
21use crate::settings::versioned::SettingsManager;
22use crate::version::get_current_version;
23use fxhash::FxBuildHasher;
24use indexmap::IndexMap;
25use sqlx::{Sqlite, Transaction};
26use std::collections::HashSet;
27use std::path::Path;
28use std::str::FromStr;
29use tokio::fs;
30use tracing::{error, info, warn};
31
32crate::migration!(
33    /// Imports book metadata from `.metadata.json` and reading progress from
34    /// `.reading-states/<fingerprint>.json` into SQLite for every library path
35    /// listed in `Settings.toml`. Covers both legacy library modes:
36    ///
37    /// - Database mode: had `.metadata.json` keyed by fingerprint.
38    /// - Filesystem mode: no `.metadata.json`; only `.reading-states/` files.
39    ///   Stub book rows are inserted to satisfy the foreign key before reading
40    ///   states are written. A follow-up migration prunes stubs for missing files.
41    ///
42    /// The migration is idempotent (all inserts use `ON CONFLICT … DO NOTHING`).
43    "v1_import_legacy_filesystem_data",
44    async fn import_legacy_filesystem_data(pool: &SqlitePool) {
45        let settings = SettingsManager::new(get_current_version()).load();
46
47        if settings.libraries.is_empty() {
48            info!("no libraries in settings, skipping legacy data import");
49            return Ok(());
50        }
51
52        for lib in &settings.libraries {
53            let library_path = &lib.path;
54            let library_name = &lib.name;
55            let path_str = library_path.to_string_lossy();
56
57            info!(path = %path_str, name = %library_name, "importing legacy data for library");
58
59            let library_id = ensure_library(pool, &path_str, library_name).await;
60
61            let library_id = match library_id {
62                Ok(id) => id,
63                Err(e) => {
64                    error!(path = %path_str, error = %e, "failed to register library, skipping");
65                    continue;
66                }
67            };
68
69            let (book_count, state_count) = import_library(pool, library_id, library_path).await;
70
71            info!(
72                path = %path_str,
73                books_imported = book_count,
74                reading_states_imported = state_count,
75                "library import complete"
76            );
77        }
78
79        Ok(())
80    }
81);
82
83/// Ensures the library row exists and returns its id.
84#[cfg_attr(feature = "otel", tracing::instrument(skip(pool), fields(path = %path, name = %name), ret(level = tracing::Level::TRACE)))]
85async fn ensure_library(
86    pool: &sqlx::SqlitePool,
87    path: &str,
88    name: &str,
89) -> Result<i64, anyhow::Error> {
90    let existing: Option<i64> =
91        sqlx::query_scalar!("SELECT id FROM libraries WHERE path = ?", path)
92            .fetch_optional(pool)
93            .await?
94            .flatten();
95
96    if let Some(id) = existing {
97        return Ok(id);
98    }
99
100    let now = UnixTimestamp::now();
101    let result = sqlx::query!(
102        "INSERT INTO libraries (path, name, created_at) VALUES (?, ?, ?)",
103        path,
104        name,
105        now
106    )
107    .execute(pool)
108    .await?;
109
110    Ok(result.last_insert_rowid())
111}
112
113/// Imports all books and reading states from a single library directory.
114///
115/// Loads `.metadata.json` and the `.reading-states/` directory, inserts all
116/// entries into the database within a single transaction, then renames the
117/// legacy files and removes `.thumbnail-previews/`.
118///
119/// Returns `(books_imported, reading_states_imported)`.
120#[cfg_attr(feature = "otel", tracing::instrument(skip(pool), fields(library_id = library_id, path = ?library_path)))]
121async fn import_library(
122    pool: &sqlx::SqlitePool,
123    library_id: i64,
124    library_path: &Path,
125) -> (usize, usize) {
126    let mut tx = match pool.begin().await {
127        Ok(tx) => tx,
128        Err(e) => {
129            error!(path = ?library_path, error = %e, "failed to begin transaction for library import");
130            return (0, 0);
131        }
132    };
133
134    let metadata_path = library_path.join(METADATA_FILENAME);
135    let metadata = load_metadata(&metadata_path).await;
136
137    let (books_imported, states_from_metadata, metadata_fps) =
138        import_metadata_entries(&mut tx, library_id, metadata).await;
139
140    let reading_states_dir = library_path.join(READING_STATES_DIRNAME);
141    let states_from_dir =
142        import_orphan_reading_states(&mut tx, library_id, &reading_states_dir, &metadata_fps).await;
143
144    if let Err(e) = tx.commit().await {
145        error!(path = ?library_path, error = %e, "failed to commit library import transaction");
146        return (0, 0);
147    }
148
149    #[cfg(not(feature = "test"))]
150    {
151        mark_library_imported(library_path).await;
152        delete_thumbnail_previews(library_path).await;
153    }
154
155    (books_imported, states_from_metadata + states_from_dir)
156}
157
158/// Imports all entries from a `.metadata.json` file into the database.
159///
160/// Returns `(books_imported, reading_states_imported, fingerprints_seen)`.
161/// The fingerprint set is passed to [`import_orphan_reading_states`] to skip
162/// books whose reading state was already written from this file.
163#[cfg_attr(feature = "otel", tracing::instrument(skip(tx, metadata), fields(library_id = library_id)))]
164async fn import_metadata_entries(
165    tx: &mut Transaction<'_, Sqlite>,
166    library_id: i64,
167    metadata: Option<IndexMap<Fp, Info, FxBuildHasher>>,
168) -> (usize, usize, HashSet<Fp>) {
169    let mut books_imported: usize = 0;
170    let mut states_imported: usize = 0;
171    let mut seen_fps: HashSet<Fp> = HashSet::new();
172
173    let entries = match metadata {
174        Some(e) => e,
175        None => return (0, 0, seen_fps),
176    };
177
178    for (fp, info) in &entries {
179        if let Err(e) = insert_book(tx, library_id, *fp, info).await {
180            error!(fp = %fp, error = %e, "failed to insert book from metadata");
181            continue;
182        }
183        books_imported += 1;
184
185        if let Some(reader_info) = info.reader_info.as_ref().or(info.reader.as_ref()) {
186            seen_fps.insert(*fp);
187            if let Err(e) = insert_reading_state(tx, *fp, reader_info).await {
188                error!(fp = %fp, error = %e, "failed to insert reading state from metadata");
189            } else {
190                states_imported += 1;
191            }
192        }
193    }
194
195    (books_imported, states_imported, seen_fps)
196}
197
198/// Imports reading states from `.reading-states/` that are not in `already_imported`.
199///
200/// The `already_imported` set contains fingerprints whose reading state was
201/// already written from `.metadata.json`. Skipping those keeps the migration
202/// idempotent and ensures the metadata file's version takes precedence.
203///
204/// For each fingerprint not yet imported, a stub `books` row is inserted first
205/// to satisfy the foreign key constraint. A follow-up migration is responsible
206/// for cleaning up any stub rows whose files are no longer on disk.
207///
208/// Returns the number of reading states imported.
209#[cfg_attr(feature = "otel", tracing::instrument(skip(tx, already_imported), fields(library_id = library_id, path = ?reading_states_dir)))]
210async fn import_orphan_reading_states(
211    tx: &mut Transaction<'_, Sqlite>,
212    library_id: i64,
213    reading_states_dir: &Path,
214    already_imported: &HashSet<Fp>,
215) -> usize {
216    if !reading_states_dir.exists() {
217        return 0;
218    }
219
220    let mut dir_entries = match fs::read_dir(reading_states_dir).await {
221        Ok(d) => d,
222        Err(e) => {
223            error!(path = ?reading_states_dir, error = %e, "failed to read .reading-states dir");
224            return 0;
225        }
226    };
227
228    let mut states_imported: usize = 0;
229
230    loop {
231        let entry = match dir_entries.next_entry().await {
232            Ok(Some(e)) => e,
233            Ok(None) => break,
234            Err(e) => {
235                error!(path = ?reading_states_dir, error = %e, "failed to read directory entry");
236                break;
237            }
238        };
239
240        let path = entry.path();
241
242        let fp = match path
243            .file_stem()
244            .and_then(|s| s.to_str())
245            .and_then(|s| Fp::from_str(s).ok())
246        {
247            Some(fp) => fp,
248            None => {
249                warn!(path = ?path, "skipping unrecognised reading-state filename");
250                continue;
251            }
252        };
253
254        if already_imported.contains(&fp) {
255            continue;
256        }
257
258        let content = match fs::read_to_string(&path).await {
259            Ok(c) => c,
260            Err(e) => {
261                error!(fp = %fp, path = ?path, error = %e, "failed to read reading-state file");
262                continue;
263            }
264        };
265
266        let reader_info: crate::metadata::ReaderInfo = match serde_json::from_str(&content) {
267            Ok(r) => r,
268            Err(e) => {
269                error!(fp = %fp, error = %e, "failed to parse reading-state JSON");
270                continue;
271            }
272        };
273
274        if let Err(e) = ensure_stub_book(tx, library_id, fp).await {
275            error!(fp = %fp, error = %e, "failed to insert stub book for orphan reading state, skipping");
276            continue;
277        }
278
279        if let Err(e) = insert_reading_state(tx, fp, &reader_info).await {
280            error!(fp = %fp, error = %e, "failed to insert orphan reading state");
281        } else {
282            states_imported += 1;
283        }
284    }
285
286    states_imported
287}
288
289/// Inserts a stub `books` row and a `library_books` association for `fp` if
290/// they do not already exist.
291///
292/// The stub has empty strings and zero for the file fields. A follow-up
293/// migration is responsible for pruning stub rows whose files are no longer
294/// present on disk. `Library::import()` will fill in the real values for files
295/// that are still present.
296#[cfg_attr(feature = "otel", tracing::instrument(skip(tx), fields(library_id = library_id, fp = %fp)))]
297async fn ensure_stub_book(
298    tx: &mut Transaction<'_, Sqlite>,
299    library_id: i64,
300    fp: Fp,
301) -> Result<(), anyhow::Error> {
302    let fp_str = fp.to_string();
303    let now = UnixTimestamp::now();
304
305    sqlx::query!(
306        r#"
307        INSERT OR IGNORE INTO books (fingerprint, absolute_path, file_kind, file_size, added_at)
308        VALUES (?, '', '', 0, ?)
309        "#,
310        fp_str,
311        now,
312    )
313    .execute(&mut **tx)
314    .await?;
315
316    sqlx::query!(
317        r#"
318        INSERT OR IGNORE INTO library_books (library_id, book_fingerprint, added_to_library_at)
319        VALUES (?, ?, ?)
320        "#,
321        library_id,
322        fp_str,
323        now,
324    )
325    .execute(&mut **tx)
326    .await?;
327
328    Ok(())
329}
330
331/// Renames `.metadata.json` and `.reading-states/` to their `.imported` suffixed
332/// equivalents so that subsequent runs of the migration skip this library.
333#[cfg(not(feature = "test"))]
334#[cfg_attr(feature = "otel", tracing::instrument(fields(path = ?library_path)))]
335async fn mark_library_imported(library_path: &Path) {
336    let metadata_src = library_path.join(METADATA_FILENAME);
337    let metadata_dst = library_path.join(format!("{}.imported", METADATA_FILENAME));
338
339    if metadata_src.exists() {
340        if let Err(e) = fs::rename(&metadata_src, &metadata_dst).await {
341            warn!(path = ?metadata_src, error = %e, "failed to rename .metadata.json after import");
342        }
343    }
344
345    let states_src = library_path.join(READING_STATES_DIRNAME);
346    let states_dst = library_path.join(format!("{}.imported", READING_STATES_DIRNAME));
347
348    if states_src.exists() {
349        if let Err(e) = fs::rename(&states_src, &states_dst).await {
350            warn!(path = ?states_src, error = %e, "failed to rename .reading-states after import");
351        }
352    }
353}
354
355/// Removes `.thumbnail-previews/` from the library directory.
356///
357/// Thumbnails will be regenerated and stored in the database, so the legacy
358/// directory is no longer needed after migration.
359#[cfg(not(feature = "test"))]
360#[cfg_attr(feature = "otel", tracing::instrument(fields(path = ?library_path)))]
361async fn delete_thumbnail_previews(library_path: &Path) {
362    let previews_dir = library_path.join(THUMBNAIL_PREVIEWS_DIRNAME);
363
364    if !previews_dir.exists() {
365        return;
366    }
367
368    if let Err(e) = fs::remove_dir_all(&previews_dir).await {
369        warn!(path = ?previews_dir, error = %e, "failed to delete .thumbnail-previews after import");
370    }
371}
372
373#[cfg_attr(feature = "otel", tracing::instrument(fields(path = ?path), ret(level = tracing::Level::TRACE)))]
374async fn load_metadata(path: &Path) -> Option<IndexMap<Fp, Info, FxBuildHasher>> {
375    if !path.exists() {
376        return None;
377    }
378
379    let content = match fs::read_to_string(path).await {
380        Ok(c) => c,
381        Err(e) => {
382            error!(path = ?path, error = %e, "failed to read .metadata.json");
383            return None;
384        }
385    };
386
387    match serde_json::from_str(&content) {
388        Ok(m) => Some(m),
389        Err(e) => {
390            error!(path = ?path, error = %e, "failed to parse .metadata.json");
391            None
392        }
393    }
394}
395
396#[cfg_attr(feature = "otel", tracing::instrument(skip(tx, info), fields(library_id = library_id, fp = %fp)))]
397async fn insert_book(
398    tx: &mut Transaction<'_, Sqlite>,
399    library_id: i64,
400    fp: Fp,
401    info: &Info,
402) -> Result<(), anyhow::Error> {
403    let book_row = info_to_book_row(fp, info);
404    let fp_str = fp.to_string();
405
406    sqlx::query!(
407        r#"
408        INSERT OR IGNORE INTO books (
409            fingerprint, title, subtitle, year, language, publisher,
410            series, edition, volume, number, identifier,
411            absolute_path, file_kind, file_size, added_at
412        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
413        "#,
414        book_row.fingerprint,
415        book_row.title,
416        book_row.subtitle,
417        book_row.year,
418        book_row.language,
419        book_row.publisher,
420        book_row.series,
421        book_row.edition,
422        book_row.volume,
423        book_row.number,
424        book_row.identifier,
425        book_row.absolute_path,
426        book_row.file_kind,
427        book_row.file_size,
428        book_row.added_at,
429    )
430    .execute(&mut **tx)
431    .await?;
432
433    sqlx::query!(
434        r#"
435        INSERT OR IGNORE INTO library_books (library_id, book_fingerprint, added_to_library_at, file_path)
436        VALUES (?, ?, ?, ?)
437        "#,
438        library_id,
439        fp_str,
440        book_row.added_at,
441        book_row.file_path,
442    )
443    .execute(&mut **tx)
444    .await?;
445
446    let authors = extract_authors(&info.author);
447    for (position, author_name) in authors.iter().enumerate() {
448        sqlx::query!(
449            r#"INSERT OR IGNORE INTO authors (name) VALUES (?)"#,
450            author_name
451        )
452        .execute(&mut **tx)
453        .await?;
454
455        let author_id: i64 =
456            sqlx::query_scalar!(r#"SELECT id FROM authors WHERE name = ?"#, author_name)
457                .fetch_one(&mut **tx)
458                .await?;
459
460        let pos = position as i64;
461        sqlx::query!(
462            r#"
463            INSERT OR IGNORE INTO book_authors (book_fingerprint, author_id, position)
464            VALUES (?, ?, ?)
465            "#,
466            fp_str,
467            author_id,
468            pos,
469        )
470        .execute(&mut **tx)
471        .await?;
472    }
473
474    for category_name in &info.categories {
475        sqlx::query!(
476            r#"INSERT OR IGNORE INTO categories (name) VALUES (?)"#,
477            category_name
478        )
479        .execute(&mut **tx)
480        .await?;
481
482        let category_id: i64 =
483            sqlx::query_scalar!(r#"SELECT id FROM categories WHERE name = ?"#, category_name)
484                .fetch_one(&mut **tx)
485                .await?;
486
487        sqlx::query!(
488            r#"
489            INSERT OR IGNORE INTO book_categories (book_fingerprint, category_id)
490            VALUES (?, ?)
491            "#,
492            fp_str,
493            category_id,
494        )
495        .execute(&mut **tx)
496        .await?;
497    }
498
499    Ok(())
500}
501
502#[cfg_attr(feature = "otel", tracing::instrument(skip(tx, reader_info), fields(fp = %fp)))]
503async fn insert_reading_state(
504    tx: &mut Transaction<'_, Sqlite>,
505    fp: Fp,
506    reader_info: &crate::metadata::ReaderInfo,
507) -> Result<(), anyhow::Error> {
508    let rs = reader_info_to_reading_state_row(fp, reader_info);
509
510    sqlx::query!(
511        r#"
512        INSERT INTO reading_states (
513            fingerprint, opened, current_page, pages_count, finished, dithered,
514            zoom_mode, scroll_mode, page_offset_x, page_offset_y, rotation,
515            cropping_margins_json, margin_width, screen_margin_width,
516            font_family, font_size, text_align, line_height,
517            contrast_exponent, contrast_gray,
518            page_names_json, bookmarks_json, annotations_json
519        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
520        ON CONFLICT(fingerprint) DO NOTHING
521        "#,
522        rs.fingerprint,
523        rs.opened,
524        rs.current_page,
525        rs.pages_count,
526        rs.finished,
527        rs.dithered,
528        rs.zoom_mode,
529        rs.scroll_mode,
530        rs.page_offset_x,
531        rs.page_offset_y,
532        rs.rotation,
533        rs.cropping_margins_json,
534        rs.margin_width,
535        rs.screen_margin_width,
536        rs.font_family,
537        rs.font_size,
538        rs.text_align,
539        rs.line_height,
540        rs.contrast_exponent,
541        rs.contrast_gray,
542        rs.page_names_json,
543        rs.bookmarks_json,
544        rs.annotations_json,
545    )
546    .execute(&mut **tx)
547    .await?;
548
549    Ok(())
550}
cadmus_core/library/migrations.rs

cadmus_core/library/
migrations.rs
