release: 7.0.13

Author: taariq

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 
 ## [7.0.12] - 2025-12-11
 
+## [7.0.13] - 2025-12-11
+
+### Fixed
+
+- **SQLite re-runs are now deterministic**: `truncate_jsonb_table()` now logs at INFO level, verifies the target table is empty, and errors early if rows remain instead of failing later with duplicate primary keys.
+- **`--drop-existing` works for SQLite**: When the flag is supplied, each JSONB table is dropped before recreation, ensuring a pristine schema even if manual changes were made to the target database.
+
+### Added
+
+- **`drop_jsonb_table()` helper**: Shared helper allows future workflows (and the SQLite path today) to explicitly remove JSONB tables when a clean rebuild is required.
+
 ### Added
 
 - **Memory-efficient SQLite migration**: SQLite to PostgreSQL migration now processes rows in batches instead of loading entire tables into memory. This enables migration of large SQLite databases (7M+ rows, multi-GB files) without OOM errors. Batch size automatically adjusts based on available system memory.

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "database-replicator"
-version = "7.0.12"
+version = "7.0.13"
 edition = "2021"
 license = "Apache-2.0"
 description = "Universal database-to-PostgreSQL replication CLI. Supports PostgreSQL, SQLite, MongoDB, and MySQL."

Cargo.toml

@@ -560,7 +560,7 @@ No. The tool opens SQLite databases in **read-only mode**. It's impossible to mo
 
 ### Can I replicate the same database twice?
 
-Re-running `init` will **drop and recreate** tables. All data will be replaced with fresh data from SQLite. This is useful for:
+Re-running `init` will **clear** previously migrated tables (via truncate by default or a full drop when `--drop-existing` is supplied). All data will be replaced with fresh data from SQLite. This is useful for:
 - Correcting errors in the first migration
 - Refreshing data from an updated SQLite snapshot
 
@@ -570,7 +570,7 @@ SQLite replications are snapshot-only. For incremental updates:
 
 1. **Option 1**: Periodic full re-migration
    - Create SQLite backup/snapshot
-   - Re-run `init` with `--drop-existing` flag (implied in init)
+   - Re-run `init` (add `--drop-existing` to drop/recreate tables instead of truncate)
 
 2. **Option 2**: Track changes in application
    - Maintain updated_at timestamps

README-SQLite.md

@@ -153,15 +153,17 @@ pub async fn init(
                 );
             }
             if drop_existing {
-                tracing::warn!("⚠ --drop-existing flag is not applicable for SQLite sources");
+                tracing::info!(
+                    "--drop-existing: existing JSONB tables on the target will be dropped"
+                );
             }
             if !enable_sync {
                 tracing::warn!(
                     "⚠ SQLite sources don't support continuous replication (one-time migration only)"
                 );
             }
 
-            return init_sqlite_to_postgres(source_url, target_url).await;
+            return init_sqlite_to_postgres(source_url, target_url, drop_existing).await;
         }
         crate::SourceType::MongoDB => {
             // MongoDB to PostgreSQL migration (simpler path)
@@ -853,6 +855,7 @@ async fn drop_database_if_exists(target_conn: &Client, db_name: &str) -> Result<
 ///
 /// * `sqlite_path` - Path to SQLite database file (.db, .sqlite, or .sqlite3)
 /// * `target_url` - PostgreSQL connection string for target (Seren) database
+/// * `drop_existing` - Drop any existing JSONB tables on the target before migrating
 ///
 /// # Returns
 ///
@@ -874,12 +877,17 @@ async fn drop_database_if_exists(target_conn: &Client, db_name: &str) -> Result<
 /// # async fn example() -> Result<()> {
 /// init_sqlite_to_postgres(
 ///     "database.db",
-///     "postgresql://user:pass@seren.example.com/targetdb"
+///     "postgresql://user:pass@seren.example.com/targetdb",
+///     false,
 /// ).await?;
 /// # Ok(())
 /// # }
 /// ```
-pub async fn init_sqlite_to_postgres(sqlite_path: &str, target_url: &str) -> Result<()> {
+pub async fn init_sqlite_to_postgres(
+    sqlite_path: &str,
+    target_url: &str,
+    drop_existing: bool,
+) -> Result<()> {
     tracing::info!("Starting SQLite to PostgreSQL migration...");
 
     // Step 1: Validate SQLite file
@@ -940,6 +948,12 @@ pub async fn init_sqlite_to_postgres(sqlite_path: &str, target_url: &str) -> Res
             row_count
         );
 
+        if drop_existing {
+            crate::jsonb::writer::drop_jsonb_table(&target_client, table_name)
+                .await
+                .with_context(|| format!("Failed to drop existing JSONB table '{}'", table_name))?;
+        }
+
         // Create JSONB table in PostgreSQL
         crate::jsonb::writer::create_jsonb_table(&target_client, table_name, "sqlite")
             .await

src/commands/init.rs

@@ -1,7 +1,7 @@
 // ABOUTME: Write JSONB data to PostgreSQL with metadata
 // ABOUTME: Handles table creation, single row inserts, and batch inserts
 
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
 use tokio_postgres::Client;
 
 /// Create a table with JSONB schema for storing non-PostgreSQL data
@@ -126,7 +126,7 @@ pub async fn truncate_jsonb_table(client: &Client, table_name: &str) -> Result<(
     crate::jsonb::validate_table_name(table_name)
         .context("Invalid table name for JSONB table truncation")?;
 
-    tracing::debug!("Truncating JSONB table '{}'", table_name);
+    tracing::info!("Truncating JSONB table '{}'", table_name);
 
     let truncate_sql = format!(
         r#"TRUNCATE TABLE "{}" RESTART IDENTITY CASCADE"#,
@@ -138,7 +138,45 @@ pub async fn truncate_jsonb_table(client: &Client, table_name: &str) -> Result<(
         .await
         .with_context(|| format!("Failed to truncate JSONB table '{}'", table_name))?;
 
-    tracing::debug!("Truncated JSONB table '{}'", table_name);
+    let verify_sql = format!(r#"SELECT COUNT(*) FROM "{}""#, table_name);
+    let remaining_rows: i64 = client
+        .query_one(&verify_sql, &[])
+        .await
+        .with_context(|| format!("Failed to verify truncate of '{}'", table_name))?
+        .get(0);
+
+    if remaining_rows > 0 {
+        bail!(
+            "Truncate verification failed: table '{}' still has {} rows after truncate",
+            table_name,
+            remaining_rows
+        );
+    }
+
+    tracing::info!(
+        "Truncated JSONB table '{}' successfully ({} rows remaining)",
+        table_name,
+        remaining_rows
+    );
+
+    Ok(())
+}
+
+/// Drop a JSONB table if it exists.
+pub async fn drop_jsonb_table(client: &Client, table_name: &str) -> Result<()> {
+    crate::jsonb::validate_table_name(table_name)
+        .context("Invalid table name for JSONB table drop")?;
+
+    tracing::info!("Dropping JSONB table '{}'", table_name);
+
+    let drop_sql = format!(r#"DROP TABLE IF EXISTS "{}" CASCADE"#, table_name);
+
+    client
+        .execute(&drop_sql, &[])
+        .await
+        .with_context(|| format!("Failed to drop JSONB table '{}'", table_name))?;
+
+    tracing::info!("Dropped JSONB table '{}' (if it existed)", table_name);
 
     Ok(())
 }