fix(2meet-data-optimizer): restore native meta without overwriting newer values or inventing defaults

- TMDO_Native_Restorer::run() gains a policy: overwrite (flat wins; the
  default in aeav_only, where writes only reached flat and native is
  stale) or only_missing (native wins, flat only fills gaps; the default in
  every other mode, where native is the source of truth and may be newer —
  on dev36 an aeav_only round trip overwrote WooCommerce stock 42 with 0).
  CLI: --overwrite / --only-missing.
- Native values are read first and each owned key is classified as
  missing / differs / same / default_skipped; unchanged values are no
  longer rewritten. A flat value equal to its field default is not
  written for an entity without a native row (dev36: 1,368 invented
  session_count / account_flags rows). --dry-run reports the counts.
- Verify: NativeRestorer* 17 OK (5 new: policy by mode, forced
  overwrite, only-missing in aeav_only, dry-run counts, no invented
  defaults); unit 687 OK, integration 490 OK, phpcs (phpcs.xml) 0,
  stan OK; dev36 `bridge-restore-native user --dry-run`: policy
  only_missing, activity 120 defaults skipped
- PR: PR-058

cli/class-tmdo-cli-v2.php

@@ -162,7 +162,15 @@ class TMDO_CLI_V2 {
 	 * : post | user | term | comment
 	 *
 	 * [--dry-run]
-	 * : Count the flat rows per group; write nothing.
+	 * : Count the flat rows per group and how their values compare with native
+	 * meta (missing / differs / same / default skipped); write nothing.
+	 *
+	 * [--overwrite]
+	 * : Flat values replace differing native values. Default in aeav_only.
+	 *
+	 * [--only-missing]
+	 * : Only fill native rows that do not exist; never replace. Default in every
+	 * other mode, where native meta is the source of truth.
 	 *
 	 * [--batch=<n>]
 	 * : Rows per batch. Default 500.
@@ -190,17 +198,34 @@ class TMDO_CLI_V2 {
 	public function bridge_restore_native( $args, $assoc_args ): void {
 		$entity = (string) ( $args[0] ?? '' );
 		if ( ! in_array( $entity, TMDO_Entity_Registry::VALID_ENTITY_TYPES, true ) ) {
-			WP_CLI::error( 'Usage: wp tmdo bridge-restore-native <post|user|term|comment> [--dry-run] [--batch=<n>] [--demote] [--group=<group> [--retire]]' );
+			WP_CLI::error( 'Usage: wp tmdo bridge-restore-native <post|user|term|comment> [--dry-run] [--overwrite|--only-missing] [--batch=<n>] [--demote] [--group=<group> [--retire]]' );
 		}
 		$dry_run = isset( $assoc_args['dry-run'] );
 		$group   = sanitize_key( (string) ( $assoc_args['group'] ?? '' ) );
 		if ( isset( $assoc_args['retire'] ) && '' === $group ) {
 			WP_CLI::error( '--retire needs --group=<group>.' );
 		}
-		$report = TMDO_Native_Restorer::run( $entity, $dry_run, (int) ( $assoc_args['batch'] ?? 500 ), isset( $assoc_args['demote'] ), $group );
+		if ( isset( $assoc_args['overwrite'] ) && isset( $assoc_args['only-missing'] ) ) {
+			WP_CLI::error( 'Use either --overwrite or --only-missing, not both.' );
+		}
+		$policy = isset( $assoc_args['overwrite'] ) ? 'overwrite' : ( isset( $assoc_args['only-missing'] ) ? 'only_missing' : '' );
+		$report = TMDO_Native_Restorer::run( $entity, $dry_run, (int) ( $assoc_args['batch'] ?? 500 ), isset( $assoc_args['demote'] ), $group, $policy );
 
+		WP_CLI::log( "Policy: {$report['policy']}" );
 		foreach ( $report['groups'] as $group => $g ) {
-			WP_CLI::log( sprintf( '%s/%s: %d row(s)%s', $entity, $group, $g['rows'], $dry_run ? '' : sprintf( ', %d native value(s) written', $g['written'] ) ) );
+			WP_CLI::log(
+				sprintf(
+					'%s/%s: %d row(s) — native missing %d, differs %d, same %d, default skipped %d%s',
+					$entity,
+					$group,
+					$g['rows'],
+					$g['missing'],
+					$g['differs'],
+					$g['same'],
+					$g['default_skipped'],
+					$dry_run ? '' : sprintf( '; %d native value(s) written', $g['written'] )
+				)
+			);
 		}
 		if ( $dry_run ) {
 			WP_CLI::success( '[dry-run] Nothing written.' );

includes/migration/class-tmdo-native-restorer.php

@@ -14,6 +14,16 @@
  *     the value Type_Caster returns); a NULL value deletes the native row.
  *   - Entities with no row in the owning group's table are not touched:
  *     leftover native rows for them are outside the flat data.
+ *   - A flat value equal to its field default is not written for an entity
+ *     that has no native row (e.g. activity.session_count = 0): that would
+ *     invent native meta the entity never had.
+ *
+ * Policy (which side wins where both hold a value):
+ *   - overwrite    — flat wins; the default in aeav_only, where writes only
+ *                    reached the flat tables and native is stale.
+ *   - only_missing — native wins, flat only fills gaps; the default in every
+ *                    other mode, where native is the source of truth (and may
+ *                    be newer, e.g. WooCommerce stock written with direct SQL).
  *
  * Batched by entity id with a cursor option so an interrupted run resumes; a
  * final pass re-copies rows updated since the first run started. With
@@ -52,14 +62,19 @@ final class TMDO_Native_Restorer {
 	 * @param int    $batch       Rows per batch.
 	 * @param bool   $demote      After a complete run, switch the entity to dual_write.
 	 * @param string $group       Only this group ('' = every group).
-	 * @return array{groups: array<string, array{rows:int, written:int}>, demoted: bool}
+	 * @param string $policy      'overwrite' | 'only_missing' | '' (by the entity's mode).
+	 * @return array{groups: array<string, array{rows:int, written:int, missing:int, differs:int, same:int, default_skipped:int}>, demoted: bool, policy: string}
 	 */
-	public static function run( string $entity_type, bool $dry_run = false, int $batch = 500, bool $demote = false, string $group = '' ): array {
+	public static function run( string $entity_type, bool $dry_run = false, int $batch = 500, bool $demote = false, string $group = '', string $policy = '' ): array {
 		global $wpdb;
 
+		if ( ! in_array( $policy, array( 'overwrite', 'only_missing' ), true ) ) {
+			$policy = TMDO_Mode_Manager::MODE_AEAV_ONLY === TMDO_Mode_Manager::get( $entity_type ) ? 'overwrite' : 'only_missing';
+		}
 		$report  = array(
 			'groups'  => array(),
 			'demoted' => false,
+			'policy'  => $policy,
 		);
 		$adapter = TMDO_Entity_Registry::get_adapter( $entity_type );
 		if ( ! $adapter ) {
@@ -83,8 +98,12 @@ final class TMDO_Native_Restorer {
 		}
 		foreach ( $tables as $group => $table ) {
 			$entry = array(
-				'rows'    => 0,
+				'rows'            => 0,
-				'written' => 0,
+				'written'         => 0,
+				'missing'         => 0,
+				'differs'         => 0,
+				'same'            => 0,
+				'default_skipped' => 0,
 			);
 			$key   = "{$entity_type}/{$group}";
 			$last  = $dry_run ? 0 : (int) ( $cursor[ $key ] ?? 0 );
@@ -94,9 +113,11 @@ final class TMDO_Native_Restorer {
 				$entry['rows'] += $fetched;
 				if ( $fetched ) {
 					$last = (int) end( $rows )[ $id_col ];
+					foreach ( self::apply_rows( $entity_type, $group, $adapter, $rows, $policy, ! $dry_run ) as $count_key => $n ) {
+						$entry[ $count_key ] += $n;
+					}
 					if ( ! $dry_run ) {
-						$entry['written'] += self::write_rows( $entity_type, $group, $adapter, $rows );
+						$cursor[ $key ] = $last;
-						$cursor[ $key ]    = $last;
 						update_option( self::CURSOR_OPTION, $cursor, false );
 					}
 				}
@@ -112,7 +133,7 @@ final class TMDO_Native_Restorer {
 		// Catch-up: writes that landed in the flat tables after a batch passed their row.
 		foreach ( $tables as $group => $table ) {
 			$rows = self::fetch( $entity_type, $table, $id_col, 't.updated_at >= %s', array( $started[ $entity_type ] ) );
-			self::write_rows( $entity_type, $group, $adapter, $rows );
+			self::apply_rows( $entity_type, $group, $adapter, $rows, $policy, true );
 		}
 
 		foreach ( array_keys( $tables ) as $group ) {
@@ -235,24 +256,87 @@ final class TMDO_Native_Restorer {
 	}
 
 	/**
-	 * Replace the native meta of the keys this group owns for each row.
+	 * Compare the native meta of the keys this group owns with the flat rows,
+	 * and (when $write) bring native in line according to $policy.
 	 *
 	 * @param string                           $entity_type Entity type.
 	 * @param string                           $group       Group the rows come from.
 	 * @param TMDO_Entity_Adapter_Interface    $adapter     Entity adapter.
 	 * @param array<int, array<string, mixed>> $rows        Flat rows.
-	 * @return int Native values written.
+	 * @param string                           $policy      'overwrite' | 'only_missing'.
+	 * @param bool                             $write       False = classify only (dry run).
+	 * @return array{written:int, missing:int, differs:int, same:int, default_skipped:int}
 	 */
-	private static function write_rows( string $entity_type, string $group, TMDO_Entity_Adapter_Interface $adapter, array $rows ): int {
+	private static function apply_rows( string $entity_type, string $group, TMDO_Entity_Adapter_Interface $adapter, array $rows, string $policy, bool $write ): array {
-		global $wpdb;
+		$counts                  = array(
-		if ( empty( $rows ) ) {
+			'written'         => 0,
-			return 0;
+			'missing'         => 0,
+			'differs'         => 0,
+			'same'            => 0,
+			'default_skipped' => 0,
+		);
+		list( $wanted, $fields ) = self::flat_values( $entity_type, $group, $adapter, $rows );
+		if ( empty( $wanted ) ) {
+			return $counts;
 		}
-		$meta_table = $adapter->get_native_meta_table();
+		$native = self::native_values( $entity_type, $adapter, array_keys( $wanted ), array_keys( $fields ) );
-		$id_col     = $adapter->get_entity_id_column();
 
 		$deletes = array();
 		$inserts = array();
+		foreach ( $wanted as $id => $values ) {
+			foreach ( $values as $key => $value ) {
+				$has = isset( $native[ $id ] ) && array_key_exists( $key, $native[ $id ] );
+				if ( null === $value ) {
+					if ( $has ) {
+						++$counts['differs'];
+						if ( 'overwrite' === $policy ) {
+							$deletes[ $key ][] = $id;
+						}
+					}
+					continue;
+				}
+				if ( ! $has ) {
+					if ( self::is_default( $value, $fields[ $key ] ) ) {
+						++$counts['default_skipped'];
+						continue;
+					}
+					++$counts['missing'];
+					$inserts[] = array( $id, $key, $value );
+					continue;
+				}
+				if ( self::same_value( $native[ $id ][ $key ], $value ) ) {
+					++$counts['same'];
+					continue;
+				}
+				++$counts['differs'];
+				if ( 'overwrite' === $policy ) {
+					$deletes[ $key ][] = $id;
+					$inserts[]         = array( $id, $key, $value );
+				}
+			}
+		}
+
+		if ( $write ) {
+			self::write_native( $entity_type, $adapter, $deletes, $inserts, array_keys( $wanted ) );
+			$counts['written'] = count( $inserts );
+		}
+		return $counts;
+	}
+
+	/**
+	 * Values this group would write to native, per entity and key (NULL = none),
+	 * restricted to the keys the group owns for that entity.
+	 *
+	 * @param string                           $entity_type Entity type.
+	 * @param string                           $group       Group name.
+	 * @param TMDO_Entity_Adapter_Interface    $adapter     Entity adapter.
+	 * @param array<int, array<string, mixed>> $rows        Flat rows.
+	 * @return array{0: array<int, array<string, string|null>>, 1: array<string, array<string, mixed>>}
+	 */
+	private static function flat_values( string $entity_type, string $group, TMDO_Entity_Adapter_Interface $adapter, array $rows ): array {
+		$id_col = $adapter->get_entity_id_column();
+		$wanted = array();
+		$fields = array();
 		foreach ( $rows as $row ) {
 			$id        = (int) $row[ $id_col ];
 			$post_type = 'post' === $entity_type ? (string) ( $row['__post_type'] ?? '' ) : '';
@@ -262,25 +346,93 @@ final class TMDO_Native_Restorer {
 				if ( null === $owner || $owner['group'] !== $group || ! array_key_exists( $col, $row ) ) {
 					continue;
 				}
-				$deletes[ $field['key'] ][] = $id;
+				$fields[ $field['key'] ]        = $field;
-				if ( null !== $row[ $col ] ) {
+				$wanted[ $id ][ $field['key'] ] = null === $row[ $col ]
-					$inserts[] = array( $id, $field['key'], (string) maybe_serialize( TMDO_Type_Caster::from_db( $row[ $col ], $field ) ) );
+					? null
-				}
+					: (string) maybe_serialize( TMDO_Type_Caster::from_db( $row[ $col ], $field ) );
 			}
 		}
+		return array( $wanted, $fields );
+	}
 
-		foreach ( $deletes as $meta_key => $ids ) {
+	/**
-			$in = implode( ',', array_map( 'intval', $ids ) );
+	 * First native value per entity and key.
+	 *
+	 * @param string                        $entity_type Entity type.
+	 * @param TMDO_Entity_Adapter_Interface $adapter     Entity adapter.
+	 * @param int[]                         $ids         Entity IDs.
+	 * @param string[]                      $keys        Meta keys.
+	 * @return array<int, array<string, string>>
+	 */
+	private static function native_values( string $entity_type, TMDO_Entity_Adapter_Interface $adapter, array $ids, array $keys ): array {
+		global $wpdb;
+		$meta_table = $adapter->get_native_meta_table();
+		$id_col     = $adapter->get_entity_id_column();
+		$pk         = 'user' === $entity_type ? 'umeta_id' : 'meta_id';
+		$in_ids     = implode( ',', array_map( 'intval', $ids ) );
+		$in_keys    = implode( ',', array_fill( 0, count( $keys ), '%s' ) );
+		$rows       = (array) $wpdb->get_results(
+			$wpdb->prepare( "SELECT `{$id_col}` AS id, meta_key, meta_value FROM `{$meta_table}` WHERE `{$id_col}` IN ({$in_ids}) AND meta_key IN ({$in_keys}) ORDER BY `{$pk}`", ...$keys ),
+			ARRAY_A
+		);
+
+		$native = array();
+		foreach ( $rows as $r ) {
+			if ( ! isset( $native[ (int) $r['id'] ][ $r['meta_key'] ] ) ) {
+				$native[ (int) $r['id'] ][ $r['meta_key'] ] = (string) $r['meta_value'];
+			}
+		}
+		return $native;
+	}
+
+	/**
+	 * Delete / insert native rows and clear the entities' meta caches.
+	 *
+	 * @param string                                       $entity_type Entity type.
+	 * @param TMDO_Entity_Adapter_Interface                $adapter     Entity adapter.
+	 * @param array<string, int[]>                         $deletes     Meta key → entity IDs to clear first.
+	 * @param array<int, array{0:int, 1:string, 2:string}> $inserts Rows to insert.
+	 * @param int[]                                        $ids         Entities touched.
+	 * @return void
+	 */
+	private static function write_native( string $entity_type, TMDO_Entity_Adapter_Interface $adapter, array $deletes, array $inserts, array $ids ): void {
+		global $wpdb;
+		$meta_table = $adapter->get_native_meta_table();
+		$id_col     = $adapter->get_entity_id_column();
+
+		foreach ( $deletes as $meta_key => $del_ids ) {
+			$in = implode( ',', array_map( 'intval', $del_ids ) );
 			$wpdb->query( $wpdb->prepare( "DELETE FROM `{$meta_table}` WHERE meta_key = %s AND `{$id_col}` IN ({$in})", $meta_key ) );
 		}
 		if ( $inserts ) {
 			$values = implode( ', ', array_fill( 0, count( $inserts ), '(%d, %s, %s)' ) );
 			$wpdb->query( $wpdb->prepare( "INSERT INTO `{$meta_table}` (`{$id_col}`, meta_key, meta_value) VALUES {$values}", ...array_merge( ...$inserts ) ) );
 		}
-
+		foreach ( $ids as $id ) {
-		foreach ( array_unique( array_map( static fn( $row ) => (int) $row[ $id_col ], $rows ) ) as $id ) {
 			wp_cache_delete( $id, "{$entity_type}_meta" );
 		}
-		return count( $inserts );
+	}
+
+	/**
+	 * Whether a stored value (as WP would store it) equals the field default.
+	 *
+	 * @param string               $value Value as it would be stored.
+	 * @param array<string, mixed> $field Field definition.
+	 * @return bool
+	 */
+	private static function is_default( string $value, array $field ): bool {
+		return isset( $field['default'] ) && (string) maybe_serialize( $field['default'] ) === $value;
+	}
+
+	/**
+	 * Native and candidate values are equal (numeric-aware: '3' equals '3.000000').
+	 *
+	 * @param string $native    Native meta value.
+	 * @param string $candidate Value from the flat table.
+	 * @return bool
+	 */
+	private static function same_value( string $native, string $candidate ): bool {
+		return $native === $candidate
+			|| ( is_numeric( $native ) && is_numeric( $candidate ) && (float) $native === (float) $candidate );
 	}
 }

tests/integration/NativeRestorerTest.php

@@ -139,4 +139,77 @@ class NativeRestorerTest extends TestCase {
 
 		$this->assertSame( array( '9' ), $this->native( 1, 'level' ) );
 	}
+
+	private function set_user_mode( string $mode ): void {
+		$GLOBALS['_wp_options']['wpdo_bridge_modes'] = array( 'user' => $mode );
+		TMDO_Mode_Manager::reset_cache();
+	}
+
+	/**
+	 * Native is the source of truth outside aeav_only (e.g. WooCommerce writes
+	 * stock with direct SQL): the default policy there only fills gaps.
+	 */
+	public function test_native_authoritative_modes_only_fill_missing_rows(): void {
+		$this->set_user_mode( 'dual_write' );
+
+		$report = TMDO_Native_Restorer::run( 'user', false );
+
+		$this->assertSame( array( '1' ), $this->native( 1, 'level' ), 'differing native value kept' );
+		$this->assertSame( array( serialize( array( 'a', 'b' ) ) ), $this->native( 1, 'tags' ), 'missing native row filled' );
+		$this->assertSame( array( '5' ), $this->native( 2, 'level' ) );
+		$this->assertSame( array( 'stale' ), $this->native( 2, 'tags' ), 'NULL flat value does not delete native here' );
+		$this->assertSame( 'only_missing', $report['policy'] );
+		$this->assertSame( 2, $report['groups']['rn_profile']['missing'] );
+		$this->assertSame( 2, $report['groups']['rn_profile']['differs'] );
+	}
+
+	public function test_overwrite_policy_replaces_native_in_any_mode(): void {
+		$this->set_user_mode( 'dual_write' );
+
+		TMDO_Native_Restorer::run( 'user', false, 500, false, '', 'overwrite' );
+
+		$this->assertSame( array( '3' ), $this->native( 1, 'level' ) );
+		$this->assertSame( array(), $this->native( 2, 'tags' ) );
+	}
+
+	public function test_only_missing_policy_keeps_native_in_aeav_only(): void {
+		TMDO_Native_Restorer::run( 'user', false, 500, false, '', 'only_missing' );
+
+		$this->assertSame( array( '1' ), $this->native( 1, 'level' ) );
+		$this->assertSame( array( serialize( array( 'a', 'b' ) ) ), $this->native( 1, 'tags' ) );
+	}
+
+	public function test_dry_run_classifies_without_writing(): void {
+		$report = TMDO_Native_Restorer::run( 'user', true );
+
+		$this->assertSame( 'overwrite', $report['policy'], 'aeav_only: flat is the source of truth' );
+		$this->assertSame( 2, $report['groups']['rn_profile']['missing'] );
+		$this->assertSame( 2, $report['groups']['rn_profile']['differs'] );
+		$this->assertSame( 0, $report['groups']['rn_profile']['same'] );
+		$this->assertSame( array( '1' ), $this->native( 1, 'level' ) );
+	}
+
+	/**
+	 * A flat column holding its field default for an entity that never had
+	 * the key (e.g. activity.session_count = 0) is not written back.
+	 */
+	public function test_defaults_are_not_invented_as_native_rows(): void {
+		global $wpdb;
+		$table = 'wp_itest_wpdo_user_rn_counts';
+		$wpdb->query( "DROP TABLE IF EXISTS `{$table}`" );
+		$wpdb->query( "CREATE TABLE `{$table}` ( id BIGINT NOT NULL AUTO_INCREMENT, user_id BIGINT NOT NULL, visits BIGINT(20) DEFAULT 0, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), UNIQUE KEY uk_entity (user_id) ) ENGINE=InnoDB" );
+		$wpdb->query( "INSERT INTO `{$table}` (user_id, visits, updated_at) VALUES (1, 0, '2020-01-01'), (2, 7, '2020-01-01')" );
+		WPDO_Entity_Registry::register_group( 'user', 'rn_counts', array( array( 'key' => 'visits', 'type' => 'integer', 'default' => 0 ) ) );
+		WPDO_Schema_Manager::flush_table_exists_cache();
+
+		try {
+			$report = TMDO_Native_Restorer::run( 'user', false, 500, false, 'rn_counts' );
+
+			$this->assertSame( array(), $this->native( 1, 'visits' ), 'default 0 not invented' );
+			$this->assertSame( array( '7' ), $this->native( 2, 'visits' ) );
+			$this->assertSame( 1, $report['groups']['rn_counts']['default_skipped'] );
+		} finally {
+			$wpdb->query( "DROP TABLE IF EXISTS `{$table}`" );
+		}
+	}
 }