From 8c049f0cf25b57a3ade17ecb26420ac7e20cf452 Mon Sep 17 00:00:00 2001 From: wpdev Date: Sat, 26 Sep 2026 19:22:34 +0800 Subject: [PATCH] fix(2meet-data-optimizer): restore native meta without overwriting newer values or inventing defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TMDO_Native_Restorer::run() gains a policy: overwrite (flat wins; the default in aeav_only, where writes only reached flat and native is stale) or only_missing (native wins, flat only fills gaps; the default in every other mode, where native is the source of truth and may be newer — on dev36 an aeav_only round trip overwrote WooCommerce stock 42 with 0). CLI: --overwrite / --only-missing. - Native values are read first and each owned key is classified as missing / differs / same / default_skipped; unchanged values are no longer rewritten. A flat value equal to its field default is not written for an entity without a native row (dev36: 1,368 invented session_count / account_flags rows). --dry-run reports the counts. - Verify: NativeRestorer* 17 OK (5 new: policy by mode, forced overwrite, only-missing in aeav_only, dry-run counts, no invented defaults); unit 687 OK, integration 490 OK, phpcs (phpcs.xml) 0, stan OK; dev36 `bridge-restore-native user --dry-run`: policy only_missing, activity 120 defaults skipped - PR: PR-058 --- cli/class-tmdo-cli-v2.php | 33 ++- .../migration/class-tmdo-native-restorer.php | 200 +++++++++++++++--- tests/integration/NativeRestorerTest.php | 73 +++++++ 3 files changed, 278 insertions(+), 28 deletions(-) diff --git a/cli/class-tmdo-cli-v2.php b/cli/class-tmdo-cli-v2.php index bdc728f..fe6c07e 100644 --- a/cli/class-tmdo-cli-v2.php +++ b/cli/class-tmdo-cli-v2.php @@ -162,7 +162,15 @@ class TMDO_CLI_V2 { * : post | user | term | comment * * [--dry-run] - * : Count the flat rows per group; write nothing. + * : Count the flat rows per group and how their values compare with native + * meta (missing / differs / same / default skipped); write nothing. + * + * [--overwrite] + * : Flat values replace differing native values. Default in aeav_only. + * + * [--only-missing] + * : Only fill native rows that do not exist; never replace. Default in every + * other mode, where native meta is the source of truth. * * [--batch=] * : Rows per batch. Default 500. @@ -190,17 +198,34 @@ class TMDO_CLI_V2 { public function bridge_restore_native( $args, $assoc_args ): void { $entity = (string) ( $args[0] ?? '' ); if ( ! in_array( $entity, TMDO_Entity_Registry::VALID_ENTITY_TYPES, true ) ) { - WP_CLI::error( 'Usage: wp tmdo bridge-restore-native [--dry-run] [--batch=] [--demote] [--group= [--retire]]' ); + WP_CLI::error( 'Usage: wp tmdo bridge-restore-native [--dry-run] [--overwrite|--only-missing] [--batch=] [--demote] [--group= [--retire]]' ); } $dry_run = isset( $assoc_args['dry-run'] ); $group = sanitize_key( (string) ( $assoc_args['group'] ?? '' ) ); if ( isset( $assoc_args['retire'] ) && '' === $group ) { WP_CLI::error( '--retire needs --group=.' ); } - $report = TMDO_Native_Restorer::run( $entity, $dry_run, (int) ( $assoc_args['batch'] ?? 500 ), isset( $assoc_args['demote'] ), $group ); + if ( isset( $assoc_args['overwrite'] ) && isset( $assoc_args['only-missing'] ) ) { + WP_CLI::error( 'Use either --overwrite or --only-missing, not both.' ); + } + $policy = isset( $assoc_args['overwrite'] ) ? 'overwrite' : ( isset( $assoc_args['only-missing'] ) ? 'only_missing' : '' ); + $report = TMDO_Native_Restorer::run( $entity, $dry_run, (int) ( $assoc_args['batch'] ?? 500 ), isset( $assoc_args['demote'] ), $group, $policy ); + WP_CLI::log( "Policy: {$report['policy']}" ); foreach ( $report['groups'] as $group => $g ) { - WP_CLI::log( sprintf( '%s/%s: %d row(s)%s', $entity, $group, $g['rows'], $dry_run ? '' : sprintf( ', %d native value(s) written', $g['written'] ) ) ); + WP_CLI::log( + sprintf( + '%s/%s: %d row(s) — native missing %d, differs %d, same %d, default skipped %d%s', + $entity, + $group, + $g['rows'], + $g['missing'], + $g['differs'], + $g['same'], + $g['default_skipped'], + $dry_run ? '' : sprintf( '; %d native value(s) written', $g['written'] ) + ) + ); } if ( $dry_run ) { WP_CLI::success( '[dry-run] Nothing written.' ); diff --git a/includes/migration/class-tmdo-native-restorer.php b/includes/migration/class-tmdo-native-restorer.php index 0fa3cf8..15f9bbd 100644 --- a/includes/migration/class-tmdo-native-restorer.php +++ b/includes/migration/class-tmdo-native-restorer.php @@ -14,6 +14,16 @@ * the value Type_Caster returns); a NULL value deletes the native row. * - Entities with no row in the owning group's table are not touched: * leftover native rows for them are outside the flat data. + * - A flat value equal to its field default is not written for an entity + * that has no native row (e.g. activity.session_count = 0): that would + * invent native meta the entity never had. + * + * Policy (which side wins where both hold a value): + * - overwrite — flat wins; the default in aeav_only, where writes only + * reached the flat tables and native is stale. + * - only_missing — native wins, flat only fills gaps; the default in every + * other mode, where native is the source of truth (and may + * be newer, e.g. WooCommerce stock written with direct SQL). * * Batched by entity id with a cursor option so an interrupted run resumes; a * final pass re-copies rows updated since the first run started. With @@ -52,14 +62,19 @@ final class TMDO_Native_Restorer { * @param int $batch Rows per batch. * @param bool $demote After a complete run, switch the entity to dual_write. * @param string $group Only this group ('' = every group). - * @return array{groups: array, demoted: bool} + * @param string $policy 'overwrite' | 'only_missing' | '' (by the entity's mode). + * @return array{groups: array, demoted: bool, policy: string} */ - public static function run( string $entity_type, bool $dry_run = false, int $batch = 500, bool $demote = false, string $group = '' ): array { + public static function run( string $entity_type, bool $dry_run = false, int $batch = 500, bool $demote = false, string $group = '', string $policy = '' ): array { global $wpdb; + if ( ! in_array( $policy, array( 'overwrite', 'only_missing' ), true ) ) { + $policy = TMDO_Mode_Manager::MODE_AEAV_ONLY === TMDO_Mode_Manager::get( $entity_type ) ? 'overwrite' : 'only_missing'; + } $report = array( 'groups' => array(), 'demoted' => false, + 'policy' => $policy, ); $adapter = TMDO_Entity_Registry::get_adapter( $entity_type ); if ( ! $adapter ) { @@ -83,8 +98,12 @@ final class TMDO_Native_Restorer { } foreach ( $tables as $group => $table ) { $entry = array( - 'rows' => 0, - 'written' => 0, + 'rows' => 0, + 'written' => 0, + 'missing' => 0, + 'differs' => 0, + 'same' => 0, + 'default_skipped' => 0, ); $key = "{$entity_type}/{$group}"; $last = $dry_run ? 0 : (int) ( $cursor[ $key ] ?? 0 ); @@ -94,9 +113,11 @@ final class TMDO_Native_Restorer { $entry['rows'] += $fetched; if ( $fetched ) { $last = (int) end( $rows )[ $id_col ]; + foreach ( self::apply_rows( $entity_type, $group, $adapter, $rows, $policy, ! $dry_run ) as $count_key => $n ) { + $entry[ $count_key ] += $n; + } if ( ! $dry_run ) { - $entry['written'] += self::write_rows( $entity_type, $group, $adapter, $rows ); - $cursor[ $key ] = $last; + $cursor[ $key ] = $last; update_option( self::CURSOR_OPTION, $cursor, false ); } } @@ -112,7 +133,7 @@ final class TMDO_Native_Restorer { // Catch-up: writes that landed in the flat tables after a batch passed their row. foreach ( $tables as $group => $table ) { $rows = self::fetch( $entity_type, $table, $id_col, 't.updated_at >= %s', array( $started[ $entity_type ] ) ); - self::write_rows( $entity_type, $group, $adapter, $rows ); + self::apply_rows( $entity_type, $group, $adapter, $rows, $policy, true ); } foreach ( array_keys( $tables ) as $group ) { @@ -235,24 +256,87 @@ final class TMDO_Native_Restorer { } /** - * Replace the native meta of the keys this group owns for each row. + * Compare the native meta of the keys this group owns with the flat rows, + * and (when $write) bring native in line according to $policy. * * @param string $entity_type Entity type. * @param string $group Group the rows come from. * @param TMDO_Entity_Adapter_Interface $adapter Entity adapter. * @param array> $rows Flat rows. - * @return int Native values written. + * @param string $policy 'overwrite' | 'only_missing'. + * @param bool $write False = classify only (dry run). + * @return array{written:int, missing:int, differs:int, same:int, default_skipped:int} */ - private static function write_rows( string $entity_type, string $group, TMDO_Entity_Adapter_Interface $adapter, array $rows ): int { - global $wpdb; - if ( empty( $rows ) ) { - return 0; + private static function apply_rows( string $entity_type, string $group, TMDO_Entity_Adapter_Interface $adapter, array $rows, string $policy, bool $write ): array { + $counts = array( + 'written' => 0, + 'missing' => 0, + 'differs' => 0, + 'same' => 0, + 'default_skipped' => 0, + ); + list( $wanted, $fields ) = self::flat_values( $entity_type, $group, $adapter, $rows ); + if ( empty( $wanted ) ) { + return $counts; } - $meta_table = $adapter->get_native_meta_table(); - $id_col = $adapter->get_entity_id_column(); + $native = self::native_values( $entity_type, $adapter, array_keys( $wanted ), array_keys( $fields ) ); $deletes = array(); $inserts = array(); + foreach ( $wanted as $id => $values ) { + foreach ( $values as $key => $value ) { + $has = isset( $native[ $id ] ) && array_key_exists( $key, $native[ $id ] ); + if ( null === $value ) { + if ( $has ) { + ++$counts['differs']; + if ( 'overwrite' === $policy ) { + $deletes[ $key ][] = $id; + } + } + continue; + } + if ( ! $has ) { + if ( self::is_default( $value, $fields[ $key ] ) ) { + ++$counts['default_skipped']; + continue; + } + ++$counts['missing']; + $inserts[] = array( $id, $key, $value ); + continue; + } + if ( self::same_value( $native[ $id ][ $key ], $value ) ) { + ++$counts['same']; + continue; + } + ++$counts['differs']; + if ( 'overwrite' === $policy ) { + $deletes[ $key ][] = $id; + $inserts[] = array( $id, $key, $value ); + } + } + } + + if ( $write ) { + self::write_native( $entity_type, $adapter, $deletes, $inserts, array_keys( $wanted ) ); + $counts['written'] = count( $inserts ); + } + return $counts; + } + + /** + * Values this group would write to native, per entity and key (NULL = none), + * restricted to the keys the group owns for that entity. + * + * @param string $entity_type Entity type. + * @param string $group Group name. + * @param TMDO_Entity_Adapter_Interface $adapter Entity adapter. + * @param array> $rows Flat rows. + * @return array{0: array>, 1: array>} + */ + private static function flat_values( string $entity_type, string $group, TMDO_Entity_Adapter_Interface $adapter, array $rows ): array { + $id_col = $adapter->get_entity_id_column(); + $wanted = array(); + $fields = array(); foreach ( $rows as $row ) { $id = (int) $row[ $id_col ]; $post_type = 'post' === $entity_type ? (string) ( $row['__post_type'] ?? '' ) : ''; @@ -262,25 +346,93 @@ final class TMDO_Native_Restorer { if ( null === $owner || $owner['group'] !== $group || ! array_key_exists( $col, $row ) ) { continue; } - $deletes[ $field['key'] ][] = $id; - if ( null !== $row[ $col ] ) { - $inserts[] = array( $id, $field['key'], (string) maybe_serialize( TMDO_Type_Caster::from_db( $row[ $col ], $field ) ) ); - } + $fields[ $field['key'] ] = $field; + $wanted[ $id ][ $field['key'] ] = null === $row[ $col ] + ? null + : (string) maybe_serialize( TMDO_Type_Caster::from_db( $row[ $col ], $field ) ); } } + return array( $wanted, $fields ); + } - foreach ( $deletes as $meta_key => $ids ) { - $in = implode( ',', array_map( 'intval', $ids ) ); + /** + * First native value per entity and key. + * + * @param string $entity_type Entity type. + * @param TMDO_Entity_Adapter_Interface $adapter Entity adapter. + * @param int[] $ids Entity IDs. + * @param string[] $keys Meta keys. + * @return array> + */ + private static function native_values( string $entity_type, TMDO_Entity_Adapter_Interface $adapter, array $ids, array $keys ): array { + global $wpdb; + $meta_table = $adapter->get_native_meta_table(); + $id_col = $adapter->get_entity_id_column(); + $pk = 'user' === $entity_type ? 'umeta_id' : 'meta_id'; + $in_ids = implode( ',', array_map( 'intval', $ids ) ); + $in_keys = implode( ',', array_fill( 0, count( $keys ), '%s' ) ); + $rows = (array) $wpdb->get_results( + $wpdb->prepare( "SELECT `{$id_col}` AS id, meta_key, meta_value FROM `{$meta_table}` WHERE `{$id_col}` IN ({$in_ids}) AND meta_key IN ({$in_keys}) ORDER BY `{$pk}`", ...$keys ), + ARRAY_A + ); + + $native = array(); + foreach ( $rows as $r ) { + if ( ! isset( $native[ (int) $r['id'] ][ $r['meta_key'] ] ) ) { + $native[ (int) $r['id'] ][ $r['meta_key'] ] = (string) $r['meta_value']; + } + } + return $native; + } + + /** + * Delete / insert native rows and clear the entities' meta caches. + * + * @param string $entity_type Entity type. + * @param TMDO_Entity_Adapter_Interface $adapter Entity adapter. + * @param array $deletes Meta key → entity IDs to clear first. + * @param array $inserts Rows to insert. + * @param int[] $ids Entities touched. + * @return void + */ + private static function write_native( string $entity_type, TMDO_Entity_Adapter_Interface $adapter, array $deletes, array $inserts, array $ids ): void { + global $wpdb; + $meta_table = $adapter->get_native_meta_table(); + $id_col = $adapter->get_entity_id_column(); + + foreach ( $deletes as $meta_key => $del_ids ) { + $in = implode( ',', array_map( 'intval', $del_ids ) ); $wpdb->query( $wpdb->prepare( "DELETE FROM `{$meta_table}` WHERE meta_key = %s AND `{$id_col}` IN ({$in})", $meta_key ) ); } if ( $inserts ) { $values = implode( ', ', array_fill( 0, count( $inserts ), '(%d, %s, %s)' ) ); $wpdb->query( $wpdb->prepare( "INSERT INTO `{$meta_table}` (`{$id_col}`, meta_key, meta_value) VALUES {$values}", ...array_merge( ...$inserts ) ) ); } - - foreach ( array_unique( array_map( static fn( $row ) => (int) $row[ $id_col ], $rows ) ) as $id ) { + foreach ( $ids as $id ) { wp_cache_delete( $id, "{$entity_type}_meta" ); } - return count( $inserts ); + } + + /** + * Whether a stored value (as WP would store it) equals the field default. + * + * @param string $value Value as it would be stored. + * @param array $field Field definition. + * @return bool + */ + private static function is_default( string $value, array $field ): bool { + return isset( $field['default'] ) && (string) maybe_serialize( $field['default'] ) === $value; + } + + /** + * Native and candidate values are equal (numeric-aware: '3' equals '3.000000'). + * + * @param string $native Native meta value. + * @param string $candidate Value from the flat table. + * @return bool + */ + private static function same_value( string $native, string $candidate ): bool { + return $native === $candidate + || ( is_numeric( $native ) && is_numeric( $candidate ) && (float) $native === (float) $candidate ); } } diff --git a/tests/integration/NativeRestorerTest.php b/tests/integration/NativeRestorerTest.php index 436f65e..bf81167 100644 --- a/tests/integration/NativeRestorerTest.php +++ b/tests/integration/NativeRestorerTest.php @@ -139,4 +139,77 @@ class NativeRestorerTest extends TestCase { $this->assertSame( array( '9' ), $this->native( 1, 'level' ) ); } + + private function set_user_mode( string $mode ): void { + $GLOBALS['_wp_options']['wpdo_bridge_modes'] = array( 'user' => $mode ); + TMDO_Mode_Manager::reset_cache(); + } + + /** + * Native is the source of truth outside aeav_only (e.g. WooCommerce writes + * stock with direct SQL): the default policy there only fills gaps. + */ + public function test_native_authoritative_modes_only_fill_missing_rows(): void { + $this->set_user_mode( 'dual_write' ); + + $report = TMDO_Native_Restorer::run( 'user', false ); + + $this->assertSame( array( '1' ), $this->native( 1, 'level' ), 'differing native value kept' ); + $this->assertSame( array( serialize( array( 'a', 'b' ) ) ), $this->native( 1, 'tags' ), 'missing native row filled' ); + $this->assertSame( array( '5' ), $this->native( 2, 'level' ) ); + $this->assertSame( array( 'stale' ), $this->native( 2, 'tags' ), 'NULL flat value does not delete native here' ); + $this->assertSame( 'only_missing', $report['policy'] ); + $this->assertSame( 2, $report['groups']['rn_profile']['missing'] ); + $this->assertSame( 2, $report['groups']['rn_profile']['differs'] ); + } + + public function test_overwrite_policy_replaces_native_in_any_mode(): void { + $this->set_user_mode( 'dual_write' ); + + TMDO_Native_Restorer::run( 'user', false, 500, false, '', 'overwrite' ); + + $this->assertSame( array( '3' ), $this->native( 1, 'level' ) ); + $this->assertSame( array(), $this->native( 2, 'tags' ) ); + } + + public function test_only_missing_policy_keeps_native_in_aeav_only(): void { + TMDO_Native_Restorer::run( 'user', false, 500, false, '', 'only_missing' ); + + $this->assertSame( array( '1' ), $this->native( 1, 'level' ) ); + $this->assertSame( array( serialize( array( 'a', 'b' ) ) ), $this->native( 1, 'tags' ) ); + } + + public function test_dry_run_classifies_without_writing(): void { + $report = TMDO_Native_Restorer::run( 'user', true ); + + $this->assertSame( 'overwrite', $report['policy'], 'aeav_only: flat is the source of truth' ); + $this->assertSame( 2, $report['groups']['rn_profile']['missing'] ); + $this->assertSame( 2, $report['groups']['rn_profile']['differs'] ); + $this->assertSame( 0, $report['groups']['rn_profile']['same'] ); + $this->assertSame( array( '1' ), $this->native( 1, 'level' ) ); + } + + /** + * A flat column holding its field default for an entity that never had + * the key (e.g. activity.session_count = 0) is not written back. + */ + public function test_defaults_are_not_invented_as_native_rows(): void { + global $wpdb; + $table = 'wp_itest_wpdo_user_rn_counts'; + $wpdb->query( "DROP TABLE IF EXISTS `{$table}`" ); + $wpdb->query( "CREATE TABLE `{$table}` ( id BIGINT NOT NULL AUTO_INCREMENT, user_id BIGINT NOT NULL, visits BIGINT(20) DEFAULT 0, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), UNIQUE KEY uk_entity (user_id) ) ENGINE=InnoDB" ); + $wpdb->query( "INSERT INTO `{$table}` (user_id, visits, updated_at) VALUES (1, 0, '2020-01-01'), (2, 7, '2020-01-01')" ); + WPDO_Entity_Registry::register_group( 'user', 'rn_counts', array( array( 'key' => 'visits', 'type' => 'integer', 'default' => 0 ) ) ); + WPDO_Schema_Manager::flush_table_exists_cache(); + + try { + $report = TMDO_Native_Restorer::run( 'user', false, 500, false, 'rn_counts' ); + + $this->assertSame( array(), $this->native( 1, 'visits' ), 'default 0 not invented' ); + $this->assertSame( array( '7' ), $this->native( 2, 'visits' ) ); + $this->assertSame( 1, $report['groups']['rn_counts']['default_skipped'] ); + } finally { + $wpdb->query( "DROP TABLE IF EXISTS `{$table}`" ); + } + } }