mustal utf8fix

tools/database_compare/database_compare.php

@@ -123,6 +123,12 @@ if ($argc > 1) {
       $clean = false;
     } 
 
+    if (in_array('-utf8fix', $argv)) {
+      $utf8fix = true;
+    } else {
+      $utf8fix = false;
+    } 
+
     $connection_info_contents = file_get_contents($connection_info_file_name);
     if (!$connection_info_contents) {
         echo("Unable to load $connection_info_file_name\n");
@@ -197,7 +203,16 @@ if ($argc > 1) {
         }*/
 
         echo("--------------- Comparing database '$schema@$host' vs. JSON '".$compare_def['database']."@".$compare_def['host']."' ---------------\n");
-        $compare_differences = mustal_compare_table_array($compare_def,"in JSON",$db_def,"in DB",true);
+
+        if($utf8fix) {
+            $column_collation_aliases = array(
+                ['utf8mb3_general_ci','utf8_general_ci']
+            );
+        } else {
+            $column_collation_aliases = array();
+        }
+
+        $compare_differences = mustal_compare_table_array($compare_def,"in JSON",$db_def,"in DB",true,$column_collation_aliases);
         echo((empty($compare_differences)?0:count($compare_differences))." differences.\n");
 
         if ($verbose) {
@@ -319,6 +334,7 @@ function info() {
     echo("\t-e: export database structure to files\n");
     echo("\t-c: compare content of files with database structure\n");
     echo("\t-i: ignore column definitions\n");
+    echo("\t-utf8fix: apply fix for 'utf8' != 'utf8mb3'\n");
     echo("\t-upgrade: Create the needed SQL to upgrade the database to match the JSON\n");
     echo("\t-do: Execute the SQL to upgrade the database to match the JSON (risky!)\n");
     echo("\t-clean: (not yet implemented) Create the needed SQL to remove items from the database not in the JSON\n");

tools/database_compare/mustal_mysql_upgrade_tool.php

@@ -201,7 +201,8 @@ function mustal_load_tables_from_json(string $path, string $tables_file_name) :
 // Compare two definitions
 // Report based on the first array
 // Return Array
-function mustal_compare_table_array(array $nominal, string $nominal_name, array $actual, string $actual_name, bool $check_column_definitions) : array {
+// $column_collation_aliases may contain synonyms for collations e.g. utf8mb3_general_ci vs utf8_general_ci
+function mustal_compare_table_array(array $nominal, string $nominal_name, array $actual, string $actual_name, bool $check_column_definitions, array $column_collation_aliases = array()) : array {
 
     $compare_differences = array();
 
@@ -254,8 +255,19 @@ function mustal_compare_table_array(array $nominal, string $nominal_name, array
                     if ($check_column_definitions) {
                         $found_column = $found_table['columns'][$column_key];
                         foreach ($column as $key => $value) {                            
-                            if ($found_column[$key] != $value) {
 
+                            // Apply aliases                                
+                            if (!empty($column_collation_aliases)) {
+                                foreach($column_collation_aliases as $column_collation_alias) {
+                                    if ($value == $column_collation_alias[0]) {
+                                        $value = $column_collation_alias[1];
+                                    }
+                                    if ($found_column[$key] == $column_collation_alias[0]) {                              
+                                        $found_column[$key] = $column_collation_alias[1];
+                                    }
+                                }
+                            } 
+                            if ($found_column[$key] != $value) {
                                 if ($key != 'Key') { // Keys will be handled separately
                                     $compare_difference = array();
                                     $compare_difference['type'] = "Column definition";

upgrade/upgrade.php

@@ -37,6 +37,7 @@ function git(string $command, &$output, bool $verbose, string $error_text) : int
 }
 
 // -------------------------------- START
+
 // Check for correct call method
 $directory = "";
 if (php_sapi_name() == "cli") {
@@ -51,6 +52,15 @@ if (php_sapi_name() == "cli") {
 }
 
 if ($cli) {
+
+    if ($argc > 1) {
+
+        if (in_array('-c', $argv)) {
+          $check = true;
+        } else {
+          $check = false;
+        } 
+
         if (in_array('-v', $argv)) {
           $verbose = true;
         } else {
@@ -64,11 +74,28 @@ if ($cli) {
         } 
 
         if (in_array('-do', $argv)) {
+          $check = true;
           $do_upgrade = true;
         } else {
           $do_upgrade = false;
         } 
+
+        if (in_array('-utf8fix', $argv)) {
+          $utf8fix = true;
+        } else {
+          $utf8fix = false;
+        } 
+
+        if ($check) {
             upgrade_main($directory, $verbose, $do_upgrade, $force);
+        } else {
+            info();
+        }
+
+    } else {
+        info();
+    }
+
 } 
 // -------------------------------- END
 
@@ -90,7 +117,7 @@ function upgrade_main(string $directory,bool $verbose, bool $do_upgrade, bool $f
 
     echo("--------------- OpenXE upgrade ---------------\n");
 
-    require_once($directory.'/../cronjobs/githash.php');
+    //require_once($directory.'/../cronjobs/githash.php');
 
     $remote_info_contents = file_get_contents($remote_file_name);
     if (!$remote_info_contents) {
@@ -136,12 +163,19 @@ function upgrade_main(string $directory,bool $verbose, bool $do_upgrade, bool $f
 
 
     if ($verbose) {
-        echo("--------------- Update history ---------------\n");
+        echo("--------------- Upgrade history ---------------\n");
         $retval = git("log --date=short-local --pretty=\"%cd (%h): %s\" HEAD --not HEAD~4",$output,$verbose,"Error while showing history!");
         if ($retval != 0) {
             abort("");
             return(-1);
         }
+    } else {
+        echo("--------------- Current version ---------------\n");
+        $retval = git("log -1 --date=short-local --pretty=\"%cd (%h): %s\" HEAD",$output,$verbose,"Error while showing history!");
+        if ($retval != 0) {
+            abort("");
+            return(-1);
+        }
     }
 
     if ($do_upgrade) { 
@@ -212,7 +246,18 @@ function upgrade_main(string $directory,bool $verbose, bool $do_upgrade, bool $f
             return(-1);
         }
         echo("--------------- Comparing database '$schema@$host' vs. JSON '".$compare_def['database']."@".$compare_def['host']."' ---------------\n");
-        $compare_differences = mustal_compare_table_array($compare_def,"in JSON",$db_def,"in DB",true);
+
+        if($utf8fix) {
+            $column_collation_aliases = array(
+                ['utf8mb3_general_ci','utf8_general_ci']
+            );
+        } else {
+            $column_collation_aliases = array();
+        }
+
+        
+        $compare_differences = mustal_compare_table_array($compare_def,"in JSON",$db_def,"in DB",true,$column_collation_aliases);
+
         echo((empty($compare_differences)?0:count($compare_differences))." differences.\n");
 
         echo("--------------- Calculating database upgrade for '$schema@$host'... ---------------\n");
@@ -273,7 +318,7 @@ function upgrade_main(string $directory,bool $verbose, bool $do_upgrade, bool $f
             $db_def = mustal_load_tables_from_db($host, $schema, $user, $passwd, $mustal_replacers);
 
             echo("--------------- Comparing database '$schema@$host' vs. JSON '".$compare_def['database']."@".$compare_def['host']."' ---------------\n");
-            $compare_differences = mustal_compare_table_array($compare_def,"in JSON",$db_def,"in DB",true);
+            $compare_differences = mustal_compare_table_array($compare_def,"in JSON",$db_def,"in DB",true,$column_collation_aliases);
             echo((empty($compare_differences)?0:count($compare_differences))." differences.\n");
 
         }
@@ -309,4 +354,19 @@ function upgrade_main(string $directory,bool $verbose, bool $do_upgrade, bool $f
     return(0);
 }
 
+function info() {
+    echo("OpenXE upgrade tool\n");
+    echo("Copyright 2022 (c) OpenXE project\n");
+    echo("\n");
+    echo("Upgrade files and database\n");
+    echo("Options:\n");
+    echo("\t-c: check for upgrades\n");
+    echo("\t-v: verbose output\n");
+    echo("\t-f: force override of existing files\n");
+    echo("\t-utf8fix: apply fix for 'utf8' != 'utf8mb3'\n");
+    echo("\t-do: execute the upgrade\n");
+    echo("\t-clean: (not yet implemented) create the needed SQL to remove items from the database not in the JSON\n");
+    echo("\n");
+}
+