From e6baa612a53cc684a566489861ff9564b7ae8ae3 Mon Sep 17 00:00:00 2001 From: OpenXE <> Date: Wed, 23 Nov 2022 15:15:56 +0000 Subject: [PATCH] Database tool optimized --- tools/database_compare/database_export.php | 365 +++++++++++++++------ 1 file changed, 270 insertions(+), 95 deletions(-) diff --git a/tools/database_compare/database_export.php b/tools/database_compare/database_export.php index 0cdebf79..5930edd7 100644 --- a/tools/database_compare/database_export.php +++ b/tools/database_compare/database_export.php @@ -1,7 +1,7 @@ = 1) { +if ($argc > 1) { if (in_array('-v', $argv)) { $verbose = true; @@ -74,6 +80,18 @@ if ($argc >= 1) { $force = false; } + if (in_array('-e', $argv)) { + $export = true; + } else { + $export = false; + } + + if (in_array('-c', $argv)) { + $compare = true; + } else { + $compare = false; + } + /* if (strpos($argv[1],'-') == 0) { $module_name = $argv[1]; @@ -89,125 +107,282 @@ if ($argc >= 1) { if (!$mysqli) { echo "Connection failed\n"; echo "Error number: " . mysqli_connect_errno() . "\n"; - echo "Error message: " . mysqli_connect_error() . "\n"; + echo "Error message: '" . mysqli_connect_error() . "'\n\n"; exit; } - echo "Successfully connected!\n"; - - // Prepare files - mkdir($target_folder); - - $tables_file_name = $target_folder."/".$tables_file_name_wo_folder; - - if (!$force && file_exists($tables_file_name)) { - echo("File exists: " .$tables_file_name . "\n"); - echo("Use -f to force overwrite.\n"); - exit; - } - - $tables_file = fopen($tables_file_name, "w"); - if (empty($tables_file)) { - echo ("Failed to write to " . $tables_file_name."\n"); - exit(); - } + echo "--------------- Successfully connected! --------------- \n"; + echo("--------------- Loading from database... ---------------\n"); // Get tables and views - $tables = array(); - $sql = "SHOW FULL TABLES"; $result = mysqli_query($mysqli, $sql); - if (!$result) { - echo "Query error: " . mysqli_error($mysqli); + echo "Query error: '" . mysqli_error($mysqli)."'"; exit; } - - $colwidth = 0; - + $tables = array(); while ($row = mysqli_fetch_assoc($result)) { - $table = array(); $table['name'] = $row['Tables_in_'.$schema]; $table['type'] = $row['Table_type']; $tables[] = $table; // Add table to list of tables - - if (strlen($table['name']) > $colwidth) { - $colwidth = strlen($table['name']); - } - } - - $pre_text = "Table name"; - echo(" | ".$pre_text); - for ($filler = strlen($pre_text); $filler < $colwidth; $filler++) { - echo(" "); } - echo(" | ".'type'."\n"); - echo(" | "); - for ($filler = 0; $filler < $colwidth; $filler++) { - echo("-"); - } - echo("\n"); - - fwrite($tables_file, 'tablename'.$delimiter."type\n"); - - foreach ($tables as $table) { - echo(" | ".$table['name']); - for ($filler = strlen($table['name']); $filler < $colwidth; $filler++) { - echo(" "); - } - echo(" | ".$table['type']." |\n"); - fwrite($tables_file, $table['name'].$delimiter.$table['type']."\n"); - } - fclose($tables_file); - echo(" | "); - for ($filler = 0; $filler < $colwidth; $filler++) { - echo("-"); - } - echo("\n"); - - // Now export all colums of the tables - - $tablecount = 0; - - foreach ($tables as $table) { - $table_file_name = $target_folder."/".$table['name'].".txt"; - - if (!$force && file_exists($table_file_name)) { - echo("File exists: " .$table_file_name . "\n"); - echo("Use -f to force overwrite.\n"); - exit; - } - - $table_file = fopen($table_file_name, "w"); - if (empty($table_file)) { - echo ("Failed to write to " . $table_file_name."\n"); - exit(); - } + // Get and add columns of the table + foreach ($tables as &$table) { $sql = "SHOW FULL COLUMNS FROM ".$table['name']; $result = mysqli_query($mysqli, $sql); if (!$result) { - echo "Query error: " . mysqli_error($mysqli); + echo "Query error: '" . mysqli_error($mysqli)."'\n\n"; exit; } - $first = true; + $columns = array(); while ($column = mysqli_fetch_assoc($result)) { - - if ($first) { - fwrite($table_file,implode_with_quote($quote,$delimiter,array_keys($column))."\n"); - $first = false; - } - fwrite($table_file,implode_with_quote($quote,$delimiter,array_values($column))."\n"); + $columns[] = $column; // Add column to list of columns } - $tablecount++; + + $table['columns'] = $columns; + + if ($verbose) { + echo("Table '".$table['name']."' of type '".$table['type']."'\n"); + } + + } + unset($table); + // ------ IMPORT COMPLETE ------ + + echo("--------------- Loading from database complete. ---------------\n"); + + + if ($export) { + + echo("--------------- Export to CSV... ---------------\n"); + + // Prepare tables file + if (!is_dir($target_folder)) { + mkdir($target_folder); + } + if (!$force && file_exists($tables_file_name)) { + echo("File exists: '" .$tables_file_name . "'!\n"); + echo("Use -f to force overwrite.\n\n"); + exit; + } + + $tables_file = fopen($tables_file_name, "w"); + if (empty($tables_file)) { + echo ("Failed to write to '" . $tables_file_name."'!\n\n"); + exit(); + } + + $first_table = true; + // Now export all colums of the tables + foreach ($tables as $export_table) { + + if ($verbose) { + echo("Table '".$export_table['name']."' of type '".$export_table['type']."' loaded from database.\n"); + } + if ($first_table) { + $first_table = false; + fwrite($tables_file,$quote.'name'.$quote.$delimiter.$quote.'type'.$quote."\n"); + } + fwrite($tables_file,$quote.$export_table['name'].$quote.$delimiter.$quote.$export_table['type'].$quote."\n"); + + // Prepare export_table file + $table_file_name = $target_folder."/".$export_table['name'].".txt"; + if (!$force && file_exists($table_file_name)) { + echo("File exists: '" .$table_file_name . "'!\n"); + echo("Use -f to force overwrite.\n\n"); + exit; + } + $table_file = fopen($table_file_name, "w"); + if (empty($table_file)) { + echo ("Failed to write to '" . $table_file_name."'!\n\n"); + exit(); + } + + $first_column = true; + + foreach ($export_table['columns'] as $column) { + if ($first_column) { + $first_column = false; + fwrite($table_file,implode_with_quote($quote,$delimiter,array_keys($column))."\n"); + } + fwrite($table_file,implode_with_quote($quote,$delimiter,array_values($column))."\n"); + } + unset($column); + + fclose($table_file); + } + unset($export_table); + + echo("Exported ".count($tables)." tables.\n"); + fwrite($tables_file,"\n"); + fclose($tables_file); + echo("--------------- Export to CSV complete. ---------------\n"); } - echo("\nExported $tablecount tables.\n"); - echo("\n\nDone!\n"); + if ($compare) { + + // Results here as ['text'] ['diff'] + $compare_differences = array(); + + echo("--------------- Loading from CSV... ---------------\n"); + + $compare_tables = array(); + $first_table = true; + $tables_file = fopen($tables_file_name, "r"); + + if (!$tables_file) { + echo("File not found: '" .$tables_file_name . "'\n"); + echo("\n"); + exit; + } + + while (($csv_line = fgetcsv($tables_file,0,$delimiter,$quote)) !== FALSE) { + + if ($first_table) { + $first_table = false; + } else if (count($csv_line) == 2) { + $new_compare_table = array(); + $new_compare_table['name'] = $csv_line['0']; + $new_compare_table['type'] = $csv_line['1']; + $compare_tables[] = $new_compare_table; + + if ($verbose) { + echo("Table '".$new_compare_table['name']."' loaded from CSV '$tables_file_name'.\n"); + } + + } else { + + } + } + fclose($tables_file); + + // Get columns for each compare_table + + foreach ($compare_tables as &$compare_table) { + + $table_file_name = $target_folder."/".$compare_table['name'].".txt"; + if (!file_exists($table_file_name)) { + echo("File not found: '" .$table_file_name . "'\n"); + echo("\n"); + exit; + } + $table_file = fopen($table_file_name, "r"); + if (empty($table_file)) { + echo ("Failed to open '" . $table_file_name."'\n\n"); + exit(); + } + + $first_column = true; + $column_headers = array(); + $columns = array(); + $column = array(); + while (($csv_line = fgetcsv($table_file,0,$delimiter,$quote)) !== FALSE) { + + if ($first_column) { + $first_column = false; + $column_headers = $csv_line; + } else { + for ($cr = 0;$cr < count($csv_line);$cr++) { + $column[$column_headers[$cr]] = $csv_line[$cr]; + } + $columns[] = $column; + } + } + + $compare_table['columns'] = $columns; + + if ($verbose) { + echo("Colums loaded for '".$compare_table['name']."' from CSV $table_file_name. \n"); + } + } + unset($compare_table); + + echo("--------------- Loading from CSV complete. ---------------\n"); + + // Do the comparison + + echo("--------------- Comparison... ---------------\n"); + + echo("Number of tables: ".count($tables)." in Database, ".count($compare_tables)." in CSV.\n"); + + if (count($tables) != count($compare_tables)) { + + } + + + foreach ($tables as $database_table) { + + $found_table = array(); + foreach ($compare_tables as $compare_table) { + if ($database_table['name'] == $compare_table['name']) { + $found_table = $compare_table; + break; + } + } + unset($compare_table); + + if ($found_table) { + + if ($verbose) { + echo("Table '".$database_table['name']."' found in CSV '$tables_file_name'.\n"); + } + + // Check columns + $compare_table_columns = array_column($found_table['columns'],'Field'); + + foreach ($database_table['columns'] as $column) { + + $column_name_to_find = $column['Field']; + $column_key = array_search($column_name_to_find,$compare_table_columns,true); + if ($column_key !== false) { + + if ($verbose) { + echo("Column '".$column['Field']."' from table '".$database_table['name']."' in table '".$found_table['name']."' found in CSV.\n"); + } + + $column_diff = array_diff($column,$found_table['columns'][$column_key]); + + if (!empty($column_diff)) { + $compare_difference = array(); + $compare_difference['text'] = $color_red."Difference:".$color_default." Column '".$column['Field']."' from table '".$database_table['name']."' is different from '".$found_table['name']."' in CSV.\n"; + $compare_difference['diff'] = $column_diff; + $compare_differences[] = $compare_difference; + echo($compare_difference['text']); + } + } else { + $compare_difference = array(); + $compare_difference['text'] = $color_red."Difference:".$color_default." Column '".$column['Field']."' from table '".$database_table['name']."' in table '".$found_table['name']."' not found in CSV.\n"; + $compare_difference['diff'] = $column_diff; + $compare_differences[] = $compare_difference; + echo($compare_difference['text']); + } + } + unset($column); + } else { + $compare_difference = array(); + $compare_difference['text'] = $color_red."Difference:".$color_default." Table '".$database_table['name']."' not found in CSV '$tables_file_name'.\n"; + $compare_difference['diff'] = $column_diff; + $compare_differences[] = $compare_difference; + echo($compare_difference['text']); + } + } + unset($database_table); + + echo("\nComparison found ".(empty($compare_differences)?0:count($compare_differences))." differences.\n"); + + echo("--------------- Comparison complete. ---------------\n"); + + } + + echo("--------------- Done! ---------------\n"); + + echo("\n"); } else { info(); @@ -219,11 +394,11 @@ function info() { echo("Copyright 2022 (c) OpenXE project\n\n"); echo("\n"); echo("Export database structures in a defined format for database comparison / upgrade\n"); - echo("arg1: ..\n"); echo("Options\n"); echo("\t-v: verbose output\n"); echo("\t-f: force override of existing files\n"); - echo("\t-c: select columns like this: -c col1,col2,col3,col3\n"); + echo("\t-e: export database structure to files\n"); + echo("\t-c: compare content of files with database structure\n"); echo("\n"); }