From 04d6d5ca99ebfd1cebb8ce06618fb3811fc1a8aa Mon Sep 17 00:00:00 2001 From: Charles Date: Thu, 9 Jan 2020 10:55:03 +0100 Subject: phpmyadmin working --- .../classes/Plugins/Import/AbstractImportCsv.php | 94 +++ .../libraries/classes/Plugins/Import/ImportCsv.php | 818 +++++++++++++++++++++ .../libraries/classes/Plugins/Import/ImportLdi.php | 176 +++++ .../classes/Plugins/Import/ImportMediawiki.php | 604 +++++++++++++++ .../libraries/classes/Plugins/Import/ImportOds.php | 427 +++++++++++ .../libraries/classes/Plugins/Import/ImportShp.php | 335 +++++++++ .../libraries/classes/Plugins/Import/ImportSql.php | 200 +++++ .../libraries/classes/Plugins/Import/ImportXml.php | 375 ++++++++++ .../libraries/classes/Plugins/Import/README | 156 ++++ .../classes/Plugins/Import/ShapeFileImport.php | 46 ++ .../classes/Plugins/Import/Upload/UploadApc.php | 83 +++ .../Plugins/Import/Upload/UploadNoplugin.php | 60 ++ .../Plugins/Import/Upload/UploadProgress.php | 97 +++ .../Plugins/Import/Upload/UploadSession.php | 95 +++ 14 files changed, 3566 insertions(+) create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/AbstractImportCsv.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportCsv.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportLdi.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportMediawiki.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportOds.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportShp.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportSql.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportXml.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/README create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/ShapeFileImport.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/Upload/UploadApc.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/Upload/UploadNoplugin.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/Upload/UploadProgress.php create mode 100644 srcs/phpmyadmin/libraries/classes/Plugins/Import/Upload/UploadSession.php (limited to 'srcs/phpmyadmin/libraries/classes/Plugins/Import') diff --git a/srcs/phpmyadmin/libraries/classes/Plugins/Import/AbstractImportCsv.php b/srcs/phpmyadmin/libraries/classes/Plugins/Import/AbstractImportCsv.php new file mode 100644 index 0000000..5c5a250 --- /dev/null +++ b/srcs/phpmyadmin/libraries/classes/Plugins/Import/AbstractImportCsv.php @@ -0,0 +1,94 @@ +setOptionsText(__('Options')); + + // create the root group that will be the options field for + // $importPluginProperties + // this will be shown as "Format specific options" + $importSpecificOptions = new OptionsPropertyRootGroup( + "Format Specific Options" + ); + + // general options main group + $generalOptions = new OptionsPropertyMainGroup("general_opts"); + + // create common items and add them to the group + $leaf = new BoolPropertyItem( + "replace", + __( + 'Update data when duplicate keys found on import (add ON DUPLICATE ' + . 'KEY UPDATE)' + ) + ); + $generalOptions->addProperty($leaf); + $leaf = new TextPropertyItem( + "terminated", + __('Columns separated with:') + ); + $leaf->setSize(2); + $generalOptions->addProperty($leaf); + $leaf = new TextPropertyItem( + "enclosed", + __('Columns enclosed with:') + ); + $leaf->setSize(2); + $leaf->setLen(2); + $generalOptions->addProperty($leaf); + $leaf = new TextPropertyItem( + "escaped", + __('Columns escaped with:') + ); + $leaf->setSize(2); + $leaf->setLen(2); + $generalOptions->addProperty($leaf); + $leaf = new TextPropertyItem( + "new_line", + __('Lines terminated with:') + ); + $leaf->setSize(2); + $generalOptions->addProperty($leaf); + + // add the main group to the root group + $importSpecificOptions->addProperty($generalOptions); + + // set the options for the import plugin property item + $importPluginProperties->setOptions($importSpecificOptions); + $this->properties = $importPluginProperties; + + return $generalOptions; + } +} diff --git a/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportCsv.php b/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportCsv.php new file mode 100644 index 0000000..e5494a9 --- /dev/null +++ b/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportCsv.php @@ -0,0 +1,818 @@ +setProperties(); + } + + /** + * Sets the import plugin properties. + * Called in the constructor. + * + * @return void + */ + protected function setProperties() + { + $this->_setAnalyze(false); + + if ($GLOBALS['plugin_param'] !== 'table') { + $this->_setAnalyze(true); + } + + $generalOptions = parent::setProperties(); + $this->properties->setText('CSV'); + $this->properties->setExtension('csv'); + + if ($GLOBALS['plugin_param'] !== 'table') { + $leaf = new TextPropertyItem( + "new_tbl_name", + __( + 'Name of the new table (optional):' + ) + ); + $generalOptions->addProperty($leaf); + + if ($GLOBALS['plugin_param'] === 'server') { + $leaf = new TextPropertyItem( + "new_db_name", + __( + 'Name of the new database (optional):' + ) + ); + $generalOptions->addProperty($leaf); + } + + $leaf = new NumberPropertyItem( + "partial_import", + __( + 'Import these many number of rows (optional):' + ) + ); + $generalOptions->addProperty($leaf); + + $leaf = new BoolPropertyItem( + "col_names", + __( + 'The first line of the file contains the table column names' + . ' (if this is unchecked, the first line will become part' + . ' of the data)' + ) + ); + $generalOptions->addProperty($leaf); + } else { + $leaf = new NumberPropertyItem( + "partial_import", + __( + 'Import these many number of rows (optional):' + ) + ); + $generalOptions->addProperty($leaf); + + $hint = new Message( + __( + 'If the data in each row of the file is not' + . ' in the same order as in the database, list the corresponding' + . ' column names here. Column names must be separated by commas' + . ' and not enclosed in quotations.' + ) + ); + $leaf = new TextPropertyItem( + "columns", + __('Column names:') . ' ' . Util::showHint($hint) + ); + $generalOptions->addProperty($leaf); + } + + $leaf = new BoolPropertyItem( + "ignore", + __('Do not abort on INSERT error') + ); + $generalOptions->addProperty($leaf); + } + + /** + * Handles the whole import logic + * + * @param array $sql_data 2-element array with sql data + * + * @return void + */ + public function doImport(array &$sql_data = []) + { + global $db, $table, $csv_terminated, $csv_enclosed, $csv_escaped, + $csv_new_line, $csv_columns, $err_url, $import_file_name; + // $csv_replace and $csv_ignore should have been here, + // but we use directly from $_POST + global $error, $timeout_passed, $finished, $message; + + $import_file_name = basename($import_file_name, ".csv"); + $import_file_name = mb_strtolower($import_file_name); + $import_file_name = preg_replace("/[^a-zA-Z0-9_]/", "_", $import_file_name); + + $replacements = [ + '\\n' => "\n", + '\\t' => "\t", + '\\r' => "\r", + ]; + $csv_terminated = strtr($csv_terminated, $replacements); + $csv_enclosed = strtr($csv_enclosed, $replacements); + $csv_escaped = strtr($csv_escaped, $replacements); + $csv_new_line = strtr($csv_new_line, $replacements); + + $param_error = false; + if (strlen($csv_terminated) === 0) { + $message = Message::error( + __('Invalid parameter for CSV import: %s') + ); + $message->addParam(__('Columns terminated with')); + $error = true; + $param_error = true; + // The default dialog of MS Excel when generating a CSV produces a + // semi-colon-separated file with no chance of specifying the + // enclosing character. Thus, users who want to import this file + // tend to remove the enclosing character on the Import dialog. + // I could not find a test case where having no enclosing characters + // confuses this script. + // But the parser won't work correctly with strings so we allow just + // one character. + } elseif (mb_strlen($csv_enclosed) > 1) { + $message = Message::error( + __('Invalid parameter for CSV import: %s') + ); + $message->addParam(__('Columns enclosed with')); + $error = true; + $param_error = true; + // I could not find a test case where having no escaping characters + // confuses this script. + // But the parser won't work correctly with strings so we allow just + // one character. + } elseif (mb_strlen($csv_escaped) > 1) { + $message = Message::error( + __('Invalid parameter for CSV import: %s') + ); + $message->addParam(__('Columns escaped with')); + $error = true; + $param_error = true; + } elseif (mb_strlen($csv_new_line) != 1 + && $csv_new_line != 'auto' + ) { + $message = Message::error( + __('Invalid parameter for CSV import: %s') + ); + $message->addParam(__('Lines terminated with')); + $error = true; + $param_error = true; + } + + // If there is an error in the parameters entered, + // indicate that immediately. + if ($param_error) { + Util::mysqlDie( + $message->getMessage(), + '', + false, + $err_url + ); + } + + $buffer = ''; + $required_fields = 0; + $sql_template = ''; + $fields = []; + if (! $this->_getAnalyze()) { + $sql_template = 'INSERT'; + if (isset($_POST['csv_ignore'])) { + $sql_template .= ' IGNORE'; + } + $sql_template .= ' INTO ' . Util::backquote($table); + + $tmp_fields = $GLOBALS['dbi']->getColumns($db, $table); + + if (empty($csv_columns)) { + $fields = $tmp_fields; + } else { + $sql_template .= ' ('; + $fields = []; + $tmp = preg_split('/,( ?)/', $csv_columns); + foreach ($tmp as $key => $val) { + if (count($fields) > 0) { + $sql_template .= ', '; + } + /* Trim also `, if user already included backquoted fields */ + $val = trim($val, " \t\r\n\0\x0B`"); + $found = false; + foreach ($tmp_fields as $field) { + if ($field['Field'] == $val) { + $found = true; + break; + } + } + if (! $found) { + $message = Message::error( + __( + 'Invalid column (%s) specified! Ensure that columns' + . ' names are spelled correctly, separated by commas' + . ', and not enclosed in quotes.' + ) + ); + $message->addParam($val); + $error = true; + break; + } + if (isset($field)) { + $fields[] = $field; + } + $sql_template .= Util::backquote($val); + } + $sql_template .= ') '; + } + + $required_fields = count($fields); + + $sql_template .= ' VALUES ('; + } + + // Defaults for parser + $i = 0; + $len = 0; + $lastlen = null; + $line = 1; + $lasti = -1; + $values = []; + $csv_finish = false; + $max_lines = 0; // defaults to 0 (get all the lines) + + // If we get a negative value, probably someone changed min value attribute in DOM or there is an integer overflow, whatever be the case, get all the lines + if (isset($_REQUEST['csv_partial_import']) && $_REQUEST['csv_partial_import'] > 0) { + $max_lines = $_REQUEST['csv_partial_import']; + } + $max_lines_constraint = $max_lines+1; + // if the first row has to be counted as column names, include one more row in the max lines + if (isset($_REQUEST['csv_col_names'])) { + $max_lines_constraint++; + } + + $tempRow = []; + $rows = []; + $col_names = []; + $tables = []; + + $col_count = 0; + $max_cols = 0; + $csv_terminated_len = mb_strlen($csv_terminated); + while (! ($finished && $i >= $len) && ! $error && ! $timeout_passed) { + $data = $this->import->getNextChunk(); + if ($data === false) { + // subtract data we didn't handle yet and stop processing + $GLOBALS['offset'] -= strlen($buffer); + break; + } elseif ($data !== true) { + // Append new data to buffer + $buffer .= $data; + unset($data); + + // Force a trailing new line at EOF to prevent parsing problems + if ($finished && $buffer) { + $finalch = mb_substr($buffer, -1); + if ($csv_new_line == 'auto' + && $finalch != "\r" + && $finalch != "\n" + ) { + $buffer .= "\n"; + } elseif ($csv_new_line != 'auto' + && $finalch != $csv_new_line + ) { + $buffer .= $csv_new_line; + } + } + + // Do not parse string when we're not at the end + // and don't have new line inside + if (($csv_new_line == 'auto' + && mb_strpos($buffer, "\r") === false + && mb_strpos($buffer, "\n") === false) + || ($csv_new_line != 'auto' + && mb_strpos($buffer, $csv_new_line) === false) + ) { + continue; + } + } + + // Current length of our buffer + $len = mb_strlen($buffer); + // Currently parsed char + + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 && $ch == $csv_terminated[0]) { + $ch = $this->readCsvTerminatedString( + $buffer, + $ch, + $i, + $csv_terminated_len + ); + $i += $csv_terminated_len - 1; + } + while ($i < $len) { + // Deadlock protection + if ($lasti == $i && $lastlen == $len) { + $message = Message::error( + __('Invalid format of CSV input on line %d.') + ); + $message->addParam($line); + $error = true; + break; + } + $lasti = $i; + $lastlen = $len; + + // This can happen with auto EOL and \r at the end of buffer + if (! $csv_finish) { + // Grab empty field + if ($ch == $csv_terminated) { + if ($i == $len - 1) { + break; + } + $values[] = ''; + $i++; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 && $ch == $csv_terminated[0]) { + $ch = $this->readCsvTerminatedString( + $buffer, + $ch, + $i, + $csv_terminated_len + ); + $i += $csv_terminated_len - 1; + } + continue; + } + + // Grab one field + $fallbacki = $i; + if ($ch == $csv_enclosed) { + if ($i == $len - 1) { + break; + } + $need_end = true; + $i++; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 && $ch == $csv_terminated[0]) { + $ch = $this->readCsvTerminatedString( + $buffer, + $ch, + $i, + $csv_terminated_len + ); + $i += $csv_terminated_len - 1; + } + } else { + $need_end = false; + } + $fail = false; + $value = ''; + while (($need_end + && ($ch != $csv_enclosed + || $csv_enclosed == $csv_escaped)) + || (! $need_end + && ! ($ch == $csv_terminated + || $ch == $csv_new_line + || ($csv_new_line == 'auto' + && ($ch == "\r" || $ch == "\n")))) + ) { + if ($ch == $csv_escaped) { + if ($i == $len - 1) { + $fail = true; + break; + } + $i++; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 + && $ch == $csv_terminated[0] + ) { + $ch = $this->readCsvTerminatedString( + $buffer, + $ch, + $i, + $csv_terminated_len + ); + $i += $csv_terminated_len - 1; + } + if ($csv_enclosed == $csv_escaped + && ($ch == $csv_terminated + || $ch == $csv_new_line + || ($csv_new_line == 'auto' + && ($ch == "\r" || $ch == "\n"))) + ) { + break; + } + } + $value .= $ch; + if ($i == $len - 1) { + if (! $finished) { + $fail = true; + } + break; + } + $i++; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 && $ch == $csv_terminated[0]) { + $ch = $this->readCsvTerminatedString( + $buffer, + $ch, + $i, + $csv_terminated_len + ); + $i += $csv_terminated_len - 1; + } + } + + // unquoted NULL string + if (false === $need_end && $value === 'NULL') { + $value = null; + } + + if ($fail) { + $i = $fallbacki; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 && $ch == $csv_terminated[0]) { + $i += $csv_terminated_len - 1; + } + break; + } + // Need to strip trailing enclosing char? + if ($need_end && $ch == $csv_enclosed) { + if ($finished && $i == $len - 1) { + $ch = null; + } elseif ($i == $len - 1) { + $i = $fallbacki; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 + && $ch == $csv_terminated[0] + ) { + $i += $csv_terminated_len - 1; + } + break; + } else { + $i++; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 + && $ch == $csv_terminated[0] + ) { + $ch = $this->readCsvTerminatedString( + $buffer, + $ch, + $i, + $csv_terminated_len + ); + $i += $csv_terminated_len - 1; + } + } + } + // Are we at the end? + if ($ch == $csv_new_line + || ($csv_new_line == 'auto' && ($ch == "\r" || $ch == "\n")) + || ($finished && $i == $len - 1) + ) { + $csv_finish = true; + } + // Go to next char + if ($ch == $csv_terminated) { + if ($i == $len - 1) { + $i = $fallbacki; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 + && $ch == $csv_terminated[0] + ) { + $i += $csv_terminated_len - 1; + } + break; + } + $i++; + $ch = mb_substr($buffer, $i, 1); + if ($csv_terminated_len > 1 + && $ch == $csv_terminated[0] + ) { + $ch = $this->readCsvTerminatedString( + $buffer, + $ch, + $i, + $csv_terminated_len + ); + $i += $csv_terminated_len - 1; + } + } + // If everything went okay, store value + $values[] = $value; + } + + // End of line + if ($csv_finish + || $ch == $csv_new_line + || ($csv_new_line == 'auto' && ($ch == "\r" || $ch == "\n")) + ) { + if ($csv_new_line == 'auto' && $ch == "\r") { // Handle "\r\n" + if ($i >= ($len - 2) && ! $finished) { + break; // We need more data to decide new line + } + if (mb_substr($buffer, $i + 1, 1) == "\n") { + $i++; + } + } + // We didn't parse value till the end of line, so there was + // empty one + if (! $csv_finish) { + $values[] = ''; + } + + if ($this->_getAnalyze()) { + foreach ($values as $val) { + $tempRow[] = $val; + ++$col_count; + } + + if ($col_count > $max_cols) { + $max_cols = $col_count; + } + $col_count = 0; + + $rows[] = $tempRow; + $tempRow = []; + } else { + // Do we have correct count of values? + if (count($values) != $required_fields) { + // Hack for excel + if ($values[count($values) - 1] == ';') { + unset($values[count($values) - 1]); + } else { + $message = Message::error( + __( + 'Invalid column count in CSV input' + . ' on line %d.' + ) + ); + $message->addParam($line); + $error = true; + break; + } + } + + $first = true; + $sql = $sql_template; + foreach ($values as $key => $val) { + if (! $first) { + $sql .= ', '; + } + if ($val === null) { + $sql .= 'NULL'; + } else { + $sql .= '\'' + . $GLOBALS['dbi']->escapeString($val) + . '\''; + } + + $first = false; + } + $sql .= ')'; + if (isset($_POST['csv_replace'])) { + $sql .= " ON DUPLICATE KEY UPDATE "; + foreach ($fields as $field) { + $fieldName = Util::backquote( + $field['Field'] + ); + $sql .= $fieldName . " = VALUES(" . $fieldName + . "), "; + } + $sql = rtrim($sql, ', '); + } + + /** + * @todo maybe we could add original line to verbose + * SQL in comment + */ + $this->import->runQuery($sql, $sql, $sql_data); + } + + $line++; + $csv_finish = false; + $values = []; + $buffer = mb_substr($buffer, $i + 1); + $len = mb_strlen($buffer); + $i = 0; + $lasti = -1; + $ch = mb_substr($buffer, 0, 1); + if ($max_lines > 0 && $line == $max_lines_constraint) { + $finished = 1; + break; + } + } + } // End of parser loop + if ($max_lines > 0 && $line == $max_lines_constraint) { + $finished = 1; + break; + } + } // End of import loop + + if ($this->_getAnalyze()) { + /* Fill out all rows */ + $num_rows = count($rows); + for ($i = 0; $i < $num_rows; ++$i) { + for ($j = count($rows[$i]); $j < $max_cols; ++$j) { + $rows[$i][] = 'NULL'; + } + } + + if (isset($_REQUEST['csv_col_names'])) { + $col_names = array_splice($rows, 0, 1); + $col_names = $col_names[0]; + // MySQL column names can't end with a space character. + foreach ($col_names as $key => $col_name) { + $col_names[$key] = rtrim($col_name); + } + } + + if ((isset($col_names) && count($col_names) != $max_cols) + || ! isset($col_names) + ) { + // Fill out column names + for ($i = 0; $i < $max_cols; ++$i) { + $col_names[] = 'COL ' . ($i + 1); + } + } + + // get new table name, if user didn't provide one, set the default name + if (isset($_REQUEST['csv_new_tbl_name']) + && strlen($_REQUEST['csv_new_tbl_name']) > 0 + ) { + $tbl_name = $_REQUEST['csv_new_tbl_name']; + } elseif (mb_strlen((string) $db)) { + $result = $GLOBALS['dbi']->fetchResult('SHOW TABLES'); + + // logic to get table name from filename + // if no table then use filename as table name + if (count($result) === 0) { + $tbl_name = $import_file_name; + } else { + // check to see if {filename} as table exist + $name_array = preg_grep("/{$import_file_name}/isU", $result); + // if no use filename as table name + if (count($name_array) === 0) { + $tbl_name = $import_file_name; + } else { + // check if {filename}_ as table exist + $name_array = preg_grep("/{$import_file_name}_/isU", $result); + $tbl_name = $import_file_name . "_" . (count($name_array) + 1); + } + } + } else { + $tbl_name = $import_file_name; + } + + $tables[] = [ + $tbl_name, + $col_names, + $rows, + ]; + + /* Obtain the best-fit MySQL types for each column */ + $analyses = []; + $analyses[] = $this->import->analyzeTable($tables[0]); + + /** + * string $db_name (no backquotes) + * + * array $table = array(table_name, array() column_names, array()() rows) + * array $tables = array of "$table"s + * + * array $analysis = array(array() column_types, array() column_sizes) + * array $analyses = array of "$analysis"s + * + * array $create = array of SQL strings + * + * array $options = an associative array of options + */ + + /* Set database name to the currently selected one, if applicable, + * Otherwise, check if user provided the database name in the request, + * if not, set the default name + */ + if (isset($_REQUEST['csv_new_db_name']) + && strlen($_REQUEST['csv_new_db_name']) > 0 + ) { + $newDb = $_REQUEST['csv_new_db_name']; + } else { + $result = $GLOBALS['dbi']->fetchResult('SHOW DATABASES'); + if (! is_array($result)) { + $result = []; + } + $newDb = 'CSV_DB ' . (count($result) + 1); + } + list($db_name, $options) = $this->getDbnameAndOptions($db, $newDb); + + /* Non-applicable parameters */ + $create = null; + + /* Created and execute necessary SQL statements from data */ + $this->import->buildSql($db_name, $tables, $analyses, $create, $options, $sql_data); + + unset($tables); + unset($analyses); + } + + // Commit any possible data in buffers + $this->import->runQuery('', '', $sql_data); + + if (count($values) != 0 && ! $error) { + $message = Message::error( + __('Invalid format of CSV input on line %d.') + ); + $message->addParam($line); + $error = true; + } + } + + /** + * Read the expected column_separated_with String of length + * $csv_terminated_len from the $buffer + * into variable $ch and return the read string $ch + * + * @param string $buffer The original string buffer read from + * csv file + * @param string $ch Partially read "column Separated with" + * string, also used to return after + * reading length equal $csv_terminated_len + * @param int $i Current read counter of buffer string + * @param int $csv_terminated_len The length of "column separated with" + * String + * + * @return string + */ + public function readCsvTerminatedString($buffer, $ch, $i, $csv_terminated_len) + { + for ($j = 0; $j < $csv_terminated_len - 1; $j++) { + $i++; + $ch .= mb_substr($buffer, $i, 1); + } + + return $ch; + } + /* ~~~~~~~~~~~~~~~~~~~~ Getters and Setters ~~~~~~~~~~~~~~~~~~~~ */ + + /** + * Returns true if the table should be analyzed, false otherwise + * + * @return bool + */ + private function _getAnalyze() + { + return $this->_analyze; + } + + /** + * Sets to true if the table should be analyzed, false otherwise + * + * @param bool $analyze status + * + * @return void + */ + private function _setAnalyze($analyze) + { + $this->_analyze = $analyze; + } +} diff --git a/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportLdi.php b/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportLdi.php new file mode 100644 index 0000000..91260c5 --- /dev/null +++ b/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportLdi.php @@ -0,0 +1,176 @@ +setProperties(); + } + + /** + * Sets the import plugin properties. + * Called in the constructor. + * + * @return void + */ + protected function setProperties() + { + if ($GLOBALS['cfg']['Import']['ldi_local_option'] == 'auto') { + $GLOBALS['cfg']['Import']['ldi_local_option'] = false; + + $result = $GLOBALS['dbi']->tryQuery( + 'SELECT @@local_infile;' + ); + if ($result != false && $GLOBALS['dbi']->numRows($result) > 0) { + $tmp = $GLOBALS['dbi']->fetchRow($result); + if ($tmp[0] == 'ON') { + $GLOBALS['cfg']['Import']['ldi_local_option'] = true; + } + } + $GLOBALS['dbi']->freeResult($result); + unset($result); + } + + $generalOptions = parent::setProperties(); + $this->properties->setText('CSV using LOAD DATA'); + $this->properties->setExtension('ldi'); + + $leaf = new TextPropertyItem( + "columns", + __('Column names: ') + ); + $generalOptions->addProperty($leaf); + + $leaf = new BoolPropertyItem( + "ignore", + __('Do not abort on INSERT error') + ); + $generalOptions->addProperty($leaf); + + $leaf = new BoolPropertyItem( + "local_option", + __('Use LOCAL keyword') + ); + $generalOptions->addProperty($leaf); + } + + /** + * Handles the whole import logic + * + * @param array $sql_data 2-element array with sql data + * + * @return void + */ + public function doImport(array &$sql_data = []) + { + global $finished, $import_file, $charset_conversion, $table; + global $ldi_local_option, $ldi_replace, $ldi_ignore, $ldi_terminated, + $ldi_enclosed, $ldi_escaped, $ldi_new_line, $skip_queries, $ldi_columns; + + $compression = $GLOBALS['import_handle']->getCompression(); + + if ($import_file == 'none' + || $compression != 'none' + || $charset_conversion + ) { + // We handle only some kind of data! + $GLOBALS['message'] = Message::error( + __('This plugin does not support compressed imports!') + ); + $GLOBALS['error'] = true; + + return; + } + + $sql = 'LOAD DATA'; + if (isset($ldi_local_option)) { + $sql .= ' LOCAL'; + } + $sql .= ' INFILE \'' . $GLOBALS['dbi']->escapeString($import_file) + . '\''; + if (isset($ldi_replace)) { + $sql .= ' REPLACE'; + } elseif (isset($ldi_ignore)) { + $sql .= ' IGNORE'; + } + $sql .= ' INTO TABLE ' . Util::backquote($table); + + if (strlen((string) $ldi_terminated) > 0) { + $sql .= ' FIELDS TERMINATED BY \'' . $ldi_terminated . '\''; + } + if (strlen((string) $ldi_enclosed) > 0) { + $sql .= ' ENCLOSED BY \'' + . $GLOBALS['dbi']->escapeString($ldi_enclosed) . '\''; + } + if (strlen((string) $ldi_escaped) > 0) { + $sql .= ' ESCAPED BY \'' + . $GLOBALS['dbi']->escapeString($ldi_escaped) . '\''; + } + if (strlen((string) $ldi_new_line) > 0) { + if ($ldi_new_line == 'auto') { + $ldi_new_line + = (PHP_EOL == "\n") + ? '\n' + : '\r\n'; + } + $sql .= ' LINES TERMINATED BY \'' . $ldi_new_line . '\''; + } + if ($skip_queries > 0) { + $sql .= ' IGNORE ' . $skip_queries . ' LINES'; + $skip_queries = 0; + } + if (strlen((string) $ldi_columns) > 0) { + $sql .= ' ('; + $tmp = preg_split('/,( ?)/', $ldi_columns); + $cnt_tmp = count($tmp); + for ($i = 0; $i < $cnt_tmp; $i++) { + if ($i > 0) { + $sql .= ', '; + } + /* Trim also `, if user already included backquoted fields */ + $sql .= Util::backquote( + trim($tmp[$i], " \t\r\n\0\x0B`") + ); + } // end for + $sql .= ')'; + } + + $this->import->runQuery($sql, $sql, $sql_data); + $this->import->runQuery('', '', $sql_data); + $finished = true; + } +} diff --git a/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportMediawiki.php b/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportMediawiki.php new file mode 100644 index 0000000..9723daf --- /dev/null +++ b/srcs/phpmyadmin/libraries/classes/Plugins/Import/ImportMediawiki.php @@ -0,0 +1,604 @@ +setProperties(); + } + + /** + * Sets the import plugin properties. + * Called in the constructor. + * + * @return void + */ + protected function setProperties() + { + $this->_setAnalyze(false); + if ($GLOBALS['plugin_param'] !== 'table') { + $this->_setAnalyze(true); + } + + $importPluginProperties = new ImportPluginProperties(); + $importPluginProperties->setText(__('MediaWiki Table')); + $importPluginProperties->setExtension('txt'); + $importPluginProperties->setMimeType('text/plain'); + $importPluginProperties->setOptions([]); + $importPluginProperties->setOptionsText(__('Options')); + + $this->properties = $importPluginProperties; + } + + /** + * Handles the whole import logic + * + * @param array $sql_data 2-element array with sql data + * + * @return void + */ + public function doImport(array &$sql_data = []) + { + global $error, $timeout_passed, $finished; + + // Defaults for parser + + // The buffer that will be used to store chunks read from the imported file + $buffer = ''; + + // Used as storage for the last part of the current chunk data + // Will be appended to the first line of the next chunk, if there is one + $last_chunk_line = ''; + + // Remembers whether the current buffer line is part of a comment + $inside_comment = false; + // Remembers whether the current buffer line is part of a data comment + $inside_data_comment = false; + // Remembers whether the current buffer line is part of a structure comment + $inside_structure_comment = false; + + // MediaWiki only accepts "\n" as row terminator + $mediawiki_new_line = "\n"; + + // Initialize the name of the current table + $cur_table_name = ""; + + while (! $finished && ! $error && ! $timeout_passed) { + $data = $this->import->getNextChunk(); + + if ($data === false) { + // Subtract data we didn't handle yet and stop processing + $GLOBALS['offset'] -= mb_strlen($buffer); + break; + } elseif ($data !== true) { + // Append new data to buffer + $buffer = $data; + unset($data); + // Don't parse string if we're not at the end + // and don't have a new line inside + if (mb_strpos($buffer, $mediawiki_new_line) === false) { + continue; + } + } + + // Because of reading chunk by chunk, the first line from the buffer + // contains only a portion of an actual line from the imported file. + // Therefore, we have to append it to the last line from the previous + // chunk. If we are at the first chunk, $last_chunk_line should be empty. + $buffer = $last_chunk_line . $buffer; + + // Process the buffer line by line + $buffer_lines = explode($mediawiki_new_line, $buffer); + + $full_buffer_lines_count = count($buffer_lines); + // If the reading is not finalised, the final line of the current chunk + // will not be complete + if (! $finished) { + $last_chunk_line = $buffer_lines[--$full_buffer_lines_count]; + } + + for ($line_nr = 0; $line_nr < $full_buffer_lines_count; ++$line_nr) { + $cur_buffer_line = trim($buffer_lines[$line_nr]); + + // If the line is empty, go to the next one + if ($cur_buffer_line === '') { + continue; + } + + $first_character = $cur_buffer_line[0]; + $matches = []; + + // Check beginning of comment + if (! strcmp(mb_substr($cur_buffer_line, 0, 4), "") + ) { + // Only data comments are closed. The structure comments + // will be closed when a data comment begins (in order to + // skip structure tables) + if ($inside_data_comment) { + $inside_data_comment = false; + } + + // End comments that are not related to table structure + if (! $inside_structure_comment) { + $inside_comment = false; + } + } else { + // Check table name + $match_table_name = []; + if (preg_match( + "/^Table data for `(.*)`$/", + $cur_buffer_line, + $match_table_name + ) + ) { + $cur_table_name = $match_table_name[1]; + $inside_data_comment = true; + + $inside_structure_comment + = $this->_mngInsideStructComm( + $inside_structure_comment + ); + } elseif (preg_match( + "/^Table structure for `(.*)`$/", + $cur_buffer_line, + $match_table_name + ) + ) { + // The structure comments will be ignored + $inside_structure_comment = true; + } + } + continue; + } elseif (preg_match('/^\{\|(.*)$/', $cur_buffer_line, $matches)) { + // Check start of table + + // This will store all the column info on all rows from + // the current table read from the buffer + $cur_temp_table = []; + + // Will be used as storage for the current row in the buffer + // Once all its columns are read, it will be added to + // $cur_temp_table and then it will be emptied + $cur_temp_line = []; + + // Helps us differentiate the header columns + // from the normal columns + $in_table_header = false; + // End processing because the current line does not + // contain any column information + } elseif (mb_substr($cur_buffer_line, 0, 2) === '|-' + || mb_substr($cur_buffer_line, 0, 2) === '|+' + || mb_substr($cur_buffer_line, 0, 2) === '|}' + ) { + // Check begin row or end table + + // Add current line to the values storage + if (! empty($cur_temp_line)) { + // If the current line contains header cells + // ( marked with '!' ), + // it will be marked as table header + if ($in_table_header) { + // Set the header columns + $cur_temp_table_headers = $cur_temp_line; + } else { + // Normal line, add it to the table + $cur_temp_table[] = $cur_temp_line; + } + } + + // Empty the temporary buffer + $cur_temp_line = []; + + // No more processing required at the end of the table + if (mb_substr($cur_buffer_line, 0, 2) === '|}') { + $current_table = [ + $cur_table_name, + $cur_temp_table_headers, + $cur_temp_table, + ]; + + // Import the current table data into the database + $this->_importDataOneTable($current_table, $sql_data); + + // Reset table name + $cur_table_name = ""; + } + // What's after the row tag is now only attributes + } elseif (($first_character === '|') || ($first_character === '!')) { + // Check cell elements + + // Header cells + if ($first_character === '!') { + // Mark as table header, but treat as normal row + $cur_buffer_line = str_replace('!!', '||', $cur_buffer_line); + // Will be used to set $cur_temp_line as table header + $in_table_header = true; + } else { + $in_table_header = false; + } + + // Loop through each table cell + $cells = $this->_explodeMarkup($cur_buffer_line); + foreach ($cells as $cell) { + $cell = $this->_getCellData($cell); + + // Delete the beginning of the column, if there is one + $cell = trim($cell); + $col_start_chars = [ + "|", + "!", + ]; + foreach ($col_start_chars as $col_start_char) { + $cell = $this->_getCellContent($cell, $col_start_char); + } + + // Add the cell to the row + $cur_temp_line[] = $cell; + } // foreach $cells + } else { + // If it's none of the above, then the current line has a bad + // format + $message = Message::error( + __('Invalid format of mediawiki input on line:
%s.') + ); + $message->addParam($cur_buffer_line); + $error = true; + } + } // End treating full buffer lines + } // while - finished parsing buffer + } + + /** + * Imports data from a single table + * + * @param array $table containing all table info: + * $table[0] - string + * containing table name + * $table[1] - array[] of + * table headers $table[2] - + * array[][] of table content + * rows + * + * @param array $sql_data 2-element array with sql data + * + * @global bool $analyze whether to scan for column types + * + * @return void + */ + private function _importDataOneTable(array $table, array &$sql_data) + { + $analyze = $this->_getAnalyze(); + if ($analyze) { + // Set the table name + $this->_setTableName($table[0]); + + // Set generic names for table headers if they don't exist + $this->_setTableHeaders($table[1], $table[2][0]); + + // Create the tables array to be used in Import::buildSql() + $tables = []; + $tables[] = [ + $table[0], + $table[1], + $table[2], + ]; + + // Obtain the best-fit MySQL types for each column + $analyses = []; + $analyses[] = $this->import->analyzeTable($tables[0]); + + $this->_executeImportTables($tables, $analyses, $sql_data); + } + + // Commit any possible data in buffers + $this->import->runQuery('', '', $sql_data); + } + + /** + * Sets the table name + * + * @param string $table_name reference to the name of the table + * + * @return void + */ + private function _setTableName(&$table_name) + { + if (empty($table_name)) { + $result = $GLOBALS['dbi']->fetchResult('SHOW TABLES'); + // todo check if the name below already exists + $table_name = 'TABLE ' . (count($result) + 1); + } + } + + /** + * Set generic names for table headers, if they don't exist + * + * @param array $table_headers reference to the array containing the headers + * of a table + * @param array $table_row array containing the first content row + * + * @return void + */ + private function _setTableHeaders(array &$table_headers, array $table_row) + { + if (empty($table_headers)) { + // The first table row should contain the number of columns + // If they are not set, generic names will be given (COL 1, COL 2, etc) + $num_cols = count($table_row); + for ($i = 0; $i < $num_cols; ++$i) { + $table_headers[$i] = 'COL ' . ($i + 1); + } + } + } + + /** + * Sets the database name and additional options and calls Import::buildSql() + * Used in PMA_importDataAllTables() and $this->_importDataOneTable() + * + * @param array $tables structure: + * array( + * array(table_name, array() column_names, array()() + * rows) + * ) + * @param array $analyses structure: + * $analyses = array( + * array(array() column_types, array() column_sizes) + * ) + * @param array $sql_data 2-element array with sql data + * + * @global string $db name of the database to import in + * + * @return void + */ + private function _executeImportTables(array &$tables, array &$analyses, array &$sql_data) + { + global $db; + + // $db_name : The currently selected database name, if applicable + // No backquotes + // $options : An associative array of options + list($db_name, $options) = $this->getDbnameAndOptions($db, 'mediawiki_DB'); + + // Array of SQL strings + // Non-applicable parameters + $create = null; + + // Create and execute necessary SQL statements from data + $this->import->buildSql($db_name, $tables, $analyses, $create, $options, $sql_data); + } + + /** + * Replaces all instances of the '||' separator between delimiters + * in a given string + * + * @param string $replace the string to be replaced with + * @param string $subject the text to be replaced + * + * @return string with replacements + */ + private function _delimiterReplace($replace, $subject) + { + // String that will be returned + $cleaned = ""; + // Possible states of current character + $inside_tag = false; + $inside_attribute = false; + // Attributes can be declared with either " or ' + $start_attribute_character = false; + + // The full separator is "||"; + // This remembers if the previous character was '|' + $partial_separator = false; + + // Parse text char by char + for ($i = 0, $iMax = strlen($subject); $i < $iMax; $i++) { + $cur_char = $subject[$i]; + // Check for separators + if ($cur_char == '|') { + // If we're not inside a tag, then this is part of a real separator, + // so we append it to the current segment + if (! $inside_attribute) { + $cleaned .= $cur_char; + if ($partial_separator) { + $inside_tag = false; + $inside_attribute = false; + } + } elseif ($partial_separator) { + // If we are inside a tag, we replace the current char with + // the placeholder and append that to the current segment + $cleaned .= $replace; + } + + // If the previous character was also '|', then this ends a + // full separator. If not, this may be the beginning of one + $partial_separator = ! $partial_separator; + } else { + // If we're inside a tag attribute and the current character is + // not '|', but the previous one was, it means that the single '|' + // was not appended, so we append it now + if ($partial_separator && $inside_attribute) { + $cleaned .= "|"; + } + // If the char is different from "|", no separator can be formed + $partial_separator = false; + + // any other character should be appended to the current segment + $cleaned .= $cur_char; + + if ($cur_char == '<' && ! $inside_attribute) { + // start of a tag + $inside_tag = true; + } elseif ($cur_char == '>' && ! $inside_attribute) { + // end of a tag + $inside_tag = false; + } elseif (($cur_char == '"' || $cur_char == "'") && $inside_tag) { + // start or end of an attribute + if (! $inside_attribute) { + $inside_attribute = true; + // remember the attribute`s declaration character (" or ') + $start_attribute_character = $cur_char; + } else { + if ($cur_char == $start_attribute_character) { + $inside_attribute = false; + // unset attribute declaration character + $start_attribute_character = false; + } + } + } + } + } // end for each character in $subject + + return $cleaned; + } + + /** + * Separates a string into items, similarly to explode + * Uses the '||' separator (which is standard in the mediawiki format) + * and ignores any instances of it inside markup tags + * Used in parsing buffer lines containing data cells + * + * @param string $text text to be split + * + * @return array + */ + private function _explodeMarkup($text) + { + $separator = "||"; + $placeholder = "\x00"; + + // Remove placeholder instances + $text = str_replace($placeholder, '', $text); + + // Replace instances of the separator inside HTML-like + // tags with the placeholder + $cleaned = $this->_delimiterReplace($placeholder, $text); + // Explode, then put the replaced separators back in + $items = explode($separator, $cleaned); + foreach ($items as $i => $str) { + $items[$i] = str_replace($placeholder, $separator, $str); + } + + return $items; + } + + + /* ~~~~~~~~~~~~~~~~~~~~ Getters and Setters ~~~~~~~~~~~~~~~~~~~~ */ + + /** + * Returns true if the table should be analyzed, false otherwise + * + * @return bool + */ + private function _getAnalyze() + { + return $this->_analyze; + } + + /** + * Sets to true if the table should be analyzed, false otherwise + * + * @param bool $analyze status + * + * @return void + */ + private function _setAnalyze($analyze) + { + $this->_analyze = $analyze; + } + + /** + * Get cell + * + * @param string $cell Cell + * + * @return mixed + */ + private function _getCellData($cell) + { + // A cell could contain both parameters and data + $cell_data = explode('|', $cell, 2); + + // A '|' inside an invalid link should not + // be mistaken as delimiting cell parameters +