From bb60df07096073ba591a9cef82bd1c8a3c39b982 Mon Sep 17 00:00:00 2001 From: Michael Kaufmann Date: Fri, 28 Jul 2023 12:20:06 +0200 Subject: [PATCH] more work on backup feature Signed-off-by: Michael Kaufmann --- lib/Froxlor/Backup/Storages/Ftp.php | 16 +- lib/Froxlor/Backup/Storages/Local.php | 12 +- lib/Froxlor/Backup/Storages/Rsync.php | 9 +- lib/Froxlor/Backup/Storages/S3.php | 9 +- lib/Froxlor/Backup/Storages/Sftp.php | 9 +- lib/Froxlor/Backup/Storages/Storage.php | 144 ++++++++++++++++-- .../Backup/Storages/StorageFactory.php | 27 ++++ .../admin/tablelisting.backup_storages.php | 2 + 8 files changed, 200 insertions(+), 28 deletions(-) diff --git a/lib/Froxlor/Backup/Storages/Ftp.php b/lib/Froxlor/Backup/Storages/Ftp.php index 3e63065e..1108ec46 100644 --- a/lib/Froxlor/Backup/Storages/Ftp.php +++ b/lib/Froxlor/Backup/Storages/Ftp.php @@ -35,19 +35,23 @@ class Ftp extends Storage } /** + * Move/Upload file from tmp-source-directory. The file should be moved or deleted afterward. + * Must return the (relative) path including filename to the backup. + * * @param string $filename * @param string $tmp_source_directory - * @return bool + * @return string * @throws Exception */ - protected function putFile(string $filename, string $tmp_source_directory): bool + protected function putFile(string $filename, string $tmp_source_directory): string { $source = FileDir::makeCorrectFile($tmp_source_directory . "/" . $filename); - $target = basename($filename); - if (file_exists($source) && !file_exists($target)) { - return ftp_put($this->ftp_conn, $target, $source, FTP_BINARY); + if (file_exists($source) && ftp_size($this->ftp_conn, $filename) == -1) { + if (ftp_put($this->ftp_conn, $filename, $source, FTP_BINARY)) { + return FileDir::makeCorrectFile($this->getDestinationDirectory() . '/' . $filename); + } } - return false; + return ""; } /** diff --git a/lib/Froxlor/Backup/Storages/Local.php b/lib/Froxlor/Backup/Storages/Local.php index b43df89f..7c5d0bfb 100644 --- a/lib/Froxlor/Backup/Storages/Local.php +++ b/lib/Froxlor/Backup/Storages/Local.php @@ -21,19 +21,23 @@ class Local extends Storage } /** + * Move/Upload file from tmp-source-directory. The file should be moved or deleted afterward. + * Must return the (relative) path including filename to the backup. + * * @param string $filename * @param string $tmp_source_directory - * @return bool + * @return string * @throws Exception */ - protected function putFile(string $filename, string $tmp_source_directory): bool + protected function putFile(string $filename, string $tmp_source_directory): string { $source = FileDir::makeCorrectFile($tmp_source_directory . "/" . $filename); $target = FileDir::makeCorrectFile($this->getDestinationDirectory() . "/" . $filename); if (file_exists($source) && !file_exists($target)) { - return rename($source, $target); + rename($source, $target); + return $target; } - return false; + return ""; } /** diff --git a/lib/Froxlor/Backup/Storages/Rsync.php b/lib/Froxlor/Backup/Storages/Rsync.php index de37a1cb..687a372a 100644 --- a/lib/Froxlor/Backup/Storages/Rsync.php +++ b/lib/Froxlor/Backup/Storages/Rsync.php @@ -14,13 +14,16 @@ class Rsync extends Storage } /** + * Move/Upload file from tmp-source-directory. The file should be moved or deleted afterward. + * Must return the (relative) path including filename to the backup. + * * @param string $filename * @param string $tmp_source_directory - * @return bool + * @return string */ - protected function putFile(string $filename, string $tmp_source_directory): bool + protected function putFile(string $filename, string $tmp_source_directory): string { - // TODO: Implement putFiles() method. + return ""; } /** diff --git a/lib/Froxlor/Backup/Storages/S3.php b/lib/Froxlor/Backup/Storages/S3.php index 9c7001e3..2e45e262 100644 --- a/lib/Froxlor/Backup/Storages/S3.php +++ b/lib/Froxlor/Backup/Storages/S3.php @@ -14,13 +14,16 @@ class S3 extends Storage } /** + * Move/Upload file from tmp-source-directory. The file should be moved or deleted afterward. + * Must return the (relative) path including filename to the backup. + * * @param string $filename * @param string $tmp_source_directory - * @return bool + * @return string */ - protected function putFile(string $filename, string $tmp_source_directory): bool + protected function putFile(string $filename, string $tmp_source_directory): string { - // TODO: Implement putFiles() method. + return ""; } /** diff --git a/lib/Froxlor/Backup/Storages/Sftp.php b/lib/Froxlor/Backup/Storages/Sftp.php index 17bd0e6a..eff8a1af 100644 --- a/lib/Froxlor/Backup/Storages/Sftp.php +++ b/lib/Froxlor/Backup/Storages/Sftp.php @@ -14,13 +14,16 @@ class Sftp extends Storage } /** + * Move/Upload file from tmp-source-directory. The file should be moved or deleted afterward. + * Must return the (relative) path including filename to the backup. + * * @param string $filename * @param string $tmp_source_directory - * @return bool + * @return string */ - protected function putFile(string $filename, string $tmp_source_directory): bool + protected function putFile(string $filename, string $tmp_source_directory): string { - // TODO: Implement putFiles() method. + return ""; } /** diff --git a/lib/Froxlor/Backup/Storages/Storage.php b/lib/Froxlor/Backup/Storages/Storage.php index d5f72442..24f04519 100644 --- a/lib/Froxlor/Backup/Storages/Storage.php +++ b/lib/Froxlor/Backup/Storages/Storage.php @@ -13,10 +13,13 @@ abstract class Storage protected array $filesToStore; + /** + * @throws Exception + */ public function __construct(array $storage_data) { $this->sData = $storage_data; - $this->tmpDirectory = sys_get_temp_dir(); + $this->tmpDirectory = FileDir::makeCorrectDir(sys_get_temp_dir() . '/backup-' . $this->sData['loginname']); } /** @@ -37,22 +40,126 @@ abstract class Storage * prepare files to back up (e.g. create archive or similar) and fill $filesToStore * * @return void + * @throws Exception */ public function prepareFiles(): void { $this->filesToStore = []; + $tmpdir = FileDir::makeCorrectDir($this->tmpDirectory . '/.tmp/'); + FileDir::safe_exec('mkdir -p ' . escapeshellarg($tmpdir)); + // create archive of web, mail and database data + $this->prepareWebData(); + $this->prepareDatabaseData(); + $this->prepareMailData(); // create json-info-file } /** + * @throws Exception + */ + private function prepareWebData(): void + { + $tmpdir = FileDir::makeCorrectDir($this->tmpDirectory . '/.tmp/web'); + FileDir::safe_exec('mkdir -p ' . escapeshellarg($tmpdir)); + FileDir::safe_exec('tar cfz ' . escapeshellarg(FileDir::makeCorrectFile($tmpdir . '/' . $this->sData['loginname'] . '-web.tar.gz')) . ' -C ' . escapeshellarg($this->sData['documentroot']) . ' .'); + $this->filesToStore[] = FileDir::makeCorrectFile($tmpdir . '/' . $this->sData['loginname'] . '-web.tar.gz'); + } + + /** + * @throws Exception + */ + private function prepareDatabaseData(): void + { + $tmpdir = FileDir::makeCorrectDir($this->tmpDirectory . '/.tmp/mysql'); + FileDir::safe_exec('mkdir -p ' . escapeshellarg($tmpdir)); + + // get all customer database-names + $sel_stmt = Database::prepare(" + SELECT `databasename`, `dbserver` FROM `" . TABLE_PANEL_DATABASES . "` + WHERE `customerid` = :cid ORDER BY `dbserver` + "); + Database::pexecute($sel_stmt, [ + 'cid' => $this->sData['customerid'] + ]); + + $has_dbs = false; + $current_dbserver = -1; + while ($row = $sel_stmt->fetch()) { + // Get sql_root data for the specific database-server the database resides on + if ($current_dbserver != $row['dbserver']) { + Database::needRoot(true, $row['dbserver']); + Database::needSqlData(); + $sql_root = Database::getSqlData(); + Database::needRoot(false); + // create temporary mysql-defaults file for the connection-credentials/details + $mysqlcnf_file = tempnam("/tmp", "frx"); + $mysqlcnf = "[mysqldump]\npassword=" . $sql_root['passwd'] . "\nhost=" . $sql_root['host'] . "\n"; + if (!empty($sql_root['port'])) { + $mysqlcnf .= "port=" . $sql_root['port'] . "\n"; + } elseif (!empty($sql_root['socket'])) { + $mysqlcnf .= "socket=" . $sql_root['socket'] . "\n"; + } + file_put_contents($mysqlcnf_file, $mysqlcnf); + } + $bool_false = false; + FileDir::safe_exec('mysqldump --defaults-file=' . escapeshellarg($mysqlcnf_file) . ' -u ' . escapeshellarg($sql_root['user']) . ' ' . $row['databasename'] . ' > ' . FileDir::makeCorrectFile($tmpdir . '/' . $row['databasename'] . '_' . date('YmdHi', time()) . '.sql'), $bool_false, [ + '>' + ]); + $has_dbs = true; + $current_dbserver = $row['dbserver']; + } + + if ($has_dbs) { + $this->filesToStore[] = $tmpdir; + } + + if (@file_exists($mysqlcnf_file)) { + @unlink($mysqlcnf_file); + } + } + + private function prepareMailData(): void + { + $tmpdir = FileDir::makeCorrectDir($this->tmpDirectory . '/.tmp/mail'); + FileDir::safe_exec('mkdir -p ' . escapeshellarg($tmpdir)); + + // get all customer mail-accounts + $sel_stmt = Database::prepare(" + SELECT `homedir`, `maildir` FROM `" . TABLE_MAIL_USERS . "` + WHERE `customerid` = :cid + "); + Database::pexecute($sel_stmt, [ + 'cid' => $this->sData['customerid'] + ]); + + $tar_file_list = ""; + $mail_homedir = ""; + while ($row = $sel_stmt->fetch()) { + $tar_file_list .= escapeshellarg("./" . $row['maildir']) . " "; + if (empty($mail_homedir)) { + // this should be equal for all entries + $mail_homedir = $row['homedir']; + } + } + + if (!empty($tar_file_list)) { + FileDir::safe_exec('tar cfz ' . escapeshellarg(FileDir::makeCorrectFile($tmpdir . '/' . $this->sData['loginname'] . '-mail.tar.gz')) . ' -C ' . escapeshellarg($mail_homedir) . ' ' . trim($tar_file_list)); + $this->filesToStore[] = FileDir::makeCorrectFile($tmpdir . '/' . $this->sData['loginname'] . '-mail.tar.gz'); + } + } + + /** + * Move/Upload file from tmp-source-directory. The file should be moved or deleted afterward. + * Must return the (relative) path including filename to the backup. + * * @param string $filename * @param string $tmp_source_directory - * @return bool + * @return string */ - abstract protected function putFile(string $filename, string $tmp_source_directory): bool; + abstract protected function putFile(string $filename, string $tmp_source_directory): string; /** * @param string $filename @@ -91,6 +198,8 @@ abstract class Storage } /** + * Returns the storage configured destination path for all backups + * * @return string * @throws Exception */ @@ -111,18 +220,35 @@ abstract class Storage return false; } - $filename = FileDir::makeCorrectFile("/backup-" . $this->sData['loginname'] . "-" . date('c') . ".tar.gz"); + $filename = FileDir::makeCorrectFile($this->tmpDirectory . "/backup-" . $this->sData['loginname'] . "-" . date('c') . ".tar.gz"); + $tmpdir = FileDir::makeCorrectDir($this->tmpDirectory . '/.tmp/'); + $create_export_tar_data = implode(" ", $this->filesToStore); + FileDir::safe_exec('chown -R ' . (int)$this->sData['guid'] . ':' . (int)$this->sData['guid'] . ' ' . escapeshellarg($tmpdir)); - // @todo create archive $filename from $filesToStore + if (!empty($data['pgp_public_key'])) { + // pack all archives in tmp-dir to one archive and encrypt it with gpg + $recipient_file = FileDir::makeCorrectFile($this->tmpDirectory . '/' . $this->sData['loginname'] . '-recipients.gpg'); + file_put_contents($recipient_file, $data['pgp_public_key']); + $return_value = []; + FileDir::safe_exec('tar cfz - -C ' . escapeshellarg($tmpdir) . ' ' . trim($create_export_tar_data) . ' | gpg --encrypt --recipient-file ' . escapeshellarg($recipient_file) . ' --output ' . escapeshellarg($filename) . ' --trust-model always --batch --yes', $return_value, ['|']); + } else { + // pack all archives in tmp-dir to one archive + FileDir::safe_exec('tar cfz ' . escapeshellarg($filename) . ' -C ' . escapeshellarg($tmpdir) . ' ' . trim($create_export_tar_data)); + } // determine filesize (use stat locally here b/c files are possibly large and php's filesize() can't handle them) - $sizeCheckFile = FileDir::makeCorrectFile($this->tmpDirectory . "/" . $filename); - $fileSizeOutput = FileDir::safe_exec('/usr/bin/stat -c "%s" ' . escapeshellarg($sizeCheckFile)); + $fileSizeOutput = FileDir::safe_exec('/usr/bin/stat -c "%s" ' . escapeshellarg($filename)); $fileSize = (int)array_shift($fileSizeOutput); // add entry to database and upload/store file - $this->addEntry($filename, $fileSize); - return $this->putFile($filename, $this->tmpDirectory); + + FileDir::safe_exec('rm -rf ' . escapeshellarg($tmpdir)); + $fileDest = $this->putFile(basename($filename), $this->tmpDirectory); + if (!empty($fileDest)) { + $this->addEntry($fileDest, $fileSize); + return true; + } + return false; } /** diff --git a/lib/Froxlor/Backup/Storages/StorageFactory.php b/lib/Froxlor/Backup/Storages/StorageFactory.php index bd2b7f4e..73e28a90 100644 --- a/lib/Froxlor/Backup/Storages/StorageFactory.php +++ b/lib/Froxlor/Backup/Storages/StorageFactory.php @@ -2,6 +2,9 @@ namespace Froxlor\Backup\Storages; +use Exception; +use Froxlor\Database\Database; + class StorageFactory { public static function fromType(string $type, array $storage_data): Storage @@ -9,4 +12,28 @@ class StorageFactory $type = "\\Froxlor\\Backup\\Storages\\" . ucfirst($type); return new $type($storage_data); } + + /** + * @throws Exception + */ + public static function fromStorageId(int $storage_id, array $user_data): Storage + { + $storage = self::readStorageData($storage_id); + $storage_data = $user_data; + $storage_data['storage'] = $storage; + return self::fromType($storage['type'], $storage_data); + } + + /** + * @throws Exception + */ + private static function readStorageData(int $storage_id): array + { + $stmt = Database::prepare("SELECT * FROM `" . TABLE_PANEL_BACKUP_STORAGES . "` WHERE `id` = :bid"); + $storage = Database::pexecute_first($stmt, ['bid' => $storage_id]); + if (empty($storage)) { + throw new Exception("Invalid/empty backup-storage. Unable to continue"); + } + return $storage; + } } diff --git a/lib/tablelisting/admin/tablelisting.backup_storages.php b/lib/tablelisting/admin/tablelisting.backup_storages.php index 716fa1fd..c1c32bd0 100644 --- a/lib/tablelisting/admin/tablelisting.backup_storages.php +++ b/lib/tablelisting/admin/tablelisting.backup_storages.php @@ -26,6 +26,7 @@ use Froxlor\UI\Callbacks\Admin; use Froxlor\UI\Callbacks\Customer; use Froxlor\UI\Callbacks\Impersonate; +use Froxlor\UI\Callbacks\PHPConf; use Froxlor\UI\Callbacks\ProgressBar; use Froxlor\UI\Callbacks\Style; use Froxlor\UI\Callbacks\Text; @@ -115,6 +116,7 @@ return [ 'action' => 'delete', 'id' => ':id' ], + 'visible' => [PHPConf::class, 'isNotDefault'] ], ], ]