add backup settings and update cron fork

Signed-off-by: Maurice Preuß (envoyr) <envoyr@froxlor.org>

actions/admin/settings/110.accounts.php

@@ -236,7 +236,7 @@ return [
 					'varname' => 'backupenabled',
 					'type' => 'checkbox',
 					'default' => false,
-					'cronmodule' => 'froxlor/backup',
+					'cronmodule' => 'froxlor/export',
 					'save_method' => 'storeSettingField'
 				],
 				'system_createstdsubdom_default' => [

actions/admin/settings/230.backup.php

@@ -32,19 +32,20 @@ return [
 			'fields' => [
 				'system_backup_enabled' => [
 					'label' => lng('serversettings.backup_enabled'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'diskquota_enabled',
+					'varname' => 'enabled',
 					'type' => 'checkbox',
 					'default' => false,
 					'save_method' => 'storeSettingField',
-					'overview_option' => true
+					'overview_option' => true,
+					'cronmodule' => 'froxlor/backup'
 				],
 				'system_backup_type' => [
 					'label' => lng('serversettings.backup_type'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_type',
+					'varname' => 'type',
 					'type' => 'select',
-					'default' => 'S3',
+					'default' => 'Local',
 					'select_var' => [
 						'Local' => lng('serversettings.local'),
 						'SFTP' => lng('serversettings.sftp'),
@@ -56,66 +57,72 @@ return [
 				],
 				'system_backup_region' => [
 					'label' => lng('serversettings.backup_region'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_region',
+					'varname' => 'region',
 					'type' => 'text',
 					'default' => 'eu-central-1',
 					'save_method' => 'storeSettingField',
 				],
 				'system_backup_bucket' => [
 					'label' => lng('serversettings.backup_bucket'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_bucket',
+					'varname' => 'bucket',
 					'type' => 'text',
 					'default' => '',
 					'save_method' => 'storeSettingField',
 				],
 				'system_backup_destination_path' => [
 					'label' => lng('serversettings.backup_destination_path'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_destination_path',
+					'varname' => 'destination_path',
 					'type' => 'text',
-					'default' => 'backups',
+					'string_type' => 'confdir',
+					'default' => '/srv/backups/',
 					'save_method' => 'storeSettingField',
 				],
 				'system_backup_hostname' => [
 					'label' => lng('serversettings.backup_hostname'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_hostname',
+					'varname' => 'hostname',
 					'type' => 'text',
 					'default' => '',
 					'save_method' => 'storeSettingField',
 				],
 				'system_backup_username' => [
 					'label' => lng('serversettings.backup_username'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_username',
+					'varname' => 'username',
 					'type' => 'text',
 					'default' => '',
 					'save_method' => 'storeSettingField',
 				],
 				'system_backup_password' => [
 					'label' => lng('serversettings.backup_password'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_password',
+					'varname' => 'password',
 					'type' => 'password',
 					'default' => '',
 					'save_method' => 'storeSettingField',
 				],
 				'system_backup_pgp_public_key' => [
 					'label' => lng('serversettings.backup_pgp_public_key'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_pgp_public_key',
+					'varname' => 'pgp_public_key',
 					'type' => 'textarea',
 					'default' => '',
 					'save_method' => 'storeSettingField',
+					'plausibility_check_method' => [
+						'\\Froxlor\\Validate\\Check',
+						'checkPgpPublicKeySetting'
+					],
 				],
 				'system_backup_retention' => [
 					'label' => lng('serversettings.backup_retention'),
-					'settinggroup' => 'system',
+					'settinggroup' => 'backup',
-					'varname' => 'backup_retention',
+					'varname' => 'retention',
 					'type' => 'number',
 					'default' => 3,
+					'min' => 0,
 					'save_method' => 'storeSettingField',
 				],
 			]

install/froxlor.sql.php

@@ -700,6 +700,16 @@ opcache.validate_timestamps'),
 	('system', 'traffictool', 'goaccess'),
 	('system', 'req_limit_per_interval', 60),
 	('system', 'req_limit_interval', 60),
+	('backup', 'enabled', 0),
+	('backup', 'type', 'Local'),
+	('backup', 'region', ''),
+	('backup', 'bucket', ''),
+	('backup', 'destination_path', '/srv/backups/'),
+	('backup', 'hostname', ''),
+	('backup', 'username', ''),
+	('backup', 'password', ''),
+	('backup', 'pgp_public_key', ''),
+	('backup', 'retention', '3'),
 	('api', 'enabled', '0'),
 	('api', 'customer_default', '1'),
 	('2fa', 'enabled', '1'),
@@ -913,7 +923,8 @@ INSERT INTO `cronjobs_run` (`id`, `module`, `cronfile`, `cronclass`, `interval`,
 	(3, 'froxlor/reports', 'usage_report', '\\Froxlor\\Cron\\Traffic\\ReportsCron', '1 DAY', '1', 'cron_usage_report'),
 	(4, 'froxlor/core', 'mailboxsize', '\\Froxlor\\Cron\\System\\MailboxsizeCron', '6 HOUR', '1', 'cron_mailboxsize'),
 	(5, 'froxlor/letsencrypt', 'letsencrypt', '\\Froxlor\\Cron\\Http\\LetsEncrypt\\AcmeSh', '5 MINUTE', '0', 'cron_letsencrypt'),
-	(6, 'froxlor/backup', 'backup', '\\Froxlor\\Cron\\System\\BackupCron', '1 DAY', '0', 'cron_backup');
+	(6, 'froxlor/export', 'export', '\\Froxlor\\Cron\\System\\ExportCron', '1 DAY', '0', 'cron_export');
+	(7, 'froxlor/backup', 'backup', '\\Froxlor\\Cron\\Backup\\BackupCron', '1 DAY', '0', 'cron_backup');
 
 
 DROP TABLE IF EXISTS `ftp_quotalimits`;

install/updates/froxlor/update_2.x.inc.php

@@ -524,5 +524,52 @@ if (Froxlor::isDatabaseVersion('202304260')) {
 		Update::lastStepStatus(1, 'Customized setting, not changing');
 	}
 
+	Update::showUpdateStep("Creating new tables and fields for backups");
+	Database::query("DROP TABLE IF EXISTS `panel_backups`;");
+	$sql = "CREATE TABLE `panel_backups` (
+	  `id` int(11) NOT NULL AUTO_INCREMENT,
+	  `adminid` int(11) NOT NULL,
+	  `customerid` int(11) NOT NULL,
+	  `loginname` varchar(255) NOT NULL,
+	  `size` bigint(20) NOT NULL,
+	  `created_at` int(15) NOT NULL,
+	  PRIMARY KEY (`id`)
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;";
+	Database::query($sql);
+	Update::lastStepStatus(0);
+
+	Update::showUpdateStep("Adding new backup settings");
+	Settings::AddNew('backup.enabled', 0);
+	Settings::AddNew('backup.type', 'Local');
+	Settings::AddNew('backup.region', '');
+	Settings::AddNew('backup.bucket', '');
+	Settings::AddNew('backup.destination_path', '/srv/backups/');
+	Settings::AddNew('backup.hostname', '');
+	Settings::AddNew('backup.username', '');
+	Settings::AddNew('backup.password', '');
+	Settings::AddNew('backup.pgp_public_key', '');
+	Settings::AddNew('backup.retention', 3);
+	Update::lastStepStatus(0);
+
+	Update::showUpdateStep("Adjusting cronjobs");
+	Database::query("
+        UPDATE `" . TABLE_PANEL_CRONRUNS . "` SET
+        `module`= 'froxlor/export',
+        `cronfile` = 'export',
+        `cronclass` = '\\Froxlor\\Cron\\System\\ExportCron',
+        `desc_lng_key` = 'cron_export'
+        WHERE `module` = 'froxlor/backup'
+    ");
+	Database::query("
+        INSERT INTO `" . TABLE_PANEL_CRONRUNS . "` SET
+        `module`= 'froxlor/backup',
+        `cronfile` = 'backup',
+        `cronclass` = '\\Froxlor\\Cron\\Backup\\BackupCron',
+        `interval` = '1 DAY',
+        `isactive` = '0',
+        `desc_lng_key` = 'cron_backup'
+    ");
+	Update::lastStepStatus(0);
+
 	Froxlor::updateToDbVersion('202305240');
 }

lib/Froxlor/Cli/MasterCron.php

@@ -52,7 +52,7 @@ final class MasterCron extends CliCommand
 		$this->setDescription('Regulary perform tasks created by froxlor');
 		$this->addArgument('job', InputArgument::IS_ARRAY, 'Job(s) to run');
 		$this->addOption('run-task', 'r', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Run a specific task [1 = re-generate configs, 4 = re-generate dns zones, 10 = re-set quotas, 99 = re-create cron.d-file]')
-			->addOption('force', 'f', InputOption::VALUE_NONE, 'Forces re-generating of config-files (webserver, nameserver, etc.)')
+			->addOption('force', 'f', InputOption::VALUE_NONE, 'Forces given job or, if none given, forces re-generating of config-files (webserver, nameserver, etc.)')
 			->addOption('debug', 'd', InputOption::VALUE_NONE, 'Output debug information about what is going on to STDOUT.')
 			->addOption('no-fork', 'N', InputOption::VALUE_NONE, 'Do not fork to background (traffic cron only).');
 	}
@@ -71,12 +71,13 @@ final class MasterCron extends CliCommand
 
 		// handle force option
 		if ($input->getOption('force')) {
-			// rebuild all config files
+			if (empty($jobs) || in_array('tasks', $jobs)) {
-			Cronjob::inserttask(TaskId::REBUILD_VHOST);
+				Cronjob::inserttask(TaskId::REBUILD_VHOST);
-			Cronjob::inserttask(TaskId::REBUILD_DNS);
+				Cronjob::inserttask(TaskId::REBUILD_DNS);
-			Cronjob::inserttask(TaskId::CREATE_QUOTA);
+				Cronjob::inserttask(TaskId::CREATE_QUOTA);
-			Cronjob::inserttask(TaskId::REBUILD_CRON);
+				Cronjob::inserttask(TaskId::REBUILD_CRON);
-			array_push($jobs, 'tasks');
+				array_push($jobs, 'tasks');
+			}
 			define('CRON_IS_FORCED', 1);
 		}
 		// handle debug option
@@ -91,7 +92,7 @@ final class MasterCron extends CliCommand
 		if ($input->getOption('run-task')) {
 			$tasks_to_run = $input->getOption('run-task');
 			foreach ($tasks_to_run as $ttr) {
-				if (in_array($ttr, [1, 4, 10, 99])) {
+				if (in_array($ttr, [TaskId::REBUILD_VHOST, TaskId::REBUILD_DNS, TaskId::CREATE_QUOTA, TaskId::REBUILD_CRON])) {
 					Cronjob::inserttask($ttr);
 					array_push($jobs, 'tasks');
 				} else {

lib/Froxlor/Cron/Backup/BackupCron.php

@@ -0,0 +1,61 @@
+<?php
+
+/**
+ * This file is part of the Froxlor project.
+ * Copyright (c) 2010 the Froxlor Team (see authors).
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can also view it online at
+ * https://files.froxlor.org/misc/COPYING.txt
+ *
+ * @copyright  the authors
+ * @author     Froxlor team <team@froxlor.org>
+ * @license    https://files.froxlor.org/misc/COPYING.txt GPLv2
+ */
+
+namespace Froxlor\Cron\Backup;
+
+use Froxlor\Cron\Forkable;
+use Froxlor\Cron\FroxlorCron;
+
+class BackupCron extends FroxlorCron
+{
+	use Forkable;
+
+	public static function run()
+	{
+		$users = ['web1', 'web2', 'web3', 'web4', 'web5', 'web6', 'web7', 'web8', 'web9', 'web10'];
+
+		self::runFork([self::class, 'handle'], [
+			[
+				'user' => '1',
+				'data' => 'value1',
+			],
+			[
+				'user' => '2',
+				'data' => 'value2',
+			]
+		]);
+	}
+
+	private static function handle($user, $data)
+	{
+		echo "BackupCron: started - creating customer backup for user $user\n";
+
+		echo $data . "\n";
+
+		sleep(rand(1, 3));
+
+		echo "BackupCron: finished - creating customer backup for user $user\n";
+	}
+}

lib/Froxlor/Cron/Forkable.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace Froxlor\Cron;
+
+use Froxlor\Database\Database;
+use Froxlor\FroxlorLogger;
+
+trait Forkable
+{
+	public static function runFork($closure, array $attributes = [], int $concurrentChildren = 3)
+	{
+		$childrenPids = [];
+
+		// We only fork if pcntl_fork is available and nofork flag is not set
+		if (function_exists('pcntl_fork') && !defined('CRON_NOFORK_FLAG')) {
+			foreach ($attributes as $closureAttributes) {
+				// We close the database - connection before we fork, so we don't share resources with the child
+				Database::needRoot(false); // this forces the connection to be set to null
+				$pid = pcntl_fork();
+
+				if ($pid == -1) {
+					exit("Error forking...\n");
+				} else if ($pid == 0) {
+					// re-create db
+					Database::needRoot(false);
+					$closure(...$closureAttributes);
+					exit();
+				} else {
+					$childrenPids[] = $pid;
+					while(count($childrenPids) >= $concurrentChildren) {
+						foreach($childrenPids as $key => $pid) {
+							$res = pcntl_waitpid($pid, $status, WNOHANG);
+
+							// If the process has already exited
+							if($res == -1 || $res > 0)
+								unset($childrenPids[$key]);
+						}
+
+						sleep(1);
+					}
+				}
+			}
+			while(pcntl_waitpid(0, $status) != -1);
+		} else {
+			if (!defined('CRON_NOFORK_FLAG')) {
+				if (extension_loaded('pcntl')) {
+					$msg = "PHP compiled with pcntl but pcntl_fork function is not available.";
+				} else {
+					$msg = "PHP compiled without pcntl.";
+				}
+				FroxlorLogger::getInstanceOf()->logAction(FroxlorLogger::CRON_ACTION, LOG_WARNING, $msg . " Not forking " . self::class . ", this may take a long time!");
+			}
+			foreach ($attributes as $closureAttributes) {
+				$closure(...$closureAttributes);
+			}
+		}
+	}
+}

lib/Froxlor/Cron/System/ExportCron.php

@@ -25,59 +25,25 @@
 
 namespace Froxlor\Cron\System;
 
+use Exception;
+use Froxlor\Cron\Forkable;
 use Froxlor\Cron\FroxlorCron;
 use Froxlor\Database\Database;
 use Froxlor\FileDir;
 use Froxlor\FroxlorLogger;
 use Froxlor\Settings;
 
-class BackupCron extends FroxlorCron
+class ExportCron extends FroxlorCron
 {
+	use Forkable;
 
 	public static function run()
 	{
-		// Check Backup-Lock
+		self::runFork([self::class, 'handle']);
-		if (function_exists('pcntl_fork') && !defined('CRON_NOFORK_FLAG')) {
+	}
-			$BackupLock = FileDir::makeCorrectFile("/var/run/froxlor_cron_backup.lock");
-			if (file_exists($BackupLock) && is_numeric($BackupPid = file_get_contents($BackupLock))) {
-				if (function_exists('posix_kill')) {
-					$BackupPidStatus = @posix_kill($BackupPid, 0);
-				} else {
-					system("kill -CHLD " . $BackupPid . " 1> /dev/null 2> /dev/null", $BackupPidStatus);
-					$BackupPidStatus = !$BackupPidStatus;
-				}
-				if ($BackupPidStatus) {
-					FroxlorLogger::getInstanceOf()->logAction(FroxlorLogger::CRON_ACTION, LOG_INFO, 'Backup run already in progress');
-					return 1;
-				}
-			}
-			// Create Backup Log and Fork
-			// We close the database - connection before we fork, so we don't share resources with the child
-			Database::needRoot(false); // this forces the connection to be set to null
-			$BackupPid = pcntl_fork();
-			// Parent
-			if ($BackupPid) {
-				file_put_contents($BackupLock, $BackupPid);
-				// unnecessary to recreate database connection here
-				return 0;
-			} elseif ($BackupPid == 0) {
-				// Child
-				posix_setsid();
-				// re-create db
-				Database::needRoot(false);
-			} else {
-				// Fork failed
-				return 1;
-			}
-		} elseif (!defined('CRON_NOFORK_FLAG')) {
-			if (extension_loaded('pcntl')) {
-				$msg = "PHP compiled with pcntl but pcntl_fork function is not available.";
-			} else {
-				$msg = "PHP compiled without pcntl.";
-			}
-			FroxlorLogger::getInstanceOf()->logAction(FroxlorLogger::CRON_ACTION, LOG_WARNING, $msg . " Not forking backup-cron, this may take a long time!");
-		}
 
+	public static function handle()
+	{
 		FroxlorLogger::getInstanceOf()->logAction(FroxlorLogger::CRON_ACTION, LOG_INFO, 'BackupCron: started - creating customer backup');
 
 		$result_tasks_stmt = Database::query("
@@ -113,12 +79,6 @@ class BackupCron extends FroxlorCron
 				'id' => $row['id']
 			]);
 		}
-
-
-		if (function_exists('pcntl_fork') && !defined('CRON_NOFORK_FLAG')) {
-			@unlink($BackupLock);
-			die();
-		}
 	}
 
 	/**
@@ -128,6 +88,7 @@ class BackupCron extends FroxlorCron
 	 *
 	 * @return void
 	 *
+	 * @throws Exception
 	 */
 	private static function createCustomerBackup($data = null, $customerdocroot = null, &$cronlog = null)
 	{

lib/Froxlor/Cron/Traffic/TrafficCron.php

@@ -30,6 +30,7 @@ namespace Froxlor\Cron\Traffic;
  * @author        Froxlor team <team@froxlor.org> (2010-)
  */
 
+use Froxlor\Cron\Forkable;
 use Froxlor\Cron\FroxlorCron;
 use Froxlor\Database\Database;
 use Froxlor\FileDir;
@@ -42,51 +43,15 @@ use PDO;
 
 class TrafficCron extends FroxlorCron
 {
+	use Forkable;
 
 	public static function run()
 	{
-		// Check Traffic-Lock
+		self::runFork([self::class, 'handle']);
-		if (function_exists('pcntl_fork') && !defined('CRON_NOFORK_FLAG')) {
+	}
-			$TrafficLock = FileDir::makeCorrectFile("/var/run/froxlor_cron_traffic.lock");
-			if (file_exists($TrafficLock) && is_numeric($TrafficPid = file_get_contents($TrafficLock))) {
-				if (function_exists('posix_kill')) {
-					$TrafficPidStatus = @posix_kill($TrafficPid, 0);
-				} else {
-					system("kill -CHLD " . $TrafficPid . " 1> /dev/null 2> /dev/null", $TrafficPidStatus);
-					$TrafficPidStatus = !$TrafficPidStatus;
-				}
-				if ($TrafficPidStatus) {
-					FroxlorLogger::getInstanceOf()->logAction(FroxlorLogger::CRON_ACTION, LOG_INFO, 'Traffic Run already in progress');
-					return 1;
-				}
-			}
-			// Create Traffic Log and Fork
-			// We close the database - connection before we fork, so we don't share resources with the child
-			Database::needRoot(false); // this forces the connection to be set to null
-			$TrafficPid = pcntl_fork();
-			// Parent
-			if ($TrafficPid) {
-				file_put_contents($TrafficLock, $TrafficPid);
-				// unnecessary to recreate database connection here
-				return 0;
-			} elseif ($TrafficPid == 0) {
-				// Child
-				posix_setsid();
-				// re-create db
-				Database::needRoot(false);
-			} else {
-				// Fork failed
-				return 1;
-			}
-		} elseif (!defined('CRON_NOFORK_FLAG')) {
-			if (extension_loaded('pcntl')) {
-				$msg = "PHP compiled with pcntl but pcntl_fork function is not available.";
-			} else {
-				$msg = "PHP compiled without pcntl.";
-			}
-			FroxlorLogger::getInstanceOf()->logAction(FroxlorLogger::CRON_ACTION, LOG_INFO, $msg . " Not forking traffic-cron, this may take a long time!");
-		}
 
+	public static function handle()
+	{
 		/**
 		 * TRAFFIC AND DISKUSAGE MEASURE
 		 */
@@ -611,11 +576,6 @@ class TrafficCron extends FroxlorCron
 		}
 
 		Database::query("UPDATE `" . TABLE_PANEL_SETTINGS . "` SET `value` = UNIX_TIMESTAMP() WHERE `settinggroup` = 'system' AND `varname` = 'last_traffic_run'");
-
-		if (function_exists('pcntl_fork') && !defined('CRON_NOFORK_FLAG')) {
-			@unlink($TrafficLock);
-			die();
-		}
 	}
 
 	/**

lib/Froxlor/Validate/Check.php

@@ -314,4 +314,30 @@ class Check
 		}
 		return $returnvalue;
 	}
+
+	public static function checkPgpPublicKeySetting($fieldname, $fielddata, $newfieldvalue, $allnewfieldvalues)
+	{
+		// if the field is empty, we don't need to check anything
+		if ($newfieldvalue === '') {
+			return [self::FORMFIELDS_PLAUSIBILITY_CHECK_OK];
+		}
+
+		// check if gnupg extension is loaded
+		if (!extension_loaded('gnupg')) {
+			return [
+				self::FORMFIELDS_PLAUSIBILITY_CHECK_ERROR,
+				'gnupgextensionnotavailable'
+			];
+		}
+		// check if the pgp public key is a valid key
+		putenv('GNUPGHOME='.sys_get_temp_dir());
+		if (gnupg_import(gnupg_init(), $newfieldvalue) === false) {
+			return [
+				self::FORMFIELDS_PLAUSIBILITY_CHECK_ERROR,
+				'invalidpgppublickey'
+			];
+		}
+
+		return [self::FORMFIELDS_PLAUSIBILITY_CHECK_OK];
+	}
 }