diff --git a/.gitignore b/.gitignore index 8fad738f..90644df9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ install/update.log .settings/ *.diff *~ + +parse.php diff --git a/actions/admin/settings/150.mail.php b/actions/admin/settings/150.mail.php index bc2d39ea..b5ce0522 100644 --- a/actions/admin/settings/150.mail.php +++ b/actions/admin/settings/150.mail.php @@ -93,6 +93,54 @@ return array( 'default' => true, 'save_method' => 'storeSettingResetCatchall', ), + 'system_mailtraffic_enabled' => array( + 'label' => $lng['serversettings']['mailtraffic_enabled'], + 'settinggroup' => 'system', + 'varname' => 'mailtraffic_enabled', + 'type' => 'bool', + 'default' => true, + 'save_method' => 'storeSettingField', + ), + 'system_mdaserver' => array( + 'label' => $lng['serversettings']['mdaserver'], + 'settinggroup' => 'system', + 'varname' => 'mdaserver', + 'type' => 'option', + 'option_mode' => 'one', + 'default' => 'dovecot', + 'option_options' => array('courier' => 'Courier', 'dovecot' => 'Dovecot'), + 'save_method' => 'storeSettingField', + ), + 'system_mdalog' => array( + 'label' => $lng['serversettings']['mdalog'], + 'settinggroup' => 'system', + 'varname' => 'mdalog', + 'type' => 'string', + 'string_type' => 'file', + 'default' => '/var/log/mail.log', + 'string_emptyallowed' => true, + 'save_method' => 'storeSettingField', + ), + 'system_mtaserver' => array( + 'label' => $lng['serversettings']['mtaserver'], + 'settinggroup' => 'system', + 'varname' => 'mtaserver', + 'type' => 'option', + 'option_mode' => 'one', + 'default' => 'postfix', + 'option_options' => array('exim4' => 'Exim4', 'postfix' => 'Postfix'), + 'save_method' => 'storeSettingField', + ), + 'system_mtalog' => array( + 'label' => $lng['serversettings']['mtalog'], + 'settinggroup' => 'system', + 'varname' => 'mtalog', + 'type' => 'string', + 'string_type' => 'file', + 'default' => '/var/log/mail.log', + 'string_emptyallowed' => true, + 'save_method' => 'storeSettingField', + ), ), ), ), diff --git a/install/froxlor.sql b/install/froxlor.sql index 042172a3..e1dfaf37 100644 --- a/install/froxlor.sql +++ b/install/froxlor.sql @@ -494,6 +494,11 @@ INSERT INTO `panel_settings` (`settinggroup`, `varname`, `value`) VALUES ('system', 'customer_ssl_path', '/etc/ssl/froxlor-custom/'), ('system', 'allow_error_report_admin', '1'), ('system', 'allow_error_report_customer', '0'), + ('system', 'mdalog' '/var/log/mail.log'), + ('system', 'mtalog' '/var/log/mail.log'), + ('system', 'mdaserver' 'dovecot'), + ('system', 'mtaserver' 'postfix'), + ('system', 'mailtraffic_enabled', '1'), ('panel', 'decimal_places', '4'), ('panel', 'adminmail', 'admin@SERVERNAME'), ('panel', 'phpmyadmin_url', ''), @@ -521,7 +526,7 @@ INSERT INTO `panel_settings` (`settinggroup`, `varname`, `value`) VALUES ('panel', 'phpconfigs_hidestdsubdomain', '0'), ('panel', 'allow_theme_change_admin', '1'), ('panel', 'allow_theme_change_customer', '1'), - ('panel', 'version', '0.9.32-dev1'); + ('panel', 'version', '0.9.32-dev2'); DROP TABLE IF EXISTS `panel_tasks`; diff --git a/install/updates/froxlor/0.9/update_0.9.inc.php b/install/updates/froxlor/0.9/update_0.9.inc.php index e06c7367..b466532f 100644 --- a/install/updates/froxlor/0.9/update_0.9.inc.php +++ b/install/updates/froxlor/0.9/update_0.9.inc.php @@ -2625,3 +2625,23 @@ if (isFroxlorVersion('0.9.31.1')) { updateToVersion('0.9.32-dev1'); } + +if (isFroxlorVersion('0.9.32-dev1')) { + + showUpdateStep("Updating from 0.9.32-dev1 to 0.9.32-dev2"); + lastStepStatus(0); + + showUpdateStep("Adding mailserver - settings for traffic analysis"); + $ins_stmt = Database::prepare(" + INSERT INTO `".TABLE_PANEL_SETTINGS."` SET `settinggroup` = 'system', `varname` = :varname, `value` = :value + "); + + Database::pexecute($ins_stmt, array('varname' => 'mailtraffic_enabled', 'value' => isset($_POST['mailtraffic_enabled']) ? (int)$_POST['mailtraffic_enabled'] : '1')); + Database::pexecute($ins_stmt, array('varname' => 'mdalog', 'value' => isset($_POST['mdalog']) ? $_POST['mdalog'] : '/var/log/mail.log')); + Database::pexecute($ins_stmt, array('varname' => 'mtalog', 'value' => isset($_POST['mtalog']) ? $_POST['mtalog'] : '/var/log/mail.log')); + Database::pexecute($ins_stmt, array('varname' => 'mdaserver', 'value' => isset($_POST['mdaserver']) ? $_POST['mdaserver'] : 'dovecot')); + Database::pexecute($ins_stmt, array('varname' => 'mtaserver', 'value' => isset($_POST['mtaserver']) ? $_POST['mtaserver'] : 'postfix')); + lastStepStatus(0); + + updateToVersion('0.9.32-dev2'); +} diff --git a/install/updates/preconfig/0.9/preconfig_0.9.inc.php b/install/updates/preconfig/0.9/preconfig_0.9.inc.php index 95396007..df0df94b 100644 --- a/install/updates/preconfig/0.9/preconfig_0.9.inc.php +++ b/install/updates/preconfig/0.9/preconfig_0.9.inc.php @@ -610,4 +610,28 @@ function parseAndOutputPreconfig(&$has_preconfig, &$return, $current_version) { eval("\$return.=\"" . getTemplate("update/preconfigitem") . "\";"); } + if (versionInUpdate($current_version, '0.9.32-dev2')) { + $has_preconfig = true; + $description = 'To enable logging of the mail-traffic, you need to set the following settings accordingly

'; + $question = 'Do you want to enable the traffic collection for mail? (default: yes): '; + $question.= makeyesno('mailtraffic_enabled', '1', '0', '1').'
'; + $question.= 'Mail Transfer Agent
'; + $question.= 'Type of your MTA: '; + $question.= '
'; + $question.= 'Logfile for your MTA: '; + $question.= '
'; + $question.= 'Mail Delivery Agent
'; + $question.= 'Type of your MDA: '; + $question.= '

'; + $question.= 'Logfile for your MDA: '; + $question.= '
'; + eval("\$return.=\"" . getTemplate("update/preconfigitem") . "\";"); + } + } diff --git a/lib/classes/mail/class.mailLogParser.php b/lib/classes/mail/class.mailLogParser.php new file mode 100644 index 00000000..4b48c8fc --- /dev/null +++ b/lib/classes/mail/class.mailLogParser.php @@ -0,0 +1,289 @@ + + * @author Froxlor team (2010-) + * @license GPLv2 http://files.froxlor.org/misc/COPYING.txt + * @package Cron + * + * @since 0.9.32 + * + */ + +class MailLogParser { + + private $startTime; + private $domainTraffic = array(); + private $myDomains = array(); + private $mails = array(); + + /** + * constructor + * @param string logFile + * @param int startTime + * @param string logFileExim + */ + public function __construct($startTime = 0) { + $this->startTime = $startTime; + + // Get all domains from Database + $stmt = Database::prepare("SELECT domain FROM `" . TABLE_PANEL_DOMAINS . "`"); + Database::pexecute($stmt, array()); + while ($domain_row = $stmt->fetch(PDO::FETCH_ASSOC)) { + $this->myDomains[] = $domain_row["domain"]; + } + + // Parse MTA traffic + if (Settings::Get("system.mtaserver") == "postfix") { + $this->_parsePostfixLog(Settings::Get("system.mtalog")); + $this->_parsePostfixLog(Settings::Get("system.mtalog") . ".1"); + } elseif (Settings::Get("system.mtaserver") == "exim4") { + $this->_parseExim4Log(Settings::Get("system.mtalog")); + } + + // Parse MDA traffic + if (Settings::Get("system.mdaserver") == "dovecot") { + $this->_parseDovecotLog(Settings::Get("system.mdalog")); + $this->_parsePostfixLog(Settings::Get("system.mdalog") . ".1"); + } elseif (Settings::Get("system.mdaserver") == "courier") { + $this->_parseCourierLog(Settings::Get("system.mdalog")); + $this->_parsePostfixLog(Settings::Get("system.mdalog") . ".1"); + } + } + + + /** + * parsePostfixLog + * parses the traffic from a postfix logfile + * @param logFile + */ + private function _parsePostfixLog($logFile) { + // Check if file exists + if (!file_exists($logFile)) { + return false; + } + + // Open the log file + try { + $file_handle = fopen($logFile, "r"); + if (!$file_handle) { + throw new Exception("Could not open the file!"); + } + } catch (Exception $e) { + echo "Error (File: ".$e->getFile().", line ". $e->getLine()."): ".$e->getMessage(); + return false; + } + + while (!feof($file_handle)) { + unset($matches); + $line = fgets($file_handle); + + $timestamp = $this->_getLogTimestamp($line); + if ($this->startTime < $timestamp) { + if (preg_match("/postfix\/qmgr.*(?::|\])\s([A-Z\d]+).*from=?, size=(\d+),/", $line, $matches)) { + // Postfix from + $this->mails[$matches[1]] = array("domainFrom" => strtolower($matches[2]), "size" => $matches[3]); + } elseif (preg_match("/postfix\/(?:pipe|smtp).*(?::|\])\s([A-Z\d]+).*to=?,/", $line, $matches)) { + // Postfix to + if (array_key_exists($matches[1], $this->mails)) { + $this->mails[$matches[1]]["domainTo"] = strtolower($matches[2]); + + // Only mails from/to outside the system should be added + $mail = $this->mails[$matches[1]]; + if (in_array($mail["domainFrom"], $this->myDomains) || in_array($mail["domainTo"], $this->myDomains)) { + // Outgoing traffic + if (array_key_exists("domainFrom", $mail)) { + $this->_addDomainTraffic($mail["domainFrom"], $mail["size"], $timestamp); + } + + // Incoming traffic + if (array_key_exists("domainTo", $mail) && in_array($mail["domainTo"], $this->myDomains)) { + $this->_addDomainTraffic($mail["domainTo"], $mail["size"], $timestamp); + } + } + unset($mail); + } + } + } + } + fclose($file_handle); + return true; + } + + + /** + * parseExim4Log + * parses the smtp traffic from a exim4 logfile + * @param logFile + */ + private function _parseExim4Log($logFile) { + // Check if file exists + if (!file_exists($logFile)) { + return false; + } + + // Open the log file + try { + $file_handle = fopen($logFile, "r"); + if (!$file_handle) { + throw new Exception("Could not open the file!"); + } + } catch (Exception $e) { + echo "Error (File: ".$e->getFile().", line ". $e->getLine()."): ".$e->getMessage(); + return false; + } + + while (!feof($file_handle)) { + unset($matches); + $line = fgets($file_handle); + + $timestamp = $this->_getLogTimestamp($line); + if ($this->startTime < $timestamp) { + if (preg_match("/<= .*@([a-z0-9.\-]+) .*S=(\d+)/i", $line, $matches)) { + // Outgoing traffic + $this->_addDomainTraffic($matches[1], $matches[2], $timestamp); + } elseif (preg_match("/=> .*? .*S=(\d+)/i" , $line, $matches)) { + // Incoming traffic + $this->_addDomainTraffic($matches[1], $matches[2], $timestamp); + } + } + } + } + + + /** + * parseDovecotLog + * parses the dovecot imap/pop3 traffic from logfile + * @param logFile + */ + private function _parseDovecotLog($logFile) { + // Check if file exists + if (!file_exists($logFile)) { + return false; + } + + // Open the log file + try { + $file_handle = fopen($logFile, "r"); + if (!$file_handle) { + throw new Exception("Could not open the file!"); + } + } catch (Exception $e) { + echo "Error (File: ".$e->getFile().", line ". $e->getLine()."): ".$e->getMessage(); + return false; + } + + while (!feof($file_handle)) { + unset($matches); + $line = fgets($file_handle); + + $timestamp = $this->_getLogTimestamp($line); + if ($this->startTime < $timestamp) { + if (preg_match("/dovecot.*(?::|\]) imap\(.*@([a-z0-9\.\-]+)\):.*in=(\d+) out=(\d+)/i", $line, $matches)) { + // Dovecot IMAP + $this->_addDomainTraffic($matches[1], (int)$matches[2] + (int)$matches[3], $timestamp); + } elseif (preg_match("/dovecot.*(?::|\]) pop3\(.*@([a-z0-9\.\-]+)\):.*size=(\d+)/i", $line, $matches)) { + // Dovecot POP3 + $this->_addDomainTraffic($matches[1], $matches[2], $timestamp); + } + } + } + } + + + /** + * parseCourierLog + * parses the dovecot imap/pop3 traffic from logfile + * @param logFile + */ + private function _parseCourierLog($logFile) { + // Check if file exists + if (!file_exists($logFile)) { + return false; + } + + // Open the log file + try { + $file_handle = fopen($logFile, "r"); + if (!$file_handle) { + throw new Exception("Could not open the file!"); + } + } catch (Exception $e) { + echo "Error (File: ".$e->getFile().", line ". $e->getLine()."): ".$e->getMessage(); + return false; + } + + while (!feof($file_handle)) { + unset($matches); + $line = fgets($file_handle); + + $timestamp = $this->_getLogTimestamp($line); + if ($this->startTime < $timestamp) { + if (preg_match("/(?:imapd|pop3d)(?:-ssl)?.*(?::|\]).*user=.*@([a-z0-9\.\-]+),.*rcvd=(\d+), sent=(\d+),/i", $line, $matches)) { + // Courier IMAP & POP3 + $this->_addDomainTraffic($matches[1], (int)$matches[2] + (int)$matches[3], $timestamp); + } + } + } + } + + + /** + * _addDomainTraffic + * adds the traffic to the domain array if we own the domain + * @param string domain + * @param int traffic + */ + private function _addDomainTraffic($domain, $traffic, $timestamp) { + $date = date("Y-m-d", $timestamp); + if (in_array($domain, $this->myDomains)) { + if (array_key_exists($domain, $this->domainTraffic) && array_key_exists($date, $this->domainTraffic[$domain])) { + $this->domainTraffic[$domain][$date] += (int)$traffic; + } else { + if (!array_key_exists($domain, $this->domainTraffic)) { + $this->domainTraffic[$domain] = array(); + } + $this->domainTraffic[$domain][$date] = (int)$traffic; + } + } + } + + + /** + * getLogTimestamp + * @param string line + * return int + */ + private function _getLogTimestamp($line) { + if (preg_match("/((?:[A-Z]{3} |\d{4}-\d{2}-)\d{2} \d{2}:\d{2}:\d{2})/i", $line, $matches)) { + return strtotime($matches[1]); + } else { + return 0; + } + } + + + /** + * getDomainTraffic + * returns the traffic of a given domain or 0 if the domain has no traffic + * @param string domain + * return array + */ + public function getDomainTraffic($domain) { + if (array_key_exists($domain, $this->domainTraffic)) { + return $this->domainTraffic[$domain]; + } else { + return 0; + } + } + + +} \ No newline at end of file diff --git a/lib/tables.inc.php b/lib/tables.inc.php index c9565432..e9d0e812 100644 --- a/lib/tables.inc.php +++ b/lib/tables.inc.php @@ -51,6 +51,6 @@ define('TABLE_PANEL_DOMAIN_SSL_SETTINGS', 'domain_ssl_settings'); define('TABLE_DOMAINTOIP', 'panel_domaintoip'); // VERSION INFO -$version = '0.9.32-dev1'; +$version = '0.9.32-dev2'; $dbversion = '2'; $branding = ''; diff --git a/lng/english.lng.php b/lng/english.lng.php index c9ddfffc..3674a51f 100644 --- a/lng/english.lng.php +++ b/lng/english.lng.php @@ -1787,3 +1787,13 @@ $lng['logger']['cron'] = "Cronjob"; $lng['logger']['login'] = "Login"; $lng['logger']['intern'] = "Internal"; $lng['logger']['unknown'] = "Unknown"; +$lng['serversettings']['mailtraffic_enabled']['title'] = "Analyse mail traffic"; +$lng['serversettings']['mailtraffic_enabled']['description'] = "Enable analysing of mailserver logs to calculate the traffic"; +$lng['serversettings']['mdaserver']['title'] = "MDA type"; +$lng['serversettings']['mdaserver']['description'] = "Type of the Mail Delivery Server"; +$lng['serversettings']['mdalog']['title'] = "MDA log"; +$lng['serversettings']['mdalog']['description'] = "Logfile of the Mail Delivery Server"; +$lng['serversettings']['mtaserver']['title'] = "MTA type"; +$lng['serversettings']['mtaserver']['description'] = "Type of the Mail Transfer Agent"; +$lng['serversettings']['mtalog']['title'] = "MTA log"; +$lng['serversettings']['mtalog']['description'] = "Logfile of the Mail Transfer Agent"; diff --git a/lng/german.lng.php b/lng/german.lng.php index 70562462..3f63369a 100644 --- a/lng/german.lng.php +++ b/lng/german.lng.php @@ -1513,3 +1513,13 @@ $lng['logger']['cron'] = "Cronjob"; $lng['logger']['login'] = "Login"; $lng['logger']['intern'] = "Intern"; $lng['logger']['unknown'] = "Unbekannt"; +$lng['serversettings']['mailtraffic_enabled']['title'] = "Analysiere Mailtraffic"; +$lng['serversettings']['mailtraffic_enabled']['description'] = "Aktiviere das analysieren der Logdateien des Mailsystems um den verbrauchten Traffic zu berechnen"; +$lng['serversettings']['mdaserver']['title'] = "Typ des MDA"; +$lng['serversettings']['mdaserver']['description'] = "Der eingesetzte Mail Delivery Server"; +$lng['serversettings']['mdalog']['title'] = "Logdatei des MDA"; +$lng['serversettings']['mdalog']['description'] = "Die Logdatei des Mail Delivery Server"; +$lng['serversettings']['mtaserver']['title'] = "Typ des MTA"; +$lng['serversettings']['mtaserver']['description'] = "Der eingesetzte Mail Transfer Agent"; +$lng['serversettings']['mtalog']['title'] = "Logdatei des MTA"; +$lng['serversettings']['mtalog']['description'] = "Die Logdatei des Mail Transfer Agent"; diff --git a/scripts/jobs/cron_traffic.php b/scripts/jobs/cron_traffic.php index b43008ec..25ed5d3d 100644 --- a/scripts/jobs/cron_traffic.php +++ b/scripts/jobs/cron_traffic.php @@ -147,6 +147,15 @@ if (Settings::Get('system.diskquota_enabled')) { $usedquota = getFilesystemQuota(); } +/** + * MAIL-Traffic + */ +if (Settings::Get("system.mailtraffic_enabled")) { + $stmt = Database::prepare("SELECT lastrun FROM `" . TABLE_PANEL_CRONRUNS . "` WHERE `cronfile` = 'cron_traffic.php'"); + $result = Database::pexecute_first($stmt, array()); + $mailTrafficCalc = new MailLogParser(0); +} + $result_stmt = Database::query("SELECT * FROM `" . TABLE_PANEL_CUSTOMERS . "` ORDER BY `customerid` ASC"); while ($row = $result_stmt->fetch(PDO::FETCH_ASSOC)) { @@ -237,6 +246,48 @@ while ($row = $result_stmt->fetch(PDO::FETCH_ASSOC)) { * Mail-Traffic */ $mailtraffic = 0; + if (Settings::Get("system.mailtraffic_enabled")) { + fwrite($debugHandler, 'mail traffic usage for ' . $row['loginname'] . " started...\n"); + + $currentDay = date("Y-m-d"); + + $domains_stmt = Database::prepare("SELECT domain FROM `" . TABLE_PANEL_DOMAINS . "` WHERE `customerid` = :cid"); + Database::pexecute($domains_stmt, array("cid" => $row['customerid'])); + while ($domainRow = $domains_stmt->fetch(PDO::FETCH_ASSOC)) { + $domainMailTraffic = $mailTrafficCalc->getDomainTraffic($domainRow["domain"]); + if (!is_array($domainMailTraffic)) { continue; } + + foreach ($domainMailTraffic as $day => $dayTraffic) { + $dayTraffic = floatval($dayTraffic / 1024); + + list($year, $month, $day) = explode("-", $day); + if ($day == $currentDay) { + $mailtraffic = $dayTraffic; + } else { + // Check if an entry for the given day exists + $stmt = Database::prepare("SELECT * FROM `" . TABLE_PANEL_TRAFFIC . "` + WHERE `customerid` = :cid + AND `year` = :year + AND `month` = :month + AND `day` = :day"); + $params = array( + "cid" => $row['customerid'], + "year" => $year, + "month" => $month, + "day" => $day + ); + Database::pexecute($stmt, $params); + if ($stmt->rowCount() > 0) { + $updRow = $stmt->fetch(PDO::FETCH_ASSOC); + $upd_stmt = Database::prepare("UPDATE `" . TABLE_PANEL_TRAFFIC . "` SET + `mail` = :mail + WHERE `id` = :id"); + Database::pexecute($upd_stmt, array("mail" => $updRow['mail'] + $dayTraffic, "id" => $updRow['id'])); + } + } + } + } + } /** * Total Traffic diff --git a/templates/Sparkle/admin/admins/admins_admin.tpl b/templates/Sparkle/admin/admins/admins_admin.tpl index 75132afd..19775762 100644 --- a/templates/Sparkle/admin/admins/admins_admin.tpl +++ b/templates/Sparkle/admin/admins/admins_admin.tpl @@ -19,11 +19,11 @@ Webspace: -
+
-
+
@@ -37,11 +37,11 @@ Traffic: -
+
-
+
diff --git a/templates/Sparkle/admin/customers/customers_customer.tpl b/templates/Sparkle/admin/customers/customers_customer.tpl index 78c1c75c..1c5016a1 100644 --- a/templates/Sparkle/admin/customers/customers_customer.tpl +++ b/templates/Sparkle/admin/customers/customers_customer.tpl @@ -26,11 +26,11 @@ Webspace: -
+
-
+
@@ -44,11 +44,11 @@ Traffic: -
+
-
+