Merge pull request #3 from dmamontov/master

fix memory usage

retailcrm/app.php

@@ -10,7 +10,7 @@ if (
 
 require_once 'bootstrap.php';
 
-$options = getopt('luce:m:p:r:h:');
+$options = getopt('dluce:m:p:r:h:');
 
 if (isset($options['e'])) {
     $command = new Command($options);

retailcrm/src/Components/Command.php

@@ -10,6 +10,7 @@ class Command
     private $limit;
     private $update;
     private $container;
+    private $debug;
 
     public function __construct($arguments)
     {
@@ -22,6 +23,7 @@ class Command
         $this->limit   = isset($arguments['l']);
         $this->update  = isset($arguments['u']);
         $this->custom  = isset($arguments['c']);
+        $this->debug   = isset($arguments['d']);
 
         $this->container = Container::getInstance();
 
@@ -40,9 +42,17 @@ class Command
             return;
         }
 
-        $command = 'run' . ucfirst($this->run);
+        $debug = new DebugHelper();
+        if ($this->debug) {
+            $debug->write(sprintf('Start %s', ucfirst($this->run)));
+        }
 
-        return $this->$command();
+        $command = sprintf('run%s', ucfirst($this->run));
+        $this->$command();
+
+        if ($this->debug) {
+            $debug->write(sprintf('End %s', ucfirst($this->run)));
+        }
     }
 
     public function runDump()
@@ -52,9 +62,12 @@ class Command
         $dbName = $this->container->settings['db']['dbname'];
         $dbHost = $this->container->settings['db']['host'];
 
-        $dumpfile = $this->container->saveDir . "dbdump.sql.gz";
+        $dumpfile = sprintf('%sdbdump.sql.gz', $this->container->saveDir);
 
-        $cmd = "mysqldump -u $dbUser --password=$dbPass --host=$dbHost $dbName | gzip --best > $dumpfile";
+        $cmd = sprintf(
+            'mysqldump -u %s --password=%s --host=%s %s | gzip --best > %s',
+            $dbUser, $dbPass, $dbHost, $dbName, $dumpfile
+        );
         passthru($cmd);
     }
 
@@ -254,12 +267,10 @@ class Command
         $handler = $rule->getHandler('AmoHandler');
         $data = $handler->prepare($amo);
 
-        if (!empty($data)) {
+        if (!empty($data) && !empty($data['customers'])) {
-            if (!empty($data['customers'])) {
+            $this->requestHelper->uploadCustomers($data['customers']);
-                $this->requestHelper->uploadCustomers($data['customers']);
+            if (!empty($data['orders'])) {
-                if (!empty($data['orders'])) {
+                $this->requestHelper->uploadOrders($data['orders'], true);
-                   $this->requestHelper->uploadOrders($data['orders'], true);
-                }
             }
         }
     }

retailcrm/src/Helpers/DebugHelper.php

@@ -0,0 +1,48 @@
+<?php
+
+class DebugHelper
+{
+    private $baseMemoryUsage;
+    private $tusage;
+    private $rusage;
+
+    public function __construct()
+    {
+        $this->baseMemoryUsage = memory_get_usage(true);
+
+        $proc = getrusage();
+        $this->tusage = microtime(true);
+        $this->rusage = $proc['ru_utime.tv_sec'] * 1e6 + $proc['ru_utime.tv_usec'];
+    }
+
+    private function formatSize($size)
+    {
+        $postfix = array('b', 'Kb', 'Mb', 'Gb', 'Tb');
+        $position = 0;
+        while ($size >= 1024 && $position < 4) {
+            $size /= 1024;
+            $position++;
+        }
+
+        return sprintf('%s %s', round($size, 2), $postfix[$position]);
+    }
+
+    public function getMemoryUsage()
+    {
+        return $this->formatSize(memory_get_usage(true) - $this->baseMemoryUsage);
+    }
+
+    public function getCpuUsage()
+    {
+        $proc = getrusage();
+        $proc["ru_utime.tv_usec"] = ($proc["ru_utime.tv_sec"] * 1e6 + $proc["ru_utime.tv_usec"]) - $this->rusage;
+        $time = (microtime(true) - $this->tusage) * 1000000;
+
+        return $time > 0 ? sprintf("%01.2f", ($proc["ru_utime.tv_usec"] / $time) * 100) : '0.00';
+    }
+
+    public function write($string)
+    {
+        echo sprintf("%s\t%s\t%s%s", $string, $this->getCpuUsage(), $this->getMemoryUsage(), PHP_EOL);
+    }
+}

retailcrm/src/Helpers/IcmlHelper.php

@@ -4,6 +4,7 @@ class IcmlHelper
 {
     protected $shop;
     protected $file;
+    protected $tmpFile;
 
     protected $properties = array(
             'name',
@@ -17,48 +18,93 @@ class IcmlHelper
             'productActivity'
     );
 
-    protected $document;
+    protected $xml;
     protected $categories;
     protected $offers;
 
+    protected $chunk = 500;
+
     public function __construct($shop, $file)
     {
         $this->shop = $shop;
         $this->file = $file;
+        $this->tmpFile = sprintf('%s.tmp', $file);
     }
 
     public function generate($categories, $offers)
     {
-        $string = '<?xml version="1.0" encoding="UTF-8"?>
+        if (!file_exists($this->tmpFile)) {
-            <yml_catalog date="'.date('Y-m-d H:i:s').'">
+            $this->writeHead();
-                <shop>
+        }
-                    <name>'.$this->shop.'</name>
-                    <categories/>
-                    <offers/>
-                </shop>
-            </yml_catalog>
-        ';
 
-        $xml = new SimpleXMLElement(
+        if (!empty($categories)) {
-            $string,
+            $this->writeCategories($categories);
-            LIBXML_NOENT |LIBXML_NOCDATA | LIBXML_COMPACT | LIBXML_PARSEHUGE
+            unset($categories);
+        }
+
+        if (!empty($offers)) {
+            $this->writeOffers($offers);
+            unset($offers);
+        }
+
+        $dom = dom_import_simplexml(simplexml_load_file($this->tmpFile))->ownerDocument;
+        $dom->formatOutput = true;
+        $formatted = $dom->saveXML();
+
+        unset($dom, $this->xml);
+
+        file_put_contents($this->tmpFile, $formatted);
+        rename($this->tmpFile, $this->file);
+    }
+
+    private function loadXml()
+    {
+        return new SimpleXMLElement(
+                $this->tmpFile,
+                LIBXML_NOENT | LIBXML_NOCDATA | LIBXML_COMPACT | LIBXML_PARSEHUGE,
+                true
+        );
+    }
+
+    private function writeHead()
+    {
+        $string = sprintf(
+            '<?xml version="1.0" encoding="UTF-8"?><yml_catalog date="%s"><shop><name>%s</name><categories/><offers/></shop></yml_catalog>',
+            date('Y-m-d H:i:s'),
+            $this->shop
         );
 
-        $this->document = new DOMDocument();
+        file_put_contents($this->tmpFile, $string, LOCK_EX);
-        $this->document->preserveWhiteSpace = false;
+    }
-        $this->document->formatOutput = true;
-        $this->document->loadXML($xml->asXML());
 
-        $this->categories = $this->document
+    private function writeCategories($categories)
-            ->getElementsByTagName('categories')->item(0);
+    {
-        $this->offers = $this->document
+        $chunkCategories = array_chunk($categories, $this->chunk);
-            ->getElementsByTagName('offers')->item(0);
+        foreach ($chunkCategories as $categories) {
+            $this->xml = $this->loadXml();
 
-        $this->addCategories($categories);
+            $this->categories = $this->xml->shop->categories;
-        $this->addOffers($offers);
+            $this->addCategories($categories);
 
-        $this->document->saveXML();
+            $this->xml->asXML($this->tmpFile);
-        $this->document->save($this->file);
+        }
+
+        unset($this->categories);
+    }
+
+    private function writeOffers($offers)
+    {
+        $chunkOffers = array_chunk($offers, $this->chunk);
+        foreach ($chunkOffers as $offers) {
+            $this->xml = $this->loadXml();
+
+            $this->offers = $this->xml->shop->offers;
+            $this->addOffers($offers);
+
+            $this->xml->asXML($this->tmpFile);
+        }
+
+        unset($this->offers);
     }
 
     private function addCategories($categories)
@@ -70,16 +116,12 @@ class IcmlHelper
                 continue;
             }
 
-            $e = $this->categories->appendChild(
+            $e = $this->categories->addChild('category', $category['name']);
-                $this->document->createElement(
-                    'category', $category['name']
-                )
-            );
 
-            $e->setAttribute('id', $category['id']);
+            $e->addAttribute('id', $category['id']);
 
             if (array_key_exists('parentId', $category) && $category['parentId'] > 0) {
-                $e->setAttribute('parentId', $category['parentId']);
+                $e->addAttribute('parentId', $category['parentId']);
             }
         }
      }
@@ -88,39 +130,33 @@ class IcmlHelper
     {
         $offers = DataHelper::filterRecursive($offers);
 
-        foreach ($offers as $offer) {
+        foreach ($offers as $key => $offer) {
 
             if (!array_key_exists('id', $offer)) {
                 continue;
             }
 
-            $e = $this->offers->appendChild(
+            $e = $this->offers->addChild('offer');
-                $this->document->createElement('offer')
-            );
 
-            $e->setAttribute('id', $offer['id']);
+            $e->addAttribute('id', $offer['id']);
 
             if (!array_key_exists('productId', $offer) || empty($offer['productId'])) {
                 $offer['productId'] = $offer['id'];
             }
-            $e->setAttribute('productId', $offer['productId']);
+            $e->addAttribute('productId', $offer['productId']);
 
             if (!empty($offer['quantity'])) {
-                $e->setAttribute('quantity', (int) $offer['quantity']);
+                $e->addAttribute('quantity', (int) $offer['quantity']);
             } else {
-                $e->setAttribute('quantity', 0);
+                $e->addAttribute('quantity', 0);
             }
 
             if (is_array($offer['categoryId'])) {
                 foreach ($offer['categoryId'] as $categoryId) {
-                    $e->appendChild(
+                    $e->addChild('categoryId', $categoryId);
-                        $this->document->createElement('categoryId', $categoryId)
-                    );
                 }
             } else {
-                $e->appendChild(
+                $e->addChild('categoryId', $offer['categoryId']);
-                    $this->document->createElement('categoryId', $offer['categoryId'])
-                );
             }
 
             if (!array_key_exists('name', $offer) || empty($offer['name'])) {
@@ -137,16 +173,14 @@ class IcmlHelper
             if (array_key_exists('params', $offer) && !empty($offer['params'])) {
                 array_walk($offer['params'], array($this, 'setOffersParams'), $e);
             }
+
+            unset($offers[$key]);
         }
     }
 
     private function setOffersProperties($value, $key, &$e) {
         if (in_array($key, $this->properties) && $key != 'params') {
-            $e->appendChild(
+            $e->addChild($key, htmlspecialchars($value));
-                $this->document->createElement($key)
-            )->appendChild(
-                $this->document->createTextNode($value)
-            );
         }
     }
 
@@ -159,14 +193,9 @@ class IcmlHelper
             !empty($value['name']) &&
             !empty($value['value'])
         ) {
-            $param = $this->document->createElement('param');
+            $param = $e->addChild('param', htmlspecialchars($value['value']));
-            $param->setAttribute('code', $value['code']);
+            $param->addAttribute('code', $value['code']);
-            $param->setAttribute('name', $value['name']);
+            $param->addAttribute('name', htmlspecialchars($value['name']));
-            $param->appendChild(
-                $this->document->createTextNode($value['value'])
-            );
-
-            $e->appendChild($param);
         }
     }
 }