Implement archive extraction and file handling and note in NEWS and INSTALL

that PHP's zip extension is now needed.

INSTALL.mdtext

@@ -6,9 +6,15 @@ the installation of InDefero by itself.
 
 ## PHP modules for indefero
 
-Indefero need the GD module for PHP. It's named "php5-gd" in debian.
+Indefero needs additional PHP modules to function correctly, namely
 
-    $ apt-get install php5-gd
+ - gd (for graphic operations)
+ - zip (for upload archive processing)
+
+The package names of these modules might vary between distributions,
+for Debian they are
+
+    $ apt-get install php5-gd php5-zip
 
 ## Recommended Layout of the Files
 

NEWS.mdtext

@@ -11,6 +11,7 @@ by the friendly folks from Scilab <http://www.scilab.org/>!
   `$cfg['webhook_processing']` flag to "compat", we urge you to change the
   implementations of this web hook as this setting is likely to be removed
   in future versions of Indefero.
+- Indefero now needs PHP's zip module which is not enabled by default.
 
 ## New Features
 
@@ -19,6 +20,9 @@ by the friendly folks from Scilab <http://www.scilab.org/>!
 - It is now possible to configure a web hook that informs an external URL about
   new and updated downloads for a specific project, similar to the available 
   post-commit web hook
+- One can now upload multiple files at once by using a special archive format
+  which Indefero processes in the background and for which individual upload
+  records are created
 
 # InDefero 1.2 - xxx xxx xx xx:xx 2011 UTC
 

src/IDF/Form/UploadArchive.php

@@ -29,6 +29,7 @@ class IDF_Form_UploadArchive extends Pluf_Form
 {
     public $user = null;
     public $project = null;
+    private $archiveHelper = null;
 
     public function initFields($extra=array())
     {
@@ -40,26 +41,34 @@ class IDF_Form_UploadArchive extends Pluf_Form
                                             'label' => __('Archive file'),
                                             'initial' => '',
                                             'max_size' => Pluf::f('max_upload_archive_size', 20971520),
-                                            ));
+                                            'move_function_params' => array(
+                                                'upload_path' => Pluf::f('upload_path').'/'.$this->project->shortname.'/archives',
+                                                'upload_path_create' => true,
+                                                'upload_overwrite' => true,
+                                            )));
     }
 
 
     public function clean_archive()
     {
-        $extra = strtolower(implode('|', explode(' ', Pluf::f('idf_extra_upload_ext'))));
+        $this->archiveHelper = new IDF_Form_UploadArchiveHelper(
-        if (strlen($extra)) $extra .= '|';
+            Pluf::f('upload_path').'/'.$this->project->shortname.'/archives/'.$this->cleaned_data['archive']);
-        if (!preg_match('/\.('.$extra.'png|jpg|jpeg|gif|bmp|psd|tif|aiff|asf|avi|bz2|css|doc|eps|gz|jar|mdtext|mid|mov|mp3|mpg|ogg|pdf|ppt|ps|qt|ra|ram|rm|rtf|sdd|sdw|sit|sxi|sxw|swf|tgz|txt|wav|xls|xml|war|wmv|zip)$/i', $this->cleaned_data['file'])) {
-            @unlink(Pluf::f('upload_path').'/'.$this->project->shortname.'/files/'.$this->cleaned_data['file']);
-            throw new Pluf_Form_Invalid(__('For security reasons, you cannot upload a file with this extension.'));
-        }
-        return $this->cleaned_data['file'];
-    }
 
-    /**
+        // basic archive validation
-     * Validate the interconnection in the form.
+        $this->archiveHelper->validate();
-     */
+
-    public function clean()
+        // extension validation
-    {
+        $names = $this->archiveHelper->getEntryNames();
+        foreach ($names as $name) {
+            $extra = strtolower(implode('|', explode(' ', Pluf::f('idf_extra_upload_ext'))));
+            if (strlen($extra)) $extra .= '|';
+            if (!preg_match('/\.('.$extra.'png|jpg|jpeg|gif|bmp|psd|tif|aiff|asf|avi|bz2|css|doc|eps|gz|jar|mdtext|mid|mov|mp3|mpg|ogg|pdf|ppt|ps|qt|ra|ram|rm|rtf|sdd|sdw|sit|sxi|sxw|swf|tgz|txt|wav|xls|xml|war|wmv|zip)$/i', $name)) {
+                @unlink(Pluf::f('upload_path').'/'.$this->project->shortname.'/files/'.$this->cleaned_data['file']);
+                throw new Pluf_Form_Invalid(sprintf(__('For security reasons, you cannot upload a file (%s) with this extension.'), $name));
+            }
+        }
+
+        // label and file name validation
         $conf = new IDF_Conf();
         $conf->setProject($this->project);
         $onemax = array();
@@ -68,26 +77,39 @@ class IDF_Form_UploadArchive extends Pluf_Form
                 $onemax[] = mb_strtolower(trim($class));
             }
         }
-        $count = array();
+
-        for ($i=1;$i<7;$i++) {
+        foreach ($names as $name) {
-            $this->cleaned_data['label'.$i] = trim($this->cleaned_data['label'.$i]);
+            $meta = $this->archiveHelper->getMetaData($name);
-            if (strpos($this->cleaned_data['label'.$i], ':') !== false) {
+            $count = array();
-                list($class, $name) = explode(':', $this->cleaned_data['label'.$i], 2);
+            foreach ($meta['labels'] as $label) {
-                list($class, $name) = array(mb_strtolower(trim($class)),
+                $label = trim($label);
-                                            trim($name));
+                if (strpos($label, ':') !== false) {
-            } else {
+                    list($class, $name) = explode(':', $label, 2);
-                $class = 'other';
+                    list($class, $name) = array(mb_strtolower(trim($class)),
-                $name = $this->cleaned_data['label'.$i];
+                    trim($name));
+                } else {
+                    $class = 'other';
+                    $name = $label;
+                }
+                if (!isset($count[$class])) $count[$class] = 1;
+                else $count[$class] += 1;
+                if (in_array($class, $onemax) and $count[$class] > 1) {
+                    throw new Pluf_Form_Invalid(
+                       sprintf(__('You cannot provide more than label from the %s class to a download (%s).'), $class, $name)
+                    );
+                }
             }
-            if (!isset($count[$class])) $count[$class] = 1;
+
-            else $count[$class] += 1;
+            $sql = new Pluf_SQL('file=%s AND project=%s', array($name, $this->project->id));
-            if (in_array($class, $onemax) and $count[$class] > 1) {
+            $upload = Pluf::factory('IDF_Upload')->getOne(array('filter' => $sql->gen()));
-                if (!isset($this->errors['label'.$i])) $this->errors['label'.$i] = array();
+
-                $this->errors['label'.$i][] = sprintf(__('You cannot provide more than label from the %s class to an issue.'), $class);
+            if ($upload) {
-                throw new Pluf_Form_Invalid(__('You provided an invalid label.'));
+                throw new Pluf_Form_Invalid(
+                    sprintf(__('A file with the name "%s" has already been uploaded.'), $name));
             }
         }
-        return $this->cleaned_data;
+
+        return $this->cleaned_data['archive'];
     }
 
     /**
@@ -96,9 +118,9 @@ class IDF_Form_UploadArchive extends Pluf_Form
      */
     function failed()
     {
-        if (!empty($this->cleaned_data['file'])
+        if (!empty($this->cleaned_data['archive'])
-            and file_exists(Pluf::f('upload_path').'/'.$this->project->shortname.'/files/'.$this->cleaned_data['file'])) {
+            and file_exists(Pluf::f('upload_path').'/'.$this->project->shortname.'/archives/'.$this->cleaned_data['archive'])) {
-            @unlink(Pluf::f('upload_path').'/'.$this->project->shortname.'/files/'.$this->cleaned_data['file']);
+            @unlink(Pluf::f('upload_path').'/'.$this->project->shortname.'/archives/'.$this->cleaned_data['archive']);
         }
     }
 
@@ -107,65 +129,93 @@ class IDF_Form_UploadArchive extends Pluf_Form
      *
      * @param bool Commit in the database or not. If not, the object
      *             is returned but not saved in the database.
-     * @return Object Model with data set from the form.
      */
     function save($commit=true)
     {
         if (!$this->isValid()) {
             throw new Exception(__('Cannot save the model from an invalid form.'));
         }
-        // Add a tag for each label
+
-        $tags = array();
+        $uploadDir = Pluf::f('upload_path').'/'.$this->project->shortname.'/files/';
-        for ($i=1;$i<7;$i++) {
+        $fileNames = $this->archiveHelper->getEntryNames();
-            if (strlen($this->cleaned_data['label'.$i]) > 0) {
+
-                if (strpos($this->cleaned_data['label'.$i], ':') !== false) {
+        foreach ($fileNames as $fileName) {
-                    list($class, $name) = explode(':', $this->cleaned_data['label'.$i], 2);
+            $meta = $this->archiveHelper->getMetaData($fileName);
-                    list($class, $name) = array(trim($class), trim($name));
+
-                } else {
+            // add a tag for each label
-                    $class = 'Other';
+            $tags = array();
-                    $name = trim($this->cleaned_data['label'.$i]);
+            foreach ($meta['labels'] as $label) {
+                $label = trim($label);
+                if (strlen($label) > 0) {
+                    if (strpos($label, ':') !== false) {
+                        list($class, $name) = explode(':', $label, 2);
+                        list($class, $name) = array(trim($class), trim($name));
+                    } else {
+                        $class = 'Other';
+                        $name = $label;
+                    }
+                    $tags[] = IDF_Tag::add($name, $this->project, $class);
                 }
-                $tags[] = IDF_Tag::add($name, $this->project, $class);
             }
+
+            // extract the file
+            $this->archiveHelper->extract($fileName, $uploadDir);
+
+            // create the upload
+            $upload = new IDF_Upload();
+            $upload->project = $this->project;
+            $upload->submitter = $this->user;
+            $upload->summary = trim($meta['summary']);
+            $upload->changelog = trim($meta['description']);
+            $upload->file = $fileName;
+            $upload->filesize = filesize($uploadDir.$fileName);
+            $upload->downloads = 0;
+            $upload->create();
+            foreach ($tags as $tag) {
+                $upload->setAssoc($tag);
+            }
+
+            // process a possible replacement
+            if (!empty($meta['replaces'])) {
+                $sql = new Pluf_SQL('file=%s AND project=%s', array($meta['replaces'], $this->project->id));
+                $oldUpload = Pluf::factory('IDF_Upload')->getOne(array('filter' => $sql->gen()));
+                if ($oldUpload) {
+                    $tags = $this->project->getTagsFromConfig('labels_download_predefined',
+                                                              IDF_Form_UploadConf::init_predefined);
+                    // the deprecate tag is - by definition - always the last one
+                    $deprecatedTag = array_pop($tags);
+                    $oldUpload->setAssoc($deprecatedTag);
+                }
+            }
+
+            // send the notification
+            $upload->notify($this->project->getConf());
+            /**
+             * [signal]
+             *
+             * IDF_Upload::create
+             *
+             * [sender]
+             *
+             * IDF_Form_Upload
+             *
+             * [description]
+             *
+             * This signal allows an application to perform a set of tasks
+             * just after the upload of a file and after the notification run.
+             *
+             * [parameters]
+             *
+             * array('upload' => $upload);
+             *
+             */
+            $params = array('upload' => $upload);
+            Pluf_Signal::send('IDF_Upload::create', 'IDF_Form_Upload',
+                              $params);
         }
-        // Create the upload
+
-        $upload = new IDF_Upload();
+        // finally unlink the uploaded archive
-        $upload->project = $this->project;
+        @unlink(Pluf::f('upload_path').'/'.$this->project->shortname.'/archives/'.$this->cleaned_data['archive']);
-        $upload->submitter = $this->user;
-        $upload->summary = trim($this->cleaned_data['summary']);
-        $upload->changelog = trim($this->cleaned_data['changelog']);
-        $upload->file = $this->cleaned_data['file'];
-        $upload->filesize = filesize(Pluf::f('upload_path').'/'.$this->project->shortname.'/files/'.$this->cleaned_data['file']);
-        $upload->downloads = 0;
-        $upload->create();
-        foreach ($tags as $tag) {
-            $upload->setAssoc($tag);
-        }
-        // Send the notification
-        $upload->notify($this->project->getConf());
-        /**
-         * [signal]
-         *
-         * IDF_Upload::create
-         *
-         * [sender]
-         *
-         * IDF_Form_Upload
-         *
-         * [description]
-         *
-         * This signal allows an application to perform a set of tasks
-         * just after the upload of a file and after the notification run.
-         *
-         * [parameters]
-         *
-         * array('upload' => $upload);
-         *
-         */
-        $params = array('upload' => $upload);
-        Pluf_Signal::send('IDF_Upload::create', 'IDF_Form_Upload',
-                          $params);
-        return $upload;
     }
 }
 

src/IDF/Form/UploadArchiveHelper.php

@@ -0,0 +1,152 @@
+<?php
+/* -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+# ***** BEGIN LICENSE BLOCK *****
+# This file is part of InDefero, an open source project management application.
+# Copyright (C) 2008-2011 Céondo Ltd and contributors.
+#
+# InDefero is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# InDefero is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# ***** END LICENSE BLOCK ***** */
+
+class IDF_Form_UploadArchiveHelper
+{
+    private $file = null;
+    private $entries = array();
+
+    public function __construct($file)
+    {
+        $this->file = $file;
+    }
+
+    /**
+     * Validates the archive; throws a invalid form exception in case the
+     * archive contains invalid data or cannot be read.
+     */
+    public function validate()
+    {
+        if (!file_exists($this->file)) {
+            throw new Pluf_Form_Invalid(__('The archive does not exist.'));
+        }
+
+        $za = new ZipArchive();
+        $res = $za->open($this->file);
+        if ($res !== true) {
+            throw new Pluf_Form_Invalid(
+                sprintf(__('The archive could not be read (code %d).'), $res));
+        }
+
+        $manifest = $za->getFromName('manifest.xml');
+        if ($manifest === false) {
+            throw new Pluf_Form_Invalid(__('The archive does not contain a manifest.xml.'));
+        }
+
+        libxml_use_internal_errors(true);
+        $xml = @simplexml_load_string($manifest);
+        if ($xml === false) {
+            $error = libxml_get_last_error();
+            throw new Pluf_Form_Invalid(
+                sprintf(__('The archive\'s manifest is invalid: %s'), $error->message));
+        }
+
+        foreach (@$xml->file as $idx => $file)
+        {
+            $entry = array(
+                'name' => (string)@$file->name,
+                'summary' => (string)@$file->summary,
+                'description' => (string)@$file->description,
+                'replaces' => (string)@$file->replaces,
+                'labels' => array(),
+                'stream' => null
+            );
+
+            if (empty($entry['name'])) {
+                throw new Pluf_Form_Invalid(
+                    sprintf(__('The entry %d in the manifest is missing a file name.'), $idx));
+            }
+
+            if (empty($entry['summary'])) {
+                throw new Pluf_Form_Invalid(
+                    sprintf(__('The entry %d in the manifest is missing a summary.'), $idx));
+            }
+
+            if ($entry['name'] === 'manifest.xml') {
+                throw new Pluf_Form_Invalid(__('The manifest must not reference itself.'));
+            }
+
+            if ($za->locateName($entry['name']) === false) {
+                throw new Pluf_Form_Invalid(
+                    sprintf(__('The entry %s in the manifest does not exist in the archive.'), $entry['name']));
+            }
+
+            if (in_array($entry['name'], $this->entries)) {
+                throw new Pluf_Form_Invalid(
+                    sprintf(__('The entry %s in the manifest is referenced more than once.'), $entry['name']));
+            }
+
+            if ($file->labels) {
+                foreach (@$file->labels->label as $label) {
+                    $entry['labels'][] = (string)$label;
+                }
+            }
+
+            $this->entries[$entry['name']] = $entry;
+        }
+
+        $za->close();
+    }
+
+    /**
+     * Returns all entry names
+     *
+     * @return array of string
+     */
+    public function getEntryNames()
+    {
+        return array_keys($this->entries);
+    }
+
+    /**
+     * Returns meta data for the given entry
+     *
+     * @param string $name
+     * @throws Exception
+     */
+    public function getMetaData($name)
+    {
+        if (!array_key_exists($name, $this->entries)) {
+            throw new Exception('unknown file ' . $name);
+        }
+        return $this->entries[$name];
+    }
+
+    /**
+     * Extracts the file entry $name at $path
+     *
+     * @param string $name
+     * @param string $path
+     * @throws Exception
+     */
+    public function extract($name, $path)
+    {
+        if (!array_key_exists($name, $this->entries)) {
+            throw new Exception('unknown file ' . $name);
+        }
+        $za = new ZipArchive();
+        $za->open($this->file);
+        $za->extractTo($path, $name);
+        $za->close();
+    }
+}