Added a search engine in the issues.

src/IDF/Issue.php

@@ -132,7 +132,12 @@ class IDF_Issue extends Pluf_Model
 
     function _toIndex()
     {
-        return '';
+        $r = array();
+        foreach ($this->get_comments_list() as $c) {
+            $r[] = $c->_toIndex();
+        }
+        $str = str_repeat($this->summary.' ', 4).' '.implode(' ', $r);
+        return Pluf_Text::cleanString(html_entity_decode($str, ENT_QUOTES, 'UTF-8'));
     }
 
 
@@ -146,13 +151,6 @@ class IDF_Issue extends Pluf_Model
 
     function postSave($create=false)
     {
-        // This will be used to fire the indexing or send a
+        print IDF_Search::index($this);
-        // notification email to the interested people, etc.
-        $q = new Pluf_Queue();
-        $q->model_class = __CLASS__;
-        $q->model_id = $this->id;
-        $q->action = 'updated';
-        $q->lock = 0;
-        $q->create();
     }
 }

src/IDF/IssueComment.php

@@ -107,13 +107,5 @@ class IDF_IssueComment extends Pluf_Model
 
     function postSave($create=false)
     {
-        // This will be used to fire the indexing or send a
-        // notification email to the interested people, etc.
-        $q = new Pluf_Queue();
-        $q->model_class = __CLASS__;
-        $q->model_id = $this->id;
-        $q->action = 'updated';
-        $q->lock = 0;
-        $q->create();
     }
 }

src/IDF/Migrations/2Search.php

@@ -0,0 +1,52 @@
+<?php
+/* -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+# ***** BEGIN LICENSE BLOCK *****
+# This file is part of InDefero, an open source project management application.
+# Copyright (C) 2008 Céondo Ltd and contributors.
+#
+# InDefero is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# InDefero is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# ***** END LICENSE BLOCK ***** */
+
+/**
+ * Add the download of files.
+ */
+
+function IDF_Migrations_2Search_up($params=null)
+{
+    $models = array(
+                    'IDF_Search_Occ',
+                    );
+    $db = Pluf::db();
+    $schema = new Pluf_DB_Schema($db);
+    foreach ($models as $model) {
+        $schema->model = new $model();
+        $schema->createTables();
+    }
+}
+
+function IDF_Migrations_2Search_down($params=null)
+{
+    $models = array(
+                    'IDF_Search_Occ',
+                    );
+    $db = Pluf::db();
+    $schema = new Pluf_DB_Schema($db);
+    foreach ($models as $model) {
+        $schema->model = new $model();
+        $schema->dropTables();
+    }
+}

src/IDF/Migrations/Install.php

@@ -36,6 +36,7 @@ function IDF_Migrations_Install_setup($params=null)
                     'IDF_IssueComment',
                     'IDF_Conf',
                     'IDF_Upload',
+                    'IDF_Search_Occ',
                     );
     $db = Pluf::db();
     $schema = new Pluf_DB_Schema($db);
@@ -65,6 +66,7 @@ function IDF_Migrations_Install_teardown($params=null)
     $perm = Pluf_Permission::getFromString('IDF.project-owner');
     if ($perm) $perm->delete();
     $models = array(
+                    'IDF_Search_Occ',
                     'IDF_Upload',
                     'IDF_Conf',
                     'IDF_IssueComment',

src/IDF/Search.php

@@ -0,0 +1,160 @@
+<?php
+/* -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+# ***** BEGIN LICENSE BLOCK *****
+# This file is part of InDefero, an open source project management application.
+# Copyright (C) 2008 Céondo Ltd and contributors.
+#
+# InDefero is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# InDefero is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# Based on work under GNU LGPL copyright, from the Pluf Framework
+# Copyright (C) 2001-2007 Loic d'Anterroches and contributors.
+#
+# ***** END LICENSE BLOCK ***** */
+
+/**
+ * Class implementing the search engine
+ *
+ * It is a modified version of the Pluf_Search class to be able to
+ * cluster the results by project.
+ */
+class IDF_Search extends Pluf_Search
+{
+    /**
+     * Search.
+     *
+     * Returns an array of array with model_class, model_id and
+     * score. The list is already sorted by score descending.
+     *
+     * You can then filter the list as you wish with another set of
+     * weights.
+     *
+     * @param string Query string.
+     * @param int Project id to limit the results (null)
+     * @param string Stemmer class.
+     * @return array Results.
+     */
+    public static function mySearch($query, $project=null, $stemmer='Pluf_Text_Stemmer_Porter')
+    {
+        $query = Pluf_Text::cleanString(html_entity_decode($query, ENT_QUOTES, 'UTF-8'));
+        $words = Pluf_Text::tokenize($query);
+        if ($stemmer != null) {
+            $words = self::stem($words, $stemmer);
+        }
+        $words_flat = array();
+        foreach ($words as $word=>$c) {
+            $words_flat[] = $word;
+        }
+        $word_ids = self::getWordIds($words_flat);
+        if (in_array(null, $word_ids)) {
+            return array();
+        }
+        return self::mySearchDocuments($word_ids, $project);
+    }
+
+    /**
+     * Search documents.
+     *
+     * Only the total of the ponderated occurences is used to sort the
+     * results.
+     *
+     * @param array Ids.
+     * @param IDF_Project Project to limit the search.
+     * @return array Sorted by score, returns model_class, model_id and score.
+     */
+    public static function mySearchDocuments($wids, $project)
+    {
+        $db =& Pluf::db();
+        $gocc = new IDF_Search_Occ();
+        $where = array();
+        foreach ($wids as $id) {
+            $where[] = $db->qn('word').'='.(int)$id;
+        }
+        $prj = (is_null($project)) ? '' : ' AND project='.(int)$project->id;
+        $select = 'SELECT model_class, model_id, SUM(pondocc) AS score FROM '.$gocc->getSqlTable().' WHERE '.implode(' OR ', $where).$prj.' GROUP BY model_class, model_id HAVING COUNT(*)='.count($wids).' ORDER BY score DESC';
+        return $db->select($select);
+    }
+
+    /**
+     * Index a document.
+     *
+     * See Pluf_Search for the disclaimer and informations.
+     *
+     * @param Pluf_Model Document to index.
+     * @param Stemmer used. ('Pluf_Text_Stemmer_Porter')
+     * @return array Statistics.
+     */
+    public static function index($doc, $stemmer='Pluf_Text_Stemmer_Porter')
+    {
+        $words = Pluf_Text::tokenize($doc->_toIndex());
+        if ($stemmer != null) {
+            $words = self::stem($words, $stemmer);
+        }
+        // Get the total number of words.
+        $total = 0.0;
+        $words_flat = array();
+        foreach ($words as $word => $occ) {
+            $total += (float) $occ;
+            $words_flat[] = $word;
+        }
+        // Drop the last indexation.
+        $gocc = new IDF_Search_Occ();
+        $sql = new Pluf_SQL('DELETE FROM '.$gocc->getSqlTable().' WHERE model_class=%s AND model_id=%s', array($doc->_model, $doc->id));
+        $db =& Pluf::db();
+        $db->execute($sql->gen());
+        // Get the ids for each word.
+        $ids = self::getWordIds($words_flat);
+        // Insert a new word for the missing words and add the occ.
+        $n = count($ids);
+        $new_words = 0;
+        $done = array();
+        for ($i=0;$i<$n;$i++) {
+            if ($ids[$i] === null) {
+                $word = new Pluf_Search_Word();
+                $word->word = $words_flat[$i];
+                $word->create();
+                $ids[$i] = $word->id;
+                $new_words++;
+            }
+            if (isset($done[$ids[$i]])) {
+                continue;
+            }
+            $done[$ids[$i]] = true;
+            $occ = new IDF_Search_Occ();
+            $occ->word = new Pluf_Search_Word($ids[$i]);
+            $occ->model_class = $doc->_model;
+            $occ->model_id = $doc->id;
+            $occ->project = $doc->get_project();
+            $occ->occ = $words[$words_flat[$i]];
+            $occ->pondocc = $words[$words_flat[$i]]/$total;
+            $occ->create();
+        }
+        // update the stats
+        $sql = new Pluf_SQL('model_class=%s AND model_id=%s',
+                            array($doc->_model, $doc->id));
+        $last_index = Pluf::factory('Pluf_Search_Stats')->getList(array('filter' => $sql->gen()));
+        if ($last_index->count() == 0) {
+            $stats = new Pluf_Search_Stats();
+            $stats->model_class = $doc->_model;
+            $stats->model_id = $doc->id;
+            $stats->indexations = 1;
+            $stats->create();
+        } else {
+            $last_index[0]->indexations += 1;
+            $last_index[0]->update();
+        }
+        return array('total' => $total, 'new' => $new_words, 'unique'=>$n);
+    }
+}

src/IDF/Search/Occ.php

@@ -0,0 +1,99 @@
+<?php
+/* -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+# ***** BEGIN LICENSE BLOCK *****
+# This file is part of InDefero, an open source project management application.
+# Copyright (C) 2008 Céondo Ltd and contributors.
+#
+# InDefero is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# InDefero is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# ***** END LICENSE BLOCK ***** */
+
+/**
+ * Storage of the occurence of the words.
+ */
+class IDF_Search_Occ extends Pluf_Model
+{
+    public $_model = 'IDF_Search_Occ';
+
+    function init()
+    {
+        $this->_a['verbose'] = __('occurence');
+        $this->_a['table'] = 'idf_search_occs';
+        $this->_a['model'] = 'IDF_Search_Occ';
+        $this->_a['cols'] = array(
+                             // It is mandatory to have an "id" column.
+                            'id' =>
+                            array(
+                                  'type' => 'Pluf_DB_Field_Sequence',
+                                  //It is automatically added.
+                                  'blank' => true, 
+                                  ),
+                            'word' => 
+                            array(
+                                  'type' => 'Pluf_DB_Field_Foreignkey',
+                                  'model' => 'Pluf_Search_Word',
+                                  'blank' => false,
+                                  'verbose' => __('word'),
+                                   ),
+                            'model_class' =>
+                            array(
+                                  'type' => 'Pluf_DB_Field_Varchar',
+                                  'blank' => false,
+                                  'size' => 150,
+                                  'verbose' => __('model class'),
+                                  ),
+                            'model_id' =>
+                            array(
+                                  'type' => 'Pluf_DB_Field_Integer',
+                                  'blank' => false,
+                                  'verbose' => __('model id'),
+                                  ),
+                            'project' => 
+                            array(
+                                  'type' => 'Pluf_DB_Field_Foreignkey',
+                                  'model' => 'IDF_Project',
+                                  'blank' => false,
+                                  'verbose' => __('project'),
+                                   ),
+                            'occ' =>
+                            array(
+                                  'type' => 'Pluf_DB_Field_Integer',
+                                  'blank' => false,
+                                  'verbose' => __('occurences'),
+                                  ),
+                            'pondocc' =>
+                            array(
+                                  'type' => 'Pluf_DB_Field_Float',
+                                  'blank' => false,
+                                  'verbose' => __('ponderated occurence'),
+                                  ),
+                            );
+        $this->_a['idx'] = array(                           
+                            'model_class_id_combo_word_idx' =>
+                            array(
+                                  'type' => 'unique',
+                                  'col' => 'model_class, model_id, word',
+                                  ),
+                            );
+
+    }
+
+    function __toString()
+    {
+        return $this->word;
+    }
+}
+

src/IDF/Views/Issue.php

@@ -183,6 +183,47 @@ class IDF_Views_Issue
                                                $request);
     }
 
+    public $search_precond = array('IDF_Precondition::accessIssues');
+    public function search($request, $match)
+    {
+        if (!isset($request->REQUEST['q']) or trim($request->REQUEST['q']) == '') {
+            $url =  Pluf_HTTP_URL_urlForView('IDF_Views_Issue::index');
+            return new Pluf_HTTP_Response_Redirect($url);
+        }
+        $prj = $request->project;
+        $q = $request->REQUEST['q'];
+        $title = sprintf(__('Search Issues - %s'), Pluf_esc($q));
+        $issues = new Pluf_Search_ResultSet(IDF_Search::mySearch($q, $prj));
+        if (count($issues) > 100) {
+            // no more than 100 results as we do not care
+            $issues->results = array_slice($issues->results, 0, 100);
+        }
+        $pag = new Pluf_Paginator();
+        $pag->items = $issues;
+        $pag->class = 'recent-issues';
+        $pag->item_extra_props = array('project_m' => $prj,
+                                       'shortname' => $prj->shortname,
+                                       'current_user' => $request->user);
+        $pag->summary = __('This table shows the found issues.');
+        $pag->action = array('IDF_Views_Issue::search', array($prj->shortname), array('q'=> $q));
+        $list_display = array(
+                              'id' => __('Id'),
+                              array('summary', 'IDF_Views_Issue_SummaryAndLabels', __('Summary')),
+                              array('status', 'IDF_Views_Issue_ShowStatus', __('Status')),
+                              array('modif_dtime', 'Pluf_Paginator_DateAgo', __('Last Updated')),
+                              );
+        $pag->configure($list_display);
+        $pag->items_per_page = 100;
+        $pag->no_results_text = __('No issues were found.');
+        $pag->setFromRequest($request);
+        $params = array('page_title' => $title,
+                        'issues' => $pag,
+                        'q' => $q,
+                        );
+        return Pluf_Shortcuts_RenderToResponse('issues/search.html', $params, $request);
+
+    }
+
     public $view_precond = array('IDF_Precondition::accessIssues');
     public function view($request, $match)
     {

src/IDF/conf/views.php

@@ -91,6 +91,12 @@ $ctl[] = array('regex' => '#^/p/(\w+)/issues/$#',
                'model' => 'IDF_Views_Issue',
                'method' => 'index');
 
+$ctl[] = array('regex' => '#^/p/(\w+)/issues/search/$#',
+               'base' => $base,
+               'priority' => 4,
+               'model' => 'IDF_Views_Issue',
+               'method' => 'search');
+
 $ctl[] = array('regex' => '#^/p/(\w+)/issues/(\d+)/$#',
                'base' => $base,
                'priority' => 4,

src/IDF/relations.php

@@ -28,5 +28,6 @@ $m['IDF_Issue'] = array('relate_to' => array('IDF_Project', 'Pluf_User', 'IDF_Ta
 $m['IDF_IssueComment'] = array('relate_to' => array('IDF_Issue', 'Pluf_User'));
 $m['IDF_Upload'] = array('relate_to' => array('IDF_Project', 'Pluf_User'),
                          'relate_to_many' => array('IDF_Tag'));
+$m['IDF_Search_Occ'] = array('relate_to' => array('IDF_Project'),);
 
 return $m;

src/IDF/templates/issues/base.html

@@ -3,7 +3,11 @@
 {block subtabs}
 <div id="sub-tabs">
 <a href="{url 'IDF_Views_Issue::index', array($project.shortname)}">{trans 'Open issues'}</a>
-{if !$user.isAnonymous()} | <a href="{url 'IDF_Views_Issue::create', array($project.shortname)}">{trans 'New Issue'}</a> | <a href="{url 'IDF_Views_Issue::myIssues', array($project.shortname, 'submit')}">{trans 'My Issues'}</a>{/if}
+{if !$user.isAnonymous()} | <a href="{url 'IDF_Views_Issue::create', array($project.shortname)}">{trans 'New Issue'}</a> | <a href="{url 'IDF_Views_Issue::myIssues', array($project.shortname, 'submit')}">{trans 'My Issues'}</a>{/if} |
+<form class="star" action="{url 'IDF_Views_Issue::search', array($project.shortname)}" method="get">
+<input type="text" value="{$q}" name="q" size="20" />
+<input type="submit" name="s" value="{trans 'Search'}" />
+</form>
 {superblock}
 </div>
 {/block}

src/IDF/templates/issues/search.html

@@ -0,0 +1,12 @@
+{extends "issues/base.html"}
+{block docclass}yui-t2{/block}
+{block body}
+{$issues.render}
+{if !$user.isAnonymous()}
+{aurl 'url', 'IDF_Views_Issue::create', array($project.shortname)}
+<p><a href="{$url}"><img style="vertical-align: text-bottom;" src="{media '/idf/img/add.png'}" alt="+" align="bottom" /></a> <a href="{$url}">{trans 'New Issue'}</a></p>{/if}
+
+{/block}
+{block context}
+<p><strong>{trans 'Found issues:'}</strong> {$issues.nb_items}</p>
+{/block}