Installation¶

Use your favorite Python package manager to install the app from PyPI, e.g.

Example:

pip install django-haystack

Configuration¶

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',

    # Added.
    'haystack',

    # Then your usual apps...
    'blog',
]

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.solr_backend.SolrEngine',
        'URL': 'http://127.0.0.1:8983/solr'
        # ...or for multicore...
        # 'URL': 'http://127.0.0.1:8983/solr/mysite',
    },
}

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
        'URL': 'http://127.0.0.1:9200/',
        'INDEX_NAME': 'haystack',
    },
}

import os
HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
        'PATH': os.path.join(os.path.dirname(__file__), 'whoosh_index'),
    },
}

import os
HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'xapian_backend.XapianEngine',
        'PATH': os.path.join(os.path.dirname(__file__), 'xapian_index'),
    },
}

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
    },
}

Handling Data¶

import datetime
from haystack import indexes
from myapp.models import Note


class NoteIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    author = indexes.CharField(model_attr='user')
    pub_date = indexes.DateTimeField(model_attr='pub_date')

    def get_model(self):
        return Note

    def index_queryset(self, using=None):
        """Used when the entire index for model is updated."""
        return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now())

{{ object.title }}
{{ object.user.get_full_name }}
{{ object.body }}

Setting Up The Views¶

(r'^search/', include('haystack.urls')),

{% extends 'base.html' %}

{% block content %}
    <h2>Search</h2>

    <form method="get" action=".">
        <table>
            {{ form.as_table }}
            <tr>
                <td>&nbsp;</td>
                <td>
                    <input type="submit" value="Search">
                </td>
            </tr>
        </table>

        {% if query %}
            <h3>Results</h3>

            {% for result in page.object_list %}
                <p>
                    <a href="{{ result.object.get_absolute_url }}">{{ result.object.title }}</a>
                </p>
            {% empty %}
                <p>No results found.</p>
            {% endfor %}

            {% if page.has_previous or page.has_next %}
                <div>
                    {% if page.has_previous %}<a href="?q={{ query }}&amp;page={{ page.previous_page_number }}">{% endif %}&laquo; Previous{% if page.has_previous %}</a>{% endif %}
                    |
                    {% if page.has_next %}<a href="?q={{ query }}&amp;page={{ page.next_page_number }}">{% endif %}Next &raquo;{% if page.has_next %}</a>{% endif %}
                </div>
            {% endif %}
        {% else %}
            {# Show some example queries to run, maybe query syntax, something else? #}
        {% endif %}
    </form>
{% endblock %}

Complete!¶

You can now visit the search section of your site, enter a search query and receive search results back for the query! Congratulations!

What’s Next?¶

This tutorial just scratches the surface of what Haystack provides. The SearchQuerySet is the underpinning of all search in Haystack and provides a powerful, QuerySet-like API (see SearchQuerySet API). You can use much more complicated SearchForms/SearchViews to give users a better UI (see Views & Forms). And the Best Practices provides insight into non-obvious or advanced usages of Haystack.

Forms¶

from django import forms
from haystack.forms import SearchForm


class DateRangeSearchForm(SearchForm):
    start_date = forms.DateField(required=False)
    end_date = forms.DateField(required=False)

    def search(self):
        # First, store the SearchQuerySet received from other processing.
        sqs = super(DateRangeSearchForm, self).search()

        if not self.is_valid():
            return self.no_query_found()

        # Check to see if a start_date was chosen.
        if self.cleaned_data['start_date']:
            sqs = sqs.filter(pub_date__gte=self.cleaned_data['start_date'])

        # Check to see if an end_date was chosen.
        if self.cleaned_data['end_date']:
            sqs = sqs.filter(pub_date__lte=self.cleaned_data['end_date'])

        return sqs

Views¶

# views.py
from datetime import date

from haystack.generic_views import SearchView

class MySearchView(SearchView):
    """My custom search view."""

    def get_queryset(self):
        queryset = super(MySearchView, self).get_queryset()
        # further filter queryset based on some set of criteria
        return queryset.filter(pub_date__gte=date(2015, 1, 1))

    def get_context_data(self, *args, **kwargs):
        context = super(MySearchView, self).get_context_data(*args, **kwargs)
        # do something
        return context

# urls.py

urlpatterns = patterns('',
    url(r'^/search/?$', MySearchView.as_view(), name='search_view'),
)

### old-style views...
# urls.py

sqs = SearchQuerySet().filter(author='john')

urlpatterns = patterns('haystack.views',
    url(r'^$', SearchView(
        template='my/special/path/john_search.html',
        searchqueryset=sqs,
        form_class=SearchForm
    ), name='haystack_search'),
)

### new-style views...
# views.py

class JohnSearchView(SearchView):
    template_name = 'my/special/path/john_search.html'
    queryset = SearchQuerySet().filter(author='john')
    form_class = SearchForm

# urls.py
from myapp.views import JohnSearchView

urlpatterns = patterns('',
    url(r'^$', JohnSearchView.as_view(), name='haystack_search'),
)

from django.conf.urls.defaults import *
from haystack.forms import ModelSearchForm
from haystack.query import SearchQuerySet
from haystack.views import SearchView

sqs = SearchQuerySet().filter(author='john')

# Without threading...
urlpatterns = patterns('haystack.views',
    url(r'^$', SearchView(
        template='my/special/path/john_search.html',
        searchqueryset=sqs,
        form_class=SearchForm
    ), name='haystack_search'),
)

# With threading...
from haystack.views import SearchView, search_view_factory

urlpatterns = patterns('haystack.views',
    url(r'^$', search_view_factory(
        view_class=SearchView,
        template='my/special/path/john_search.html',
        searchqueryset=sqs,
        form_class=ModelSearchForm
    ), name='haystack_search'),
)

class FacetedSearchView(SearchView):
    def extra_context(self):
        extra = super(FacetedSearchView, self).extra_context()

        if self.results == []:
            extra['facets'] = self.form.search().facet_counts()
        else:
            extra['facets'] = self.results.facet_counts()

        return extra

highlight¶

Takes a block of text and highlights words from a provided query within that block of text. Optionally accepts arguments to provide the HTML tag to wrap highlighted word in, a CSS class to use with the tag and a maximum length of the blurb in characters.

The defaults are span for the HTML tag, highlighted for the CSS class and 200 characters for the excerpt.

Syntax:

{% highlight <text_block> with <query> [css_class "class_name"] [html_tag "span"] [max_length 200] %}

Example:

# Highlight summary with default behavior.
{% highlight result.summary with query %}

# Highlight summary but wrap highlighted words with a div and the
# following CSS class.
{% highlight result.summary with query html_tag "div" css_class "highlight_me_please" %}

# Highlight summary but only show 40 words.
{% highlight result.summary with query max_length 40 %}

The highlighter used by this tag can be overridden as needed. See the Highlighting documentation for more information.

more_like_this¶

Fetches similar items from the search index to find content that is similar to the provided model’s content.

Syntax:

{% more_like_this model_instance as varname [for app_label.model_name,app_label.model_name,...] [limit n] %}

Example:

# Pull a full SearchQuerySet (lazy loaded) of similar content.
{% more_like_this entry as related_content %}

# Pull just the top 5 similar pieces of content.
{% more_like_this entry as related_content limit 5  %}

# Pull just the top 5 similar entries or comments.
{% more_like_this entry as related_content for "blog.entry,comments.comment" limit 5  %}

This tag behaves exactly like SearchQuerySet.more_like_this, so all notes in that regard apply here as well.

clear_index¶

The clear_index command wipes out your entire search index. Use with caution. In addition to the standard management command options, it accepts the following arguments:

``--noinput``:
    If provided, the interactive prompts are skipped and the index is
    uncerimoniously wiped out.
``--verbosity``:
    Accepted but ignored.
``--using``:
    If provided, determines which connection should be used. Default is
    ``default``.
``--nocommit``:
    If provided, it will pass commit=False to the backend.  This means that the
    update will not become immediately visible and will depend on another explicit commit
    or the backend's commit strategy to complete the update.

By default, this is an INTERACTIVE command and assumes that you do NOT wish to delete the entire index.

update_index¶

The update_index command will freshen all of the content in your index. It iterates through all indexed models and updates the records in the index. In addition to the standard management command options, it accepts the following arguments:

``--age``:
    Number of hours back to consider objects new. Useful for nightly
    reindexes (``--age=24``). Requires ``SearchIndexes`` to implement
    the ``get_updated_field`` method. Default is ``None``.
``--start``:
    The start date for indexing within. Can be any dateutil-parsable string,
    recommended to be YYYY-MM-DDTHH:MM:SS. Requires ``SearchIndexes`` to
    implement the ``get_updated_field`` method. Default is ``None``.
``--end``:
    The end date for indexing within. Can be any dateutil-parsable string,
    recommended to be YYYY-MM-DDTHH:MM:SS. Requires ``SearchIndexes`` to
    implement the ``get_updated_field`` method. Default is ``None``.
``--batch-size``:
    Number of items to index at once. Default is 1000.
``--remove``:
    Remove objects from the index that are no longer present in the
    database.
``--workers``:
    Allows for the use multiple workers to parallelize indexing. Requires
    ``multiprocessing``.
``--verbosity``:
    If provided, dumps out more information about what's being done.

      * ``0`` = No output
      * ``1`` = Minimal output describing what models were indexed
        and how many records.
      * ``2`` = Full output, including everything from ``1`` plus output
        on each batch that is indexed, which is useful when debugging.
``--using``:
    If provided, determines which connection should be used. Default is
    ``default``.
``--nocommit``:
    If provided, it will pass commit=False to the backend.  This means that the
    updates will not become immediately visible and will depend on another explicit commit
    or the backend's commit strategy to complete the update.

Examples:

# Update everything.
./manage.py update_index --settings=settings.prod

# Update everything with lots of information about what's going on.
./manage.py update_index --settings=settings.prod --verbosity=2

# Update everything, cleaning up after deleted models.
./manage.py update_index --remove --settings=settings.prod

# Update everything changed in the last 2 hours.
./manage.py update_index --age=2 --settings=settings.prod

# Update everything between Dec. 1, 2011 & Dec 31, 2011
./manage.py update_index --start='2011-12-01T00:00:00' --end='2011-12-31T23:59:59' --settings=settings.prod

# Update just a couple apps.
./manage.py update_index blog auth comments --settings=settings.prod

# Update just a single model (in a complex app).
./manage.py update_index auth.User --settings=settings.prod

# Crazy Go-Nuts University
./manage.py update_index events.Event media news.Story --start='2011-01-01T00:00:00 --remove --using=hotbackup --workers=12 --verbosity=2 --settings=settings.prod

rebuild_index¶

A shortcut for clear_index followed by update_index. It accepts any/all of the arguments of the following arguments:

``--age``:
    Number of hours back to consider objects new. Useful for nightly
    reindexes (``--age=24``). Requires ``SearchIndexes`` to implement
    the ``get_updated_field`` method.
``--batch-size``:
    Number of items to index at once. Default is 1000.
``--site``:
    The site object to use when reindexing (like `search_sites.mysite`).
``--noinput``:
    If provided, the interactive prompts are skipped and the index is
    uncerimoniously wiped out.
``--remove``:
    Remove objects from the index that are no longer present in the
    database.
``--verbosity``:
    If provided, dumps out more information about what's being done.

      * ``0`` = No output
      * ``1`` = Minimal output describing what models were indexed
        and how many records.
      * ``2`` = Full output, including everything from ``1`` plus output
        on each batch that is indexed, which is useful when debugging.
``--using``:
    If provided, determines which connection should be used. Default is
    ``default``.
``--nocommit``:
    If provided, it will pass commit=False to the backend.  This means that the
    update will not become immediately visible and will depend on another explicit commit
    or the backend's commit strategy to complete the update.

For when you really, really want a completely rebuilt index.

build_solr_schema¶

Once all of your SearchIndex classes are in place, this command can be used to generate the XML schema Solr needs to handle the search data. It accepts the following arguments:

``--filename``:
    If provided, directs output to a file instead of stdout.
``--using``:
    If provided, determines which connection should be used. Default is
    ``default``.

haystack_info¶

Provides some basic information about how Haystack is setup and what models it is handling. It accepts no arguments. Useful when debugging or when using Haystack-enabled third-party apps.

What is Haystack?¶

Haystack is meant to be a portable interface to a search engine of your choice. Some might call it a search framework, an abstraction layer or what have you. The idea is that you write your search code once and should be able to freely switch between backends as your situation necessitates.

Why should I consider using Haystack?¶

Haystack is targeted at the following use cases:

• If you want to feature search on your site and search solutions like Google or Yahoo search don’t fit your needs.
• If you want to be able to customize your search and search on more than just the main content.
• If you want to have features like drill-down (faceting) or “More Like This”.
• If you want a interface that is non-search engine specific, allowing you to change your mind later without much rewriting.

When should I not be using Haystack?¶

• Non-Model-based data. If you just want to index random data (flat files, alternate sources, etc.), Haystack isn’t a good solution. Haystack is very Model-based and doesn’t work well outside of that use case.
• Ultra-high volume. Because of the very nature of Haystack (abstraction layer), there’s more overhead involved. This makes it portable, but as with all abstraction layers, you lose a little performance. You also can’t take full advantage of the exact feature-set of your search engine. This is the price of pluggable backends.

Why was Haystack created when there are so many other search options?¶

The proliferation of search options in Django is a relatively recent development and is actually one of the reasons for Haystack’s existence. There are too many options that are only partial solutions or are too engine specific.

Further, most use an unfamiliar API and documentation is lacking in most cases.

Haystack is an attempt to unify these efforts into one solution. That’s not to say there should be no alternatives, but Haystack should provide a good solution to 80%+ of the search use cases out there.

What’s the history behind Haystack?¶

Haystack started because of my frustration with the lack of good search options (before many other apps came out) and as the result of extensive use of Djangosearch. Djangosearch was a decent solution but had a number of shortcomings, such as:

• Tied to the models.py, so you’d have to modify the source of third-party ( or django.contrib) apps in order to effectively use it.
• All or nothing approach to indexes. So all indexes appear on all sites and in all places.
• Lack of tests.
• Lack of documentation.
• Uneven backend implementations.

The initial idea was to simply fork Djangosearch and improve on these (and other issues). However, after stepping back, I decided to overhaul the entire API (and most of the underlying code) to be more representative of what I would want as an end-user. The result was starting afresh and reusing concepts (and some code) from Djangosearch as needed.

As a result of this heritage, you can actually still find some portions of Djangosearch present in Haystack (especially in the SearchIndex and SearchBackend classes) where it made sense. The original authors of Djangosearch are aware of this and thus far have seemed to be fine with this reuse.

Why doesn’t <search engine X> have a backend included in Haystack?¶

Several possibilities on this.

1. Licensing

   A common problem is that the Python bindings for a specific engine may have been released under an incompatible license. The goal is for Haystack to remain BSD licensed and importing bindings with an incompatible license can technically convert the entire codebase to that license. This most commonly occurs with GPL’ed bindings.

2. Lack of time

   The search engine in question may be on the list of backends to add and we simply haven’t gotten to it yet. We welcome patches for additional backends.

3. Incompatible API

   In order for an engine to work well with Haystack, a certain baseline set of features is needed. This is often an issue when the engine doesn’t support ranged queries or additional attributes associated with a search record.

4. We’re not aware of the engine

   If you think we may not be aware of the engine you’d like, please tell us about it (preferably via the group - http://groups.google.com/group/django-haystack/). Be sure to check through the backends (in case it wasn’t documented) and search the history on the group to minimize duplicates.

LJWorld/Lawrence.com/KUSports¶

For all things search-related.

Using: Solr

• http://www2.ljworld.com/search/
• http://www2.ljworld.com/search/vertical/news.story/
• http://www2.ljworld.com/marketplace/
• http://www.lawrence.com/search/
• http://www.kusports.com/search/

AltWeeklies¶

Providing an API to story aggregation.

Using: Whoosh

• http://www.northcoastjournal.com/altweeklies/documentation/

Trapeze¶

Various projects.

Using: Xapian

• http://www.trapeze.com/
• http://www.windmobile.ca/
• http://www.bonefishgrill.com/
• http://www.canadiantire.ca/ (Portions of)

Vickerey.com¶

For (really well done) search & faceting.

Using: Solr

• http://store.vickerey.com/products/search/

Eldarion¶

Various projects.

Using: Solr

• http://eldarion.com/

Sunlight Labs¶

For general search.

Using: Whoosh & Solr

• http://sunlightlabs.com/
• http://subsidyscope.com/

NASA¶

For general search.

Using: Solr

• An internal site called SMD Spacebook 1.1.
• http://science.nasa.gov/

AllForLocal¶

For general search.

• http://www.allforlocal.com/

HUGE¶

Various projects.

Using: Solr

• http://hugeinc.com/
• http://houselogic.com/

Brick Design¶

For search on Explore.

Using: Solr

• http://bricksf.com/
• http://explore.org/

Winding Road¶

For general search.

Using: Solr

• http://www.windingroad.com/

Reddit¶

For Reddit Gifts.

Using: Whoosh

• http://redditgifts.com/

Pegasus News¶

For general search.

Using: Xapian

• http://www.pegasusnews.com/

Rampframe¶

For general search.

Using: Xapian

• http://www.rampframe.com/

Forkinit¶

For general search, model-specific search and suggestions via MLT.

Using: Solr

• http://forkinit.com/

Structured Abstraction¶

For general search.

Using: Xapian

• http://www.structuredabstraction.com/
• http://www.delivergood.org/

CustomMade¶

For general search.

Using: Solr

• http://www.custommade.com/

University of the Andes, Dept. of Political Science¶

For general search & section-specific search. Developed by Monoku.

Using: Solr

• http://www.congresovisible.org/
• http://www.monoku.com/

Christchurch Art Gallery¶

For general search & section-specific search.

Using: Solr

• http://christchurchartgallery.org.nz/search/
• http://christchurchartgallery.org.nz/collection/browse/

DevCheatSheet.com¶

For general search.

Using: Xapian

• http://devcheatsheet.com/

TodasLasRecetas¶

For search, faceting & More Like This.

Using: Solr

• http://www.todaslasrecetas.es/receta/s/?q=langostinos
• http://www.todaslasrecetas.es/receta/9526/brochetas-de-langostinos

AstroBin¶

For general search.

Using: Solr

• http://www.astrobin.com/

European Paper Company¶

For general search.

Using: ???

• http://europeanpaper.com/

mtn-op¶

For general search.

Using: ???

• http://mountain-op.com/

Crate¶

Crate is a PyPI mirror/replacement. It’s using Haystack to power all search & faceted navigation on the site.

Using: Elasticsearch

• https://crate.io/

Pix Populi¶

Pix Populi is a popular French photo sharing site.

Using: Solr

• http://www.pix-populi.fr/

LocalWiki¶

LocalWiki is a tool for collaborating in local, geographic communities. It’s using Haystack to power search on every LocalWiki instance.

Using: Solr

• http://localwiki.org/

Pitchup¶

For faceting, geo and autocomplete.

Using: ???

• http://www.pitchup.com/search/

Gidsy¶

Gidsy makes it easy for anyone to organize and find exciting things to do everywhere in the world.

For activity search, area pages, forums and private messages.

Using: Elasticsearch

• https://gidsy.com/
• https://gidsy.com/search/
• https://gidsy.com/forum/

GroundCity¶

Groundcity is a Romanian dynamic real estate site.

For real estate, forums and comments.

Using: Whoosh

• http://groundcity.ro/cautare/

Docket Alarm¶

Docket Alarm allows people to search court dockets across the country. With it, you can search court dockets in the International Trade Commission (ITC), the Patent Trial and Appeal Board (PTAB) and All Federal Courts.

Using: Elasticsearch

• https://www.docketalarm.com/search/ITC
• https://www.docketalarm.com/search/PTAB
• https://www.docketalarm.com/search/dockets

Educreations¶

Educreations makes it easy for anyone to teach what they know and learn what they don’t with a recordable whiteboard. Haystack is used to provide search across users and lessons.

Using: Solr

• http://www.educreations.com/browse/

Sub Apps¶

These are apps that build on top of the infrastructure provided by Haystack. Useful for essentially extending what Haystack can do.

Haystack-Enabled Apps¶

These are reusable apps that ship with SearchIndexes, suitable for quick integration with Haystack.

• django-faq (freq. asked questions app) - http://github.com/benspaulding/django-faq
• django-essays (blog-like essay app) - http://github.com/bkeating/django-essays
• gtalug (variety of apps) - http://github.com/myles/gtalug
• sciencemuseum (science museum open data) - http://github.com/simonw/sciencemuseum
• vz-wiki (wiki) - http://github.com/jobscry/vz-wiki
• ffmff (events app) - http://github.com/stefreak/ffmff
• Dinette (forums app) - http://github.com/uswaretech/Dinette
• fiftystates_site (site) - http://github.com/sunlightlabs/fiftystates_site
• Open-Knesset (site) - http://github.com/ofri/Open-Knesset

Solr¶

Official Download Location: http://www.apache.org/dyn/closer.cgi/lucene/solr/

Solr is Java but comes in a pre-packaged form that requires very little other than the JRE and Jetty. It’s very performant and has an advanced featureset. Haystack suggests using Solr 3.5+, though it’s possible to get it working on Solr 1.4 with a little effort. Installation is relatively simple:

curl -LO https://archive.apache.org/dist/lucene/solr/4.10.2/solr-4.10.2.tgz
tar xvzf solr-4.10.2.tgz
cd solr-4.10.2
cd example
java -jar start.jar

You’ll need to revise your schema. You can generate this from your application (once Haystack is installed and setup) by running ./manage.py build_solr_schema. Take the output from that command and place it in solr-4.10.2/example/solr/collection1/conf/schema.xml. Then restart Solr.

/myproj/myapp/templates/search_configuration/solr.xml
# ...or...
/myproj/templates/search_configuration/solr.xml

You’ll also need a Solr binding, pysolr. The official pysolr package, distributed via PyPI, is the best version to use (2.1.0+). Place pysolr.py somewhere on your PYTHONPATH.

<requestHandler name="/mlt" class="solr.MoreLikeThisHandler" />

class MySearchIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    # ... normal fields then...
    suggestions = indexes.FacetCharField()

    def prepare(self, obj):
        prepared_data = super(MySearchIndex, self).prepare(obj)
        prepared_data['suggestions'] = prepared_data['text']
        return prepared_data

<searchComponent name="spellcheck" class="solr.SpellCheckComponent">

    <str name="queryAnalyzerFieldType">textSpell</str>

    <lst name="spellchecker">
      <str name="name">default</str>
      <str name="field">suggestions</str>
      <str name="spellcheckIndexDir">./spellchecker1</str>
      <str name="buildOnCommit">true</str>
    </lst>
</searchComponent>

<requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />

<requestHandler name="standard" class="solr.StandardRequestHandler" default="true">
    <arr name="last-components">
        <str>spellcheck</str>
    </arr>
</requestHandler>

Elasticsearch¶

Official Download Location: http://www.elasticsearch.org/download/

Elasticsearch is Java but comes in a pre-packaged form that requires very little other than the JRE. It’s also very performant, scales easily and has an advanced featureset. Haystack requires at least version 0.90.0+. Installation is best done using a package manager:

# On Mac OS X...
brew install elasticsearch

# On Ubuntu...
apt-get install elasticsearch

# Then start via:
elasticsearch -f -D es.config=<path to YAML config>

# Example:
elasticsearch -f -D es.config=/usr/local/Cellar/elasticsearch/0.90.0/config/elasticsearch.yml

You may have to alter the configuration to run on localhost when developing locally. Modifications should be done in a YAML file, the stock one being config/elasticsearch.yml:

# Unicast Discovery (disable multicast)
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["127.0.0.1"]

# Name your cluster here to whatever.
# My machine is called "Venus", so...
cluster:
  name: venus

network:
  host: 127.0.0.1

path:
  logs: /usr/local/var/log
  data: /usr/local/var/data

You’ll also need an Elasticsearch binding: elasticsearch (NOT pyes). Place elasticsearch somewhere on your PYTHONPATH (usually python setup.py install or pip install elasticsearch).

Whoosh¶

Official Download Location: http://bitbucket.org/mchaput/whoosh/

Whoosh is pure Python, so it’s a great option for getting started quickly and for development, though it does work for small scale live deployments. The current recommended version is 1.3.1+. You can install via PyPI using sudo easy_install whoosh or sudo pip install whoosh.

Note that, while capable otherwise, the Whoosh backend does not currently support “More Like This” or faceting. Support for these features has recently been added to Whoosh itself & may be present in a future release.

Xapian¶

Official Download Location: http://xapian.org/download

Xapian is written in C++ so it requires compilation (unless your OS has a package for it). Installation looks like:

curl -O http://oligarchy.co.uk/xapian/1.2.18/xapian-core-1.2.18.tar.xz
curl -O http://oligarchy.co.uk/xapian/1.2.18/xapian-bindings-1.2.18.tar.xz

unxz xapian-core-1.2.18.tar.xz
unxz xapian-bindings-1.2.18.tar.xz

tar xvf xapian-core-1.2.18.tar
tar xvf xapian-bindings-1.2.18.tar

cd xapian-core-1.2.18
./configure
make
sudo make install

cd ..
cd xapian-bindings-1.2.18
./configure
make
sudo make install

Xapian is a third-party supported backend. It is not included in Haystack proper due to licensing. To use it, you need both Haystack itself as well as xapian-haystack. You can download the source from http://github.com/notanumber/xapian-haystack/tree/master. Installation instructions can be found on that page as well. The backend, written by David Sauve (notanumber), fully implements the SearchQuerySet API and is an excellent alternative to Solr.

“No module named haystack.”¶

This problem usually occurs when first adding Haystack to your project.

• Are you using the haystack directory within your django-haystack checkout/install?
• Is the haystack directory on your PYTHONPATH? Alternatively, is haystack symlinked into your project?
• Start a Django shell (./manage.py shell) and try import haystack. You may receive a different, more descriptive error message.
• Double-check to ensure you have no circular imports. (i.e. module A tries importing from module B which is trying to import from module A.)

“No results found.” (On the web page)¶

Several issues can cause no results to be found. Most commonly it is either not running a rebuild_index to populate your index or having a blank document=True field, resulting in no content for the engine to search on.

• Do you have a search_indexes.py located within an installed app?

• Do you have data in your database?

• Have you run a ./manage.py rebuild_index to index all of your content?

• Try running ./manage.py rebuild_index -v2 for more verbose output to ensure data is being processed/inserted.

• Start a Django shell (./manage.py shell) and try:

  >>> from haystack.query import SearchQuerySet
  >>> sqs = SearchQuerySet().all()
  >>> sqs.count()
  

• You should get back an integer > 0. If not, check the above and reindex.

  >>> sqs[0] # Should get back a SearchResult object.
  >>> sqs[0].id # Should get something back like 'myapp.mymodel.1'.
  >>> sqs[0].text # ... or whatever your document=True field is.
  

• If you get back either u'' or None, it means that your data isn’t making it into the main field that gets searched. You need to check that the field either has a template that uses the model data, a model_attr that pulls data directly from the model or a prepare/prepare_FOO method that populates the data at index time.

• Check the template for your search page and ensure it is looping over the results properly. Also ensure that it’s either accessing valid fields coming back from the search engine or that it’s trying to access the associated model via the {{ result.object.foo }} lookup.

“LockError: [Errno 17] File exists: ‘/path/to/whoosh_index/_MAIN_LOCK’”¶

This is a Whoosh-specific traceback. It occurs when the Whoosh engine in one process/thread is locks the index files for writing while another process/thread tries to access them. This is a common error when using RealtimeSignalProcessor with Whoosh under any kind of load, which is why it’s only recommended for small sites or development.

The only real solution is to set up a cron job that runs ./manage.py rebuild_index (optionally with --age=24) that runs nightly (or however often you need) to refresh the search indexes. Then disable the use of the RealtimeSignalProcessor within your settings.

The downside to this is that you lose real-time search. For many people, this isn’t an issue and this will allow you to scale Whoosh up to a much higher traffic. If this is not acceptable, you should investigate either the Solr or Xapian backends.

“Failed to add documents to Solr: [Reason: None]”¶

This is a Solr-specific traceback. It generally occurs when there is an error with your HAYSTACK_CONNECTIONS[<alias>]['URL']. Since Solr acts as a webservice, you should test the URL in your web browser. If you receive an error, you may need to change your URL.

This can also be caused when using old versions of pysolr (2.0.9 and before) with httplib2 and including a trailing slash in your HAYSTACK_CONNECTIONS[<alias>]['URL']. If this applies to you, please upgrade to the current version of pysolr.

“Got an unexpected keyword argument ‘boost’”¶

This is a Solr-specific traceback. This can also be caused when using old versions of pysolr (2.0.12 and before). Please upgrade your version of pysolr (2.0.13+).

Settings¶

Most prominently, the old way of specifying a backend & its settings has changed to support the multiple index feature. A complete Haystack 1.X example might look like:

HAYSTACK_SEARCH_ENGINE = 'solr'
HAYSTACK_SOLR_URL = 'http://localhost:9001/solr/default'
HAYSTACK_SOLR_TIMEOUT = 60 * 5
HAYSTACK_INCLUDE_SPELLING = True
HAYSTACK_BATCH_SIZE = 100

# Or...
HAYSTACK_SEARCH_ENGINE = 'whoosh'
HAYSTACK_WHOOSH_PATH = '/home/search/whoosh_index'
HAYSTACK_WHOOSH_STORAGE = 'file'
HAYSTACK_WHOOSH_POST_LIMIT = 128 * 1024 * 1024
HAYSTACK_INCLUDE_SPELLING = True
HAYSTACK_BATCH_SIZE = 100

# Or...
HAYSTACK_SEARCH_ENGINE = 'xapian'
HAYSTACK_XAPIAN_PATH = '/home/search/xapian_index'
HAYSTACK_INCLUDE_SPELLING = True
HAYSTACK_BATCH_SIZE = 100

In Haystack 2.X, you can now supply as many backends as you like, so all of the above settings can now be active at the same time. A translated set of settings would look like:

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.solr_backend.SolrEngine',
        'URL': 'http://localhost:9001/solr/default',
        'TIMEOUT': 60 * 5,
        'INCLUDE_SPELLING': True,
        'BATCH_SIZE': 100,
    },
    'autocomplete': {
        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
        'PATH': '/home/search/whoosh_index',
        'STORAGE': 'file',
        'POST_LIMIT': 128 * 1024 * 1024,
        'INCLUDE_SPELLING': True,
        'BATCH_SIZE': 100,
    },
    'slave': {
        'ENGINE': 'xapian_backend.XapianEngine',
        'PATH': '/home/search/xapian_index',
        'INCLUDE_SPELLING': True,
        'BATCH_SIZE': 100,
    },
}

You are required to have at least one connection listed within HAYSTACK_CONNECTIONS, it must be named default & it must have a valid ENGINE within it. Bare minimum looks like:

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.simple_backend.SimpleEngine'
    }
}

The key for each backend is an identifier you use to describe the backend within your app. You should refer to the Multiple Indexes documentation for more information on using the new multiple indexes & routing features.

Also note that the ENGINE setting has changed from a lowercase “short name” of the engine to a full path to a new Engine class within the backend. Available options are:

• haystack.backends.solr_backend.SolrEngine
• haystack.backends.whoosh_backend.WhooshEngine
• haystack.backends.simple_backend.SimpleEngine

Additionally, the following settings were outright removed & will generate an exception if found:

• HAYSTACK_SITECONF - Remove this setting & the file it pointed to.
• HAYSTACK_ENABLE_REGISTRATIONS
• HAYSTACK_INCLUDE_SPELLING

Backends¶

The dummy backend was outright removed from Haystack, as it served very little use after the simple (pure-ORM-powered) backend was introduced.

If you wrote a custom backend, please refer to the “Custom Backends” section below.

Indexes¶

The other major changes affect the SearchIndex class. As the concept of haystack.site & SearchSite are gone, you’ll need to modify your indexes.

A Haystack 1.X index might’ve looked like:

import datetime
from haystack.indexes import *
from haystack import site
from myapp.models import Note


class NoteIndex(SearchIndex):
    text = CharField(document=True, use_template=True)
    author = CharField(model_attr='user')
    pub_date = DateTimeField(model_attr='pub_date')

    def get_queryset(self):
        """Used when the entire index for model is updated."""
        return Note.objects.filter(pub_date__lte=datetime.datetime.now())


site.register(Note, NoteIndex)

A converted Haystack 2.X index should look like:

import datetime
from haystack import indexes
from myapp.models import Note


class NoteIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    author = indexes.CharField(model_attr='user')
    pub_date = indexes.DateTimeField(model_attr='pub_date')

    def get_model(self):
        return Note

    def index_queryset(self, using=None):
        """Used when the entire index for model is updated."""
        return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now())

Note the import on site & the registration statements are gone. Newly added are is the NoteIndex.get_model method. This is a required method & should simply return the Model class the index is for.

There’s also a new, additional class added to the class definition. The indexes.Indexable class is a simple mixin that serves to identify the classes Haystack should automatically discover & use. If you have a custom base class (say QueuedSearchIndex) that other indexes inherit from, simply leave the indexes.Indexable off that declaration & Haystack won’t try to use it.

Additionally, the name of the document=True field is now enforced to be text across all indexes. If you need it named something else, you should set the HAYSTACK_DOCUMENT_FIELD setting. For example:

HAYSTACK_DOCUMENT_FIELD = 'pink_polka_dot'

Finally, the index_queryset method should supplant the get_queryset method. This was present in the Haystack 1.2.X series (with a deprecation warning in 1.2.4+) but has been removed in Haystack v2.

Finally, if you were unregistering other indexes before, you should make use of the new EXCLUDED_INDEXES setting available in each backend’s settings. It should be a list of strings that contain the Python import path to the indexes that should not be loaded & used. For example:

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.solr_backend.SolrEngine',
        'URL': 'http://localhost:9001/solr/default',
        'EXCLUDED_INDEXES': [
            # Imagine that these indexes exist. They don't.
            'django.contrib.auth.search_indexes.UserIndex',
            'third_party_blog_app.search_indexes.EntryIndex',
        ]
    }
}

This allows for reliable swapping of the index that handles a model without relying on correct import order.

Removal of RealTimeSearchIndex¶

Use of the haystack.indexes.RealTimeSearchIndex is no longer valid. It has been removed in favor of RealtimeSignalProcessor. To migrate, first change the inheritance of all your RealTimeSearchIndex subclasses to use SearchIndex instead:

# Old.
class MySearchIndex(indexes.RealTimeSearchIndex, indexes.Indexable):
    # ...


# New.
class MySearchIndex(indexes.SearchIndex, indexes.Indexable):
    # ...

Then update your settings to enable use of the RealtimeSignalProcessor:

HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'

Done!¶

For most basic uses of Haystack, this is all that is necessary to work with Haystack 2.X. You should rebuild your index if needed & test your new setup.

Advanced Uses¶

from haystack import connections
backend = connections['default'].get_backend()
query = connections['default'].get_query()

from haystack.backends import BaseEngine
from haystack.backends.solr_backend import SolrSearchQuery

# Code then...

class MyCustomSolrEngine(BaseEngine):
    # Use our custom backend.
    backend = MySolrBackend
    # Use the built-in Solr query.
    query = SolrSearchQuery

Supported Backends¶

The following backends are fully supported under Python 3. However, you may need to update these dependencies if you have a pre-existing setup.

• Solr (pysolr>=3.1.0)
• Elasticsearch

Notes¶

Philosophy¶

• Haystack is BSD-licensed. All contributed code must be either
  • the original work of the author, contributed under the BSD, or...
  • work taken from another project released under a BSD-compatible license.
• GPL’d (or similar) works are not eligible for inclusion.
• Haystack’s git master branch should always be stable, production-ready & passing all tests.
• Major releases (1.x.x) are commitments to backward-compatibility of the public APIs. Any documented API should ideally not change between major releases. The exclusion to this rule is in the event of either a security issue or to accommodate changes in Django itself.
• Minor releases (x.3.x) are for the addition of substantial features or major bugfixes.
• Patch releases (x.x.4) are for minor features or bugfixes.

Guidelines For Reporting An Issue/Feature¶

So you’ve found a bug or have a great idea for a feature. Here’s the steps you should take to help get it added/fixed in Haystack:

• First, check to see if there’s an existing issue/pull request for the bug/feature. All issues are at https://github.com/toastdriven/django-haystack/issues and pull reqs are at https://github.com/toastdriven/django-haystack/pulls.
• If there isn’t one there, please file an issue. The ideal report includes:
  • A description of the problem/suggestion.
  • How to recreate the bug.
  • If relevant, including the versions of your:
    • Python interpreter
    • Django
    • Haystack
    • Search engine used (as well as bindings)
    • Optionally of the other dependencies involved
  • Ideally, creating a pull request with a (failing) test case demonstrating what’s wrong. This makes it easy for us to reproduce & fix the problem. Instructions for running the tests are at Welcome to Haystack!

You might also hop into the IRC channel (#haystack on irc.freenode.net) & raise your question there, as there may be someone who can help you with a work-around.

Guidelines For Contributing Code¶

If you’re ready to take the plunge & contribute back some code/docs, the process should look like:

• Fork the project on GitHub into your own account.
• Clone your copy of Haystack.
• Make a new branch in git & commit your changes there.
• Push your new branch up to GitHub.
• Again, ensure there isn’t already an issue or pull request out there on it. If there is & you feel you have a better fix, please take note of the issue number & mention it in your pull request.
• Create a new pull request (based on your branch), including what the problem/feature is, versions of your software & referencing any related issues/pull requests.

In order to be merged into Haystack, contributions must have the following:

• A solid patch that:
  • is clear.
  • works across all supported versions of Python/Django.
  • follows the existing style of the code base (mostly PEP-8).
  • comments included as needed.
• A test case that demonstrates the previous flaw that now passes with the included patch.
• If it adds/changes a public API, it must also include documentation for those changes.
• Must be appropriately licensed (see “Philosophy”).
• Adds yourself to the AUTHORS file.

If your contribution lacks any of these things, they will have to be added by a core contributor before being merged into Haystack proper, which may take substantial time for the all-volunteer team to get to.

Guidelines For Core Contributors¶

If you’ve been granted the commit bit, here’s how to shepherd the changes in:

• Any time you go to work on Haystack, please use git pull --rebase to fetch the latest changes.

• Any new features/bug fixes must meet the above guidelines for contributing code (solid patch/tests passing/docs included).

• Commits are typically cherry-picked onto a branch off master.

  • This is done so as not to include extraneous commits, as some people submit pull reqs based on their git master that has other things applied to it.

• A set of commits should be squashed down to a single commit.

  • git merge --squash is a good tool for performing this, as is git rebase -i HEAD~N.
  • This is done to prevent anyone using the git repo from accidently pulling work-in-progress commits.

• Commit messages should use past tense, describe what changed & thank anyone involved. Examples:

  """Added support for the latest version of Whoosh (v2.3.2)."""
  """Fixed a bug in ``solr_backend.py``. Thanks to joeschmoe for the report!"""
  """BACKWARD-INCOMPATIBLE: Altered the arguments passed to ``SearchBackend``.
  
  Further description appears here if the change warrants an explanation
  as to why it was done."""
  

• For any patches applied from a contributor, please ensure their name appears in the AUTHORS file.

• When closing issues or pull requests, please reference the SHA in the closing message (i.e. Thanks! Fixed in SHA: 6b93f6). GitHub will automatically link to it.

Good Search Needs Good Content¶

Most search engines work best when they’re given corpuses with predominantly text (as opposed to other data like dates, numbers, etc.) in decent quantities (more than a couple words). This is in stark contrast to the databases most people are used to, which rely heavily on non-text data to create relationships and for ease of querying.

To this end, if search is important to you, you should take the time to carefully craft your SearchIndex subclasses to give the search engine the best information you can. This isn’t necessarily hard but is worth the investment of time and thought. Assuming you’ve only ever used the BasicSearchIndex, in creating custom SearchIndex classes, there are some easy improvements to make that will make your search better:

• For your document=True field, use a well-constructed template.
• Add fields for data you might want to be able to filter by.
• If the model has related data, you can squash good content from those related models into the parent model’s SearchIndex.
• Similarly, if you have heavily de-normalized models, it may be best represented by a single indexed model rather than many indexed models.

Avoid Hitting The Database¶

A very easy but effective thing you can do to drastically reduce hits on the database is to pre-render your search results using stored fields then disabling the load_all aspect of your SearchView.

To do this, you setup one or more stored fields (indexed=False) on your SearchIndex classes. You should specify a template for the field, filling it with the data you’d want to display on your search results pages. When the model attached to the SearchIndex is placed in the index, this template will get rendered and stored in the index alongside the record.

The second aspect is customizing your SearchView and its templates. First, pass the load_all=False to your SearchView, ideally in your URLconf. This prevents the SearchQuerySet from loading all models objects for results ahead of time. Then, in your template, simply display the stored content from your SearchIndex as the HTML result.

Content-Type Specific Templates¶

Frequently, when displaying results, you’ll want to customize the HTML output based on what model the result represents.

In practice, the best way to handle this is through the use of include along with the data on the SearchResult.

Your existing loop might look something like:

{% for result in page.object_list %}
    <p>
        <a href="{{ result.object.get_absolute_url }}">{{ result.object.title }}</a>
    </p>
{% empty %}
    <p>No results found.</p>
{% endfor %}

An improved version might look like:

{% for result in page.object_list %}
    {% if result.content_type == "blog.post" %}
    {% include "search/includes/blog/post.html" %}
    {% endif %}
    {% if result.content_type == "media.photo" %}
    {% include "search/includes/media/photo.html" %}
    {% endif %}
{% empty %}
    <p>No results found.</p>
{% endfor %}

Those include files might look like:

# search/includes/blog/post.html
<div class="post_result">
    <h3><a href="{{ result.object.get_absolute_url }}">{{ result.object.title }}</a></h3>

    <p>{{ result.object.tease }}</p>
</div>

# search/includes/media/photo.html
<div class="photo_result">
    <a href="{{ result.object.get_absolute_url }}">
    <img src="http://your.media.example.com/media/{{ result.object.photo.url }}"></a>
    <p>Taken By {{ result.object.taken_by }}</p>
</div>

You can make this even better by standardizing on an includes layout, then writing a template tag or filter that generates the include filename. Usage might looks something like:

{% for result in page.object_list %}
    {% with result|search_include as fragment %}
    {% include fragment %}
    {% endwith %}
{% empty %}
    <p>No results found.</p>
{% endfor %}

Real-Time Search¶

If your site sees heavy search traffic and up-to-date information is very important, Haystack provides a way to constantly keep your index up to date.

You can enable the RealtimeSignalProcessor within your settings, which will allow Haystack to automatically update the index whenever a model is saved/deleted.

You can find more information within the Signal Processors documentation.

Use Of A Queue For A Better User Experience¶

By default, you have to manually reindex content, Haystack immediately tries to merge it into the search index. If you have a write-heavy site, this could mean your search engine may spend most of its time churning on constant merges. If you can afford a small delay between when a model is saved and when it appears in the search results, queuing these merges is a good idea.

You gain a snappier interface for users as updates go into a queue (a fast operation) and then typical processing continues. You also get a lower churn rate, as most search engines deal with batches of updates better than many single updates. You can also use this to distribute load, as the queue consumer could live on a completely separate server from your webservers, allowing you to tune more efficiently.

Implementing this is relatively simple. There are two parts, creating a new QueuedSignalProcessor class and creating a queue processing script to handle the actual updates.

For the QueuedSignalProcessor, you should inherit from haystack.signals.BaseSignalProcessor, then alter the setup/teardown methods to call an enqueuing method instead of directly calling handle_save/handle_delete. For example:

from haystack import signals


class QueuedSignalProcessor(signals.BaseSignalProcessor):
    # Override the built-in.
    def setup(self):
        models.signals.post_save.connect(self.enqueue_save)
        models.signals.post_delete.connect(self.enqueue_delete)

    # Override the built-in.
    def teardown(self):
        models.signals.post_save.disconnect(self.enqueue_save)
        models.signals.post_delete.disconnect(self.enqueue_delete)

    # Add on a queuing method.
    def enqueue_save(self, sender, instance, **kwargs):
        # Push the save & information onto queue du jour here...

    # Add on a queuing method.
    def enqueue_delete(self, sender, instance, **kwargs):
        # Push the delete & information onto queue du jour here...

For the consumer, this is much more specific to the queue used and your desired setup. At a minimum, you will need to periodically consume the queue, fetch the correct index from the SearchSite for your application, load the model from the message and pass that model to the update_object or remove_object methods on the SearchIndex. Proper grouping, batching and intelligent handling are all additional things that could be applied on top to further improve performance.

Highlighter¶

The Highlighter class is a pure-Python implementation included with Haystack that’s designed for flexibility. If you use the {% highlight %} template tag, you’ll be automatically using this class. You can also use it manually in your code. For example:

>>> from haystack.utils import Highlighter

>>> my_text = 'This is a sample block that would be more meaningful in real life.'
>>> my_query = 'block meaningful'

>>> highlight = Highlighter(my_query)
>>> highlight.highlight(my_text)
u'...<span class="highlighted">block</span> that would be more <span class="highlighted">meaningful</span> in real life.'

The default implementation takes three optional kwargs: html_tag, css_class and max_length. These allow for basic customizations to the output, like so:

>>> from haystack.utils import Highlighter

>>> my_text = 'This is a sample block that would be more meaningful in real life.'
>>> my_query = 'block meaningful'

>>> highlight = Highlighter(my_query, html_tag='div', css_class='found', max_length=35)
>>> highlight.highlight(my_text)
u'...<div class="found">block</div> that would be more <div class="found">meaningful</div>...'

Further, if this implementation doesn’t suit your needs, you can define your own custom highlighter class. As long as it implements the API you’ve just seen, it can highlight however you choose. For example:

# In ``myapp/utils.py``...
from haystack.utils import Highlighter

class BorkHighlighter(Highlighter):
    def render_html(self, highlight_locations=None, start_offset=None, end_offset=None):
        highlighted_chunk = self.text_block[start_offset:end_offset]

        for word in self.query_words:
            highlighted_chunk = highlighted_chunk.replace(word, 'Bork!')

        return highlighted_chunk

Then set the HAYSTACK_CUSTOM_HIGHLIGHTER setting to myapp.utils.BorkHighlighter. Usage would then look like:

>>> highlight = BorkHighlighter(my_query)
>>> highlight.highlight(my_text)
u'Bork! that would be more Bork! in real life.'

Now the {% highlight %} template tag will also use this highlighter.

What Is Faceting?¶

Faceting is a way to provide users with feedback about the number of documents which match terms they may be interested in. At its simplest, it gives document counts based on words in the corpus, date ranges, numeric ranges or even advanced queries.

Faceting is particularly useful when trying to provide users with drill-down capabilities. The general workflow in this regard is:

1. You can choose what you want to facet on.
2. The search engine will return the counts it sees for that match.
3. You display those counts to the user and provide them with a link.
4. When the user chooses a link, you narrow the search query to only include those conditions and display the results, potentially with further facets.

Haystack provides functionality so that all of the above steps are possible. From the ground up, let’s build a faceted search setup. This assumes that you have been to work through the Getting Started with Haystack and have a working Haystack installation. The same setup from the Getting Started with Haystack applies here.

1. Determine Facets And SearchQuerySet¶

Determining what you want to facet on isn’t always easy. For our purposes, we’ll facet on the author field.

In order to facet effectively, the search engine should store both a standard representation of your data as well as exact version to facet on. This is generally accomplished by duplicating the field and storing it via two different types. Duplication is suggested so that those fields are still searchable in the standard ways.

To inform Haystack of this, you simply pass along a faceted=True parameter on the field(s) you wish to facet on. So to modify our existing example:

class NoteIndex(SearchIndex, indexes.Indexable):
    text = CharField(document=True, use_template=True)
    author = CharField(model_attr='user', faceted=True)
    pub_date = DateTimeField(model_attr='pub_date')

Haystack quietly handles all of the backend details for you, creating a similar field to the type you specified with _exact appended. Our example would now have both a author and author_exact field, though this is largely an implementation detail.

To pull faceting information out of the index, we’ll use the SearchQuerySet.facet method to setup the facet and the SearchQuerySet.facet_counts method to retrieve back the counts seen.

Experimenting in a shell (./manage.py shell) is a good way to get a feel for what various facets might look like:

>>> from haystack.query import SearchQuerySet
>>> sqs = SearchQuerySet().facet('author')
>>> sqs.facet_counts()
{
    'dates': {},
    'fields': {
        'author': [
            ('john', 4),
            ('daniel', 2),
            ('sally', 1),
            ('terry', 1),
        ],
    },
    'queries': {}
}

As you can see, we get back a dictionary which provides access to the three types of facets available: fields, dates and queries. Since we only faceted on the author field (which actually facets on the author_exact field managed by Haystack), only the fields key has any data associated with it. In this case, we have a corpus of eight documents with four unique authors.

>>> from haystack.query import SearchQuerySet
>>> sqs = SearchQuerySet().facet('author', limit=-1)
>>> sqs.facet_counts()
{
    'dates': {},
    'fields': {
        'author': [
            ('abraham', 1),
            ('benny', 2),
            ('cindy', 1),
            ('diana', 5),
        ],
    },
    'queries': {}
}

>>> sqs = SearchQuerySet().facet('author', limit=2)
>>> sqs.facet_counts()
{
    'dates': {},
    'fields': {
        'author': [
            ('abraham', 1),
            ('benny', 2),
        ],
    },
    'queries': {}
}

>>> from haystack.query import SearchQuerySet
>>> sqs = SearchQuerySet().facet('author', sort='index', )
>>> sqs.facet_counts()
{
    'dates': {},
    'fields': {
        'author': [
            ('abraham', 1),
            ('benny', 2),
            ('cindy', 1),
            ('diana', 5),
        ],
    },
    'queries': {}
}

>>> sqs = SearchQuerySet().facet('author', sort='count', )
>>> sqs.facet_counts()
{
    'dates': {},
    'fields': {
        'author': [
            ('diana', 5),
            ('benny', 2),
            ('abraham', 1),
            ('cindy', 1),
        ],
    },
    'queries': {}
}

2. Switch to the FacetedSearchView and FacetedSearchForm¶

There are three things that we’ll need to do to expose facets to our frontend. The first is construct the SearchQuerySet we want to use. We should have that from the previous step. The second is to switch to the FacetedSearchView. This view is useful because it prepares the facet counts and provides them in the context as facets.

Optionally, the third step is to switch to the FacetedSearchForm. As it currently stands, this is only useful if you want to provide drill-down, though it may provide more functionality in the future. We’ll do it for the sake of having it in place but know that it’s not required.

In your URLconf, you’ll need to switch to the FacetedSearchView. Your URLconf should resemble:

from django.conf.urls.defaults import *
from haystack.forms import FacetedSearchForm
from haystack.views import FacetedSearchView


urlpatterns = patterns('haystack.views',
    url(r'^$', FacetedSearchView(form_class=FacetedSearchForm, facet_fields=['author']), name='haystack_search'),
)

The FacetedSearchView will now instantiate the FacetedSearchForm. The specified facet_fields will be present in the context variable facets. This is added in an overridden extra_context method.

3. Display The Facets In The Template¶

Templating facets involves simply adding an extra bit of processing to display the facets (and optionally to link to provide drill-down). An example template might look like this:

<form method="get" action=".">
    <table>
        <tbody>
            {{ form.as_table }}
            <tr>
                <td>&nbsp;</td>
                <td><input type="submit" value="Search"></td>
            </tr>
        </tbody>
    </table>
</form>

{% if query %}
    <!-- Begin faceting. -->
    <h2>By Author</h2>

    <div>
        <dl>
            {% if facets.fields.author %}
                <dt>Author</dt>
                {# Provide only the top 5 authors #}
                {% for author in facets.fields.author|slice:":5" %}
                    <dd><a href="{{ request.get_full_path }}&amp;selected_facets=author_exact:{{ author.0|urlencode }}">{{ author.0 }}</a> ({{ author.1 }})</dd>
                {% endfor %}
            {% else %}
                <p>No author facets.</p>
            {% endif %}
        </dl>
    </div>
    <!-- End faceting -->

    <!-- Display results... -->
    {% for result in page.object_list %}
        <div class="search_result">
            <h3><a href="{{ result.object.get_absolute_url }}">{{ result.object.title }}</a></h3>

            <p>{{ result.object.body|truncatewords:80 }}</p>
        </div>
    {% empty %}
        <p>Sorry, no results found.</p>
    {% endfor %}
{% endif %}

Displaying the facets is a matter of looping through the facets you want and providing the UI to suit. The author.0 is the facet text from the backend and the author.1 is the facet count.

4. Narrowing The Search¶

We’ve also set ourselves up for the last bit, the drill-down aspect. By appending on the selected_facets to the URLs, we’re informing the FacetedSearchForm that we want to narrow our results to only those containing the author we provided.

For a concrete example, if the facets on author come back as:

{
    'dates': {},
    'fields': {
        'author': [
            ('john', 4),
            ('daniel', 2),
            ('sally', 1),
            ('terry', 1),
        ],
    },
    'queries': {}
}

You should present a list similar to:

<ul>
    <li><a href="/search/?q=Haystack&selected_facets=author_exact:john">john</a> (4)</li>
    <li><a href="/search/?q=Haystack&selected_facets=author_exact:daniel">daniel</a> (2)</li>
    <li><a href="/search/?q=Haystack&selected_facets=author_exact:sally">sally</a> (1)</li>
    <li><a href="/search/?q=Haystack&selected_facets=author_exact:terry">terry</a> (1)</li>
</ul>

This is simply the default behavior but it is possible to override or provide your own form which does additional processing. You could also write your own faceted SearchView, which could provide additional/different facets based on facets chosen. There is a wide range of possibilities available to help the user navigate your content.

Step 1. Setup The Data¶

To do autocomplete effectively, the search backend uses n-grams (essentially a small window passed over the string). Because this alters the way your data needs to be stored, the best approach is to add a new field to your SearchIndex that contains the text you want to autocomplete on.

You have two choices: NgramField and EdgeNgramField. Though very similar, the choice of field is somewhat important.

• If you’re working with standard text, EdgeNgramField tokenizes on whitespace. This prevents incorrect matches when part of two different words are mashed together as one n-gram. This is what most users should use.
• If you’re working with Asian languages or want to be able to autocomplete across word boundaries, NgramField should be what you use.

Example (continuing from the tutorial):

import datetime
from haystack import indexes
from myapp.models import Note


class NoteIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    author = indexes.CharField(model_attr='user')
    pub_date = indexes.DateTimeField(model_attr='pub_date')
    # We add this for autocomplete.
    content_auto = indexes.EdgeNgramField(model_attr='content')

    def get_model(self):
        return Note

    def index_queryset(self, using=None):
        """Used when the entire index for model is updated."""
        return Note.objects.filter(pub_date__lte=datetime.datetime.now())

As with all schema changes, you’ll need to rebuild/update your index after making this change.

Step 2. Performing The Query¶

Haystack ships with a convenience method to perform most autocomplete searches. You simply provide a field and the query you wish to search on to the SearchQuerySet.autocomplete method. Given the previous example, an example search would look like:

from haystack.query import SearchQuerySet

SearchQuerySet().autocomplete(content_auto='old')
# Result match things like 'goldfish', 'cuckold' and 'older'.

The results from the SearchQuerySet.autocomplete method are full search results, just like any regular filter.

If you need more control over your results, you can use standard SearchQuerySet.filter calls. For instance:

from haystack.query import SearchQuerySet

sqs = SearchQuerySet().filter(content_auto=request.GET.get('q', ''))

This can also be extended to use SQ for more complex queries (and is what’s being done under the hood in the SearchQuerySet.autocomplete method).

Example Implementation¶

The above is the low-level backend portion of how you implement autocomplete. To make it work in browser, you need both a view to run the autocomplete and some Javascript to fetch the results.

Since it comes up often, here is an example implementation of those things.

A stripped-down view might look like:

# views.py
import simplejson as json
from django.http import HttpResponse
from haystack.query import SearchQuerySet


def autocomplete(request):
    sqs = SearchQuerySet().autocomplete(content_auto=request.GET.get('q', ''))[:5]
    suggestions = [result.title for result in sqs]
    # Make sure you return a JSON object, not a bare list.
    # Otherwise, you could be vulnerable to an XSS attack.
    the_data = json.dumps({
        'results': suggestions
    })
    return HttpResponse(the_data, content_type='application/json')

The template might look like:

<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <title>Autocomplete Example</title>
</head>
<body>
  <h1>Autocomplete Example</h1>

  <form method="post" action="/search/" class="autocomplete-me">
    <input type="text" id="id_q" name="q">
    <input type="submit" value="Search!">
  </form>

  <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.8.3/jquery.min.js"></script>
  <script type="text/javascript">
    // In a perfect world, this would be its own library file that got included
    // on the page and only the ``$(document).ready(...)`` below would be present.
    // But this is an example.
    var Autocomplete = function(options) {
      this.form_selector = options.form_selector
      this.url = options.url || '/search/autocomplete/'
      this.delay = parseInt(options.delay || 300)
      this.minimum_length = parseInt(options.minimum_length || 3)
      this.form_elem = null
      this.query_box = null
    }

    Autocomplete.prototype.setup = function() {
      var self = this

      this.form_elem = $(this.form_selector)
      this.query_box = this.form_elem.find('input[name=q]')

      // Watch the input box.
      this.query_box.on('keyup', function() {
        var query = self.query_box.val()

        if(query.length < self.minimum_length) {
          return false
        }

        self.fetch(query)
      })

      // On selecting a result, populate the search field.
      this.form_elem.on('click', '.ac-result', function(ev) {
        self.query_box.val($(this).text())
        $('.ac-results').remove()
        return false
      })
    }

    Autocomplete.prototype.fetch = function(query) {
      var self = this

      $.ajax({
        url: this.url
      , data: {
          'q': query
        }
      , success: function(data) {
          self.show_results(data)
        }
      })
    }

    Autocomplete.prototype.show_results = function(data) {
      // Remove any existing results.
      $('.ac-results').remove()

      var results = data.results || []
      var results_wrapper = $('<div class="ac-results"></div>')
      var base_elem = $('<div class="result-wrapper"><a href="#" class="ac-result"></a></div>')

      if(results.length > 0) {
        for(var res_offset in results) {
          var elem = base_elem.clone()
          // Don't use .html(...) here, as you open yourself to XSS.
          // Really, you should use some form of templating.
          elem.find('.ac-result').text(results[res_offset])
          results_wrapper.append(elem)
        }
      }
      else {
        var elem = base_elem.clone()
        elem.text("No results found.")
        results_wrapper.append(elem)
      }

      this.query_box.after(results_wrapper)
    }

    $(document).ready(function() {
      window.autocomplete = new Autocomplete({
        form_selector: '.autocomplete-me'
      })
      window.autocomplete.setup()
    })
  </script>
</body>
</html>

Term Boost¶

Term boosting is achieved by using SearchQuerySet.boost. You provide it the term you want to boost on & a floating point value (based around 1.0 as 100% - no boost).

Example:

# Slight increase in relevance for documents that include "banana".
sqs = SearchQuerySet().boost('banana', 1.1)

# Big decrease in relevance for documents that include "blueberry".
sqs = SearchQuerySet().boost('blueberry', 0.8)

See the SearchQuerySet API docs for more details on using this method.

Document Boost¶

Document boosting is done by adding a boost field to the prepared data SearchIndex creates. The best way to do this is to override SearchIndex.prepare:

from haystack import indexes
from notes.models import Note


class NoteSearchIndex(indexes.SearchIndex, indexes.Indexable):
    # Your regular fields here then...

    def prepare(self, obj):
        data = super(NoteSearchIndex, self).prepare(obj)
        data['boost'] = 1.1
        return data

Another approach might be to add a new field called boost. However, this can skew your schema and is not encouraged.

Field Boost¶

Field boosting is enabled by setting the boost kwarg on the desired field. An example of this might be increasing the significance of a title:

from haystack import indexes
from notes.models import Note


class NoteSearchIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    title = indexes.CharField(model_attr='title', boost=1.125)

    def get_model(self):
        return Note

from haystack.forms import SearchForm

class CustomSearchForm(SearchForm):

    def search(self):
        if not self.is_valid():
            return self.no_query_found()

        if not self.cleaned_data.get('q'):
            return self.no_query_found()

        q = self.cleaned_data['q']
        sqs = self.searchqueryset.filter(SQ(content=AutoQuery(q)) | SQ(title=AutoQuery(q)))

        if self.load_all:
            sqs = sqs.load_all()

        return sqs.highlight()

Default - BaseSignalProcessor¶

The default setup is configured to use the haystack.signals.BaseSignalProcessor class, which includes all the underlying code necessary to handle individual updates/deletes, BUT DOES NOT HOOK UP THE SIGNALS.

This means that, by default, NO ACTION IS TAKEN BY HAYSTACK when a model is saved or deleted. The BaseSignalProcessor.setup & BaseSignalProcessor.teardown methods are both empty to prevent anything from being setup at initialization time.

This usage is configured very simply (again, by default) with the HAYSTACK_SIGNAL_PROCESSOR setting. An example of manually setting this would look like:

HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.BaseSignalProcessor'

This class forms an excellent base if you’d like to override/extend for more advanced behavior. Which leads us to...

Realtime - RealtimeSignalProcessor¶

The other included SignalProcessor is the haystack.signals.RealtimeSignalProcessor class. It is an extremely thin extension of the BaseSignalProcessor class, differing only in that in implements the setup/teardown methods, tying ANY Model save/delete to the signal processor.

If the model has an associated SearchIndex, the RealtimeSignalProcessor will then trigger an update/delete of that model instance within the search index proper.

Configuration looks like:

HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'

This causes all SearchIndex classes to work in a realtime fashion.

Custom SignalProcessors¶

The BaseSignalProcessor & RealtimeSignalProcessor classes are fairly simple/straightforward to customize or extend. Rather than forking Haystack to implement your modifications, you should create your own subclass within your codebase (anywhere that’s importable is usually fine, though you should avoid models.py files).

For instance, if you only wanted User saves to be realtime, deferring all other updates to the management commands, you’d implement the following code:

from django.contrib.auth.models import User
from django.db import models
from haystack import signals


class UserOnlySignalProcessor(signals.BaseSignalProcessor):
    def setup(self):
        # Listen only to the ``User`` model.
        models.signals.post_save.connect(self.handle_save, sender=User)
        models.signals.post_delete.connect(self.handle_delete, sender=User)

    def teardown(self):
        # Disconnect only for the ``User`` model.
        models.signals.post_save.disconnect(self.handle_save, sender=User)
        models.signals.post_delete.disconnect(self.handle_delete, sender=User)

For other customizations (modifying how saves/deletes should work), you’ll need to override/extend the handle_save/handle_delete methods. The source code is your best option for referring to how things currently work on your version of Haystack.

Specifying Available Connections¶

You can supply as many backends as you like, each with a descriptive name. A complete setup that accesses all backends might look like:

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.solr_backend.SolrEngine',
        'URL': 'http://localhost:9001/solr/default',
        'TIMEOUT': 60 * 5,
        'INCLUDE_SPELLING': True,
        'BATCH_SIZE': 100,
        'SILENTLY_FAIL': True,
    },
    'autocomplete': {
        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
        'PATH': '/home/search/whoosh_index',
        'STORAGE': 'file',
        'POST_LIMIT': 128 * 1024 * 1024,
        'INCLUDE_SPELLING': True,
        'BATCH_SIZE': 100,
        'SILENTLY_FAIL': True,
    },
    'slave': {
        'ENGINE': 'xapian_backend.XapianEngine',
        'PATH': '/home/search/xapian_index',
        'INCLUDE_SPELLING': True,
        'BATCH_SIZE': 100,
        'SILENTLY_FAIL': True,
    },
    'db': {
        'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
        'SILENTLY_FAIL': True,
    }
}

You are required to have at least one connection listed within HAYSTACK_CONNECTIONS, it must be named default & it must have a valid ENGINE within it.

Management Commands¶

All management commands that manipulate data use ONLY one connection at a time. By default, they use the default index but accept a --using flag to specify a different connection. For example:

./manage.py rebuild_index --noinput --using=whoosh

Automatic Routing¶

To make the selection of the correct index easier, Haystack (like Django) has the concept of “routers”. All provided routers are checked whenever a read or write happens, stopping at the first router that knows how to handle it.

Haystack ships with a DefaultRouter enabled. It looks like:

class DefaultRouter(BaseRouter):
    def for_read(self, **hints):
        return DEFAULT_ALIAS

    def for_write(self, **hints):
        return DEFAULT_ALIAS

On a read (when a search query is executed), the DefaultRouter.for_read method is checked & returns the DEFAULT_ALIAS (which is default), telling whatever requested it that it should perform the query against the default connection. The same process is followed for writes.

If the for_read or for_write method returns None, that indicates that the current router can’t handle the data. The next router is then checked.

The hints passed can be anything that helps the router make a decision. This data should always be considered optional & be guarded against. At current, for_write receives an index option (pointing to the SearchIndex calling it) while for_read may receive models (being a list of Model classes the SearchQuerySet may be looking at).

You may provide as many routers as you like by overriding the HAYSTACK_ROUTERS setting. For example:

HAYSTACK_ROUTERS = ['myapp.routers.MasterRouter', 'myapp.routers.SlaveRouter', 'haystack.routers.DefaultRouter']

from haystack import routers


class MasterRouter(routers.BaseRouter):
    def for_write(self, **hints):
        return 'master'

    def for_read(self, **hints):
        return None


class SlaveRouter(routers.BaseRouter):
    def for_write(self, **hints):
        return None

    def for_read(self, **hints):
        return 'slave'

from haystack import routers


class MasterSlaveRouter(routers.BaseRouter):
    def for_write(self, **hints):
        return 'master'

    def for_read(self, **hints):
        return 'slave'

Manually Selecting¶

There may be times when automatic selection of the correct index is undesirable, such as when fixing erroneous data in an index or when you know exactly where data should be located.

For this, the SearchQuerySet class allows for manually selecting the index via the SearchQuerySet.using method:

from haystack.query import SearchQuerySet

# Uses the routers' opinion.
sqs = SearchQuerySet().auto_query('banana')

# Forces the default.
sqs = SearchQuerySet().using('default').auto_query('banana')

# Forces the slave connection (presuming it was setup).
sqs = SearchQuerySet().using('slave').auto_query('banana')

Custom Index Selection¶

If a specific backend has been selected, the SearchIndex.index_queryset and SearchIndex.read_queryset will receive the backend name, giving indexes the opportunity to customize the returned queryset.

For example, a site which uses separate indexes for recent items and older content might define index_queryset to filter the items based on date:

def index_queryset(self, using=None):
    qs = Note.objects.all()
    archive_limit = datetime.datetime.now() - datetime.timedelta(days=90)

    if using == "archive":
        return qs.filter(pub_date__lte=archive_limit)
    else:
        return qs.filter(pub_date__gte=archive_limit)

sqs = SearchQuerySet.using(lang).auto_query(…)

def index_queryset(self, using=None):
    return Post.objects.filter(language=using)

Extracting Content¶

SearchBackend.extract_file_contents() accepts a file or file-like object and returns a dictionary containing two keys: metadata and contents. The contents value will be a string containing all of the text which the backend managed to extract from the file contents. metadata will always be a dictionary but the keys and values will vary based on the underlying extraction engine and the type of file provided.

Indexing Extracted Content¶

Generally you will want to include the extracted text in your main document field along with everything else specified in your search template. This example shows how to override a hypothetical FileIndex‘s prepare method to include the extract content along with information retrieved from the database:

def prepare(self, obj):
    data = super(FileIndex, self).prepare(obj)

    # This could also be a regular Python open() call, a StringIO instance
    # or the result of opening a URL. Note that due to a library limitation
    # file_obj must have a .name attribute even if you need to set one
    # manually before calling extract_file_contents:
    file_obj = obj.the_file.open()

    extracted_data = self.backend.extract_file_contents(file_obj)

    # Now we'll finally perform the template processing to render the
    # text field with *all* of our metadata visible for templating:
    t = loader.select_template(('search/indexes/myapp/file_text.txt', ))
    data['text'] = t.render(Context({'object': obj,
                                     'extracted': extracted_data}))

    return data

This allows you to insert the extracted text at the appropriate place in your template, modified or intermixed with database content as appropriate:

{{ object.title }}
{{ object.owner.name }}

…

{% for k, v in extracted.metadata.items %}
    {% for val in v %}
        {{ k }}: {{ val|safe }}
    {% endfor %}
{% endfor %}

{{ extracted.contents|striptags|safe }}

Additional Requirements¶

The spatial functionality has only one non-included, non-available-in-Django dependency:

• geopy - pip install geopy

If you do not ever need distance information, you may be able to skip installing geopy.

Support¶

You need the latest & greatest of either Solr or Elasticsearch. None of the other backends (specifially the engines) support this kind of search.

For Solr, you’ll need at least v3.5+. In addition, if you have an existing install of Haystack & Solr, you’ll need to upgrade the schema & reindex your data. If you’re adding geospatial data, you would have to reindex anyhow.

For Elasticsearch, you’ll need at least v0.17.7, preferably v0.18.6 or better. If you’re adding geospatial data, you’ll have to reindex as well.

For more details, you can inspect http://wiki.apache.org/solr/SpatialSearch or http://www.elasticsearch.org/guide/reference/query-dsl/geo-bounding-box-filter.html.

Geospatial Assumptions¶

# Using positional arguments.
from haystack.utils.geo import Point
pnt = Point(-95.23592948913574, 38.97127105172941)

# Using WKT.
from django.contrib.gis.geos import GEOSGeometry
pnt = GEOSGeometry('POINT(-95.23592948913574 38.97127105172941)')

from haystack.utils.geo import D

# Start at 5 miles.
imperial_d = D(mi=5)

# Convert to fathoms...
fathom_d = imperial_d.fathom

# Now to kilometers...
km_d = imperial_d.km

# And back to miles.
mi = imperial_d.mi

Indexing¶

Indexing is relatively simple. Simply add a LocationField (or several) onto your SearchIndex class(es) & provide them a Point object. For example:

from haystack import indexes
from shops.models import Shop


class ShopIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    # ... the usual, then...
    location = indexes.LocationField(model_attr='coordinates')

    def get_model(self):
        return Shop

If you must manually prepare the data, you have to do something slightly less convenient, returning a string-ified version of the coordinates in WGS-84 as lat,long:

from haystack import indexes
from shops.models import Shop


class ShopIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    # ... the usual, then...
    location = indexes.LocationField()

    def get_model(self):
        return Shop

    def prepare_location(self, obj):
        # If you're just storing the floats...
        return "%s,%s" % (obj.latitude, obj.longitude)

Alternatively, you could build a method/property onto the Shop model that returns a Point based on those coordinates:

# shops/models.py
from django.contrib.gis.geos import Point
from django.db import models


class Shop(models.Model):
    # ... the usual, then...
    latitude = models.FloatField()
    longitude = models.FloatField()

    # Usual methods, then...
    def get_location(self):
        # Remember, longitude FIRST!
        return Point(self.longitude, self.latitude)


# shops/search_indexes.py
from haystack import indexes
from shops.models import Shop


class ShopIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, use_template=True)
    location = indexes.LocationField(model_attr='get_location')

    def get_model(self):
        return Shop

Querying¶

There are two types of geospatial queries you can run, within & dwithin. Like their GeoDjango counterparts (within & dwithin), these methods focus on finding results within an area.

from haystack.query import SearchQuerySet
from haystack.utils.geo import Point

downtown_bottom_left = Point(-95.23947, 38.9637903)
downtown_top_right = Point(-95.23362278938293, 38.973081081164715)

# 'location' is the fieldname from our ``SearchIndex``...

# Do the bounding box query.
sqs = SearchQuerySet().within('location', downtown_bottom_left, downtown_top_right)

# Can be chained with other Haystack calls.
sqs = SearchQuerySet().auto_query('coffee').within('location', downtown_bottom_left, downtown_top_right).order_by('-popularity')

from shops.models import Shop
Shop.objects.filter(location__within=(downtown_bottom_left, downtown_top_right))

from haystack.query import SearchQuerySet
from haystack.utils.geo import Point, D

ninth_and_mass = Point(-95.23592948913574, 38.96753407043678)
# Within a two miles.
max_dist = D(mi=2)

# 'location' is the fieldname from our ``SearchIndex``...

# Do the radius query.
sqs = SearchQuerySet().dwithin('location', ninth_and_mass, max_dist)

# Can be chained with other Haystack calls.
sqs = SearchQuerySet().auto_query('coffee').dwithin('location', ninth_and_mass, max_dist).order_by('-popularity')

from shops.models import Shop
Shop.objects.filter(location__dwithin=(ninth_and_mass, D(mi=2)))

from haystack.query import SearchQuerySet
from haystack.utils.geo import Point, D

ninth_and_mass = Point(-95.23592948913574, 38.96753407043678)

# On a bounding box...
downtown_bottom_left = Point(-95.23947, 38.9637903)
downtown_top_right = Point(-95.23362278938293, 38.973081081164715)

sqs = SearchQuerySet().within('location', downtown_bottom_left, downtown_top_right).distance('location', ninth_and_mass)

# ...Or on a radius query.
sqs = SearchQuerySet().dwithin('location', ninth_and_mass, D(mi=2)).distance('location', ninth_and_mass)

from haystack.query import SearchQuerySet
from haystack.utils.geo import Point, D

ninth_and_mass = Point(-95.23592948913574, 38.96753407043678)
user_loc = Point(-95.23455619812012, 38.97240128290697)

sqs = SearchQuerySet().dwithin('location', ninth_and_mass, D(mi=2)).distance('location', user_loc)

from shops.models import Shop
Shop.objects.filter(location__dwithin=(ninth_and_mass, D(mi=2))).distance(user_loc)

Ordering¶

Because you’re dealing with search, even with geospatial queries, results still come back in RELEVANCE order. If you want to offer the user ordering results by distance, there’s a simple way to enable this ordering.

Using the standard Haystack order_by method, if you specify distance or -distance ONLY, you’ll get geographic ordering. Additionally, you must have a call to .distance() somewhere in the chain, otherwise there is no distance information on the results & nothing to sort by.

Examples:

from haystack.query import SearchQuerySet
from haystack.utils.geo import Point, D

ninth_and_mass = Point(-95.23592948913574, 38.96753407043678)
downtown_bottom_left = Point(-95.23947, 38.9637903)
downtown_top_right = Point(-95.23362278938293, 38.973081081164715)

# Non-geo ordering.
sqs = SearchQuerySet().within('location', downtown_bottom_left, downtown_top_right).order_by('title')
sqs = SearchQuerySet().within('location', downtown_bottom_left, downtown_top_right).distance('location', ninth_and_mass).order_by('-created')

# Geo ordering, closest to farthest.
sqs = SearchQuerySet().within('location', downtown_bottom_left, downtown_top_right).distance('location', ninth_and_mass).order_by('distance')
# Geo ordering, farthest to closest.
sqs = SearchQuerySet().dwithin('location', ninth_and_mass, D(mi=2)).distance('location', ninth_and_mass).order_by('-distance')

# May blow up!
sqs = SearchQuerySet().dwithin('location', ninth_and_mass, D(mi=2)).distance('location', ninth_and_mass).order_by('distance', 'title')

Caveats¶

In all cases, you may call the within/dwithin/distance methods as many times as you like. However, the LAST call is the information that will be used. No combination logic is available, as this is largely a backend limitation.

Combining calls to both within & dwithin may yield unexpected or broken results. They don’t overlap when performing queries, so it may be possible to construct queries that work. Your Mileage May Vary.

Why Follow QuerySet?¶

A couple reasons to follow (at least in part) the QuerySet API:

1. Consistency with Django
2. Most Django programmers have experience with the ORM and can use this knowledge with SearchQuerySet.

And from a high-level perspective, QuerySet and SearchQuerySet do very similar things: given certain criteria, provide a set of results. Both are powered by multiple backends, both are abstractions on top of the way a query is performed.

Quick Start¶

For the impatient:

from haystack.query import SearchQuerySet
all_results = SearchQuerySet().all()
hello_results = SearchQuerySet().filter(content='hello')
hello_world_results = SearchQuerySet().filter(content='hello world')
unfriendly_results = SearchQuerySet().exclude(content='hello').filter(content='world')
recent_results = SearchQuerySet().order_by('-pub_date')[:5]

# Using the new input types...
from haystack.inputs import AutoQuery, Exact, Clean
sqs = SearchQuerySet().filter(content=AutoQuery(request.GET['q']), product_type=Exact('ancient book'))

if request.GET['product_url']:
    sqs = sqs.filter(product_url=Clean(request.GET['product_url']))

For more on the AutoQuery, Exact, Clean classes & friends, see the Input Types documentation.

SearchQuerySet¶

By default, SearchQuerySet provide the documented functionality. You can extend with your own behavior by simply subclassing from SearchQuerySet and adding what you need, then using your subclass in place of SearchQuerySet.

Most methods in SearchQuerySet “chain” in a similar fashion to QuerySet. Additionally, like QuerySet, SearchQuerySet is lazy (meaning it evaluates the query as late as possible). So the following is valid:

from haystack.query import SearchQuerySet
results = SearchQuerySet().exclude(content='hello').filter(content='world').order_by('-pub_date').boost('title', 0.5)[10:20]

The content Shortcut¶

Searching your document fields is a very common activity. To help mitigate possible differences in SearchField names (and to help the backends deal with search queries that inspect the main corpus), there is a special field called content. You may use this in any place that other fields names would work (e.g. filter, exclude, etc.) to indicate you simply want to search the main documents.

For example:

from haystack.query import SearchQuerySet

# This searches whatever fields were marked ``document=True``.
results = SearchQuerySet().exclude(content='hello')

This special pseudo-field works best with the exact lookup and may yield strange or unexpected results with the other lookups.

SearchQuerySet Methods¶

The primary interface to search in Haystack is through the SearchQuerySet object. It provides a clean, programmatic, portable API to the search backend. Many aspects are also “chainable”, meaning you can call methods one after another, each applying their changes to the previous SearchQuerySet and further narrowing the search.

All SearchQuerySet objects implement a list-like interface, meaning you can perform actions like getting the length of the results, accessing a result at an offset or even slicing the result list.

sqs = SearchQuerySet().filter(content='foo')

sqs = SearchQuerySet().filter(content='foo', pub_date__lte=datetime.date(2008, 1, 1))

# Identical to the previous example.
sqs = SearchQuerySet().filter(content='foo').filter(pub_date__lte=datetime.date(2008, 1, 1))

# To send unescaped data:
from haystack.inputs import Raw
sqs = SearchQuerySet().filter(title=Raw(trusted_query))

# To use auto-query behavior on a non-``document=True`` field.
from haystack.inputs import AutoQuery
sqs = SearchQuerySet().filter(title=AutoQuery(user_query))

sqs = SearchQuerySet().exclude(content='foo')

SearchQuerySet().filter(content='foo').order_by('author', 'pub_date')

SearchQuerySet().filter(content='foo').order_by('-pub_date')

sqs = SearchQuerySet().filter(content='foo').highlight()
result = sqs[0]
result.highlighted['text'][0] # u'Two computer scientists walk into a bar. The bartender says "<em>Foo</em>!".'

SearchQuerySet().filter(content='foo').models(BlogEntry, Comment)

SearchQuerySet().result_class(CustomResult)

SearchQuerySet().filter(content='foo').boost('bar', 1.5)

# For SOLR (setting f.author.facet.*; see http://wiki.apache.org/solr/SimpleFacetParameters#Parameters)
SearchQuerySet().facet('author', mincount=1, limit=10)
# For ElasticSearch (see http://www.elasticsearch.org/guide/reference/api/search/facets/terms-facet.html)
SearchQuerySet().facet('author', size=10, order='term')

# Count document hits for each author within the index.
SearchQuerySet().filter(content='foo').facet('author')

# Count document hits for each day between 2009-06-07 to 2009-07-07 within the index.
SearchQuerySet().filter(content='foo').date_facet('pub_date', start_date=datetime.date(2009, 6, 7), end_date=datetime.date(2009, 7, 7), gap_by='day')

# Count document hits for authors that start with 'jo' within the index.
SearchQuerySet().filter(content='foo').query_facet('author', 'jo*')

# Get stats on the author field.
SearchQuerySet().filter(content='foo').stats('author')

# Get stats on the author field, and stats on the author field
faceted by bookstore.
SearchQuerySet().filter(content='foo').stats_facet('author','bookstore')