FEATURE: when under extreme load disable search

FEATURE: when under extreme load disable search

The global setting disable_search_queue_threshold (DISCOURSE_DISABLE_SEARCH_QUEUE_THRESHOLD) which default to 1 second was added.

This protection ensures that when the application is unable to keep up with requests it will simply turn off search till it is not backed up.

To disable this protection set this to 0.

diff --git a/app/assets/javascripts/discourse/templates/full-page-search.hbs b/app/assets/javascripts/discourse/templates/full-page-search.hbs
index 627be51..3a72a70 100644
--- a/app/assets/javascripts/discourse/templates/full-page-search.hbs
+++ b/app/assets/javascripts/discourse/templates/full-page-search.hbs
@@ -125,6 +125,12 @@
               {{#if searchActive}}
                 <h3>{{i18n "search.no_results"}}</h3>
 
+                {{#if model.grouped_search_result.error}}
+                  <div class="warning">
+                    {{model.grouped_search_result.error}}
+                  </div>
+                {{/if}}
+
                 {{#if showSuggestion}}
                   <div class="no-results-suggestion">
                     {{i18n "search.cant_find"}}
diff --git a/app/assets/javascripts/discourse/widgets/search-menu.js.es6 b/app/assets/javascripts/discourse/widgets/search-menu.js.es6
index c963c18..4792f02 100644
--- a/app/assets/javascripts/discourse/widgets/search-menu.js.es6
+++ b/app/assets/javascripts/discourse/widgets/search-menu.js.es6
@@ -1,3 +1,4 @@
+import { popupAjaxError } from "discourse/lib/ajax-error";
 import { searchForTerm, isValidSearchTerm } from "discourse/lib/search";
 import { createWidget } from "discourse/widgets/widget";
 import { h } from "virtual-dom";
@@ -78,6 +79,7 @@ const SearchHelper = {
             searchData.topicId = null;
           }
         })
+        .catch(popupAjaxError)
         .finally(() => {
           searchData.loading = false;
           widget.scheduleRerender();
diff --git a/app/assets/stylesheets/common/base/search.scss b/app/assets/stylesheets/common/base/search.scss
index 2d645b9..9561a6b 100644
--- a/app/assets/stylesheets/common/base/search.scss
+++ b/app/assets/stylesheets/common/base/search.scss
@@ -2,6 +2,12 @@
   display: flex;
   justify-content: space-between;
 
+  .warning {
+    background-color: $danger-medium;
+    padding: 5px 8px;
+    color: $secondary;
+  }
+
   .search-bar {
     display: flex;
     justify-content: space-between;
diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb
index 6e12b36..82d8ac7 100644
--- a/app/controllers/search_controller.rb
+++ b/app/controllers/search_controller.rb
@@ -6,6 +6,8 @@ class SearchController < ApplicationController
 
   skip_before_action :check_xhr, only: :show
 
+  before_action :cancel_overloaded_search, only: [:query]
+
   def self.valid_context_types
     %w{user topic category private_messages}
   end
@@ -44,10 +46,14 @@ class SearchController < ApplicationController
     search_args[:ip_address] = request.remote_ip
     search_args[:user_id] = current_user.id if current_user.present?
 
-    search = Search.new(@search_term, search_args)
-    result = search.execute
-
-    result.find_user_data(guardian) if result
+    if site_overloaded?
+      result = Search::GroupedSearchResults.new(search_args[:type_filter], @search_term, context, false, 0)
+      result.error = I18n.t("search.extreme_load_error")
+    else
+      search = Search.new(@search_term, search_args)
+      result = search.execute
+      result.find_user_data(guardian) if result
+    end
 
     serializer = serialize_data(result, GroupedSearchResultSerializer, result: result)
 
@@ -82,8 +88,12 @@ class SearchController < ApplicationController
     search_args[:user_id] = current_user.id if current_user.present?
     search_args[:restrict_to_archetype] = params[:restrict_to_archetype] if params[:restrict_to_archetype].present?
 
-    search = Search.new(params[:term], search_args)
-    result = search.execute
+    if site_overloaded?
+      result = GroupedSearchResults.new(search_args["type_filter"], params[:term], context, false, 0)
+    else
+      search = Search.new(params[:term], search_args)
+      result = search.execute
+    end
     render_serialized(result, GroupedSearchResultSerializer, result: result)
   end
 
@@ -118,6 +128,18 @@ class SearchController < ApplicationController
 
   protected
 
+  def site_overloaded?
+    (queue_time = request.env['REQUEST_QUEUE_SECONDS']) &&
+      (GlobalSetting.disable_search_queue_threshold > 0) &&
+      (queue_time > GlobalSetting.disable_search_queue_threshold)
+  end
+
+  def cancel_overloaded_search
+    if site_overloaded?
+      render_json_error I18n.t("search.extreme_load_error"), status: 409
+    end
+  end
+
   def lookup_search_context
 
     return if params[:skip_context] == "true"
diff --git a/app/serializers/grouped_search_result_serializer.rb b/app/serializers/grouped_search_result_serializer.rb
index 15d2436..653ac70 100644
--- a/app/serializers/grouped_search_result_serializer.rb
+++ b/app/serializers/grouped_search_result_serializer.rb
@@ -6,7 +6,7 @@ class GroupedSearchResultSerializer < ApplicationSerializer
   has_many :categories, serializer: BasicCategorySerializer
   has_many :tags, serializer: TagSerializer
   has_many :groups, serializer: BasicGroupSerializer
-  attributes :more_posts, :more_users, :more_categories, :term, :search_log_id, :more_full_page_results, :can_create_topic
+  attributes :more_posts, :more_users, :more_categories, :term, :search_log_id, :more_full_page_results, :can_create_topic, :error
 
   def search_log_id
     object.search_log_id
diff --git a/config/discourse_defaults.conf b/config/discourse_defaults.conf
index e66ed46..4f409f5 100644
--- a/config/discourse_defaults.conf
+++ b/config/discourse_defaults.conf
@@ -224,6 +224,9 @@ force_anonymous_min_queue_seconds = 1
 # only trigger anon if we see more than N requests for this path in last 10 seconds
 force_anonymous_min_per_10_seconds = 3
 
+# disable search if app server is queueing for longer than this (in seconds)
+disable_search_queue_threshold = 1
+
 # maximum number of posts rebaked across the cluster in the periodical job
 # rebake process is very expensive, on multisite we have to make sure we never
 # flood the queue
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 9c231b9..c3ba80e 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -2096,6 +2096,7 @@ en:
         value: "SSO secret"
 
   search:
+    extreme_load_error: "Site is under extreme load, search is disabled, try again later"
     within_post: "#%{post_number} by %{username}"
     types:
       category: "Categories"
diff --git a/lib/search/grouped_search_results.rb b/lib/search/grouped_search_results.rb
index 1cd6dae..23a605f 100644
--- a/lib/search/grouped_search_results.rb
+++ b/lib/search/grouped_search_results.rb
@@ -24,7 +24,8 @@ class Search
       :term,
       :search_context,
       :include_blurbs,
-      :more_full_page_results
+      :more_full_page_results,
+      :error
     )
 
     attr_accessor :search_log_id
@@ -40,6 +41,11 @@ class Search
       @users = []
       @tags = []
       @groups = []
+      @error = nil
+    end
+
+    def error=(error)
+      @error = error
     end
 
     def find_user_data(guardian)
diff --git a/spec/requests/search_controller_spec.rb b/spec/requests/search_controller_spec.rb
index 0b69e39..f583be4 100644
--- a/spec/requests/search_controller_spec.rb
+++ b/spec/requests/search_controller_spec.rb
@@ -3,6 +3,21 @@
 require 'rails_helper'
 
 describe SearchController do
+
+  fab!(:awesome_post) do
+    SearchIndexer.enable
+    Fabricate(:post, raw: 'this is my really awesome post')
+  end
+
+  fab!(:user) do
+    Fabricate(:user)
+  end
+
+  fab!(:user_post) do
+    SearchIndexer.enable
+    Fabricate(:post, raw: "#{user.username} is a cool person")
+  end
+
   context "integration" do
     before do
       SearchIndexer.enable
@@ -18,6 +33,49 @@ describe SearchController do
       $redis.flushall
     end
 
+    context "when overloaded" do
+
+      before do
+        global_setting :disable_search_queue_threshold, 0.2
+      end
+
+      let! :start_time do
+        freeze_time
+        Time.now
+      end
+
+      let! :current_time do
+        freeze_time 0.3.seconds.from_now
+      end
+

[... diff too long, it was truncated ...]

GitHub sha: 4dcc5f16

not 100% sure if modal is the perfect option here. maybe we should do it inline.