FIX: add noindex header to user profile pages.

FIX: add noindex header to user profile pages.

diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb
index 8d64202630..07afe1a40d 100644
--- a/app/controllers/users_controller.rb
+++ b/app/controllers/users_controller.rb
@@ -73,6 +73,8 @@ class UsersController < ApplicationController
       track_visit_to_user_profile
     end
 
+    response.headers['X-Robots-Tag'] = 'noindex'
+
     # This is a hack to get around a Rails issue where values with periods aren't handled correctly
     # when used as part of a route.
     if params[:external_id] && params[:external_id].ends_with?('.json')
diff --git a/spec/requests/users_controller_spec.rb b/spec/requests/users_controller_spec.rb
index 8bf6256ade..f318ca22e6 100644
--- a/spec/requests/users_controller_spec.rb
+++ b/spec/requests/users_controller_spec.rb
@@ -2904,6 +2904,8 @@ describe UsersController do
       it 'returns success' do
         get "/u/#{user.username}.json"
         expect(response.status).to eq(200)
+        expect(response.headers['X-Robots-Tag']).to eq('noindex')
+
         json = JSON.parse(response.body)
 
         expect(json["user"]["has_title_badges"]).to eq(false)

GitHub sha: 13f22980

3 Likes

Yeah this is pretty much as good as we can do without a major shift and a DSL or parser for robots.txt which would be a huge pita.

3 Likes

This commit has been mentioned on Discourse Meta. There might be relevant details there:

https://meta.discourse.org/t/why-semrushbot-and-ahrefsbot-are-blocked-by-default/157682/4