[gh-227] Merging from testing into staging

git-svn-id: file:///tmp/wdn_thm_drupal/branches/drupal-7.x/staging@1116 20a16fea-79d4-4915-8869-1ea9d5ebf173

sites/all/modules/robotstxt/LICENSE.txt

+GNU GENERAL PUBLIC LICENSE
+
+              Version 2, June 1991
+
+Copyright (C) 1989, 1991 Free Software Foundation, Inc. 675 Mass Ave,
+Cambridge, MA 02139, USA. Everyone is permitted to copy and distribute
+verbatim copies of this license document, but changing it is not allowed.
+
+                  Preamble
+
+The licenses for most software are designed to take away your freedom to
+share and change it. By contrast, the GNU General Public License is
+intended to guarantee your freedom to share and change free software--to
+make sure the software is free for all its users. This General Public License
+applies to most of the Free Software Foundation's software and to any other
+program whose authors commit to using it. (Some other Free Software
+Foundation software is covered by the GNU Library General Public License
+instead.) You can apply it to your programs, too.
+
+When we speak of free software, we are referring to freedom, not price. Our
+General Public Licenses are designed to make sure that you have the
+freedom to distribute copies of free software (and charge for this service if
+you wish), that you receive source code or can get it if you want it, that you
+can change the software or use pieces of it in new free programs; and that
+you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid anyone to
+deny you these rights or to ask you to surrender the rights. These restrictions
+translate to certain responsibilities for you if you distribute copies of the
+software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether gratis or for
+a fee, you must give the recipients all the rights that you have. You must make
+sure that they, too, receive or can get the source code. And you must show
+them these terms so they know their rights.
+
+We protect your rights with two steps: (1) copyright the software, and (2)
+offer you this license which gives you legal permission to copy, distribute
+and/or modify the software.
+
+Also, for each author's protection and ours, we want to make certain that
+everyone understands that there is no warranty for this free software. If the
+software is modified by someone else and passed on, we want its recipients
+to know that what they have is not the original, so that any problems
+introduced by others will not reflect on the original authors' reputations.
+
+Finally, any free program is threatened constantly by software patents. We
+wish to avoid the danger that redistributors of a free program will individually
+obtain patent licenses, in effect making the program proprietary. To prevent
+this, we have made it clear that any patent must be licensed for everyone's
+free use or not licensed at all.
+
+The precise terms and conditions for copying, distribution and modification
+follow.
+
+           GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND
+               MODIFICATION
+
+0. This License applies to any program or other work which contains a notice
+placed by the copyright holder saying it may be distributed under the terms
+of this General Public License. The "Program", below, refers to any such
+program or work, and a "work based on the Program" means either the
+Program or any derivative work under copyright law: that is to say, a work
+containing the Program or a portion of it, either verbatim or with
+modifications and/or translated into another language. (Hereinafter, translation
+is included without limitation in the term "modification".) Each licensee is
+addressed as "you".
+
+Activities other than copying, distribution and modification are not covered
+by this License; they are outside its scope. The act of running the Program is
+not restricted, and the output from the Program is covered only if its contents
+constitute a work based on the Program (independent of having been made
+by running the Program). Whether that is true depends on what the Program
+does.
+
+1. You may copy and distribute verbatim copies of the Program's source
+code as you receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice and
+disclaimer of warranty; keep intact all the notices that refer to this License
+and to the absence of any warranty; and give any other recipients of the
+Program a copy of this License along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and you
+may at your option offer warranty protection in exchange for a fee.
+
+2. You may modify your copy or copies of the Program or any portion of it,
+thus forming a work based on the Program, and copy and distribute such
+modifications or work under the terms of Section 1 above, provided that you
+also meet all of these conditions:
+
+a) You must cause the modified files to carry prominent notices stating that
+you changed the files and the date of any change.
+
+b) You must cause any work that you distribute or publish, that in whole or in
+part contains or is derived from the Program or any part thereof, to be
+licensed as a whole at no charge to all third parties under the terms of this
+License.
+
+c) If the modified program normally reads commands interactively when run,
+you must cause it, when started running for such interactive use in the most
+ordinary way, to print or display an announcement including an appropriate
+copyright notice and a notice that there is no warranty (or else, saying that
+you provide a warranty) and that users may redistribute the program under
+these conditions, and telling the user how to view a copy of this License.
+(Exception: if the Program itself is interactive but does not normally print such
+an announcement, your work based on the Program is not required to print
+an announcement.)
+
+These requirements apply to the modified work as a whole. If identifiable
+sections of that work are not derived from the Program, and can be
+reasonably considered independent and separate works in themselves, then
+this License, and its terms, do not apply to those sections when you distribute
+them as separate works. But when you distribute the same sections as part
+of a whole which is a work based on the Program, the distribution of the
+whole must be on the terms of this License, whose permissions for other
+licensees extend to the entire whole, and thus to each and every part
+regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest your rights to
+work written entirely by you; rather, the intent is to exercise the right to
+control the distribution of derivative or collective works based on the
+Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of a
+storage or distribution medium does not bring the other work under the scope
+of this License.
+
+3. You may copy and distribute the Program (or a work based on it, under
+Section 2) in object code or executable form under the terms of Sections 1
+and 2 above provided that you also do one of the following:
+
+a) Accompany it with the complete corresponding machine-readable source
+code, which must be distributed under the terms of Sections 1 and 2 above
+on a medium customarily used for software interchange; or,
+
+b) Accompany it with a written offer, valid for at least three years, to give
+any third party, for a charge no more than your cost of physically performing
+source distribution, a complete machine-readable copy of the corresponding
+source code, to be distributed under the terms of Sections 1 and 2 above on
+a medium customarily used for software interchange; or,
+
+c) Accompany it with the information you received as to the offer to distribute
+corresponding source code. (This alternative is allowed only for
+noncommercial distribution and only if you received the program in object
+code or executable form with such an offer, in accord with Subsection b
+above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source code
+means all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation and
+installation of the executable. However, as a special exception, the source
+code distributed need not include anything that is normally distributed (in
+either source or binary form) with the major components (compiler, kernel,
+and so on) of the operating system on which the executable runs, unless that
+component itself accompanies the executable.
+
+If distribution of executable or object code is made by offering access to
+copy from a designated place, then offering equivalent access to copy the
+source code from the same place counts as distribution of the source code,
+even though third parties are not compelled to copy the source along with the
+object code.
+
+4. You may not copy, modify, sublicense, or distribute the Program except as
+expressly provided under this License. Any attempt otherwise to copy,
+modify, sublicense or distribute the Program is void, and will automatically
+terminate your rights under this License. However, parties who have received
+copies, or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+5. You are not required to accept this License, since you have not signed it.
+However, nothing else grants you permission to modify or distribute the
+Program or its derivative works. These actions are prohibited by law if you
+do not accept this License. Therefore, by modifying or distributing the
+Program (or any work based on the Program), you indicate your acceptance
+of this License to do so, and all its terms and conditions for copying,
+distributing or modifying the Program or works based on it.
+
+6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the original
+licensor to copy, distribute or modify the Program subject to these terms and
+conditions. You may not impose any further restrictions on the recipients'
+exercise of the rights granted herein. You are not responsible for enforcing
+compliance by third parties to this License.
+
+7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues), conditions
+are imposed on you (whether by court order, agreement or otherwise) that
+contradict the conditions of this License, they do not excuse you from the
+conditions of this License. If you cannot distribute so as to satisfy
+simultaneously your obligations under this License and any other pertinent
+obligations, then as a consequence you may not distribute the Program at all.
+For example, if a patent license would not permit royalty-free redistribution
+of the Program by all those who receive copies directly or indirectly through
+you, then the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply and
+the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any patents or
+other property right claims or to contest validity of any such claims; this
+section has the sole purpose of protecting the integrity of the free software
+distribution system, which is implemented by public license practices. Many
+people have made generous contributions to the wide range of software
+distributed through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing to
+distribute software through any other system and a licensee cannot impose
+that choice.
+
+This section is intended to make thoroughly clear what is believed to be a
+consequence of the rest of this License.
+
+8. If the distribution and/or use of the Program is restricted in certain
+countries either by patents or by copyrighted interfaces, the original copyright
+holder who places the Program under this License may add an explicit
+geographical distribution limitation excluding those countries, so that
+distribution is permitted only in or among countries not thus excluded. In such
+case, this License incorporates the limitation as if written in the body of this
+License.
+
+9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will be
+similar in spirit to the present version, but may differ in detail to address new
+problems or concerns.
+
+Each version is given a distinguishing version number. If the Program specifies
+a version number of this License which applies to it and "any later version",
+you have the option of following the terms and conditions either of that
+version or of any later version published by the Free Software Foundation. If
+the Program does not specify a version number of this License, you may
+choose any version ever published by the Free Software Foundation.
+
+10. If you wish to incorporate parts of the Program into other free programs
+whose distribution conditions are different, write to the author to ask for
+permission. For software which is copyrighted by the Free Software
+Foundation, write to the Free Software Foundation; we sometimes make
+exceptions for this. Our decision will be guided by the two goals of
+preserving the free status of all derivatives of our free software and of
+promoting the sharing and reuse of software generally.
+
+               NO WARRANTY
+
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE,
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT
+PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE
+STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
+WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
+PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL
+NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR
+AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR
+ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE
+LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL,
+SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OR INABILITY TO USE THE
+PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA
+OR DATA BEING RENDERED INACCURATE OR LOSSES
+SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE
+PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN
+IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGES.
+
+          END OF TERMS AND CONDITIONS

sites/all/modules/robotstxt/README.txt

+$Id: README.txt,v 1.3.2.2 2011/01/05 23:24:10 hass Exp $
+
+CONTENTS OF THIS FILE
+---------------------
+
+ * Introduction
+ * Installation
+ * Frequently Asked Questions (FAQ)
+ * Known Issues
+ * How Can You Contribute?
+
+
+INTRODUCTION
+------------
+
+Maintainer: hass <http://drupal.org/user/85918>
+Project Page: http://drupal.org/project/robotstxt
+
+Use this module when you are running multiple Drupal sites from a single code
+base (multisite) and you need a different robots.txt file for each one. This
+module generates the robots.txt file dynamically and gives you the chance to
+edit it, on a per-site basis.
+
+For developers, you can automatically add paths to the robots.txt file by
+implementing hook_robotstxt(). See robotstxt.api.php for more documentation.
+
+
+INSTALLATION
+------------
+
+See http://drupal.org/getting-started/install-contrib for instructions on
+how to install or update Drupal modules.
+
+Once you have the RobotsTxt modules installed, make sure to delete or rename
+the robots.txt file in the root of your Drupal installation. Otherwise, the
+module cannot intercept requests for the /robots.txt path.
+
+
+FREQUENTLY ASKED QUESTIONS
+--------------------------
+
+Q: Can this module work if I have clean URLs disabled?
+A: Yes it can! In the .htaccess file of your Drupal's root directory, add the 
+   following two lines to the mod_rewrite section, immediately after the line
+   that says "RewriteEngine on":
+
+     # Add redirection for the robots.txt path for use with the RobotsTxt module.
+     RewriteRule ^(robots.txt)$ index.php?q=$1
+
+
+KNOWN ISSUES
+------------
+
+There are no known issues at this time.
+
+To report new bug reports, feature requests, and support requests, visit
+http://drupal.org/project/issues/robotstxt.
+
+
+HOW CAN YOU CONTRIBUTE?
+---------------------
+
+- Report any bugs, feature requests, etc. in the issue tracker.
+  http://drupal.org/project/issues/robotstxt
+

sites/all/modules/robotstxt/robots.txt

+# $Id: robots.txt,v 1.8.2.2 2011/01/05 23:24:10 hass Exp $
+#
+# robots.txt
+#
+# This file is to prevent the crawling and indexing of certain parts
+# of your site by web crawlers and spiders run by sites like Yahoo!
+# and Google. By telling these "robots" where not to go on your site,
+# you save bandwidth and server resources.
+#
+# This file will be ignored unless it is at the root of your host:
+# Used:    http://example.com/robots.txt
+# Ignored: http://example.com/site/robots.txt
+#
+# For more information about the robots.txt standard, see:
+# http://www.robotstxt.org/wc/robots.html
+#
+# For syntax checking, see:
+# http://www.sxw.org.uk/computing/robots/check.html
+
+User-agent: *
+Crawl-delay: 10
+# Directories
+Disallow: /includes/
+Disallow: /misc/
+Disallow: /modules/
+Disallow: /profiles/
+Disallow: /scripts/
+Disallow: /themes/
+# Files
+Disallow: /CHANGELOG.txt
+Disallow: /cron.php
+Disallow: /INSTALL.mysql.txt
+Disallow: /INSTALL.pgsql.txt
+Disallow: /install.php
+Disallow: /INSTALL.txt
+Disallow: /LICENSE.txt
+Disallow: /MAINTAINERS.txt
+Disallow: /update.php
+Disallow: /UPGRADE.txt
+Disallow: /xmlrpc.php
+# Paths (clean URLs)
+Disallow: /admin/
+Disallow: /comment/reply/
+Disallow: /node/add/
+Disallow: /search/
+Disallow: /user/register/
+Disallow: /user/password/
+Disallow: /user/login/
+Disallow: /user/logout/
+# Paths (no clean URLs)
+Disallow: /?q=admin/
+Disallow: /?q=comment/reply/
+Disallow: /?q=node/add/
+Disallow: /?q=search/
+Disallow: /?q=user/password/
+Disallow: /?q=user/register/
+Disallow: /?q=user/login/
+Disallow: /?q=user/logout/

sites/all/modules/robotstxt/robotstxt.admin.inc

+<?php
+// $Id: robotstxt.admin.inc,v 1.3.2.2 2011/01/05 23:24:10 hass Exp $
+
+/**
+ * @file
+ * Administrative page callbacks for the robotstxt module.
+ */
+
+/**
+ * Administration settings form.
+ *
+ * @see system_settings_form()
+ */
+function robotstxt_admin_settings() {
+  $form['robotstxt'] = array(
+    '#type' => 'textarea',
+    '#title' => t('Contents of robots.txt'),
+    '#default_value' => _robotstxt_get_content(),
+    '#cols' => 60,
+    '#rows' => 20,
+    '#wysiwyg' => FALSE,
+  );
+
+  return system_settings_form($form, FALSE);
+}

sites/all/modules/robotstxt/robotstxt.api.php

+<?php
+// $Id: robotstxt.api.php,v 1.2.2.2 2011/01/05 23:24:10 hass Exp $
+
+/**
+ * @file
+ * Hooks provided by the robotstxt module.
+ */
+
+/**
+ * Add additional lines to the site's robots.txt file.
+ *
+ * @return
+ *   An array of strings to add to the robots.txt.
+ */
+function hook_robotstxt() {
+  return array(
+    'Disallow: /foo',
+    'Disallow: /bar',
+  );
+}

sites/all/modules/robotstxt/robotstxt.info

+; $Id: robotstxt.info,v 1.6.2.2 2011/01/05 23:24:10 hass Exp $
+name = robots.txt
+description = "Generates the robots.txt file dynamically and gives you the chance to edit it, on a per-site basis, from the web UI."
+core = 7.x
+files[] = robotstxt.module
+files[] = robotstxt.admin.inc
+files[] = robotstxt.install
+configure = admin/config/search/robotstxt
+
+; Information added by drupal.org packaging script on 2011-01-05
+version = "7.x-1.0"
+core = "7.x"
+project = "robotstxt"
+datestamp = "1294270341"
+

sites/all/modules/robotstxt/robotstxt.install

+<?php
+// $Id: robotstxt.install,v 1.13.2.2 2011/01/05 23:24:10 hass Exp $
+
+/**
+ * Implements hook_install().
+ */
+function robotstxt_install() {
+  if (file_exists(DRUPAL_ROOT . '/robots.txt')) {
+    variable_set('robotstxt', file_get_contents(DRUPAL_ROOT . '/robots.txt'));
+  }
+  elseif (file_exists(drupal_get_path('module', 'robotstxt') . '/robots.txt')) {
+    variable_set('robotstxt', file_get_contents(drupal_get_path('module', 'robotstxt') . '/robots.txt'));
+  }
+}
+
+/**
+ * Implements hook_uninstall().
+ */
+function robotstxt_uninstall() {
+  variable_del('robotstxt');
+}
+
+/**
+ * Implements hook_requirements().
+ */
+function robotstxt_requirements($phase) {
+  $requirements = array();
+  $t = get_t();
+
+  switch ($phase) {
+    case 'runtime' :
+      // Module cannot work without Clean URLs.
+      if (!variable_get('clean_url', 0)) {
+        $requirements['robotstxt_cleanurl'] = array(
+          'title' => $t('RobotsTxt'),
+          'severity' => REQUIREMENT_ERROR,
+          'value' => $t('<a href="!clean_url">Clean URLs</a> are mandatory for this module.', array('!clean_url' => url('admin/config/search/clean-urls'))),
+        );
+      }
+
+      // Webservers prefer the robots.txt file on disk and does not allow menu path overwrite.
+      if (file_exists(DRUPAL_ROOT . '/robots.txt')) {
+        $requirements['robotstxt_file'] = array(
+          'title' => $t('RobotsTxt'),
+          'severity' => REQUIREMENT_WARNING,
+          'value' => $t('RobotsTxt module works only if you remove the existing robots.txt file in your website root.'),
+        );
+      }
+  }
+  return $requirements;
+}
+
+/**
+ * Automatically add the 'administer robots.txt' permission to granted users.
+ */
+function robotstxt_update_6100() {
+  $roles = user_roles(FALSE, 'administer site configuration');
+  foreach ($roles as $rid => $role) {
+    _update_7000_user_role_grant_permissions($rid, array('administer robots.txt'), 'robotstxt');
+  }
+  return t("Added 'administer robots.txt' permission to all roles with 'administer site configuration' permission.");
+}
+
+/**
+ * #337820: Rename menu path 'logout' to 'user/logout' for consistency.
+ */
+function robotstxt_update_7000() {
+  $robotstxt = variable_get('robotstxt');
+  $robotstxt = str_replace('Disallow: /logout/', 'Disallow: /user/logout/', $robotstxt);
+  $robotstxt = str_replace('Disallow: /?q=logout/', 'Disallow: /?q=user/logout/', $robotstxt);
+  variable_set('robotstxt', $robotstxt);
+
+  return t("Renamed menu path 'logout' to 'user/logout'.");
+}
+
+/**
+ * #494462: Allow crawling of sites/default/files by search engines, don't disallow it in robots.txt.
+ */
+function robotstxt_update_7100() {
+  $robotstxt = variable_get('robotstxt');
+  $robotstxt = preg_replace("/Disallow:\s\/sites\/(\r\n?|\n)/", '', $robotstxt);
+  variable_set('robotstxt', $robotstxt);
+
+  return t("Allowed crawling of sites/default/files by search engines.");
+}

sites/all/modules/robotstxt/robotstxt.module

+<?php
+// $Id: robotstxt.module,v 1.9.2.2 2011/01/05 23:24:10 hass Exp $
+
+/**
+ * Implements hook_help().
+ */
+function robotstxt_help($path, $arg) {
+  switch ($path) {
+    case 'admin/help#robotstxt':
+      return '<p>'. t('In a multisite environment, there is no mechanism for having a separate robots.txt file for each site. This module addresses that need by letting you administer the robots.txt file from the settings interface.') .'</p>';
+      break;
+
+    case 'admin/config/search/robotstxt':
+      if (file_exists('./robots.txt')) {
+        drupal_set_message(t('One or more problems have been detected with the RobotsTxt configuration. Check the <a href="@status">status report</a> for more information.', array('@status' => url('admin/reports/status'))), 'warning');
+      }
+      return t('See <a href="http://www.robotstxt.org/">http://www.robotstxt.org/</a> for more information concerning how to write your <a href="@robotstxt">robots.txt</a> file.', array('@robotstxt' => base_path() . 'robots.txt'));
+      break;
+  }
+}
+
+/**
+ * Implements hook_permission().
+ */
+function robotstxt_permission() {
+  return array(
+    'administer robots.txt' => array(
+      'title' => t('Administer robots.txt'),
+      'description' => t('Perform maintenance tasks for robots.txt.'),
+    ),
+  );
+}
+
+/**
+ * Implements hook_menu().
+ */
+function robotstxt_menu() {
+  $items['robots.txt'] = array(
+    'page callback' => 'robotstxt_robots',
+    'access callback' => TRUE,
+    'type' => MENU_CALLBACK,
+  );
+  $items['admin/config/search/robotstxt'] = array(
+    'title' => 'RobotsTxt',
+    'description' => 'Manage your robots.txt file.',
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('robotstxt_admin_settings'),
+    'access arguments' => array('administer robots.txt'),
+    'file' => 'robotstxt.admin.inc',
+  );
+
+  return $items;
+}
+
+/**
+ * Show the robots.txt file.
+ */
+function robotstxt_robots() {
+  $content = array();
+  $content[] = _robotstxt_get_content();
+
+  // Hook other modules for adding additional lines.
+  if ($additions = module_invoke_all('robotstxt')) {
+    $content = array_merge($content, $additions);
+  }
+
+  // Trim any extra whitespace and filter out empty strings.
+  $content = array_map('trim', $content);
+  $content = array_filter($content);
+
+  drupal_add_http_header('Content-type', 'text/plain');
+  echo implode("\n", $content);
+  exit;
+}
+
+/**
+ * Retrieve contents of robots.txt from the database variable, site root, or
+ * module directory.
+ */
+function _robotstxt_get_content() {
+  $content = variable_get('robotstxt', FALSE);
+
+  if ($content === FALSE) {
+    $files = array(
+      DRUPAL_ROOT . '/robots.txt',
+      drupal_get_path('module', 'robotstxt') . '/robots.txt',
+    );
+    foreach ($files as $file) {
+      if (file_exists($file) && is_readable($file)) {
+        $content = file_get_contents($file);
+        break;
+      }
+    }
+  }
+
+  return $content;
+}