Add support for robots.txt disallow rules

Author: mvdbos

.phan/config.php

@@ -342,6 +342,7 @@
     // your application should be included in this list.
     'directory_list' => [
         'src',
+        'vendor/spatie/robots-txt/src',
         'vendor/guzzlehttp/guzzle/src',
         'vendor/guzzlehttp/psr7/src',
         'vendor/psr/http-client',

README.md

@@ -9,7 +9,7 @@ PHP-Spider Features
 - supports two traversal algorithms: breadth-first and depth-first
 - supports crawl depth limiting, queue size limiting and max downloads limiting
 - supports adding custom URI discovery logic, based on XPath, CSS selectors, or plain old PHP
-- comes with a useful set of URI filters, such as Domain limiting
+- comes with a useful set of URI filters, such as robots.txt and Domain limiting
 - supports custom URI filters, both prefetch (URI) and postfetch (Resource content)
 - supports custom request handling logic
 - supports Basic, Digest and NTLM HTTP authentication. See [example](example/example_basic_auth.php).
@@ -25,6 +25,10 @@ Installation
 ------------
 The easiest way to install PHP-Spider is with [composer](https://getcomposer.org/).  Find it on [Packagist](https://packagist.org/packages/vdb/php-spider).
 
+```bash
+$ composer require vdb/php-spider
+```
+
 Usage
 -----
 This is a very simple example. This code can be found in [example/example_simple.php](example/example_simple.php). For a more complete example with some logging, caching and filters, see [example/example_complex.php](example/example_complex.php). That file contains a more real-world example.

composer.json

@@ -22,7 +22,8 @@
         "symfony/dom-crawler": "^3.0.0||^4.0.0||^5.0.0||^6.0",
         "symfony/finder": "^3.0.0||^4.0.0||^5.0.0||^6.0",
         "symfony/event-dispatcher": "^4.0.0||^5.0.0||^6.0",
-        "vdb/uri": "^0.3.1",
+        "vdb/uri": "^0.3.2",
+        "spatie/robots-txt": "^1.0",
         "phan/phan": "^4.0||^5.0"
     },
     "require-dev": {

example/example_complex.php

@@ -9,6 +9,7 @@
 use VDB\Spider\EventListener\PolitenessPolicyListener;
 use VDB\Spider\Filter\Prefetch\AllowedHostsFilter;
 use VDB\Spider\Filter\Prefetch\AllowedSchemeFilter;
+use VDB\Spider\Filter\Prefetch\RobotsTxtDisallowFilter;
 use VDB\Spider\Filter\Prefetch\UriWithHashFragmentFilter;
 use VDB\Spider\Filter\Prefetch\UriWithQueryStringFilter;
 use VDB\Spider\PersistenceHandler\FileSerializedResourcePersistenceHandler;
@@ -36,6 +37,7 @@
 
 $queueManager->getDispatcher()->addSubscriber($statsHandler);
 $queueManager->getDispatcher()->addSubscriber($LogHandler);
+$spider->getDownloader()->getDispatcher()->addSubscriber($statsHandler);
 
 // Set some sane defaults for this example.
 // We only visit the first level of http://dmoztools.net. We stop at 10 queued resources
@@ -47,7 +49,7 @@
 $spider->setQueueManager($queueManager);
 
 // We add an URI discoverer. Without it, the spider wouldn't get past the seed resource.
-$spider->getDiscovererSet()->set(new XPathExpressionDiscoverer("//*[@id='cat-list-content-2']/div/a"));
+$spider->getDiscovererSet()->set(new XPathExpressionDiscoverer("//a"));
 
 // Let's tell the spider to save all found resources on the filesystem
 $spider->getDownloader()->setPersistenceHandler(
@@ -60,6 +62,7 @@
 $spider->getDiscovererSet()->addFilter(new AllowedHostsFilter(array($seed), $allowSubDomains));
 $spider->getDiscovererSet()->addFilter(new UriWithHashFragmentFilter());
 $spider->getDiscovererSet()->addFilter(new UriWithQueryStringFilter());
+$spider->getDiscovererSet()->addFilter(new RobotsTxtDisallowFilter($seed, 'PHP-Spider'));
 
 // We add an event listener to the crawler that implements a politeness policy.
 // We wait 100ms between every request to the same domain
@@ -131,3 +134,8 @@ function (GenericEvent $event) {
     echo "\n - " . $contentLengthString . " $title ($uri)";
 }
 echo "\n";
+
+echo "\nFAILED RESOURCES: ";
+foreach ($statsHandler->getFailed() as $uri => $message) {
+    echo "\n - " . $uri . " failed because: " . $message;
+}

example/lib/Example/StatsHandler.php

@@ -35,22 +35,22 @@ public static function getSubscribedEvents(): array
         );
     }
 
-    public function addToQueued(GenericEvent $event)
+    public function addToQueued(GenericEvent $event): void
     {
         $this->queued[] = $event->getArgument('uri');
     }
 
-    public function addToPersisted(GenericEvent $event)
+    public function addToPersisted(GenericEvent $event): void
     {
         $this->persisted[] = $event->getArgument('uri');
     }
 
-    public function addToFiltered(GenericEvent $event)
+    public function addToFiltered(GenericEvent $event): void
     {
         $this->filtered[] = $event->getArgument('uri');
     }
 
-    public function addToFailed(GenericEvent $event)
+    public function addToFailed(GenericEvent $event): void
     {
         $this->failed[$event->getArgument('uri')->toString()] = $event->getArgument('message');
     }

src/Discoverer/DiscovererSet.php

@@ -42,7 +42,7 @@ public function __construct(array $discoverers = array())
      * If it already exists, it is not overwritten, since we want to keep the
      * first depth it was found at.
      */
-    private function markSeen(DiscoveredUri $uri)
+    private function markSeen(DiscoveredUri $uri): void
     {
         $uriString = $uri->normalize()->toString();
         if (!array_key_exists($uriString, $this->alreadySeenUris)) {
@@ -97,20 +97,20 @@ public function discover(Resource $resource): array
      *
      * @param discovererInterface $discoverer The discoverer instance
      */
-    public function set(DiscovererInterface $discoverer)
+    public function set(DiscovererInterface $discoverer): void
     {
         $this->discoverers[$discoverer->getName()] = $discoverer;
     }
 
-    public function addFilter(PreFetchFilterInterface $filter)
+    public function addFilter(PreFetchFilterInterface $filter): void
     {
         $this->filters[] = $filter;
     }
 
     /**
      * @param UriInterface[] $discoveredUris
      */
-    private function normalize(array &$discoveredUris)
+    private function normalize(array &$discoveredUris): void
     {
         /** @var DiscoveredUri[] $discoveredUris */
         foreach ($discoveredUris as $k => $uri) {
@@ -121,7 +121,7 @@ private function normalize(array &$discoveredUris)
     /**
      * @param UriInterface[] $discoveredUris
      */
-    private function filterAlreadySeen(array &$discoveredUris)
+    private function filterAlreadySeen(array &$discoveredUris): void
     {
         foreach ($discoveredUris as $k => $uri) {
             if (array_key_exists($uri->toString(), $this->alreadySeenUris)) {
@@ -134,7 +134,7 @@ private function filterAlreadySeen(array &$discoveredUris)
      * Filter out any URI that matches any of the filters
      * @param UriInterface[] $discoveredUris
      */
-    private function filter(array &$discoveredUris)
+    private function filter(array &$discoveredUris): void
     {
         foreach ($discoveredUris as $k => $uri) {
             foreach ($this->filters as $filter) {
@@ -148,7 +148,7 @@ private function filter(array &$discoveredUris)
     /**
      * @param UriInterface[] $discoveredUris
      */
-    private function removeDuplicates(array &$discoveredUris)
+    private function removeDuplicates(array &$discoveredUris): void
     {
         // make sure there are no duplicates in the list
         $tmp = array();

src/Filter/Prefetch/ExtractRobotsTxtException.php

@@ -0,0 +1,9 @@
+<?php
+
+namespace VDB\Spider\Filter\Prefetch;
+
+use RuntimeException;
+
+class ExtractRobotsTxtException extends RuntimeException
+{
+}

src/Filter/Prefetch/FetchRobotsTxtException.php

@@ -0,0 +1,9 @@
+<?php
+
+namespace VDB\Spider\Filter\Prefetch;
+
+use RuntimeException;
+
+class FetchRobotsTxtException extends RuntimeException
+{
+}

src/Filter/Prefetch/RobotsTxtDisallowFilter.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace VDB\Spider\Filter\Prefetch;
+
+use ErrorException;
+use Exception;
+use Spatie\Robots\RobotsTxt;
+use VDB\Spider\Filter\PreFetchFilterInterface;
+use VDB\Uri\Exception\UriSyntaxException;
+use VDB\Uri\FileUri;
+use VDB\Uri\Http;
+use VDB\Uri\Uri;
+use VDB\Uri\UriInterface;
+
+/**
+ * @author Matthijs van den Bos <[email protected]>
+ */
+class RobotsTxtDisallowFilter implements PreFetchFilterInterface
+{
+    private RobotsTxt $parser;
+    private ?string $userAgent;
+    private Uri $seedUri;
+
+    /**
+     * @param string $seedUrl The robots.txt file will be loaded from this domain.
+     * @param string|null $userAgent
+     * @throws ErrorException
+     * @throws UriSyntaxException
+     */
+    public function __construct(string $seedUrl, string $userAgent = null)
+    {
+        $this->seedUri = new Uri($seedUrl);
+        $this->seedUri->normalize();
+        $this->userAgent = $userAgent;
+        $this->parser = new RobotsTxt(self::fetchRobotsTxt(self::extractRobotsTxtUri($seedUrl)));
+    }
+
+    /**
+     * @param string $robotsUri
+     * @return string
+     */
+    private static function fetchRobotsTxt(string $robotsUri): string
+    {
+        try {
+            $robotsTxt = file_get_contents($robotsUri);
+        } catch (Exception $e) {
+            throw new FetchRobotsTxtException("Could not fetch $robotsUri: " . $e->getMessage());
+        }
+
+        return $robotsTxt;
+    }
+
+    /**
+     * Clean up the URL and strip any parameters and fragments
+     *
+     * @param string $seedUrl
+     * @return string
+     *
+     * @throws ErrorException
+     * @throws UriSyntaxException
+     */
+    private static function extractRobotsTxtUri(string $seedUrl): string
+    {
+        $uri = new Uri($seedUrl);
+        if (in_array($uri->getScheme(), FileUri::$allowedSchemes)) {
+            return new FileUri($seedUrl . '/robots.txt');
+        } elseif (in_array($uri->getScheme(), Http::$allowedSchemes)) {
+            return $uri->toBaseUri()->toString() . '/robots.txt';
+        } else {
+            throw new ExtractRobotsTxtException(
+                "Seed URL scheme must be one of " .
+                implode(', ', array_merge(FileUri::$allowedSchemes, Http::$allowedSchemes))
+            );
+        }
+    }
+
+    public function match(UriInterface $uri): bool
+    {
+        // Make the uri relative to $this->seedUri, so it will match with the rules in the robots.txt
+        $relativeUri = str_replace($this->seedUri->toString(), '', $uri->normalize()->toString());
+        return !$this->parser->allows($relativeUri, $this->userAgent);
+    }
+}

src/Uri/DiscoveredUri.php

@@ -9,22 +9,14 @@
 
 class DiscoveredUri implements UriInterface
 {
-    /**
-     * @var UriInterface
-     */
-    protected $decorated;
-
-    /** @var int */
+    protected string|UriInterface $decorated;
     private int $depthFound;
 
     /**
-     * @param string|UriInterface $decorated
-     * @param int $depthFound
-     *
      * @throws ErrorException
      * @throws UriSyntaxException
      */
-    public function __construct($decorated, int $depthFound)
+    public function __construct(UriInterface|string $decorated, int $depthFound)
     {
         if (!$decorated instanceof UriInterface) {
             $decorated = new Uri($decorated);
@@ -42,14 +34,6 @@ public function getDepthFound(): int
         return $this->depthFound;
     }
 
-//    /**
-//     * @param int $depthFound The depth this Uri was found on
-//     */
-//    public function setDepthFound(int $depthFound)
-//    {
-//        $this->depthFound = $depthFound;
-//    }
-
     // @codeCoverageIgnoreStart
     // We ignore coverage for all proxy methods below:
     // the constructor is tested and if that is successful there is no point

tests/Discoverer/DiscovererSetTest.php

@@ -17,9 +17,11 @@
 use VDB\Spider\Discoverer\XPathExpressionDiscoverer;
 use VDB\Spider\Filter\Prefetch\AllowedHostsFilter;
 use VDB\Spider\Filter\Prefetch\AllowedPortsFilter;
+use VDB\Spider\Filter\Prefetch\RobotsTxtDisallowFilter;
 use VDB\Spider\Filter\Prefetch\UriFilter;
 use VDB\Spider\Uri\DiscoveredUri;
 use VDB\Uri\Exception\UriSyntaxException;
+use VDB\Uri\FileUri;
 
 /**
  *
@@ -86,6 +88,31 @@ public function testUriFilter()
         $this->assertCount(1, $uris);
     }
 
+    /**
+     * @covers \VDB\Spider\Discoverer\DiscovererSet
+     * @covers \VDB\Spider\Filter\Prefetch\RobotsTxtDisallowFilter
+     *
+     * @throws UriSyntaxException
+     * @throws ErrorException
+     * @throws Exception
+     */
+    public function testRobotsTxtDisallowFilter()
+    {
+        $baseUri = "file://" . __DIR__;
+        $resourceUri = new DiscoveredUri($baseUri, 0);
+        $uriInBody1 = $baseUri . '/internal';
+        $uriInBody2 = $baseUri . '/foo';
+
+        $spiderResource = self::createResourceWithLinks($resourceUri, [$uriInBody1, $uriInBody2]);
+
+        $discovererSet = new DiscovererSet([new XPathExpressionDiscoverer("//a")]);
+        $discovererSet->addFilter(new RobotsTxtDisallowFilter($baseUri));
+
+        $uris = $discovererSet->discover($spiderResource);
+        $this->assertCount(1, $uris);
+        $this->assertNotContains((new FileUri($uriInBody2))->toString(), array_map(fn($uri): string => (new FileUri($uri->toString()))->toString(), $uris));
+    }
+
     /**
      * @covers \VDB\Spider\Discoverer\DiscovererSet
      * @covers \VDB\Spider\Filter\Prefetch\AllowedPortsFilter

tests/Discoverer/robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Disallow: /foo
+
+User-agent: PHP-Spider
+Disallow: /bar