Update Web Crawling Pipeline

Author: kkamara

.env.example

@@ -16,3 +16,4 @@ EMAIL_HOST_PASSWORD=
 EMAIL_PORT=1025
 EMAIL_USE_TLS=False
 DOMAIN=localhost
+SELENIUM_HEADLESS=False

.env.testing

@@ -16,3 +16,4 @@ EMAIL_HOST_PASSWORD=
 EMAIL_PORT=1025
 EMAIL_USE_TLS=False
 DOMAIN=localhost
+SELENIUM_HEADLESS=True

.github/workflows/build.yml

@@ -47,10 +47,7 @@ jobs:
     - uses: nanasess/setup-chromedriver@v2
       with:
         chromedriver-version: '88.0.4324.96'
-    - run: |
-        export DISPLAY=:99
-        chromedriver --url-base=/wd/hub &
-        sudo Xvfb -ac :99 -screen 0 1280x1024x24 > /dev/null 2>&1 &
+    - run: chromedriver --url-base=/wd/hub &
 
     - name: Create environment file
       run: cp .env.testing .env
@@ -59,8 +56,7 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements.txt
     - name: See your Python code do web browsing on your screen with GUI
-      run: |
-        python manage.py crawl
+      run: python manage.py crawl
     - name: Upload Screenshots
       uses: actions/upload-artifact@v3
       with:

app/settings.py

@@ -18,6 +18,8 @@
   DEBUG=(bool,False)
 )
 
+SELENIUM_HEADLESS=env('SELENIUM_HEADLESS', False)
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 

seleniumpy/management/commands/crawl.py

@@ -1,4 +1,5 @@
 from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
 
 import os
 import json
@@ -21,7 +22,8 @@ def handle(self, *args, **options):
             userAgent = 'Mozilla/5.0 (BB10; Touch) AppleWebKit/537.1+ (KHTML, like Gecko) Version/10.0.0.1337 Mobile Safari/537.1+'
             chrome_options.add_argument('--user-agent='+userAgent)
             # Uncomment next line to remove GUI.
-            # chrome_options.add_argument('--headless')
+            if settings.SELENIUM_HEADLESS == True:
+                chrome_options.add_argument('--headless')
             browser = webdriver.Chrome(options=chrome_options)
 
             browser.get('https://www.kelvinkamara.com')