Interactive atom cleaning for DCR (guardian#24088)

* Clean interactives prior to DCR send

At the moment we do this cleaning in DCR but it is very slow. Jsoup
is able to do the minimal cleaning we require very quickly though
so we now prefer that.

* Log DCR request timeouts

Previously, these were identified (wrongly) as CAPI timeouts. A
separate exception type has been introduced to distinguish between
them.

Author: nicl

common/app/common/package.scala

@@ -18,7 +18,7 @@ import play.twirl.api.Html
 import model.ApplicationContext
 import http.ResultWithPreconnectPreload
 import http.HttpPreconnections
-import renderers.DCRLocalConnectException
+import renderers.{DCRLocalConnectException, DCRTimeoutException}
 
 object `package`
     extends implicits.Strings
@@ -59,13 +59,18 @@ object `package`
           log.info(s"Got a 410 while calling content api: $message, path: ${request.path}")
           NoCache(Gone)
       }
+
+    // Custom DCR exceptions to distinguish from CAPI/other backend errors.
+    case error: DCRLocalConnectException =>
+      throw error
+    case timeout: DCRTimeoutException =>
+      log.error(s"Got a timeout while calling DCR: ${timeout.getMessage}, path: ${request.path}", timeout)
+      NoCache(GatewayTimeout(timeout.getMessage))
+
     case timeout: TimeoutException =>
       log.error(s"Got a timeout while calling content api: ${timeout.getMessage}, path: ${request.path}", timeout)
       NoCache(GatewayTimeout(timeout.getMessage))
 
-    // This is thrown when locally unable to connect to DCR.
-    case error: DCRLocalConnectException => throw error
-
     case error =>
       log.error(s"Content api exception: ${error.getMessage}", error)
       Option(error.getCause).foreach { cause =>

common/app/model/dotcomrendering/DotcomRenderingUtils.scala

@@ -164,6 +164,7 @@ object DotcomRenderingUtils {
           calloutsUrl,
           article.elements.thumbnail,
           edition,
+          article.trail.webPublicationDate,
         ),
       )
       .filter(PageElement.isSupported)

common/app/model/dotcomrendering/pageElements/PageElement.scala

@@ -15,7 +15,9 @@ import common.Edition
 import conf.Configuration
 import layout.ContentWidths.DotcomRenderingImageRoleWidthByBreakpointMapping
 import model.content._
+import model.dotcomrendering.InteractiveSwitchOver
 import model.{ImageAsset, ImageElement, ImageMedia, VideoAsset}
+import org.joda.time.DateTime
 import org.jsoup.Jsoup
 import play.api.libs.json._
 import views.support.cleaner.SoundcloudHelper
@@ -793,6 +795,7 @@ object PageElement {
       calloutsUrl: Option[String],
       overrideImage: Option[ImageElement],
       edition: Edition,
+      webPublicationDate: DateTime,
   ): List[PageElement] = {
 
     def extractAtom: Option[Atom] =
@@ -1062,12 +1065,19 @@ object PageElement {
           }
 
           case Some(interactive: InteractiveAtom) => {
+            val isLegacy = InteractiveSwitchOver.date.isAfter(webPublicationDate)
             val encodedId = URLEncoder.encode(interactive.id, "UTF-8")
             Some(
               InteractiveAtomBlockElement(
                 id = interactive.id,
                 url = s"${Configuration.ajax.url}/embed/atom/interactive/$encodedId",
-                html = Some(interactive.html),
+                // Note, we parse legacy interactives to do minimal cleaning of
+                // the HTML (e.g. to ensure all tags are closed). Some break
+                // without this. E.g.
+                // https://www.theguardian.com/info/ng-interactive/2021/mar/17/make-sense-of-the-week-with-australia-weekend.
+                html =
+                  if (isLegacy) Some(Jsoup.parseBodyFragment(interactive.html).outerHtml)
+                  else Some(interactive.html),
                 css = Some(interactive.css),
                 js = interactive.mainJS,
                 placeholderUrl = interactive.placeholderUrl,

common/app/renderers/DotcomRenderingService.scala

@@ -21,9 +21,11 @@ import http.ResultWithPreconnectPreload
 import http.HttpPreconnections
 
 import java.net.ConnectException
+import java.util.concurrent.TimeoutException
 
-// Introduced as CAPI error handling elsewhere would smother a regular ConnectException.
+// Introduced as CAPI error handling elsewhere would smother these otherwise
 case class DCRLocalConnectException(message: String) extends ConnectException(message)
+case class DCRTimeoutException(message: String) extends TimeoutException(message)
 
 class DotcomRenderingService extends GuLogging with ResultWithPreconnectPreload {
 
@@ -62,7 +64,8 @@ class DotcomRenderingService extends GuLogging with ResultWithPreconnectPreload
               |To get started with dotcom-rendering, see:
               |
               |    https://github.com/guardian/dotcom-rendering""".stripMargin
-          Future.failed(new DCRLocalConnectException(msg))
+          Future.failed(DCRLocalConnectException(msg))
+        case t: TimeoutException => Future.failed(DCRTimeoutException(t.getMessage))
       })
     }