[opensource] Update home mixer with latest changes

Author: twitter-team

home-mixer/README.md

@@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
             - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
                 - Fetch Tweet Candidates
                     - ScoredTweetsInNetworkCandidatePipelineConfig
-                    - ScoredTweetsCrMixerCandidatePipelineConfig
+                    - ScoredTweetsTweetMixerCandidatePipelineConfig
                     - ScoredTweetsUtegCandidatePipelineConfig
                     - ScoredTweetsFrsCandidatePipelineConfig
                 - Feature Hydration and Scoring
@@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
         - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
         - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
         - ListTweetsAdsCandidatePipelineConfig (fetch ads)
-

home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel

@@ -21,6 +21,7 @@ scala_library(
         "finatra/inject/inject-utils/src/main/scala",
         "home-mixer/server/src/main/resources",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@@ -31,6 +32,10 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
         "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
         "src/thrift/com/twitter/timelines/render:thrift-scala",
+        "strato/config/columns/auth-context:auth-context-strato-client",
+        "strato/config/columns/gizmoduck:gizmoduck-strato-client",
+        "strato/src/main/scala/com/twitter/strato/fed",
+        "strato/src/main/scala/com/twitter/strato/fed/server",
         "stringcenter/client",
         "stringcenter/client/src/main/java",
         "stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala

@@ -2,7 +2,7 @@ package com.twitter.home_mixer
 
 import com.twitter.finatra.http.routing.HttpWarmup
 import com.twitter.finatra.httpclient.RequestBuilder._
-import com.twitter.inject.Logging
+import com.twitter.util.logging.Logging
 import com.twitter.inject.utils.Handler
 import com.twitter.util.Try
 import javax.inject.Inject

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala

@@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
 import com.twitter.finatra.thrift.filters._
 import com.twitter.finatra.thrift.routing.ThriftRouter
 import com.twitter.home_mixer.controller.HomeThriftController
+import com.twitter.home_mixer.federated.HomeMixerColumn
 import com.twitter.home_mixer.module._
 import com.twitter.home_mixer.param.GlobalParamConfigModule
 import com.twitter.home_mixer.product.HomeMixerProductModule
 import com.twitter.home_mixer.{thriftscala => st}
 import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
-import com.twitter.product_mixer.component_library.module.CrMixerClientModule
 import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
 import com.twitter.product_mixer.component_library.module.EarlybirdModule
 import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
 import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
 import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
 import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
-import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
 import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
+import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
 import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
 import com.twitter.product_mixer.core.controllers.ProductMixerController
 import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
 import com.twitter.product_mixer.core.module.ProductMixerModule
-import com.twitter.product_mixer.core.module.StratoClientModule
 import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
+import com.twitter.strato.fed.StratoFed
+import com.twitter.strato.fed.server.StratoFedServer
 
 object HomeMixerServerMain extends HomeMixerServer
 
-class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
+class HomeMixerServer
+    extends StratoFedServer
+    with ThriftServer
+    with Mtls
+    with HttpServer
+    with HttpMtls {
   override val name = "home-mixer-server"
 
   override val modules: Seq[Module] = Seq(
     AccountRecommendationsMixerModule,
     AdvertiserBrandSafetySettingsStoreModule,
+    BlenderClientModule,
     ClientSentImpressionsPublisherModule,
     ConversationServiceModule,
-    CrMixerClientModule,
     EarlybirdModule,
     ExploreRankerClientModule,
+    FeedbackHistoryClientModule,
     GizmoduckClientModule,
     GlobalParamConfigModule,
     HomeAdsCandidateSourceModule,
     HomeMixerFlagsModule,
     HomeMixerProductModule,
     HomeMixerResourcesModule,
-    HomeNaviModelClientModule,
     ImpressionBloomFilterModule,
     InjectionHistoryClientModule,
-    FeedbackHistoryClientModule,
     ManhattanClientsModule,
     ManhattanFeatureRepositoryModule,
     ManhattanTweetImpressionStoreModule,
     MemcachedFeatureRepositoryModule,
+    NaviModelClientModule,
     OnboardingTaskServiceModule,
     OptimizedStratoClientModule,
     PeopleDiscoveryServiceModule,
@@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
     SimClustersRecentEngagementsClientModule,
     SocialGraphServiceModule,
     StaleTweetsCacheModule,
-    StratoClientModule,
     ThriftFeatureRepositoryModule,
-    TimelineMixerClientModule,
     TimelineRankerClientModule,
     TimelineScorerClientModule,
     TimelineServiceClientModule,
     TimelinesPersistenceStoreClientModule,
+    TopicSocialProofClientModule,
     TweetImpressionStoreModule,
-    TweetyPieClientModule,
+    TweetMixerClientModule,
+    TweetypieClientModule,
     TweetypieStaticEntitiesCacheClientModule,
-    UserMetadataStoreModule,
     UserSessionStoreModule,
     new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
     new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
     new ProductScopeStringCenterModule()
   )
 
-  def configureThrift(router: ThriftRouter): Unit = {
+  override def configureThrift(router: ThriftRouter): Unit = {
     router
       .filter[LoggingMDCFilter]
       .filter[TraceIdMDCFilter]
@@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
         this.injector,
         st.HomeMixer.ExecutePipeline))
 
+  override val dest: String = "/s/home-mixer/home-mixer:strato"
+
+  override val columns: Seq[Class[_ <: StratoFed.Column]] =
+    Seq(classOf[HomeMixerColumn])
+
   override protected def warmup(): Unit = {
     handle[HomeMixerThriftServerWarmupHandler]()
     handle[HomeMixerHttpServerWarmupHandler]()

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala

@@ -3,7 +3,7 @@ package com.twitter.home_mixer
 import com.twitter.finagle.thrift.ClientId
 import com.twitter.finatra.thrift.routing.ThriftWarmup
 import com.twitter.home_mixer.{thriftscala => st}
-import com.twitter.inject.Logging
+import com.twitter.util.logging.Logging
 import com.twitter.inject.utils.Handler
 import com.twitter.product_mixer.core.{thriftscala => pt}
 import com.twitter.scrooge.Request

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel

@@ -5,30 +5,20 @@ scala_library(
     tags = ["bazel-compatible"],
     dependencies = [
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
-        "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
-        "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
-        "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
     ],
     exports = [
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala

@@ -1,18 +1,23 @@
 package com.twitter.home_mixer.candidate_pipeline
 
+import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
-import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
 import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
-import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
+import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
 import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
+import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
 import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
 import com.twitter.product_mixer.component_library.filter.FeatureFilter
+import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
 import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
 class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
   conversationServiceCandidateSource: ConversationServiceCandidateSource,
   tweetypieFeatureHydrator: TweetypieFeatureHydrator,
-  socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
   namesFeatureHydrator: NamesFeatureHydrator,
+  invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
   override val gates: Seq[BaseGate[Query]],
   override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
     extends DependentCandidatePipelineConfig[
@@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
     val tweetsWithConversationMetadata = candidates.map { candidate =>
       TweetWithConversationMetadata(
         tweetId = candidate.candidateIdLong,
-        userId = None,
-        sourceTweetId = None,
-        sourceUserId = None,
-        inReplyToTweetId = None,
+        userId = candidate.features.getOrElse(AuthorIdFeature, None),
+        sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
+        sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
+        inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
         conversationId = None,
         ancestors = Seq.empty
       )
@@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
 
   override val preFilterFeatureHydrationPhase1: Seq[
     BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
-  ] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
+  ] = Seq(
+    tweetypieFeatureHydrator,
+    InNetworkFeatureHydrator,
+  )
 
   override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
     RetweetDeduplicationFilter,
@@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
       FilterIdentifier(QuotedTweetDroppedFilterId),
       shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
     ),
+    invalidSubscriptionTweetFilter,
     InvalidConversationModuleFilter
   )
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala

@@ -1,9 +1,9 @@
 package com.twitter.home_mixer.candidate_pipeline
 
-import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
-import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
+import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
+import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
 import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@@ -15,7 +15,7 @@ import javax.inject.Singleton
 class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
   conversationServiceCandidateSource: ConversationServiceCandidateSource,
   tweetypieFeatureHydrator: TweetypieFeatureHydrator,
-  socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
+  invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
   namesFeatureHydrator: NamesFeatureHydrator) {
 
   def build(
@@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
     new ConversationServiceCandidatePipelineConfig(
       conversationServiceCandidateSource,
       tweetypieFeatureHydrator,
-      socialGraphServiceFeatureHydrator,
       namesFeatureHydrator,
+      invalidSubscriptionTweetFilter,
       gates,
       decorator
     )

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala

@@ -1,7 +1,7 @@
 package com.twitter.home_mixer.candidate_pipeline
 
 import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
-import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
+import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
 import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
 import com.twitter.home_mixer.service.HomeMixerAlertConfig

home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel

@@ -13,8 +13,5 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
         "snowflake/src/main/scala/com/twitter/snowflake/id",
-        "src/thrift/com/twitter/context:twitter-context-scala",
-        "src/thrift/com/twitter/timelines/render:thrift-scala",
-        "twitter-context/src/main/scala",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala

@@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
 import com.twitter.home_mixer.{thriftscala => t}
 import com.twitter.product_mixer.core.controllers.DebugTwitterContext
 import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
+import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
 import com.twitter.product_mixer.core.service.urt.UrtService
 import com.twitter.snowflake.id.SnowflakeId
 import com.twitter.stitch.Stitch

home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel

@@ -0,0 +1,24 @@
+scala_library(
+    sources = ["*.scala"],
+    compiler_option_sets = ["fatal_warnings"],
+    strict_deps = True,
+    tags = ["bazel-compatible"],
+    dependencies = [
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
+        "home-mixer/thrift/src/main/thrift:thrift-scala",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
+        "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
+        "src/thrift/com/twitter/gizmoduck:thrift-scala",
+        "src/thrift/com/twitter/timelines/render:thrift-scala",
+        "stitch/stitch-repo/src/main/scala",
+        "strato/config/columns/auth-context:auth-context-strato-client",
+        "strato/config/columns/gizmoduck:gizmoduck-strato-client",
+        "strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
+        "strato/src/main/scala/com/twitter/strato/callcontext",
+        "strato/src/main/scala/com/twitter/strato/fed",
+        "strato/src/main/scala/com/twitter/strato/fed/server",
+    ],
+)