diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD deleted file mode 100644 index daf4c63fa..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "cortex-deepbird/prediction/src/main/scala/com/twitter/cortex/deepbird/prediction", - "cortex-deepbird/thrift/src/main/thrift:thrift-java", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "src/scala/com/twitter/ml/api/util", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD.docx new file mode 100644 index 000000000..5aab1677d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.docx new file mode 100644 index 000000000..88c2fc923 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.scala deleted file mode 100644 index 8f707910b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.deepbirdv2 - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.bijection.scrooge.TBinaryProtocol -import com.twitter.conversions.DurationOps._ -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.builder.ClientBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.inject.TwitterModule - -/** - * Module that provides multiple deepbirdv2 prediction service clients - * We use the java api since data records are native java objects and we want to reduce overhead - * while serializing/deserializing data. - */ -object DeepBirdV2PredictionServiceClientModule extends TwitterModule { - - val RequestTimeout = 300.millis - - private def getDeepbirdPredictionServiceClient( - clientId: ClientId, - label: String, - dest: String, - statsReceiver: StatsReceiver, - serviceIdentifier: ServiceIdentifier - ): DeepbirdPredictionService.ServiceToClient = { - val clientStatsReceiver = statsReceiver.scope("clnt") - val mTlsClient = ThriftMux.client.withClientId(clientId).withMutualTls(serviceIdentifier) - new DeepbirdPredictionService.ServiceToClient( - ClientBuilder() - .name(label) - .stack(mTlsClient) - .dest(dest) - .requestTimeout(RequestTimeout) - .reportHostStats(NullStatsReceiver) - .build(), - RichClientParam( - new TBinaryProtocol.Factory(), - clientStats = clientStatsReceiver - ) - ) - } - - @Provides - @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) - def providesWtfProdDeepbirdV2PredictionService( - clientId: ClientId, - statsReceiver: StatsReceiver, - serviceIdentifier: ServiceIdentifier - ): DeepbirdPredictionService.ServiceToClient = { - getDeepbirdPredictionServiceClient( - clientId = clientId, - label = "WtfProdDeepbirdV2PredictionService", - dest = "/s/cassowary/deepbirdv2-hermit-wtf", - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD deleted file mode 100644 index 33d8ba841..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "src/thrift/com/twitter/onboarding/relevance/store:store-scala", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD.docx new file mode 100644 index 000000000..ec9006236 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.docx new file mode 100644 index 000000000..0760716c9 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.scala deleted file mode 100644 index 1c8c9f8fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.dismiss_store - -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Scanner -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/** - * this store gets the list of dismissed candidates since a certain time - * primarily used for filtering out accounts that a user has explicitly dismissed - * - * we fail open on timeouts, but loudly on other errors - */ -@Singleton -class DismissStore @Inject() ( - @Named(GuiceNamedConstants.DISMISS_STORE_SCANNER) - scanner: Scanner[(Long, Slice[ - (Long, Long) - ]), Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails], - stats: StatsReceiver) - extends Logging { - - private val MaxCandidatesToReturn = 100 - - // gets a list of dismissed candidates. if numCandidatesToFetchOption is none, we will fetch the default number of candidates - def get( - userId: Long, - negStartTimeMs: Long, - maxCandidatesToFetchOption: Option[Int] - ): Stitch[Seq[Long]] = { - - val maxCandidatesToFetch = maxCandidatesToFetchOption.getOrElse(MaxCandidatesToReturn) - - scanner - .scan( - ( - userId, - Slice( - from = None, - to = Some((negStartTimeMs, Long.MaxValue)), - limit = Some(maxCandidatesToFetch) - ) - ) - ) - .map { - case s: Seq[((Long, (Long, Long)), WhoToFollowDismissEventDetails)] if s.nonEmpty => - s.map { - case ((_: Long, (_: Long, candidateId: Long)), _: WhoToFollowDismissEventDetails) => - candidateId - } - case _ => Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD deleted file mode 100644 index 78a06d536..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "emailstorage/server/src/main/thrift/com/twitter/emailstorage/api:email-storage-service-scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "stitch/stitch-core", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD.docx new file mode 100644 index 000000000..14b6a981f Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.docx new file mode 100644 index 000000000..447b4b56d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.scala deleted file mode 100644 index 4cf66c658..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.email_storage_service - -import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing -import com.twitter.emailstorage.api.thriftscala.EmailStorageService -import com.twitter.emailstorage.api.thriftscala.GetUsersEmailsRequest -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class EmailStorageServiceClient @Inject() ( - val emailStorageService: EmailStorageService.MethodPerEndpoint) { - - def getVerifiedEmail( - userId: Long, - purposeOfProcessing: PurposeOfProcessing - ): Stitch[Option[String]] = { - val req = GetUsersEmailsRequest( - userIds = Seq(userId), - clientIdentifier = Some("follow-recommendations-service"), - purposesOfProcessing = Some(Seq(purposeOfProcessing)) - ) - - Stitch.callFuture(emailStorageService.getUsersEmails(req)) map { - _.usersEmails.map(_.confirmedEmail.map(_.email)).head - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.docx new file mode 100644 index 000000000..2cd43fc07 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.scala deleted file mode 100644 index 0a1be9d1b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.email_storage_service - -import com.twitter.emailstorage.api.thriftscala.EmailStorageService -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule - -object EmailStorageServiceModule - extends BaseClientModule[EmailStorageService.MethodPerEndpoint] - with MtlsClient { - override val label = "email-storage-service" - override val dest = "/s/email-server/email-server" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD deleted file mode 100644 index d67fa01d8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "src/thrift/com/twitter/geoduck:geoduck-scala", - "src/thrift/com/twitter/geoduck:geoduckpartnerplaces-thrift-scala", - "stitch/stitch-core", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD.docx new file mode 100644 index 000000000..db7f02902 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.docx new file mode 100644 index 000000000..275360995 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.scala deleted file mode 100644 index 69207537c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.geoduck.common.thriftscala.LocationSource -import com.twitter.geoduck.common.thriftscala.PlaceQuery -import com.twitter.geoduck.common.thriftscala.TransactionLocation -import com.twitter.geoduck.common.thriftscala.UserLocationRequest -import com.twitter.geoduck.thriftscala.LocationService -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class LocationServiceClient @Inject() (locationService: LocationService.MethodPerEndpoint) { - def getGeohashAndCountryCode(userId: Long): Stitch[GeohashAndCountryCode] = { - Stitch - .callFuture { - locationService - .userLocation( - UserLocationRequest( - Seq(userId), - Some(PlaceQuery(allPlaceTypes = Some(true))), - simpleReverseGeocode = true)) - .map(_.found.get(userId)).map { transactionLocationOpt => - val geohashOpt = transactionLocationOpt.flatMap(getGeohashFromTransactionLocation) - val countryCodeOpt = - transactionLocationOpt.flatMap(_.simpleRgcResult.flatMap(_.countryCodeAlpha2)) - GeohashAndCountryCode(geohashOpt, countryCodeOpt) - } - } - } - - private[this] def getGeohashFromTransactionLocation( - transactionLocation: TransactionLocation - ): Option[String] = { - transactionLocation.geohash.flatMap { geohash => - val geohashPrefixLength = transactionLocation.locationSource match { - // if location source is logical, keep the first 4 chars in geohash - case Some(LocationSource.Logical) => Some(4) - // if location source is physical, keep the prefix according to accuracy - // accuracy is the accuracy of GPS readings in the unit of meter - case Some(LocationSource.Physical) => - transactionLocation.coordinate.flatMap { coordinate => - coordinate.accuracy match { - case Some(accuracy) if (accuracy < 50) => Some(7) - case Some(accuracy) if (accuracy < 200) => Some(6) - case Some(accuracy) if (accuracy < 1000) => Some(5) - case Some(accuracy) if (accuracy < 50000) => Some(4) - case Some(accuracy) if (accuracy < 100000) => Some(3) - case _ => None - } - } - case Some(LocationSource.Model) => Some(4) - case _ => None - } - geohashPrefixLength match { - case Some(l: Int) => geohash.stringGeohash.map(_.take(l)) - case _ => None - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.docx new file mode 100644 index 000000000..5ef83be67 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.scala deleted file mode 100644 index a5fc79a80..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.geoduck.thriftscala.LocationService - -object LocationServiceModule - extends BaseClientModule[LocationService.MethodPerEndpoint] - with MtlsClient { - override val label = "geoduck_locationservice" - override val dest = "/s/geo/geoduck_locationservice" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.docx new file mode 100644 index 000000000..4f74bf3a2 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.scala deleted file mode 100644 index 576359128..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.geoduck.common.thriftscala.PlaceQuery -import com.twitter.geoduck.common.thriftscala.ReverseGeocodeIPRequest -import com.twitter.geoduck.service.thriftscala.GeoContext -import com.twitter.geoduck.thriftscala.ReverseGeocoder -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ReverseGeocodeClient @Inject() (rgcService: ReverseGeocoder.MethodPerEndpoint) { - def getGeohashAndCountryCode(ipAddress: String): Stitch[GeohashAndCountryCode] = { - Stitch - .callFuture { - rgcService - .reverseGeocodeIp( - ReverseGeocodeIPRequest( - Seq(ipAddress), - PlaceQuery(None), - simpleReverseGeocode = true - ) // note: simpleReverseGeocode means that country code will be included in response - ).map { response => - response.found.get(ipAddress) match { - case Some(location) => getGeohashAndCountryCodeFromLocation(location) - case _ => GeohashAndCountryCode(None, None) - } - } - } - } - - private def getGeohashAndCountryCodeFromLocation(location: Location): GeohashAndCountryCode = { - val countryCode: Option[String] = location.simpleRgcResult.flatMap { _.countryCodeAlpha2 } - - val geohashString: Option[String] = location.geohash.flatMap { hash => - hash.stringGeohash.flatMap { hashString => - Some(ReverseGeocodeClient.truncate(hashString)) - } - } - - GeohashAndCountryCode(geohashString, countryCode) - } - -} - -object ReverseGeocodeClient { - - val DefaultGeoduckIPRequestContext: GeoContext = - GeoContext(allPlaceTypes = true, includeGeohash = true, includeCountryCode = true) - - // All these geohashes are guessed by IP (Logical Location Source). - // So take the four letters to make sure it is consistent with LocationServiceClient - val GeohashLengthAfterTruncation = 4 - def truncate(geohash: String): String = geohash.take(GeohashLengthAfterTruncation) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.docx new file mode 100644 index 000000000..3d1df74f9 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.scala deleted file mode 100644 index 706ae5143..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.stitch.Stitch - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class UserLocationFetcher @Inject() ( - locationServiceClient: LocationServiceClient, - reverseGeocodeClient: ReverseGeocodeClient, - statsReceiver: StatsReceiver) { - - private val stats: StatsReceiver = statsReceiver.scope("user_location_fetcher") - private val totalRequestsCounter = stats.counter("requests") - private val emptyResponsesCounter = stats.counter("empty") - private val locationServiceExceptionCounter = stats.counter("location_service_exception") - private val reverseGeocodeExceptionCounter = stats.counter("reverse_geocode_exception") - - def getGeohashAndCountryCode( - userId: Option[Long], - ipAddress: Option[String] - ): Stitch[Option[GeohashAndCountryCode]] = { - totalRequestsCounter.incr() - val lscLocationStitch = Stitch - .collect { - userId.map(locationServiceClient.getGeohashAndCountryCode) - }.rescue { - case _: Exception => - locationServiceExceptionCounter.incr() - Stitch.None - } - - val ipLocationStitch = Stitch - .collect { - ipAddress.map(reverseGeocodeClient.getGeohashAndCountryCode) - }.rescue { - case _: Exception => - reverseGeocodeExceptionCounter.incr() - Stitch.None - } - - Stitch.join(lscLocationStitch, ipLocationStitch).map { - case (lscLocation, ipLocation) => { - val geohash = lscLocation.flatMap(_.geohash).orElse(ipLocation.flatMap(_.geohash)) - val countryCode = - lscLocation.flatMap(_.countryCode).orElse(ipLocation.flatMap(_.countryCode)) - (geohash, countryCode) match { - case (None, None) => - emptyResponsesCounter.incr() - None - case _ => Some(GeohashAndCountryCode(geohash, countryCode)) - } - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD deleted file mode 100644 index 77cb553c0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "stitch/stitch-gizmoduck", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD.docx new file mode 100644 index 000000000..6842c8dba Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.docx new file mode 100644 index 000000000..3f0814012 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.scala deleted file mode 100644 index 25b5f0d75..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.gizmoduck - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.PerspectiveEdge -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GizmoduckClient @Inject() (gizmoduckStitchClient: Gizmoduck, statsReceiver: StatsReceiver) { - val stats = statsReceiver.scope("gizmoduck_client") - val getByIdStats = stats.scope("get_by_id") - val getUserById = stats.scope("get_user_by_id") - - def isProtected(userId: Long): Stitch[Boolean] = { - // get latency metrics with StatsUtil.profileStitch when calling .getById - val response = StatsUtil.profileStitch( - gizmoduckStitchClient.getById(userId, Set(QueryFields.Safety)), - getByIdStats - ) - response.map { result => - result.user.flatMap(_.safety).map(_.isProtected).getOrElse(true) - } - } - - def getUserName(userId: Long, forUserId: Long): Stitch[Option[String]] = { - val queryFields = GizmoduckClient.GetUserByIdUserNameQueryFields - val lookupContext = LookupContext( - forUserId = Some(forUserId), - perspectiveEdges = Some(GizmoduckClient.DefaultPerspectiveEdges) - ) - // get latency metrics with StatsUtil.profileStitch when calling .getUserById - val response = StatsUtil.profileStitch( - gizmoduckStitchClient.getUserById(userId, queryFields, lookupContext), - getUserById - ) - response.map(_.profile.map(_.name)) - } -} - -object GizmoduckClient { - // Similar to GizmoduckUserRepository.DefaultPerspectiveEdges - val DefaultPerspectiveEdges: Set[PerspectiveEdge] = - Set( - PerspectiveEdge.Blocking, - PerspectiveEdge.BlockedBy, - PerspectiveEdge.DeviceFollowing, - PerspectiveEdge.FollowRequestSent, - PerspectiveEdge.Following, - PerspectiveEdge.FollowedBy, - PerspectiveEdge.LifelineFollowing, - PerspectiveEdge.LifelineFollowedBy, - PerspectiveEdge.Muting, - PerspectiveEdge.NoRetweetsFrom - ) - - // From GizmoduckUserRepository.DefaultQueryFields - val GetUserByIdQueryFields: Set[QueryFields] = Set( - QueryFields.Account, - QueryFields.Counts, - QueryFields.ExtendedProfile, - QueryFields.Perspective, - QueryFields.Profile, - QueryFields.ProfileDesign, - QueryFields.ProfileLocation, - QueryFields.Safety, - QueryFields.Roles, - QueryFields.Takedowns, - QueryFields.UrlEntities, - QueryFields.DirectMessageView, - QueryFields.MediaView - ) - - val GetUserByIdUserNameQueryFields: Set[QueryFields] = Set( - QueryFields.Profile - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.docx new file mode 100644 index 000000000..82e4545b9 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.scala deleted file mode 100644 index 9a5efea06..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.gizmoduck - -import com.google.inject.Provides -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.gizmoduck.thriftscala.UserService -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Singleton - -object GizmoduckModule extends BaseClientModule[UserService.MethodPerEndpoint] with MtlsClient { - override val label = "gizmoduck" - override val dest = "/s/gizmoduck/gizmoduck" - - @Provides - @Singleton - def provideExtraGizmoduckQueryFields: Set[QueryFields] = Set.empty - - @Provides - @Singleton - def providesStitchClient(futureIface: UserService.MethodPerEndpoint): Gizmoduck = { - Gizmoduck(futureIface) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD deleted file mode 100644 index ec1139cfe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", - "stitch/stitch-core", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD.docx new file mode 100644 index 000000000..8bfbfb5aa Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.docx new file mode 100644 index 000000000..41efe16fa Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.scala deleted file mode 100644 index f333d895f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.graph_feature_service - -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.graph_feature_service.thriftscala.PresetFeatureTypes.WtfTwoHop -import com.twitter.graph_feature_service.thriftscala.EdgeType -import com.twitter.graph_feature_service.thriftscala.GfsIntersectionResponse -import com.twitter.graph_feature_service.thriftscala.GfsPresetIntersectionRequest -import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService} -import com.twitter.stitch.Stitch -import javax.inject.{Inject, Singleton} - -@Singleton -class GraphFeatureServiceClient @Inject() ( - graphFeatureService: GraphFeatureService.MethodPerEndpoint) { - - import GraphFeatureServiceClient._ - def getIntersections( - userId: Long, - candidateIds: Seq[Long], - numIntersectionIds: Int - ): Stitch[Map[Long, FollowProof]] = { - Stitch - .callFuture( - graphFeatureService.getPresetIntersection( - GfsPresetIntersectionRequest(userId, candidateIds, WtfTwoHop, Some(numIntersectionIds)) - ) - ).map { - case GfsIntersectionResponse(gfsIntersectionResults) => - (for { - candidateId <- candidateIds - gfsIntersectionResultForCandidate = - gfsIntersectionResults.filter(_.candidateUserId == candidateId) - followProof <- for { - result <- gfsIntersectionResultForCandidate - intersection <- result.intersectionValues - if leftEdgeTypes.contains(intersection.featureType.leftEdgeType) - if rightEdgeTypes.contains(intersection.featureType.rightEdgeType) - intersectionIds <- intersection.intersectionIds.toSeq - } yield FollowProof(intersectionIds, intersection.count.getOrElse(0)) - } yield { - candidateId -> followProof - }).toMap - } - } -} - -object GraphFeatureServiceClient { - val leftEdgeTypes: Set[EdgeType] = Set(EdgeType.Following) - val rightEdgeTypes: Set[EdgeType] = Set(EdgeType.FollowedBy) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.docx new file mode 100644 index 000000000..d73c8e0ae Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.scala deleted file mode 100644 index 8d15358cd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.graph_feature_service - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService} - -object GraphFeatureStoreModule - extends BaseClientModule[GraphFeatureService.MethodPerEndpoint] - with MtlsClient { - override val label = "graph_feature_service" - override val dest = "/s/cassowary/graph_feature_service-server" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD deleted file mode 100644 index 7432ce040..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "stitch/stitch-socialgraph", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD.docx new file mode 100644 index 000000000..3d6adec1a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.docx new file mode 100644 index 000000000..f39b2a2c2 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.scala deleted file mode 100644 index 35b5d7885..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.impression_store - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.thriftscala.DisplayLocation -import com.twitter.inject.TwitterModule -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Client -import com.twitter.strato.thrift.ScroogeConvImplicits._ - -object ImpressionStoreModule extends TwitterModule { - - val columnPath: String = "onboarding/userrecs/wtfImpressionCountsStore" - - type PKey = (Long, DisplayLocation) - type LKey = Long - type Value = (Long, Int) - - @Provides - @Singleton - def providesImpressionStore(stratoClient: Client): WtfImpressionStore = { - new WtfImpressionStore( - stratoClient.scanner[ - (PKey, Slice[LKey]), - Unit, - (PKey, LKey), - Value - ](columnPath) - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.docx new file mode 100644 index 000000000..f0467cf61 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.scala deleted file mode 100644 index 68692dc28..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.impression_store - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.WtfImpression -import com.twitter.follow_recommendations.thriftscala.{DisplayLocation => TDisplayLocation} -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Scanner -import com.twitter.util.Time -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class WtfImpressionStore @Inject() ( - scanner: Scanner[ - ((Long, TDisplayLocation), Slice[Long]), - Unit, - ((Long, TDisplayLocation), Long), - (Long, Int) - ]) extends Logging { - def get(userId: Long, dl: DisplayLocation): Stitch[Seq[WtfImpression]] = { - val thriftDl = dl.toThrift - scanner.scan(((userId, thriftDl), Slice.all[Long])).map { impressionsPerDl => - val wtfImpressions = - for { - (((_, _), candidateId), (latestTs, counts)) <- impressionsPerDl - } yield WtfImpression( - candidateId = candidateId, - displayLocation = dl, - latestTime = Time.fromMilliseconds(latestTs), - counts = counts - ) - wtfImpressions - } rescue { - // fail open so that the request can still go through - case ex: Throwable => - logger.warn(s"$dl WtfImpressionsStore warn: " + ex.getMessage) - Stitch.Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD deleted file mode 100644 index 0835d840b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - name = "interests_service", - sources = ["InterestServiceClient.scala"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", - "interests-service/thrift/src/main/thrift:thrift-scala", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - "strato/src/main/scala/com/twitter/strato/data", - "strato/src/main/scala/com/twitter/strato/thrift", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD.docx new file mode 100644 index 000000000..eac2641cf Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.docx new file mode 100644 index 000000000..43216f547 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.scala deleted file mode 100644 index 3904cf515..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.scala +++ /dev/null @@ -1,115 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.interests_service - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.InterestedInInterestsFetchKey -import com.twitter.inject.Logging -import com.twitter.interests.thriftscala.InterestId -import com.twitter.interests.thriftscala.InterestRelationship -import com.twitter.interests.thriftscala.InterestedInInterestModel -import com.twitter.interests.thriftscala.UserInterest -import com.twitter.interests.thriftscala.UserInterestData -import com.twitter.interests.thriftscala.UserInterestsResponse -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Client -import com.twitter.strato.thrift.ScroogeConvImplicits._ - -@Singleton -class InterestServiceClient @Inject() ( - stratoClient: Client, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends Logging { - - val interestsServiceStratoColumnPath = "interests/interestedInInterests" - val stats = statsReceiver.scope("interest_service_client") - val errorCounter = stats.counter("error") - - private val interestsFetcher = - stratoClient.fetcher[InterestedInInterestsFetchKey, UserInterestsResponse]( - interestsServiceStratoColumnPath, - checkTypes = true - ) - - def fetchUttInterestIds( - userId: Long - ): Stitch[Seq[Long]] = { - fetchInterestRelationships(userId) - .map(_.toSeq.flatten.flatMap(extractUttInterest)) - } - - def extractUttInterest( - interestRelationShip: InterestRelationship - ): Option[Long] = { - interestRelationShip match { - case InterestRelationship.V1(relationshipV1) => - relationshipV1.interestId match { - case InterestId.SemanticCore(semanticCoreInterest) => Some(semanticCoreInterest.id) - case _ => None - } - case _ => None - } - } - - def fetchCustomInterests( - userId: Long - ): Stitch[Seq[String]] = { - fetchInterestRelationships(userId) - .map(_.toSeq.flatten.flatMap(extractCustomInterest)) - } - - def extractCustomInterest( - interestRelationShip: InterestRelationship - ): Option[String] = { - interestRelationShip match { - case InterestRelationship.V1(relationshipV1) => - relationshipV1.interestId match { - case InterestId.FreeForm(freeFormInterest) => Some(freeFormInterest.interest) - case _ => None - } - case _ => None - } - } - - def fetchInterestRelationships( - userId: Long - ): Stitch[Option[Seq[InterestRelationship]]] = { - interestsFetcher - .fetch( - InterestedInInterestsFetchKey( - userId = userId, - labels = None, - None - )) - .map(_.v) - .map { - case Some(response) => - response.interests.interests.map { interests => - interests.collect { - case UserInterest(_, Some(interestData)) => - getInterestRelationship(interestData) - }.flatten - } - case _ => None - } - .rescue { - case e: Throwable => // we are swallowing all errors - logger.warn(s"interests could not be retrieved for user $userId due to ${e.getCause}") - errorCounter.incr - Stitch.None - } - } - - private def getInterestRelationship( - interestData: UserInterestData - ): Seq[InterestRelationship] = { - interestData match { - case UserInterestData.InterestedIn(interestModels) => - interestModels.collect { - case InterestedInInterestModel.ExplicitModel(model) => model - } - case _ => Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD deleted file mode 100644 index 37a7f5906..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "phonestorage/server/src/main/thrift/com/twitter/phonestorage/api:phone-storage-service-scala", - "stitch/stitch-core", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD.docx new file mode 100644 index 000000000..428dd0f6c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.docx new file mode 100644 index 000000000..a6a0ca3d0 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.scala deleted file mode 100644 index 46e76b162..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.phone_storage_service - -import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing -import com.twitter.phonestorage.api.thriftscala.GetUserPhonesByUsersRequest -import com.twitter.phonestorage.api.thriftscala.PhoneStorageService -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PhoneStorageServiceClient @Inject() ( - val phoneStorageService: PhoneStorageService.MethodPerEndpoint) { - - /** - * PSS can potentially return multiple phone records. - * The current implementation of getUserPhonesByUsers returns only a single phone for a single user_id but - * we can trivially support handling multiple in case that changes in the future. - */ - def getPhoneNumbers( - userId: Long, - purposeOfProcessing: PurposeOfProcessing, - forceCarrierLookup: Option[Boolean] = None - ): Stitch[Seq[String]] = { - val req = GetUserPhonesByUsersRequest( - userIds = Seq(userId), - forceCarrierLookup = forceCarrierLookup, - purposesOfProcessing = Some(Seq(purposeOfProcessing)) - ) - - Stitch.callFuture(phoneStorageService.getUserPhonesByUsers(req)) map { - _.userPhones.map(_.phoneNumber) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.docx new file mode 100644 index 000000000..2e07a628b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.scala deleted file mode 100644 index 90767a509..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.phone_storage_service - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.phonestorage.api.thriftscala.PhoneStorageService - -object PhoneStorageServiceModule - extends BaseClientModule[PhoneStorageService.MethodPerEndpoint] - with MtlsClient { - override val label = "phone-storage-service" - override val dest = "/s/ibis-ds-api/ibis-ds-api:thrift2" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD deleted file mode 100644 index f711bc3e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD.docx new file mode 100644 index 000000000..a0ab7d937 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.docx new file mode 100644 index 000000000..bdc7d9334 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.scala deleted file mode 100644 index 98c0555e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.real_time_real_graph - -sealed trait EngagementType - -// We do not include SoftFollow since it's deprecated -object EngagementType { - object Click extends EngagementType - object Like extends EngagementType - object Mention extends EngagementType - object Retweet extends EngagementType - object ProfileView extends EngagementType -} - -case class Engagement(engagementType: EngagementType, timestamp: Long) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.docx new file mode 100644 index 000000000..e26a7007a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.scala deleted file mode 100644 index c41fcc98e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.real_time_real_graph - -import com.twitter.conversions.DurationOps._ -import com.twitter.util.Time - -object EngagementScorer { - private[real_time_real_graph] val MemoryDecayHalfLife = 24.hour - private val ScoringFunctionBase = 0.5 - - def apply( - engagements: Map[Long, Seq[Engagement]], - engagementScoreMap: Map[EngagementType, Double], - minScore: Double = 0.0 - ): Seq[(Long, Double, Seq[EngagementType])] = { - val now = Time.now - engagements - .mapValues { engags => - val totalScore = engags.map { engagement => score(engagement, now, engagementScoreMap) }.sum - val engagementProof = getEngagementProof(engags, engagementScoreMap) - (totalScore, engagementProof) - } - .collect { case (uid, (score, proof)) if score > minScore => (uid, score, proof) } - .toSeq - .sortBy(-_._2) - } - - /** - * The engagement score is the base score decayed via timestamp, loosely model the human memory forgetting - * curve, see https://en.wikipedia.org/wiki/Forgetting_curve - */ - private[real_time_real_graph] def score( - engagement: Engagement, - now: Time, - engagementScoreMap: Map[EngagementType, Double] - ): Double = { - val timeLapse = math.max(now.inMillis - engagement.timestamp, 0) - val engagementScore = engagementScoreMap.getOrElse(engagement.engagementType, 0.0) - engagementScore * math.pow( - ScoringFunctionBase, - timeLapse.toDouble / MemoryDecayHalfLife.inMillis) - } - - private def getEngagementProof( - engagements: Seq[Engagement], - engagementScoreMap: Map[EngagementType, Double] - ): Seq[EngagementType] = { - - val filteredEngagement = engagements - .collectFirst { - case engagement - if engagement.engagementType != EngagementType.Click - && engagementScoreMap.get(engagement.engagementType).exists(_ > 0.0) => - engagement.engagementType - } - - Seq(filteredEngagement.getOrElse(EngagementType.Click)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.docx new file mode 100644 index 000000000..3c7add580 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.scala deleted file mode 100644 index 16fedc8a0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.scala +++ /dev/null @@ -1,128 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.real_time_real_graph - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.ml.featureStore.TimelinesUserVertexOnUserClientColumn -import com.twitter.strato.generated.client.onboarding.userrecs.RealGraphScoresMhOnUserClientColumn -import com.twitter.util.Duration -import com.twitter.util.Time -import com.twitter.wtf.real_time_interaction_graph.thriftscala._ - -@Singleton -class RealTimeRealGraphClient @Inject() ( - timelinesUserVertexOnUserClientColumn: TimelinesUserVertexOnUserClientColumn, - realGraphScoresMhOnUserClientColumn: RealGraphScoresMhOnUserClientColumn) { - - def mapUserVertexToEngagementAndFilter(userVertex: UserVertex): Map[Long, Seq[Engagement]] = { - val minTimestamp = (Time.now - RealTimeRealGraphClient.MaxEngagementAge).inMillis - userVertex.outgoingInteractionMap.mapValues { interactions => - interactions - .flatMap { interaction => RealTimeRealGraphClient.toEngagement(interaction) }.filter( - _.timestamp >= minTimestamp) - }.toMap - } - - def getRecentProfileViewEngagements(userId: Long): Stitch[Map[Long, Seq[Engagement]]] = { - timelinesUserVertexOnUserClientColumn.fetcher - .fetch(userId).map(_.v).map { input => - input - .map { userVertex => - val targetToEngagements = mapUserVertexToEngagementAndFilter(userVertex) - targetToEngagements.mapValues { engagements => - engagements.filter(engagement => - engagement.engagementType == EngagementType.ProfileView) - } - }.getOrElse(Map.empty) - } - } - - def getUsersRecentlyEngagedWith( - userId: Long, - engagementScoreMap: Map[EngagementType, Double], - includeDirectFollowCandidates: Boolean, - includeNonDirectFollowCandidates: Boolean - ): Stitch[Seq[CandidateUser]] = { - val isNewUser = - SnowflakeId.timeFromIdOpt(userId).exists { signupTime => - (Time.now - signupTime) < RealTimeRealGraphClient.MaxNewUserAge - } - val updatedEngagementScoreMap = - if (isNewUser) - engagementScoreMap + (EngagementType.ProfileView -> RealTimeRealGraphClient.ProfileViewScore) - else engagementScoreMap - - Stitch - .join( - timelinesUserVertexOnUserClientColumn.fetcher.fetch(userId).map(_.v), - realGraphScoresMhOnUserClientColumn.fetcher.fetch(userId).map(_.v)).map { - case (Some(userVertex), Some(neighbors)) => - val engagements = mapUserVertexToEngagementAndFilter(userVertex) - - val candidatesAndScores: Seq[(Long, Double, Seq[EngagementType])] = - EngagementScorer.apply(engagements, engagementScoreMap = updatedEngagementScoreMap) - - val directNeighbors = neighbors.candidates.map(_._1).toSet - val (directFollows, nonDirectFollows) = candidatesAndScores - .partition { - case (id, _, _) => directNeighbors.contains(id) - } - - val candidates = - (if (includeNonDirectFollowCandidates) nonDirectFollows else Seq.empty) ++ - (if (includeDirectFollowCandidates) - directFollows.take(RealTimeRealGraphClient.MaxNumDirectFollow) - else Seq.empty) - - candidates.map { - case (id, score, proof) => - CandidateUser(id, Some(score)) - } - - case _ => Nil - } - } - - def getRealGraphWeights(userId: Long): Stitch[Map[Long, Double]] = - realGraphScoresMhOnUserClientColumn.fetcher - .fetch(userId) - .map( - _.v - .map(_.candidates.map(candidate => (candidate.userId, candidate.score)).toMap) - .getOrElse(Map.empty[Long, Double])) -} - -object RealTimeRealGraphClient { - private def toEngagement(interaction: Interaction): Option[Engagement] = { - // We do not include SoftFollow since it's deprecated - interaction match { - case Interaction.Retweet(Retweet(timestamp)) => - Some(Engagement(EngagementType.Retweet, timestamp)) - case Interaction.Favorite(Favorite(timestamp)) => - Some(Engagement(EngagementType.Like, timestamp)) - case Interaction.Click(Click(timestamp)) => Some(Engagement(EngagementType.Click, timestamp)) - case Interaction.Mention(Mention(timestamp)) => - Some(Engagement(EngagementType.Mention, timestamp)) - case Interaction.ProfileView(ProfileView(timestamp)) => - Some(Engagement(EngagementType.ProfileView, timestamp)) - case _ => None - } - } - - val MaxNumDirectFollow = 50 - val MaxEngagementAge: Duration = 14.days - val MaxNewUserAge: Duration = 30.days - val ProfileViewScore = 0.4 - val EngagementScoreMap = Map( - EngagementType.Like -> 1.0, - EngagementType.Retweet -> 1.0, - EngagementType.Mention -> 1.0 - ) - val StrongEngagementScoreMap = Map( - EngagementType.Like -> 1.0, - EngagementType.Retweet -> 1.0, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD deleted file mode 100644 index 58b5001fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "socialgraph/server/src/main/scala/com/twitter/socialgraph/util", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-socialgraph", - "strato/config/columns/onboarding/socialGraphService:socialGraphService-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD.docx new file mode 100644 index 000000000..284373657 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.docx new file mode 100644 index 000000000..ef52836e5 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.scala deleted file mode 100644 index 3ad90b5ed..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.scala +++ /dev/null @@ -1,421 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.socialgraph - -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.UserIdWithTimestamp -import com.twitter.inject.Logging -import com.twitter.socialgraph.thriftscala.EdgesRequest -import com.twitter.socialgraph.thriftscala.IdsRequest -import com.twitter.socialgraph.thriftscala.IdsResult -import com.twitter.socialgraph.thriftscala.LookupContext -import com.twitter.socialgraph.thriftscala.OverCapacity -import com.twitter.socialgraph.thriftscala.PageRequest -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.socialgraph.thriftscala.SrcRelationship -import com.twitter.socialgraph.util.ByteBufferUtil -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.onboarding.socialGraphService.IdsClientColumn -import com.twitter.util.Duration -import com.twitter.util.Time -import java.nio.ByteBuffer -import javax.inject.Inject -import javax.inject.Singleton - -case class RecentEdgesQuery( - userId: Long, - relations: Seq[RelationshipType], - // prefer to default value to better utilize the caching function of stitch - count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), - performUnion: Boolean = true, - recentEdgesWindowOpt: Option[Duration] = None, - targets: Option[Seq[Long]] = None) - -case class EdgeRequestQuery( - userId: Long, - relation: RelationshipType, - count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), - performUnion: Boolean = true, - recentEdgesWindowOpt: Option[Duration] = None, - targets: Option[Seq[Long]] = None) - -@Singleton -class SocialGraphClient @Inject() ( - socialGraph: SocialGraph, - idsClientColumn: IdsClientColumn, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends Logging { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val cacheStats = stats.scope("cache") - private val getIntersectionsStats = stats.scope("getIntersections") - private val getIntersectionsFromCachedColumnStats = - stats.scope("getIntersectionsFromCachedColumn") - private val getRecentEdgesStats = stats.scope("getRecentEdges") - private val getRecentEdgesCachedStats = stats.scope("getRecentEdgesCached") - private val getRecentEdgesFromCachedColumnStats = stats.scope("getRecentEdgesFromCachedColumn") - private val getRecentEdgesCachedInternalStats = stats.scope("getRecentEdgesCachedInternal") - private val getRecentEdgesWithTimeStats = stats.scope("getRecentEdgesWithTime") - - val sgsIdsFetcher: Fetcher[IdsRequest, Unit, IdsResult] = idsClientColumn.fetcher - - private val recentEdgesCache = StitchCache[RecentEdgesQuery, Seq[Long]]( - maxCacheSize = SocialGraphClient.MaxCacheSize, - ttl = SocialGraphClient.CacheTTL, - statsReceiver = cacheStats, - underlyingCall = getRecentEdges - ) - - def getRecentEdgesCached( - rq: RecentEdgesQuery, - useCachedStratoColumn: Boolean = true - ): Stitch[Seq[Long]] = { - getRecentEdgesCachedStats.counter("requests").incr() - if (useCachedStratoColumn) { - getRecentEdgesFromCachedColumn(rq) - } else { - StatsUtil.profileStitch( - getRecentEdgesCachedInternal(rq), - getRecentEdgesCachedInternalStats - ) - } - } - - def getRecentEdgesCachedInternal(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { - recentEdgesCache.readThrough(rq) - } - - def getRecentEdgesFromCachedColumn(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { - val pageRequest = rq.recentEdgesWindowOpt match { - case Some(recentEdgesWindow) => - PageRequest( - count = rq.count, - cursor = Some(getEdgeCursor(recentEdgesWindow)), - selectAll = Some(true) - ) - case _ => PageRequest(count = rq.count) - } - val idsRequest = IdsRequest( - rq.relations.map { relationshipType => - SrcRelationship( - source = rq.userId, - relationshipType = relationshipType, - targets = rq.targets - ) - }, - pageRequest = Some(pageRequest), - context = Some(LookupContext(performUnion = Some(rq.performUnion))) - ) - - val socialGraphStitch = sgsIdsFetcher - .fetch(idsRequest, Unit) - .map(_.v) - .map { result => - result - .map { idResult => - val userIds: Seq[Long] = idResult.ids - getRecentEdgesFromCachedColumnStats.stat("num_edges").add(userIds.size) - userIds - }.getOrElse(Seq.empty) - } - .rescue { - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch.Nil - } - - StatsUtil.profileStitch( - socialGraphStitch, - getRecentEdgesFromCachedColumnStats - ) - } - - def getRecentEdges(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { - val pageRequest = rq.recentEdgesWindowOpt match { - case Some(recentEdgesWindow) => - PageRequest( - count = rq.count, - cursor = Some(getEdgeCursor(recentEdgesWindow)), - selectAll = Some(true) - ) - case _ => PageRequest(count = rq.count) - } - val socialGraphStitch = socialGraph - .ids( - IdsRequest( - rq.relations.map { relationshipType => - SrcRelationship( - source = rq.userId, - relationshipType = relationshipType, - targets = rq.targets - ) - }, - pageRequest = Some(pageRequest), - context = Some(LookupContext(performUnion = Some(rq.performUnion))) - ) - ) - .map { idsResult => - val userIds: Seq[Long] = idsResult.ids - getRecentEdgesStats.stat("num_edges").add(userIds.size) - userIds - } - .rescue { - case e: OverCapacity => - stats.counter(e.getClass.getSimpleName).incr() - logger.warn("SGS Over Capacity", e) - Stitch.Nil - } - StatsUtil.profileStitch( - socialGraphStitch, - getRecentEdgesStats - ) - } - - // This method return recent edges of (userId, timeInMs) - def getRecentEdgesWithTime(rq: EdgeRequestQuery): Stitch[Seq[UserIdWithTimestamp]] = { - val pageRequest = rq.recentEdgesWindowOpt match { - case Some(recentEdgesWindow) => - PageRequest( - count = rq.count, - cursor = Some(getEdgeCursor(recentEdgesWindow)), - selectAll = Some(true) - ) - case _ => PageRequest(count = rq.count) - } - - val socialGraphStitch = socialGraph - .edges( - EdgesRequest( - SrcRelationship( - source = rq.userId, - relationshipType = rq.relation, - targets = rq.targets - ), - pageRequest = Some(pageRequest), - context = Some(LookupContext(performUnion = Some(rq.performUnion))) - ) - ) - .map { edgesResult => - val userIds = edgesResult.edges.map { socialEdge => - UserIdWithTimestamp(socialEdge.target, socialEdge.updatedAt) - } - getRecentEdgesWithTimeStats.stat("num_edges").add(userIds.size) - userIds - } - .rescue { - case e: OverCapacity => - stats.counter(e.getClass.getSimpleName).incr() - logger.warn("SGS Over Capacity", e) - Stitch.Nil - } - StatsUtil.profileStitch( - socialGraphStitch, - getRecentEdgesWithTimeStats - ) - } - - // This method returns the cursor for a time duration, such that all the edges returned by SGS will be created - // in the range (now-window, now) - def getEdgeCursor(window: Duration): ByteBuffer = { - val cursorInLong = (-(Time.now - window).inMilliseconds) << 20 - ByteBufferUtil.fromLong(cursorInLong) - } - - // notice that this is more expensive but more realtime than the GFS one - def getIntersections( - userId: Long, - candidateIds: Seq[Long], - numIntersectionIds: Int - ): Stitch[Map[Long, FollowProof]] = { - val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch - .collect(candidateIds.map { candidateId => - socialGraph - .ids( - IdsRequest( - Seq( - SrcRelationship(userId, RelationshipType.Following), - SrcRelationship(candidateId, RelationshipType.FollowedBy) - ), - pageRequest = Some(PageRequest(count = Some(numIntersectionIds))) - ) - ).map { idsResult => - getIntersectionsStats.stat("num_edges").add(idsResult.ids.size) - (candidateId -> FollowProof(idsResult.ids, idsResult.ids.size)) - } - }).map(_.toMap) - .rescue { - case e: OverCapacity => - stats.counter(e.getClass.getSimpleName).incr() - logger.warn("social graph over capacity in hydrating social proof", e) - Stitch.value(Map.empty) - } - StatsUtil.profileStitch( - socialGraphStitch, - getIntersectionsStats - ) - } - - def getIntersectionsFromCachedColumn( - userId: Long, - candidateIds: Seq[Long], - numIntersectionIds: Int - ): Stitch[Map[Long, FollowProof]] = { - val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch - .collect(candidateIds.map { candidateId => - val idsRequest = IdsRequest( - Seq( - SrcRelationship(userId, RelationshipType.Following), - SrcRelationship(candidateId, RelationshipType.FollowedBy) - ), - pageRequest = Some(PageRequest(count = Some(numIntersectionIds))) - ) - - sgsIdsFetcher - .fetch(idsRequest, Unit) - .map(_.v) - .map { resultOpt => - resultOpt.map { idsResult => - getIntersectionsFromCachedColumnStats.stat("num_edges").add(idsResult.ids.size) - candidateId -> FollowProof(idsResult.ids, idsResult.ids.size) - } - } - }).map(_.flatten.toMap) - .rescue { - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch.value(Map.empty) - } - StatsUtil.profileStitch( - socialGraphStitch, - getIntersectionsFromCachedColumnStats - ) - } - - def getInvalidRelationshipUserIds( - userId: Long, - maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship - ): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - SocialGraphClient.InvalidRelationshipTypes, - Some(maxNumRelationship) - ) - ) - } - - def getInvalidRelationshipUserIdsFromCachedColumn( - userId: Long, - maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship - ): Stitch[Seq[Long]] = { - getRecentEdgesFromCachedColumn( - RecentEdgesQuery( - userId, - SocialGraphClient.InvalidRelationshipTypes, - Some(maxNumRelationship) - ) - ) - } - - def getRecentFollowedUserIds(userId: Long): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - Seq(RelationshipType.Following) - ) - ) - } - - def getRecentFollowedUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = { - getRecentEdgesFromCachedColumn( - RecentEdgesQuery( - userId, - Seq(RelationshipType.Following) - ) - ) - } - - def getRecentFollowedUserIdsWithTime(userId: Long): Stitch[Seq[UserIdWithTimestamp]] = { - getRecentEdgesWithTime( - EdgeRequestQuery( - userId, - RelationshipType.Following - ) - ) - } - - def getRecentFollowedByUserIds(userId: Long): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - Seq(RelationshipType.FollowedBy) - ) - ) - } - - def getRecentFollowedByUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = { - getRecentEdgesFromCachedColumn( - RecentEdgesQuery( - userId, - Seq(RelationshipType.FollowedBy) - ) - ) - } - - def getRecentFollowedUserIdsWithTimeWindow( - userId: Long, - timeWindow: Duration - ): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - Seq(RelationshipType.Following), - recentEdgesWindowOpt = Some(timeWindow) - ) - ) - } -} - -object SocialGraphClient { - - val MaxQuerySize: Int = 500 - val MaxCacheSize: Int = 5000000 - // Ref: src/thrift/com/twitter/socialgraph/social_graph_service.thrift - val MaxNumInvalidRelationship: Int = 5000 - val CacheTTL: Duration = Duration.fromHours(24) - - val InvalidRelationshipTypes: Seq[RelationshipType] = Seq( - RelationshipType.HideRecommendations, - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.Muting, - RelationshipType.MutedBy, - RelationshipType.ReportedAsSpam, - RelationshipType.ReportedAsSpamBy, - RelationshipType.ReportedAsAbuse, - RelationshipType.ReportedAsAbuseBy, - RelationshipType.FollowRequestOutgoing, - RelationshipType.Following, - RelationshipType.UsedToFollow, - ) - - /** - * - * Whether to call SGS to validate each candidate based on the number of invalid relationship users - * prefetched during request building step. This aims to not omit any invalid candidates that are - * not filtered out in previous steps. - * If the number is 0, this might be a fail-opened SGS call. - * If the number is larger or equal to 5000, this could hit SGS page size limit. - * Both cases account for a small percentage of the total traffic (<5%). - * - * @param numInvalidRelationshipUsers number of invalid relationship users fetched from getInvalidRelationshipUserIds - * @return whether to enable post-ranker SGS predicate - */ - def enablePostRankerSgsPredicate(numInvalidRelationshipUsers: Int): Boolean = { - numInvalidRelationshipUsers == 0 || numInvalidRelationshipUsers >= MaxNumInvalidRelationship - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.docx new file mode 100644 index 000000000..9151f343c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.scala deleted file mode 100644 index 5a97448d1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.socialgraph - -import com.google.inject.Provides -import com.twitter.finagle.ThriftMux -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.stitch.socialgraph.SocialGraph -import javax.inject.Singleton - -object SocialGraphModule - extends BaseClientModule[SocialGraphService.MethodPerEndpoint] - with MtlsClient { - override val label = "social-graph-service" - override val dest = "/s/socialgraph/socialgraph" - - override def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = - client.withSessionQualifier.noFailFast - - @Provides - @Singleton - def providesStitchClient(futureIface: SocialGraphService.MethodPerEndpoint): SocialGraph = { - SocialGraph(futureIface) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD deleted file mode 100644 index 3661887c8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "src/scala/com/twitter/onboarding/relevance/candidate_generation/utt/models", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", - "src/thrift/com/twitter/onboarding/relevance/relatable_accounts:relatable_accounts-scala", - "src/thrift/com/twitter/onboarding/relevance/store:store-scala", - "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", - "src/thrift/com/twitter/search/account_search/extended_network:extended_network_users-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "src/thrift/com/twitter/wtf/ml:wtf-ml-output-thrift-scala", - "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-scala", - "src/thrift/com/twitter/wtf/triangular_loop:triangular_loop-scala", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD.docx new file mode 100644 index 000000000..6c39d9d62 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.docx new file mode 100644 index 000000000..6a957aa8d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.scala deleted file mode 100644 index 4046ac754..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.scala +++ /dev/null @@ -1,249 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.strato - -import com.google.inject.name.Named -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState -import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserKey -import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserVal -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.thrift.Protocols -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.constants.ServiceConstants._ -import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestEvents -import com.twitter.hermit.candidate.thriftscala.{Candidates => HermitCandidates} -import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace -import com.twitter.onboarding.relevance.relatable_accounts.thriftscala.RelatableAccounts -import com.twitter.inject.TwitterModule -import com.twitter.onboarding.relevance.candidates.thriftscala.InterestBasedUserRecommendations -import com.twitter.onboarding.relevance.candidates.thriftscala.UTTInterest -import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse -import com.twitter.service.metastore.gen.thriftscala.UserRecommendabilityFeatures -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Strato.{Client => StratoClient} -import com.twitter.strato.client.Client -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.Scanner -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.wtf.candidate.thriftscala.CandidateSeq -import com.twitter.wtf.ml.thriftscala.CandidateFeatures -import com.twitter.wtf.real_time_interaction_graph.thriftscala.Interaction -import com.twitter.wtf.triangular_loop.thriftscala.{Candidates => TriangularLoopCandidates} -import com.twitter.strato.opcontext.Attribution._ - -object StratoClientModule extends TwitterModule { - - // column paths - val CosineFollowPath = "recommendations/similarity/similarUsersByFollowGraph.User" - val CosineListPath = "recommendations/similarity/similarUsersByListGraph.User" - val CuratedCandidatesPath = "onboarding/curatedAccounts" - val CuratedFilteredAccountsPath = "onboarding/filteredAccountsFromRecommendations" - val PopUsersInPlacePath = "onboarding/userrecs/popUsersInPlace" - val ProfileSidebarBlacklistPath = "recommendations/hermit/profile-sidebar-blacklist" - val RealTimeInteractionsPath = "hmli/realTimeInteractions" - val SimsPath = "recommendations/similarity/similarUsersBySims.User" - val DBV2SimsPath = "onboarding/userrecs/newSims.User" - val TriangularLoopsPath = "onboarding/userrecs/triangularLoops.User" - val TwoHopRandomWalkPath = "onboarding/userrecs/twoHopRandomWalk.User" - val UserRecommendabilityPath = "onboarding/userRecommendabilityWithLongKeys.User" - val UTTAccountRecommendationsPath = "onboarding/userrecs/utt_account_recommendations" - val UttSeedAccountsRecommendationPath = "onboarding/userrecs/utt_seed_accounts" - val UserStatePath = "onboarding/userState.User" - val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User" - val ElectionCandidatesPath = "onboarding/electionAccounts" - val UserUserGraphPath = "recommendations/userUserGraph" - val WtfDissmissEventsPath = "onboarding/wtfDismissEvents" - val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts" - val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH" - val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User" - - @Provides - @Singleton - def stratoClient(serviceIdentifier: ServiceIdentifier): Client = { - val timeoutBudget = 500.milliseconds - StratoClient( - ThriftMux.client - .withRequestTimeout(timeoutBudget) - .withProtocolFactory(Protocols.binaryFactory( - stringLengthLimit = StringLengthLimit, - containerLengthLimit = ContainerLengthLimit))) - .withMutualTls(serviceIdentifier) - .build() - } - - // add strato putters, fetchers, scanners below: - @Provides - @Singleton - @Named(GuiceNamedConstants.COSINE_FOLLOW_FETCHER) - def cosineFollowFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](CosineFollowPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.COSINE_LIST_FETCHER) - def cosineListFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](CosineListPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) - def curatedBlacklistedAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = - stratoClient.fetcher[String, Unit, Seq[Long]](CuratedFilteredAccountsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.CURATED_CANDIDATES_FETCHER) - def curatedCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = - stratoClient.fetcher[String, Unit, Seq[Long]](CuratedCandidatesPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) - def popUsersInPlaceFetcher(stratoClient: Client): Fetcher[String, Unit, PopUsersInPlace] = - stratoClient.fetcher[String, Unit, PopUsersInPlace](PopUsersInPlacePath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.RELATABLE_ACCOUNTS_FETCHER) - def relatableAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, RelatableAccounts] = - stratoClient.fetcher[String, Unit, RelatableAccounts](RelatableAccountsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.PROFILE_SIDEBAR_BLACKLIST_SCANNER) - def profileSidebarBlacklistScanner( - stratoClient: Client - ): Scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit] = - stratoClient.scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit](ProfileSidebarBlacklistPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.REAL_TIME_INTERACTIONS_FETCHER) - def realTimeInteractionsFetcher( - stratoClient: Client - ): Fetcher[(Long, Long), Unit, Seq[Interaction]] = - stratoClient.fetcher[(Long, Long), Unit, Seq[Interaction]](RealTimeInteractionsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.SIMS_FETCHER) - def simsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](SimsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) - def dbv2SimsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](DBV2SimsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.TRIANGULAR_LOOPS_FETCHER) - def triangularLoopsFetcher(stratoClient: Client): Fetcher[Long, Unit, TriangularLoopCandidates] = - stratoClient.fetcher[Long, Unit, TriangularLoopCandidates](TriangularLoopsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) - def twoHopRandomWalkFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateSeq] = - stratoClient.fetcher[Long, Unit, CandidateSeq](TwoHopRandomWalkPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) - def userRecommendabilityFetcher( - stratoClient: Client - ): Fetcher[Long, Unit, UserRecommendabilityFeatures] = - stratoClient.fetcher[Long, Unit, UserRecommendabilityFeatures](UserRecommendabilityPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.USER_STATE_FETCHER) - def userStateFetcher(stratoClient: Client): Fetcher[Long, Unit, CondensedUserState] = - stratoClient.fetcher[Long, Unit, CondensedUserState](UserStatePath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.UTT_ACCOUNT_RECOMMENDATIONS_FETCHER) - def uttAccountRecommendationsFetcher( - stratoClient: Client - ): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = - stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( - UTTAccountRecommendationsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.UTT_SEED_ACCOUNTS_FETCHER) - def uttSeedAccountRecommendationsFetcher( - stratoClient: Client - ): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = - stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( - UttSeedAccountsRecommendationPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.ELECTION_CANDIDATES_FETCHER) - def electionCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = - stratoClient.fetcher[String, Unit, Seq[Long]](ElectionCandidatesPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER) - def userUserGraphFetcher( - stratoClient: Client - ): Fetcher[RecommendUserRequest, Unit, RecommendUserResponse] = - stratoClient.fetcher[RecommendUserRequest, Unit, RecommendUserResponse](UserUserGraphPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.POST_NUX_WTF_FEATURES_FETCHER) - def wtfPostNuxFeaturesFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateFeatures] = { - val attribution = ManhattanAppId("starbuck", "wtf_starbuck") - stratoClient - .fetcher[Long, Unit, CandidateFeatures](WTFPostNuxFeaturesPath) - .withAttribution(attribution) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.EXTENDED_NETWORK) - def extendedNetworkFetcher( - stratoClient: Client - ): Fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal] = { - stratoClient - .fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal](ExtendedNetworkCandidatesPath) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.DISMISS_STORE_SCANNER) - def dismissStoreScanner( - stratoClient: Client - ): Scanner[ - (Long, Slice[(Long, Long)]), - Unit, - (Long, (Long, Long)), - WhoToFollowDismissEventDetails - ] = - stratoClient.scanner[ - (Long, Slice[(Long, Long)]), // PKEY: userId, LKEY: (-ts, candidateId) - Unit, - (Long, (Long, Long)), - WhoToFollowDismissEventDetails - ](WtfDissmissEventsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.LABELED_NOTIFICATION_FETCHER) - def labeledNotificationFetcher( - stratoClient: Client - ): Fetcher[Long, Unit, LatestEvents] = { - stratoClient - .fetcher[Long, Unit, LatestEvents](LabeledNotificationPath) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD deleted file mode 100644 index 16a82a302..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/client", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD.docx new file mode 100644 index 000000000..4c565dcf0 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.docx new file mode 100644 index 000000000..b4392bc08 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.scala deleted file mode 100644 index fe8101261..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.user_state - -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps._ -import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient -import com.twitter.follow_recommendations.common.clients.cache.ThriftEnumOptionBijection -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton -import java.lang.{Long => JLong} - -@Singleton -class UserStateClient @Inject() ( - @Named(GuiceNamedConstants.USER_STATE_FETCHER) userStateFetcher: Fetcher[ - Long, - Unit, - CondensedUserState - ], - client: Client, - statsReceiver: StatsReceiver, - decider: Decider = Decider.False) { - - private val stats: StatsReceiver = statsReceiver.scope("user_state_client") - - // client to memcache cluster - val bijection = new ThriftEnumOptionBijection[UserState](UserState.apply) - val memcacheClient = MemcacheClient[Option[UserState]]( - client = client, - dest = "/s/cache/follow_recos_service:twemcaches", - valueBijection = bijection, - ttl = UserStateClient.CacheTTL, - statsReceiver = stats.scope("twemcache") - ) - - def getUserState(userId: Long): Stitch[Option[UserState]] = { - val deciderKey: String = DeciderKey.EnableDistributedCaching.toString - val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - val userStateStitch: Stitch[Option[UserState]] = - enableDistributedCaching match { - // read from memcache - case true => memcacheClient.readThrough( - // add a key prefix to address cache key collisions - key = "UserStateClient" + userId.toString, - underlyingCall = () => fetchUserState(userId) - ) - case false => fetchUserState(userId) - } - val userStateStitchWithTimeout: Stitch[Option[UserState]] = - userStateStitch - // set a 150ms timeout limit for user state fetches - .within(150.milliseconds)(DefaultTimer) - .rescue { - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - Stitch(None) - } - // profile the latency of stitch call and return the result - StatsUtil.profileStitch( - userStateStitchWithTimeout, - stats.scope("getUserState") - ) - } - - def fetchUserState(userId: JLong): Stitch[Option[UserState]] = { - userStateFetcher.fetch(userId).map(_.v.flatMap(_.userState)) - } -} - -object UserStateClient { - val CacheTTL: Duration = Duration.fromHours(6) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD deleted file mode 100644 index dc9335d5b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD.docx new file mode 100644 index 000000000..ecb8c1a63 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.docx new file mode 100644 index 000000000..d1aa0592a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.scala deleted file mode 100644 index 7c7892c02..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.follow_recommendations.common.constants - -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.hermit.model.Algorithm._ -import com.twitter.follow_recommendations.common.models.AlgorithmType - -object CandidateAlgorithmTypeConstants { - - /** - * Each algorithm is based on one, or more, of the 4 types of information we have on users, - * described in [[AlgorithmType]]. Assignment of algorithms to these categories are based on - */ - private val AlgorithmIdToType: Map[String, Set[AlgorithmType.Value]] = Map( - // Activity Algorithms: - AlgorithmToFeedbackTokenMap(NewFollowingSimilarUser).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(Sims).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(NewFollowingSimilarUserSalsa).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentEngagementNonDirectFollow).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentEngagementSimilarUser).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentEngagementSarusOcCur).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentSearchBasedRec).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TwistlyTweetAuthors).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(Follow2VecNearestNeighbors).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(EmailTweetClick).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RepeatedProfileVisits).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(GoodTweetClickEngagements).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TweetShareEngagements).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TweetSharerToShareRecipientEngagements).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TweetAuthorToShareRecipientEngagements).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(LinearRegressionFollow2VecNearestNeighbors).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(NUXLOHistory).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TrafficAttributionAccounts).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RealGraphOonV2).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(MagicRecsRecentEngagements).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(NotificationEngagement).toString -> Set(AlgorithmType.Activity), - // Social Algorithms: - AlgorithmToFeedbackTokenMap(TwoHopRandomWalk).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(RealTimeMutualFollow).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ForwardPhoneBook).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ForwardEmailBook).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(NewFollowingNewFollowingExpansion).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(NewFollowingSarusCoOccurSocialProof).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ReverseEmailBookIbis).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ReversePhoneBook).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(StrongTiePredictionRec).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(StrongTiePredictionRecWithSocialProof).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRec).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRecNoCaching).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(TriangularLoop).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(StrongTiePredictionPmi).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRAB).toString -> Set(AlgorithmType.Social), - // Geo Algorithms: - AlgorithmToFeedbackTokenMap(PopCountryBackFill).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PopCountry).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PopGeohash).toString -> Set(AlgorithmType.Geo), -// AlgorithmToFeedbackTokenMap(PopGeohashRealGraph).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(EngagedFollowerRatio).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(CrowdSearchAccounts).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(OrganicFollowAccounts).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PopGeohashQualityFollow).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PPMILocaleFollow).toString -> Set(AlgorithmType.Geo), - // Interest Algorithms: - AlgorithmToFeedbackTokenMap(TttInterest).toString -> Set(AlgorithmType.Interest), - AlgorithmToFeedbackTokenMap(UttInterestRelatedUsers).toString -> Set(AlgorithmType.Interest), - AlgorithmToFeedbackTokenMap(UttSeedAccounts).toString -> Set(AlgorithmType.Interest), - AlgorithmToFeedbackTokenMap(UttProducerExpansion).toString -> Set(AlgorithmType.Interest), - // Hybrid (more than one type) Algorithms: - AlgorithmToFeedbackTokenMap(UttProducerOfflineMbcgV1).toString -> Set( - AlgorithmType.Interest, - AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(CuratedAccounts).toString -> Set( - AlgorithmType.Interest, - AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(UserUserGraph).toString -> Set( - AlgorithmType.Social, - AlgorithmType.Activity), - ) - def getAlgorithmTypes(algoId: String): Set[String] = { - AlgorithmIdToType.get(algoId).map(_.map(_.toString)).getOrElse(Set.empty) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.docx new file mode 100644 index 000000000..49368a559 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.scala deleted file mode 100644 index d3d61fa43..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.follow_recommendations.common.constants - -object GuiceNamedConstants { - final val PRODUCER_SIDE_FEATURE_SWITCHES = "PRODUCER_SIDE_FEATURE_SWITCHES" - final val CLIENT_EVENT_LOGGER = "CLIENT_EVENT_LOGGER" - final val COSINE_FOLLOW_FETCHER = "cosine_follow_fetcher" - final val COSINE_LIST_FETCHER = "cosine_list_fetcher" - final val CURATED_CANDIDATES_FETCHER = "curated_candidates_fetcher" - final val CURATED_COMPETITOR_ACCOUNTS_FETCHER = "curated_competitor_accounts_fetcher" - final val POP_USERS_IN_PLACE_FETCHER = "pop_users_in_place_fetcher" - final val PROFILE_SIDEBAR_BLACKLIST_SCANNER = "profile_sidebar_blacklist_scanner" - final val REQUEST_LOGGER = "REQUEST_LOGGER" - final val FLOW_LOGGER = "FLOW_LOGGER" - final val REAL_TIME_INTERACTIONS_FETCHER = "real_time_interactions_fetcher" - final val SIMS_FETCHER = "sims_fetcher" - final val DBV2_SIMS_FETCHER = "dbv2_sims_fetcher" - - final val TRIANGULAR_LOOPS_FETCHER = "triangular_loops_fetcher" - final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher" - final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher" - final val USER_STATE_FETCHER = "user_state_fetcher" - final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recomendations_fetcher" - final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher" - - final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher" - final val POST_NUX_WTF_FEATURES_FETCHER = "post_nux_wtf_features_fetcher" - - final val USER_USER_GRAPH_FETCHER = "user_user_graph_fetcher" - final val DISMISS_STORE_SCANNER = "dismiss_store_scanner" - final val LABELED_NOTIFICATION_FETCHER = "labeled_notification_scanner" - - final val STP_EP_SCORER = "stp_ep_scorer" - final val STP_DBV2_SCORER = "stp_dbv2_scorer" - final val STP_RAB_DBV2_SCORER = "stp_rab_dbv2_scorer" - - final val EXTENDED_NETWORK = "extended_network_candidates" - - // scoring client constants - final val WTF_PROD_DEEPBIRDV2_CLIENT = "wtf_prod_deepbirdv2_client" - - // ann clients - final val RELATABLE_ACCOUNTS_FETCHER = "relatable_accounts_fetcher" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.docx new file mode 100644 index 000000000..e98981417 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.scala deleted file mode 100644 index 6aade704c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.constants - -import com.twitter.conversions.StorageUnitOps._ - -object ServiceConstants { - - /** thrift client response size limits - * these were estimated using monitoring dashboard - * 3MB network usage per second / 25 rps ~ 120KB/req << 1MB - * we give some buffer here in case some requests require more data than others - */ - val StringLengthLimit: Long = - 10.megabyte.inBytes - val ContainerLengthLimit: Long = 1.megabyte.inBytes -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD deleted file mode 100644 index b4afee590..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD +++ /dev/null @@ -1,82 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":candidate-algorithm-adapter", - ":client-context-adapter", - ":post-nux-algorithm-adapter", - ":pre-fetched-feature-adapter", - ], -) - -target( - name = "common", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/onboarding/relevance/util/metadata", - "util/util-slf4j-api/src/main/scala", - ], -) - -scala_library( - name = "candidate-algorithm-adapter", - sources = [ - "CandidateAlgorithmAdapter.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - ], -) - -scala_library( - name = "client-context-adapter", - sources = [ - "ClientContextAdapter.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - "snowflake/src/main/scala/com/twitter/snowflake/id", - ], -) - -scala_library( - name = "post-nux-algorithm-adapter", - sources = [ - "PostNuxAlgorithmAdapter.scala", - ], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - "src/scala/com/twitter/ml/featurestore/catalog/features/customer_journey:post-nux-algorithm-aggregate", - ], -) - -scala_library( - name = "pre-fetched-feature-adapter", - sources = [ - "PreFetchedFeatureAdapter.scala", - ], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD.docx new file mode 100644 index 000000000..cb071e8b6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.docx new file mode 100644 index 000000000..2635ffdee Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.scala deleted file mode 100644 index 7a487a95b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.hermit.model.Algorithm -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.hermit.model.Algorithm.UttProducerOfflineMbcgV1 -import com.twitter.hermit.model.Algorithm.UttProducerOnlineMbcgV1 -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature.SparseBinary -import com.twitter.ml.api.Feature.SparseContinuous -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.api.util.FDsl._ - -object CandidateAlgorithmAdapter - extends IRecordOneToOneAdapter[Option[UserCandidateSourceDetails]] { - - val CANDIDATE_ALGORITHMS: SparseBinary = new SparseBinary("candidate.source.algorithm_ids") - val CANDIDATE_SOURCE_SCORES: SparseContinuous = - new SparseContinuous("candidate.source.scores") - val CANDIDATE_SOURCE_RANKS: SparseContinuous = - new SparseContinuous("candidate.source.ranks") - - override val getFeatureContext: FeatureContext = new FeatureContext( - CANDIDATE_ALGORITHMS, - CANDIDATE_SOURCE_SCORES, - CANDIDATE_SOURCE_RANKS - ) - - /** list of candidate source remaps to avoid creating different features for experimental sources. - * the LHS should contain the experimental source, and the RHS should contain the prod source. - */ - def remapCandidateSource(a: Algorithm): Algorithm = a match { - case UttProducerOnlineMbcgV1 => UttProducerOfflineMbcgV1 - case _ => a - } - - // add the list of algorithm feedback tokens (integers) as a sparse binary feature - override def adaptToDataRecord( - userCandidateSourceDetailsOpt: Option[UserCandidateSourceDetails] - ): DataRecord = { - val dr = new DataRecord() - userCandidateSourceDetailsOpt.foreach { userCandidateSourceDetails => - val scoreMap = for { - (source, scoreOpt) <- userCandidateSourceDetails.candidateSourceScores - score <- scoreOpt - algo <- Algorithm.withNameOpt(source.name) - algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - } yield algoId.toString -> score - val rankMap = for { - (source, rank) <- userCandidateSourceDetails.candidateSourceRanks - algo <- Algorithm.withNameOpt(source.name) - algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - } yield algoId.toString -> rank.toDouble - - val algoIds = scoreMap.keys.toSet ++ rankMap.keys.toSet - - // hydrate if not empty - if (rankMap.nonEmpty) { - dr.setFeatureValue(CANDIDATE_SOURCE_RANKS, rankMap) - } - if (scoreMap.nonEmpty) { - dr.setFeatureValue(CANDIDATE_SOURCE_SCORES, scoreMap) - } - if (algoIds.nonEmpty) { - dr.setFeatureValue(CANDIDATE_ALGORITHMS, algoIds) - } - } - dr - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.docx new file mode 100644 index 000000000..7128f38e8 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.scala deleted file mode 100644 index 9aa4bdb0d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.ml.api.Feature.Binary -import com.twitter.ml.api.Feature.Continuous -import com.twitter.ml.api.Feature.Discrete -import com.twitter.ml.api.Feature.Text -import com.twitter.ml.api.util.FDsl._ -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.onboarding.relevance.util.metadata.LanguageUtil -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.snowflake.id.SnowflakeId - -object ClientContextAdapter extends IRecordOneToOneAdapter[(ClientContext, DisplayLocation)] { - - // we name features with `user.account` for relatively static user-related features - val USER_COUNTRY: Text = new Text("user.account.country") - val USER_LANGUAGE: Text = new Text("user.account.language") - // we name features with `user.context` for more dynamic user-related features - val USER_LANGUAGE_PREFIX: Text = new Text("user.context.language_prefix") - val USER_CLIENT: Discrete = new Discrete("user.context.client") - val USER_AGE: Continuous = new Continuous("user.context.age") - val USER_IS_RECENT: Binary = new Binary("user.is.recent") - // we name features with `meta` for meta info about the WTF recommendation request - val META_DISPLAY_LOCATION: Text = new Text("meta.display_location") - val META_POSITION: Discrete = new Discrete("meta.position") - // This indicates whether a data point is from a random serving policy - val META_IS_RANDOM: Binary = new Binary("prediction.engine.is_random") - - val RECENT_WIN_IN_DAYS: Int = 30 - val GOAL_META_POSITION: Long = 1L - val GOAL_META_IS_RANDOM: Boolean = true - - override val getFeatureContext: FeatureContext = new FeatureContext( - USER_COUNTRY, - USER_LANGUAGE, - USER_AGE, - USER_LANGUAGE_PREFIX, - USER_CLIENT, - USER_IS_RECENT, - META_DISPLAY_LOCATION, - META_POSITION, - META_IS_RANDOM - ) - - /** - * we only want to set the relevant fields iff they exist to eliminate redundant information - * we do some simple normalization on the language code - * we set META_POSITION to 1 always - * we set META_IS_RANDOM to true always to simulate a random serving distribution - * @param record ClientContext and DisplayLocation from the request - */ - override def adaptToDataRecord(target: (ClientContext, DisplayLocation)): DataRecord = { - val dr = new DataRecord() - val cc = target._1 - val dl = target._2 - cc.countryCode.foreach(countryCode => dr.setFeatureValue(USER_COUNTRY, countryCode)) - cc.languageCode.foreach(rawLanguageCode => { - val userLanguage = LanguageUtil.simplifyLanguage(rawLanguageCode) - val userLanguagePrefix = userLanguage.take(2) - dr.setFeatureValue(USER_LANGUAGE, userLanguage) - dr.setFeatureValue(USER_LANGUAGE_PREFIX, userLanguagePrefix) - }) - cc.appId.foreach(appId => dr.setFeatureValue(USER_CLIENT, appId)) - cc.userId.foreach(id => - SnowflakeId.timeFromIdOpt(id).map { signupTime => - val userAge = signupTime.untilNow.inMillis.toDouble - dr.setFeatureValue(USER_AGE, userAge) - dr.setFeatureValue(USER_IS_RECENT, signupTime.untilNow.inDays <= RECENT_WIN_IN_DAYS) - signupTime.untilNow.inDays - }) - dr.setFeatureValue(META_DISPLAY_LOCATION, dl.toFsName) - dr.setFeatureValue(META_POSITION, GOAL_META_POSITION) - dr.setFeatureValue(META_IS_RANDOM, GOAL_META_IS_RANDOM) - dr - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.docx new file mode 100644 index 000000000..6d8b647ba Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.scala deleted file mode 100644 index e8fe745a0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.Feature.Continuous -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.api.util.FDsl._ -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmFeatures -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmIdAggregateFeatureGroup -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmTypeAggregateFeatureGroup -import scala.collection.JavaConverters._ - -object PostNuxAlgorithmIdAdapter extends PostNuxAlgorithmAdapter { - override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures = - PostNuxAlgorithmIdAggregateFeatureGroup - - // To keep the length of feature names reasonable, we remove the prefix added by FeatureStore. - override val FeatureStorePrefix: String = - "wtf_algorithm_id.customer_journey.post_nux_algorithm_id_aggregate_feature_group." -} - -object PostNuxAlgorithmTypeAdapter extends PostNuxAlgorithmAdapter { - override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures = - PostNuxAlgorithmTypeAggregateFeatureGroup - - // To keep the length of feature names reasonable, we remove the prefix added by FeatureStore. - override val FeatureStorePrefix: String = - "wtf_algorithm_type.customer_journey.post_nux_algorithm_type_aggregate_feature_group." -} - -trait PostNuxAlgorithmAdapter extends IRecordOneToOneAdapter[DataRecord] { - - val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures - - // The string that is attached to the feature name when it is fetched from feature store. - val FeatureStorePrefix: String - - /** - * - * This stores transformed aggregate features for PostNux algorithm aggregate features. The - * transformation here is log-ratio, where ratio is the raw value divided by # of impressions. - */ - case class TransformedAlgorithmFeatures( - ratioLog: Continuous) { - def getFeatures: Seq[Continuous] = Seq(ratioLog) - } - - private def applyFeatureStorePrefix(feature: Continuous) = new Continuous( - s"$FeatureStorePrefix${feature.getFeatureName}") - - // The list of input features WITH the prefix assigned to them by FeatureStore. - lazy val allInputFeatures: Seq[Seq[Continuous]] = Seq( - PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures.map(applyFeatureStorePrefix), - PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures.map(applyFeatureStorePrefix) - ) - - // This is a list of the features WITHOUT the prefix assigned to them by FeatureStore. - lazy val outputBaseFeatureNames: Seq[Seq[Continuous]] = Seq( - PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures, - PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures - ) - - // We use backend impression to calculate ratio values. - lazy val ratioDenominators: Seq[Continuous] = Seq( - applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions7Days), - applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions30Days) - ) - - /** - * A mapping from an original feature's ID to the corresponding set of transformed features. - * This is used to compute the transformed features for each of the original ones. - */ - private lazy val TransformedFeaturesMap: Map[Continuous, TransformedAlgorithmFeatures] = - outputBaseFeatureNames.flatten.map { feature => - ( - // The input feature would have the FeatureStore prefix attached to it. - new Continuous(s"$FeatureStorePrefix${feature.getFeatureName}"), - // We don't keep the FeatureStore prefix to keep the length of feature names reasonable. - TransformedAlgorithmFeatures( - new Continuous(s"${feature.getFeatureName}-ratio-log") - )) - }.toMap - - /** - * Given a denominator, number of impressions, this function returns another function that adds - * transformed features (log1p and ratio) of an input feature to a DataRecord. - */ - private def addTransformedFeaturesToDataRecordFunc( - originalDr: DataRecord, - numImpressions: Double, - ): (DataRecord, Continuous) => DataRecord = { (record: DataRecord, feature: Continuous) => - { - Option(originalDr.getFeatureValue(feature)) foreach { featureValue => - TransformedFeaturesMap.get(feature).foreach { transformedFeatures => - record.setFeatureValue( - transformedFeatures.ratioLog, - // We don't use log1p here since the values are ratios and adding 1 to the _ratio_ would - // lead to logarithm of values between 1 and 2, essentially making all values the same. - math.log((featureValue + 1) / numImpressions) - ) - } - } - record - } - } - - /** - * @param record: The input record whose PostNuxAlgorithm aggregates are to be transformed. - * @return the input [[DataRecord]] with transformed aggregates added. - */ - override def adaptToDataRecord(record: DataRecord): DataRecord = { - if (record.continuousFeatures == null) { - // There are no base features available, and hence no transformations. - record - } else { - - /** - * The `foldLeft` below goes through pairs of (1) Feature groups, such as those calculated over - * 7 days or 30 days, and (2) the number of impressions for each of these groups, which is the - * denominator when ratio is calculated. - */ - ratioDenominators - .zip(allInputFeatures).foldLeft( /* initial empty DataRecord */ record)( - ( - /* DataRecord with transformed features up to here */ transformedRecord, - /* A tuple with the denominator (#impressions) and features to be transformed */ numImpressionsAndFeatures - ) => { - val (numImpressionsFeature, features) = numImpressionsAndFeatures - Option(record.getFeatureValue(numImpressionsFeature)) match { - case Some(numImpressions) if numImpressions > 0.0 => - /** - * With the number of impressions fixed, we generate a function that adds log-ratio - * for each feature in the current [[DataRecord]]. The `foldLeft` goes through all - * such features and applies that function while updating the kept DataRecord. - */ - features.foldLeft(transformedRecord)( - addTransformedFeaturesToDataRecordFunc(record, numImpressions)) - case _ => - transformedRecord - } - }) - } - } - - def getFeatures: Seq[Feature[_]] = TransformedFeaturesMap.values.flatMap(_.getFeatures).toSeq - - override def getFeatureContext: FeatureContext = - new FeatureContext() - .addFeatures(this.getFeatures.asJava) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.docx new file mode 100644 index 000000000..23d7da920 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.scala deleted file mode 100644 index f24ed1efe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.ml.api.Feature.Continuous -import com.twitter.ml.api.util.FDsl._ -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.util.Time - -/** - * This adapter mimics UserRecentWTFImpressionsAndFollowsAdapter (for user) and - * RecentWTFImpressionsFeatureAdapter (for candidate) for extracting recent impression - * and follow features. This adapter extracts user, candidate, and pair-wise features. - */ -object PreFetchedFeatureAdapter - extends IRecordOneToOneAdapter[ - (HasPreFetchedFeature, CandidateUser) - ] { - - // impression features - val USER_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous( - "user.prefetch.num_recent_impressions" - ) - val USER_LAST_IMPRESSION_DURATION: Continuous = new Continuous( - "user.prefetch.last_impression_duration" - ) - val CANDIDATE_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous( - "user-candidate.prefetch.num_recent_impressions" - ) - val CANDIDATE_LAST_IMPRESSION_DURATION: Continuous = new Continuous( - "user-candidate.prefetch.last_impression_duration" - ) - // follow features - val USER_NUM_RECENT_FOLLOWERS: Continuous = new Continuous( - "user.prefetch.num_recent_followers" - ) - val USER_NUM_RECENT_FOLLOWED_BY: Continuous = new Continuous( - "user.prefetch.num_recent_followed_by" - ) - val USER_NUM_RECENT_MUTUAL_FOLLOWS: Continuous = new Continuous( - "user.prefetch.num_recent_mutual_follows" - ) - // impression + follow features - val USER_NUM_RECENT_FOLLOWED_IMPRESSIONS: Continuous = new Continuous( - "user.prefetch.num_recent_followed_impression" - ) - val USER_LAST_FOLLOWED_IMPRESSION_DURATION: Continuous = new Continuous( - "user.prefetch.last_followed_impression_duration" - ) - - override def adaptToDataRecord( - record: (HasPreFetchedFeature, CandidateUser) - ): DataRecord = { - val (target, candidate) = record - val dr = new DataRecord() - val t = Time.now - // set impression features for user, optionally for candidate - dr.setFeatureValue(USER_NUM_RECENT_IMPRESSIONS, target.numWtfImpressions.toDouble) - dr.setFeatureValue( - USER_LAST_IMPRESSION_DURATION, - (t - target.latestImpressionTime).inMillis.toDouble) - target.getCandidateImpressionCounts(candidate.id).foreach { counts => - dr.setFeatureValue(CANDIDATE_NUM_RECENT_IMPRESSIONS, counts.toDouble) - } - target.getCandidateLatestTime(candidate.id).foreach { latestTime: Time => - dr.setFeatureValue(CANDIDATE_LAST_IMPRESSION_DURATION, (t - latestTime).inMillis.toDouble) - } - // set recent follow features for user - dr.setFeatureValue(USER_NUM_RECENT_FOLLOWERS, target.numRecentFollowedUserIds.toDouble) - dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_BY, target.numRecentFollowedByUserIds.toDouble) - dr.setFeatureValue(USER_NUM_RECENT_MUTUAL_FOLLOWS, target.numRecentMutualFollows.toDouble) - dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_IMPRESSIONS, target.numFollowedImpressions.toDouble) - dr.setFeatureValue( - USER_LAST_FOLLOWED_IMPRESSION_DURATION, - target.lastFollowedImpressionDurationMs.getOrElse(Long.MaxValue).toDouble) - dr - } - override def getFeatureContext: FeatureContext = new FeatureContext( - USER_NUM_RECENT_IMPRESSIONS, - USER_LAST_IMPRESSION_DURATION, - CANDIDATE_NUM_RECENT_IMPRESSIONS, - CANDIDATE_LAST_IMPRESSION_DURATION, - USER_NUM_RECENT_FOLLOWERS, - USER_NUM_RECENT_FOLLOWED_BY, - USER_NUM_RECENT_MUTUAL_FOLLOWS, - USER_NUM_RECENT_FOLLOWED_IMPRESSIONS, - USER_LAST_FOLLOWED_IMPRESSION_DURATION, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD deleted file mode 100644 index 93ddb1191..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "src/java/com/twitter/ml/api:api-base", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD.docx new file mode 100644 index 000000000..5652cc506 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.docx new file mode 100644 index 000000000..13c38e960 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.scala deleted file mode 100644 index 9d43f0c13..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.common - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -trait FeatureSource { - def id: FeatureSourceId - def featureContext: FeatureContext - def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.docx new file mode 100644 index 000000000..12f141206 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.scala deleted file mode 100644 index 66b883120..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.common - -sealed trait FeatureSourceId - -object FeatureSourceId { - object CandidateAlgorithmSourceId extends FeatureSourceId - object ClientContextSourceId extends FeatureSourceId - object FeatureStoreSourceId extends FeatureSourceId - object FeatureStoreTimelinesAuthorSourceId extends FeatureSourceId - object FeatureStoreGizmoduckSourceId extends FeatureSourceId - object FeatureStoreUserMetricCountsSourceId extends FeatureSourceId - object FeatureStoreNotificationSourceId extends FeatureSourceId - - object FeatureStorePrecomputedNotificationSourceId extends FeatureSourceId - object FeatureStorePostNuxAlgorithmSourceId extends FeatureSourceId - @deprecated object StratoFeatureHydrationSourceId extends FeatureSourceId - object PreFetchedFeatureSourceId extends FeatureSourceId - object UserScoringFeatureSourceId extends FeatureSourceId -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.docx new file mode 100644 index 000000000..1dabe6837 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.scala deleted file mode 100644 index 8f4bae887..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.common - -import com.twitter.follow_recommendations.common.models.HasMutualFollowedUserIds -import com.twitter.follow_recommendations.common.models.HasWtfImpressions -import com.twitter.follow_recommendations.common.models.WtfImpression -import com.twitter.util.Time - -trait HasPreFetchedFeature extends HasMutualFollowedUserIds with HasWtfImpressions { - - lazy val followedImpressions: Seq[WtfImpression] = { - for { - wtfImprList <- wtfImpressions.toSeq - wtfImpr <- wtfImprList - if recentFollowedUserIds.exists(_.contains(wtfImpr.candidateId)) - } yield wtfImpr - } - - lazy val numFollowedImpressions: Int = followedImpressions.size - - lazy val lastFollowedImpressionDurationMs: Option[Long] = { - if (followedImpressions.nonEmpty) { - Some((Time.now - followedImpressions.map(_.latestTime).max).inMillis) - } else None - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD deleted file mode 100644 index c0538240f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD +++ /dev/null @@ -1,59 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/core:socialgraph", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/core:usersource", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:mc-user-counting", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:user-wtf-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-impression", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-post-nux", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-user-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/timelines:timelines-author-features", - "src/scala/com/twitter/ml/featurestore/catalog/entities/core", - "src/scala/com/twitter/ml/featurestore/catalog/entities/onboarding", - "src/scala/com/twitter/ml/featurestore/catalog/features/core:socialgraph", - "src/scala/com/twitter/ml/featurestore/catalog/features/core:user", - "src/scala/com/twitter/ml/featurestore/catalog/features/interests_discovery:user-topic-relationships", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmaries", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmary-aggregates", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:nonmr-ntab-summaries", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:mc-user-counting", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline-edge", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:ratio", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:simcluster-user-interested-in-candidate-known-for", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:user-wtf-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:wtf-impression", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:wtf-user-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/rux:user-resurrection", - "src/scala/com/twitter/ml/featurestore/catalog/features/timelines:aggregate", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - "src/scala/com/twitter/ml/featurestore/lib/embedding", - "src/scala/com/twitter/ml/featurestore/lib/feature", - "src/scala/com/twitter/ml/featurestore/lib/online", - "src/scala/com/twitter/ml/featurestore/lib/params", - "src/scala/com/twitter/onboarding/relevance/adapters/features/featurestore", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/config/columns/ml/featureStore/onboarding:onboarding-strato-client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD.docx new file mode 100644 index 000000000..425da591b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.docx new file mode 100644 index 000000000..144a5f1ed Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.scala deleted file mode 100644 index 0838fb98d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Inject -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This source only takes features from the candidate's source, - * which is all the information we have about the candidate pre-feature-hydration - */ - -@Provides -@Singleton -class CandidateAlgorithmSource @Inject() (stats: StatsReceiver) extends FeatureSource { - - override val id: FeatureSourceId = FeatureSourceId.CandidateAlgorithmSourceId - - override val featureContext: FeatureContext = CandidateAlgorithmAdapter.getFeatureContext - - override def hydrateFeatures( - t: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, // we don't use the target here - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - val featureHydrationStats = stats.scope("candidate_alg_source") - val hasSourceDetailsStat = featureHydrationStats.counter("has_source_details") - val noSourceDetailsStat = featureHydrationStats.counter("no_source_details") - val noSourceRankStat = featureHydrationStats.counter("no_source_rank") - val hasSourceRankStat = featureHydrationStats.counter("has_source_rank") - val noSourceScoreStat = featureHydrationStats.counter("no_source_score") - val hasSourceScoreStat = featureHydrationStats.counter("has_source_score") - - val candidatesToAlgoMap = for { - candidate <- candidates - } yield { - if (candidate.userCandidateSourceDetails.nonEmpty) { - hasSourceDetailsStat.incr() - candidate.userCandidateSourceDetails.foreach { details => - if (details.candidateSourceRanks.isEmpty) { - noSourceRankStat.incr() - } else { - hasSourceRankStat.incr() - } - if (details.candidateSourceScores.isEmpty) { - noSourceScoreStat.incr() - } else { - hasSourceScoreStat.incr() - } - } - } else { - noSourceDetailsStat.incr() - } - candidate -> CandidateAlgorithmAdapter.adaptToDataRecord(candidate.userCandidateSourceDetails) - } - Stitch.value(candidatesToAlgoMap.toMap) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.docx new file mode 100644 index 000000000..ac6ca6174 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.scala deleted file mode 100644 index 718502d44..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.feature_hydration.adapters.ClientContextAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This source only takes features from the request (e.g. client context, WTF display location) - * No external calls are made. - */ -@Provides -@Singleton -class ClientContextSource() extends FeatureSource { - - override val id: FeatureSourceId = FeatureSourceId.ClientContextSourceId - - override val featureContext: FeatureContext = ClientContextAdapter.getFeatureContext - - override def hydrateFeatures( - t: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - Stitch.value( - candidates - .map(_ -> ((t.clientContext, t.displayLocation))).toMap.mapValues( - ClientContextAdapter.adaptToDataRecord)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.docx new file mode 100644 index 000000000..e0eed2747 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.scala deleted file mode 100644 index f78ec17cc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FeatureHydrationSourcesFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures, - FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures, - FeatureStoreSourceParams.EnableCandidateClientFeatures, - FeatureStoreSourceParams.EnableCandidatePrecomputedNotificationFeatures, - FeatureStoreSourceParams.EnableCandidateUserAuthorRealTimeAggregateFeatures, - FeatureStoreSourceParams.EnableCandidateUserFeatures, - FeatureStoreSourceParams.EnableCandidateUserResurrectionFeatures, - FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures, - FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors, - FeatureStoreSourceParams.EnableSeparateClientForGizmoduck, - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting, - FeatureStoreSourceParams.EnableSeparateClientForNotifications, - FeatureStoreSourceParams.EnableSimilarToUserFeatures, - FeatureStoreSourceParams.EnableTargetUserFeatures, - FeatureStoreSourceParams.EnableTargetUserResurrectionFeatures, - FeatureStoreSourceParams.EnableTargetUserWtfImpressionFeatures, - FeatureStoreSourceParams.EnableTopicAggregateFeatures, - FeatureStoreSourceParams.EnableUserCandidateEdgeFeatures, - FeatureStoreSourceParams.EnableUserCandidateWtfImpressionCandidateFeatures, - FeatureStoreSourceParams.EnableUserClientFeatures, - FeatureStoreSourceParams.EnableUserTopicFeatures, - FeatureStoreSourceParams.EnableUserWtfAlgEdgeFeatures, - ) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - FeatureStoreSourceParams.GlobalFetchTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.docx new file mode 100644 index 000000000..745af241c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.scala deleted file mode 100644 index fb2232927..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -object FeatureHydrationSourcesFeatureSwitchKeys { - val EnableAlgorithmAggregateFeatures = "feature_store_source_enable_algorithm_aggregate_features" - val EnableAuthorTopicAggregateFeatures = - "feature_store_source_enable_author_topic_aggregate_features" - val EnableCandidateClientFeatures = "feature_store_source_enable_candidate_client_features" - val EnableCandidateNotificationFeatures = - "feature_store_source_enable_candidate_notification_features" - val EnableCandidatePrecomputedNotificationFeatures = - "feature_store_source_enable_candidate_precomputed_notification_features" - val EnableCandidateUserFeatures = "feature_store_source_enable_candidate_user_features" - val EnableCandidateUserAuthorRealTimeAggregateFeatures = - "feature_store_source_enable_candidate_user_author_rta_features" - val EnableCandidateUserResurrectionFeatures = - "feature_store_source_enable_candidate_user_resurrection_features" - val EnableCandidateUserTimelinesAuthorAggregateFeatures = - "feature_store_source_enable_candidate_user_timelines_author_aggregate_features" - val EnableSimilarToUserFeatures = "feature_store_source_enable_similar_to_user_features" - val EnableTargetUserFeatures = "feature_store_source_enable_target_user_features" - val EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature = - "feature_store_source_enable_target_user_user_author_user_state_rta_features" - val EnableTargetUserResurrectionFeatures = - "feature_store_source_enable_target_user_resurrection_features" - val EnableTargetUserWtfImpressionFeatures = - "feature_store_source_enable_target_user_wtf_impression_features" - val EnableTopicAggregateFeatures = "feature_store_source_enable_topic_aggregate_features" - val EnableUserCandidateEdgeFeatures = "feature_store_source_enable_user_candidate_edge_features" - val EnableUserCandidateWtfImpressionCandidateFeatures = - "feature_store_source_enable_user_candidate_wtf_impression_features" - val EnableUserClientFeatures = "feature_store_source_enable_user_client_features" - val EnableUserNotificationFeatures = "feature_store_source_enable_user_notification_features" - val EnableUserTopicFeatures = "feature_store_source_enable_user_topic_features" - val EnableUserWtfAlgEdgeFeatures = "feature_store_source_enable_user_wtf_alg_edge_features" - val FeatureHydrationTimeout = "feature_store_source_hydration_timeout_in_millis" - val UseSeparateClientForTimelinesAuthor = - "feature_store_source_separate_client_for_timelines_author_data" - val UseSeparateClientMetricCenterUserCounting = - "feature_store_source_separate_client_for_mc_user_counting_data" - val UseSeparateClientForNotifications = "feature_store_source_separate_client_for_notifications" - val UseSeparateClientForGizmoduck = "feature_store_source_separate_client_for_gizmoduck" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.docx new file mode 100644 index 000000000..2a3cf5378 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.scala deleted file mode 100644 index 6c849f7a7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.scala +++ /dev/null @@ -1,342 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{Topic => TopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{UserCandidate => UserCandidateEntity} -import com.twitter.ml.featurestore.catalog.entities.onboarding.UserWtfAlgorithmEntity -import com.twitter.ml.featurestore.catalog.entities.onboarding.{ - WtfAlgorithm => WtfAlgorithmIdEntity -} -import com.twitter.ml.featurestore.catalog.entities.onboarding.{ - WtfAlgorithmType => WtfAlgorithmTypeEntity -} -import com.twitter.ml.featurestore.catalog.features.core.UserClients.FullPrimaryClientVersion -import com.twitter.ml.featurestore.catalog.features.core.UserClients.NumClients -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryClient -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryClientVersion -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryDeviceManufacturer -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryMobileSdkVersion -import com.twitter.ml.featurestore.catalog.features.core.UserClients.SecondaryClient -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Favorites -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Followers -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Following -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Tweets -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmIdAggregateFeatureGroup -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmTypeAggregateFeatureGroup -import com.twitter.ml.featurestore.catalog.features.customer_journey.{Utils => FeatureGroupUtils} -import com.twitter.ml.featurestore.catalog.features.interests_discovery.UserTopicRelationships.FollowedTopics -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFavorites -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFavoritesReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollowBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollows -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollowsReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumLoginDays -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumLoginTweetImpressions -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumMuteBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumMuted -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumOriginalTweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQualityFollowReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQuoteRetweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQuoteRetweetsReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumReplies -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRepliesReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRetweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRetweetsReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumSpamBlocked -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumSpamBlockedBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumTweetImpressions -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumTweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUnfollowBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUnfollows -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUserActiveMinutes -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasMutualFollowed -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasMutualUnfollowed -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasUnfollowed -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.Country -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.FollowersOverFollowingRatio -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.Language -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.MutualFollowsOverFollowersRatio -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.MutualFollowsOverFollowingRatio -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumFollowers -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumFollowings -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumMutualFollows -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.TweepCred -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.UserState -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameCountry -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameLanguage -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameUserState -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumFollowersGap -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumFollowingsGap -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumMutualFollowsGap -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.TweepCredGap -import com.twitter.ml.featurestore.catalog.features.onboarding.Ratio.FollowersFollowings -import com.twitter.ml.featurestore.catalog.features.onboarding.Ratio.MutualFollowsFollowing -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.HasIntersection -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionCandidateKnownForScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionClusterIds -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFavCandidateKnownForScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFavScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFollowCandidateKnownForScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFollowScore -import com.twitter.ml.featurestore.catalog.features.onboarding.UserWtfAlgorithmAggregate -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateCounts -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateImpressionCounts -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateImpressionLatestTimestamp -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfLatestTimestamp -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.Follows -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetFavRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetReplies -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetReplyRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetRetweetRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetRetweets -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsWithTweetFavs -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsWithTweetImpressions -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasAnyEngagements -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasForwardEngagements -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasReverseEngagements -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.Impressions -import com.twitter.ml.featurestore.catalog.features.rux.UserResurrection.DaysSinceRecentResurrection -import com.twitter.ml.featurestore.catalog.features.timelines.AuthorTopicAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.EngagementsReceivedByAuthorRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.NegativeEngagementsReceivedByAuthorRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.OriginalAuthorAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.TopicEngagementRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.TopicEngagementUserStateRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.TopicNegativeEngagementUserStateRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.UserEngagementAuthorUserStateRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.UserNegativeEngagementAuthorUserStateRealTimeAggregates -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.Feature - -object FeatureStoreFeatures { - import FeatureStoreRawFeatures._ - ///////////////////////////// Target user features //////////////////////// - val targetUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = - (userKeyedFeatures ++ userAlgorithmAggregateFeatures).map(_.bind(UserEntity)) - - val targetUserResurrectionFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userResurrectionFeatures.map(_.bind(UserEntity)) - val targetUserWtfImpressionFeatures: Set[BoundFeature[_ <: EntityId, _]] = - wtfImpressionUserFeatures.map(_.bind(UserEntity)) - val targetUserUserAuthorUserStateRealTimeAggregatesFeature: Set[BoundFeature[_ <: EntityId, _]] = - userAuthorUserStateRealTimeAggregatesFeature.map(_.bind(UserEntity)) - - val targetUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userStatusFeatures.map(_.bind(UserEntity).logarithm1p) - val targetUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = - mcFeatures.map(_.bind(UserEntity).logarithm1p) - - val targetUserClientFeatures: Set[BoundFeature[_ <: EntityId, _]] = - clientFeatures.map(_.bind(UserEntity)) - - ///////////////////////////// Candidate user features //////////////////////// - val candidateUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userKeyedFeatures.map(_.bind(CandidateUserEntity)) - val candidateUserAuthorRealTimeAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - authorAggregateFeatures.map(_.bind(CandidateUserEntity)) - val candidateUserResurrectionFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userResurrectionFeatures.map(_.bind(CandidateUserEntity)) - - val candidateUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userStatusFeatures.map(_.bind(CandidateUserEntity).logarithm1p) - val candidateUserTimelinesAuthorAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(timelinesAuthorAggregateFeatures.bind(CandidateUserEntity)) - val candidateUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = - mcFeatures.map(_.bind(CandidateUserEntity).logarithm1p) - - val candidateUserClientFeatures: Set[BoundFeature[_ <: EntityId, _]] = - clientFeatures.map(_.bind(CandidateUserEntity)) - - val similarToUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = - (userKeyedFeatures ++ authorAggregateFeatures).map(_.bind(AuthorEntity)) - - val similarToUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userStatusFeatures.map(_.bind(AuthorEntity).logarithm1p) - val similarToUserTimelinesAuthorAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(timelinesAuthorAggregateFeatures.bind(AuthorEntity)) - val similarToUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = - mcFeatures.map(_.bind(AuthorEntity).logarithm1p) - - val userCandidateEdgeFeatures: Set[BoundFeature[_ <: EntityId, _]] = - (simclusterUVIntersectionFeatures ++ userCandidatePostNuxEdgeFeatures).map( - _.bind(UserCandidateEntity)) - val userCandidateWtfImpressionCandidateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - wtfImpressionCandidateFeatures.map(_.bind(UserCandidateEntity)) - - /** - * Aggregate features based on candidate source algorithms. - */ - val postNuxAlgorithmIdAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(PostNuxAlgorithmIdAggregateFeatureGroup.FeaturesAsDataRecord) - .map(_.bind(WtfAlgorithmIdEntity)) - - /** - * Aggregate features based on candidate source algorithm types. There are 4 at the moment: - * Geo, Social, Activity and Interest. - */ - val postNuxAlgorithmTypeAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(PostNuxAlgorithmTypeAggregateFeatureGroup.FeaturesAsDataRecord) - .map(_.bind(WtfAlgorithmTypeEntity)) - - // user wtf-Algorithm features - val userWtfAlgorithmEdgeFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureGroupUtils.getTimelinesAggregationFrameworkCombinedFeatures( - UserWtfAlgorithmAggregate, - UserWtfAlgorithmEntity, - FeatureGroupUtils.getMaxSumAvgAggregate(UserWtfAlgorithmAggregate) - ) - - /** - * We have to add the max/sum/avg-aggregated features to the set of all features so that we can - * register them using FRS's [[FrsFeatureJsonExporter]]. - * - * Any additional such aggregated features that are included in [[FeatureStoreSource]] client - * should be registered here as well. - */ - val maxSumAvgAggregatedFeatureContext: FeatureContext = new FeatureContext() - .addFeatures( - UserWtfAlgorithmAggregate.getSecondaryAggregatedFeatureContext - ) - - // topic features - val topicAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set( - TopicEngagementRealTimeAggregates.FeaturesAsDataRecord, - TopicNegativeEngagementUserStateRealTimeAggregates.FeaturesAsDataRecord, - TopicEngagementUserStateRealTimeAggregates.FeaturesAsDataRecord - ).map(_.bind(TopicEntity)) - val userTopicFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set(FollowedTopics.bind(UserEntity)) - val authorTopicFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set( - AuthorTopicAggregates.FeaturesAsDataRecord.bind(AuthorTopicEntity)) - val topicFeatures = topicAggregateFeatures ++ userTopicFeatures ++ authorTopicFeatures - -} - -object FeatureStoreRawFeatures { - val mcFeatures = Set( - NumTweets, - NumRetweets, - NumOriginalTweets, - NumRetweetsReceived, - NumFavoritesReceived, - NumRepliesReceived, - NumQuoteRetweetsReceived, - NumFollowsReceived, - NumFollowBacks, - NumFollows, - NumUnfollows, - NumUnfollowBacks, - NumQualityFollowReceived, - NumQuoteRetweets, - NumFavorites, - NumReplies, - NumLoginTweetImpressions, - NumTweetImpressions, - NumLoginDays, - NumUserActiveMinutes, - NumMuted, - NumSpamBlocked, - NumMuteBacks, - NumSpamBlockedBacks, - NumWasMutualFollowed, - NumWasMutualUnfollowed, - NumWasUnfollowed - ) - // based off usersource, and each feature represents the cumulative 'sent' counts - val userStatusFeatures = Set( - Favorites, - Followers, - Following, - Tweets - ) - // ratio features created from combining other features - val userRatioFeatures = Set(MutualFollowsFollowing, FollowersFollowings) - // features related to user login history - val userResurrectionFeatures: Set[Feature[UserId, Int]] = Set( - DaysSinceRecentResurrection - ) - - // real-time aggregate features borrowed from timelines - val authorAggregateFeatures = Set( - EngagementsReceivedByAuthorRealTimeAggregates.FeaturesAsDataRecord, - NegativeEngagementsReceivedByAuthorRealTimeAggregates.FeaturesAsDataRecord, - ) - - val timelinesAuthorAggregateFeatures = OriginalAuthorAggregates.FeaturesAsDataRecord - - val userAuthorUserStateRealTimeAggregatesFeature: Set[Feature[UserId, DataRecord]] = Set( - UserEngagementAuthorUserStateRealTimeAggregates.FeaturesAsDataRecord, - UserNegativeEngagementAuthorUserStateRealTimeAggregates.FeaturesAsDataRecord - ) - // post nux per-user offline features - val userOfflineFeatures = Set( - NumFollowings, - NumFollowers, - NumMutualFollows, - TweepCred, - UserState, - Language, - Country, - MutualFollowsOverFollowingRatio, - MutualFollowsOverFollowersRatio, - FollowersOverFollowingRatio, - ) - // matched post nux offline features between user and candidate - val userCandidatePostNuxEdgeFeatures = Set( - HaveSameUserState, - HaveSameLanguage, - HaveSameCountry, - NumFollowingsGap, - NumFollowersGap, - NumMutualFollowsGap, - TweepCredGap, - ) - // user algorithm aggregate features - val userAlgorithmAggregateFeatures = Set( - Impressions, - Follows, - FollowRate, - FollowsWithTweetImpressions, - FollowsWithTweetFavs, - FollowsTweetFavRate, - FollowsTweetReplies, - FollowsTweetReplyRate, - FollowsTweetRetweets, - FollowsTweetRetweetRate, - HasForwardEngagements, - HasReverseEngagements, - HasAnyEngagements, - ) - val userKeyedFeatures = userRatioFeatures ++ userOfflineFeatures - val wtfImpressionUserFeatures = - Set(HomeTimelineWtfCandidateCounts, HomeTimelineWtfLatestTimestamp) - val wtfImpressionCandidateFeatures = - Set(HomeTimelineWtfCandidateImpressionCounts, HomeTimelineWtfCandidateImpressionLatestTimestamp) - val simclusterUVIntersectionFeatures = Set( - IntersectionClusterIds, - HasIntersection, - IntersectionUserFollowScore, - IntersectionUserFavScore, - IntersectionCandidateKnownForScore, - IntersectionUserFollowCandidateKnownForScore, - IntersectionUserFavCandidateKnownForScore - ) - - // Client features - val clientFeatures = Set( - NumClients, - PrimaryClient, - PrimaryClientVersion, - FullPrimaryClientVersion, - PrimaryDeviceManufacturer, - PrimaryMobileSdkVersion, - SecondaryClient - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.docx new file mode 100644 index 000000000..e9b85f388 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.scala deleted file mode 100644 index bb6b58857..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -class FeatureStoreGizmoduckSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreGizmoduckSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_gizmoduck") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStoreGizmoduckSourceId - override def featureContext: FeatureContext = getFeatureContext - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - UsersourceEntityDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userEntityId = UserEntity.withId(UserId(targetUserId)) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val authorTopicEntity = if (topicProof.isDefined) { - backupSourceStats.counter("candidates_with_topic_proof").incr() - Set( - AuthorTopicEntity.withId( - EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) - } else Nil - - val entities = - Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStoreGizmoduckSource { - private val DatasetCacheScope = "feature_store_local_cache_gizmoduck" - private val DefaultCacheMaxKeys = 20000 - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureStoreFeatures.candidateUserStatusFeatures ++ - FeatureStoreFeatures.similarToUserStatusFeatures ++ - FeatureStoreFeatures.targetUserStatusFeatures - - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.targetUserStatusFeatures), - gate = HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.candidateUserStatusFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.similarToUserStatusFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - )) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.docx new file mode 100644 index 000000000..e085b54ce Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.scala deleted file mode 100644 index 468a15ff6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.conversions.DurationOps._ -import com.twitter.ml.featurestore.catalog.datasets.core.UserMobileSdkDataset -import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmIdAggregateDataset -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmTypeAggregateDataset -import com.twitter.ml.featurestore.catalog.datasets.magicrecs.NotificationSummariesEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.UserWtfAlgorithmAggregateFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.WhoToFollowPostNuxFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.rux.UserRecentReactivationTimeDataset -import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset -import com.twitter.ml.featurestore.lib.dataset.DatasetParams -import com.twitter.ml.featurestore.lib.dataset.online.BatchingPolicy -import com.twitter.ml.featurestore.lib.params.FeatureStoreParams -import com.twitter.strato.opcontext.Attribution.ManhattanAppId -import com.twitter.strato.opcontext.ServeWithin - -object FeatureStoreParameters { - - private val FeatureServiceBatchSize = 100 - - val featureStoreParams = FeatureStoreParams( - global = DatasetParams( - serveWithin = Some(ServeWithin(duration = 240.millis, roundTripAllowance = None)), - attributions = Seq( - ManhattanAppId("omega", "wtf_impression_store"), - ManhattanAppId("athena", "wtf_athena"), - ManhattanAppId("starbuck", "wtf_starbuck"), - ManhattanAppId("apollo", "wtf_apollo") - ), - batchingPolicy = Some(BatchingPolicy.Isolated(FeatureServiceBatchSize)) - ), - perDataset = Map( - MetricCenterUserCountingFeaturesDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding"), - batchingPolicy = Some(BatchingPolicy.Isolated(200)) - ), - UsersourceEntityDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding") - ), - WhoToFollowPostNuxFeaturesDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding"), - batchingPolicy = Some(BatchingPolicy.Isolated(200)) - ), - AuthorFeaturesEntityDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding"), - batchingPolicy = Some(BatchingPolicy.Isolated(10)) - ), - UserRecentReactivationTimeDataset.id -> DatasetParams( - stratoSuffix = - None // removed due to low hit rate. we should use a negative cache in the future - ), - UserWtfAlgorithmAggregateFeaturesDataset.id -> DatasetParams( - stratoSuffix = None - ), - NotificationSummariesEntityDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding"), - serveWithin = Some(ServeWithin(duration = 45.millis, roundTripAllowance = None)), - batchingPolicy = Some(BatchingPolicy.Isolated(10)) - ), - UserMobileSdkDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding") - ), - PostNuxAlgorithmIdAggregateDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding") - ), - PostNuxAlgorithmTypeAggregateDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding") - ), - ), - enableFeatureGenerationStats = true - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.docx new file mode 100644 index 000000000..528f284dc Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.scala deleted file mode 100644 index 6821e6a5f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.scala +++ /dev/null @@ -1,232 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.CandidateAlgorithmTypeConstants -import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter.remapCandidateSource -import com.twitter.follow_recommendations.common.feature_hydration.adapters.PostNuxAlgorithmIdAdapter -import com.twitter.follow_recommendations.common.feature_hydration.adapters.PostNuxAlgorithmTypeAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmIdAggregateDataset -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmTypeAggregateDataset -import com.twitter.ml.featurestore.catalog.entities.onboarding.{WtfAlgorithm => OnboardingWtfAlgoId} -import com.twitter.ml.featurestore.catalog.entities.onboarding.{ - WtfAlgorithmType => OnboardingWtfAlgoType -} -import com.twitter.ml.featurestore.catalog.features.customer_journey.CombineAllFeaturesPolicy -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.WtfAlgorithmId -import com.twitter.ml.featurestore.lib.WtfAlgorithmType -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters._ - -class FeatureStorePostNuxAlgorithmSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStorePostNuxAlgorithmSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_post_nux_algorithm") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStorePostNuxAlgorithmSourceId - override def featureContext: FeatureContext = getFeatureContext - - private val dataRecordMerger = new DataRecordMerger - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - PostNuxAlgorithmIdAggregateDataset, - PostNuxAlgorithmTypeAggregateDataset, - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapterToDataRecord: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - // These two calculate the rate for each feature by dividing it by the number of impressions, then - // apply a log transformation. - private val transformAdapters = Seq(PostNuxAlgorithmIdAdapter, PostNuxAlgorithmTypeAdapter) - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { _: Long => - val candidateAlgoIdEntities = candidates.map { candidate => - candidate.id -> candidate.getAllAlgorithms - .flatMap { algo => - AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - }.map(algoId => OnboardingWtfAlgoId.withId(WtfAlgorithmId(algoId))) - }.toMap - - val candidateAlgoTypeEntities = candidateAlgoIdEntities.map { - case (candidateId, algoIdEntities) => - candidateId -> algoIdEntities - .map(_.id.algoId) - .flatMap(algoId => CandidateAlgorithmTypeConstants.getAlgorithmTypes(algoId.toString)) - .distinct - .map(algoType => OnboardingWtfAlgoType.withId(WtfAlgorithmType(algoType))) - } - - val entities = { - candidateAlgoIdEntities.values.flatten ++ candidateAlgoTypeEntities.values.flatten - }.toSeq.distinct - val requests = entities.map(entity => FeatureStoreRequest(Seq(entity))) - - val predictionRecordsFut = dynamicFeatureStoreClient(requests, target) - val candidateFeatureMap = predictionRecordsFut.map { - predictionRecords: Seq[PredictionRecord] => - val entityFeatureMap: Map[EntityWithId[_], DataRecord] = entities - .zip(predictionRecords).map { - case (entity, predictionRecord) => - entity -> adaptAdditionalFeaturesToDataRecord( - adapterToDataRecord.adaptToDataRecord(predictionRecord), - adapterStats, - transformAdapters) - }.toMap - - // In case we have more than one algorithm ID, or type, for a candidate, we merge the - // resulting DataRecords using the two merging policies below. - val algoIdMergeFn = - CombineAllFeaturesPolicy(PostNuxAlgorithmIdAdapter.getFeatures).getMergeFn - val algoTypeMergeFn = - CombineAllFeaturesPolicy(PostNuxAlgorithmTypeAdapter.getFeatures).getMergeFn - - val candidateAlgoIdFeaturesMap = candidateAlgoIdEntities.mapValues { entities => - val features = entities.flatMap(e => Option(entityFeatureMap.getOrElse(e, null))) - algoIdMergeFn(features) - } - - val candidateAlgoTypeFeaturesMap = candidateAlgoTypeEntities.mapValues { entities => - val features = entities.flatMap(e => Option(entityFeatureMap.getOrElse(e, null))) - algoTypeMergeFn(features) - } - - candidates.map { candidate => - val idDrOpt = candidateAlgoIdFeaturesMap.getOrElse(candidate.id, None) - val typeDrOpt = candidateAlgoTypeFeaturesMap.getOrElse(candidate.id, None) - - val featureDr = (idDrOpt, typeDrOpt) match { - case (None, Some(typeDataRecord)) => typeDataRecord - case (Some(idDataRecord), None) => idDataRecord - case (None, None) => new DataRecord() - case (Some(idDataRecord), Some(typeDataRecord)) => - dataRecordMerger.merge(idDataRecord, typeDataRecord) - idDataRecord - } - candidate -> featureDr - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStorePostNuxAlgorithmSource { - private val DatasetCacheScope = "feature_store_local_cache_post_nux_algorithm" - private val DefaultCacheMaxKeys = 1000 // Both of these datasets have <50 keys total. - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureStoreFeatures.postNuxAlgorithmIdAggregateFeatures ++ - FeatureStoreFeatures.postNuxAlgorithmTypeAggregateFeatures - - val algoIdFinalFeatures = CombineAllFeaturesPolicy( - PostNuxAlgorithmIdAdapter.getFeatures).outputFeaturesPostMerge.toSeq - val algoTypeFinalFeatures = CombineAllFeaturesPolicy( - PostNuxAlgorithmTypeAdapter.getFeatures).outputFeaturesPostMerge.toSeq - - val getFeatureContext: FeatureContext = - new FeatureContext().addFeatures((algoIdFinalFeatures ++ algoTypeFinalFeatures).asJava) - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = - BoundFeatureSet(FeatureStoreFeatures.postNuxAlgorithmIdAggregateFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = - BoundFeatureSet(FeatureStoreFeatures.postNuxAlgorithmTypeAggregateFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures) - ), - )) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.docx new file mode 100644 index 000000000..d37622f5a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.scala deleted file mode 100644 index 991bdee86..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.scala +++ /dev/null @@ -1,368 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter.remapCandidateSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.magicrecs.NotificationSummariesEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{Topic => TopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{UserCandidate => UserCandidateEntity} -import com.twitter.ml.featurestore.catalog.entities.onboarding.UserWtfAlgorithmEntity -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dynamic._ -import com.twitter.ml.featurestore.lib.feature._ -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.WtfAlgorithmId -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateAuthorTopicAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicEngagementRealTimeAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicEngagementUserStateRealTimeAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicNegativeEngagementUserStateRealTimeAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.FeatureStoreAdapter -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -import java.util.concurrent.TimeUnit - -@Singleton -class FeatureStoreSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreSource._ - - override val id: FeatureSourceId = FeatureSourceId.FeatureStoreSourceId - override val featureContext: FeatureContext = FeatureStoreSource.getFeatureContext - val hydrateFeaturesStats = stats.scope("hydrate_features") - val adapterStats = stats.scope("adapters") - val featureSet: BoundFeatureSet = BoundFeatureSet(FeatureStoreSource.allFeatures) - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = FeatureStoreSource.dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - MetricCenterUserCountingFeaturesDataset, - UsersourceEntityDataset, - AuthorFeaturesEntityDataset, - NotificationSummariesEntityDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - stats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(stats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userId = UserId(targetUserId) - val userEntityId = UserEntity.withId(userId) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val userCandidateEdgeEntityId = - UserCandidateEntity.withId(EdgeEntityId(userId, UserId(candidate.id))) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val topicEntities = if (topicProof.isDefined) { - hydrateFeaturesStats.counter("candidates_with_topic_proof").incr() - val topicId = topicProof.get.topicId - val topicEntityId = TopicEntity.withId(TopicId(topicId)) - val authorTopicEntityId = - AuthorTopicEntity.withId(EdgeEntityId(UserId(candidate.id), TopicId(topicId))) - Seq(topicEntityId, authorTopicEntityId) - } else Nil - - val candidateAlgorithmsWithScores = candidate.getAllAlgorithms - val userWtfAlgEdgeEntities = - candidateAlgorithmsWithScores.flatMap(algo => { - val algoId = AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - algoId.map(id => - UserWtfAlgorithmEntity.withId(EdgeEntityId(userId, WtfAlgorithmId(id)))) - }) - - val entities = Seq( - userEntityId, - candidateEntityId, - userCandidateEdgeEntityId) ++ similarToUserId ++ topicEntities ++ userWtfAlgEdgeEntities - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -// list of features that we will be fetching, even if we are only scribing but not scoring with them -object FeatureStoreSource { - - private val DatasetCacheScope = "feature_store_local_cache" - private val DefaultCacheMaxKeys = 70000 - - import FeatureStoreFeatures._ - - ///////////////////// ALL hydrated features ///////////////////// - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - //target user - targetUserFeatures ++ - targetUserUserAuthorUserStateRealTimeAggregatesFeature ++ - targetUserResurrectionFeatures ++ - targetUserWtfImpressionFeatures ++ - targetUserStatusFeatures ++ - targetUserMetricCountFeatures ++ - //candidate user - candidateUserFeatures ++ - candidateUserResurrectionFeatures ++ - candidateUserAuthorRealTimeAggregateFeatures ++ - candidateUserStatusFeatures ++ - candidateUserMetricCountFeatures ++ - candidateUserTimelinesAuthorAggregateFeatures ++ - candidateUserClientFeatures ++ - //similar to user - similarToUserFeatures ++ - similarToUserStatusFeatures ++ - similarToUserMetricCountFeatures ++ - similarToUserTimelinesAuthorAggregateFeatures ++ - //other - userCandidateEdgeFeatures ++ - userCandidateWtfImpressionCandidateFeatures ++ - topicFeatures ++ - userWtfAlgorithmEdgeFeatures ++ - targetUserClientFeatures - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(topicAggregateFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTopicAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(authorTopicFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userTopicFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserTopicFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserUserAuthorUserStateRealTimeAggregatesFeature), - gate = HasParams.paramGate( - FeatureStoreSourceParams.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserResurrectionFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserResurrectionFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserWtfImpressionFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserWtfImpressionFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserStatusFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserMetricCountFeatures), - gate = HasParams - .paramGate( - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserAuthorRealTimeAggregateFeatures), - gate = HasParams.paramGate( - FeatureStoreSourceParams.EnableCandidateUserAuthorRealTimeAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserResurrectionFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserResurrectionFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserStatusFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & - HasParams.paramGate( - FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserMetricCountFeatures), - gate = - HasParams - .paramGate( - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userCandidateEdgeFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserCandidateEdgeFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userCandidateWtfImpressionCandidateFeatures), - gate = HasParams.paramGate( - FeatureStoreSourceParams.EnableUserCandidateWtfImpressionCandidateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userWtfAlgorithmEdgeFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserWtfAlgEdgeFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserStatusFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserMetricCountFeatures), - gate = - HasParams - .paramGate( - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserClientFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateClientFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserClientFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserClientFeatures) - ), - ) - ) - // for calibrating features, e.g. add log transformed topic features - val featureAdapters: Seq[FeatureStoreAdapter] = Seq( - CandidateTopicEngagementRealTimeAggregatesAdapter, - CandidateTopicNegativeEngagementUserStateRealTimeAggregatesAdapter, - CandidateTopicEngagementUserStateRealTimeAggregatesAdapter, - CandidateAuthorTopicAggregatesAdapter - ) - val additionalFeatureContext: FeatureContext = FeatureContext.merge( - featureAdapters - .foldRight(new FeatureContext())((adapter, context) => - context - .addFeatures(adapter.getFeatureContext)) - ) - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - .addFeatures(additionalFeatureContext) - // The below are aggregated features that are aggregated for a second time over multiple keys. - .addFeatures(maxSumAvgAggregatedFeatureContext) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.docx new file mode 100644 index 000000000..434507318 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.scala deleted file mode 100644 index 15488ce90..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.scala +++ /dev/null @@ -1,148 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object FeatureStoreSourceParams { - case object EnableTopicAggregateFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTopicAggregateFeatures, - default = true - ) - case object EnableAlgorithmAggregateFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAlgorithmAggregateFeatures, - default = false - ) - case object EnableAuthorTopicAggregateFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAuthorTopicAggregateFeatures, - default = true - ) - case object EnableUserTopicFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserTopicFeatures, - default = false - ) - case object EnableTargetUserFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserFeatures, - default = true - ) - case object EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature, - default = true - ) - case object EnableTargetUserResurrectionFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserResurrectionFeatures, - default = true - ) - case object EnableTargetUserWtfImpressionFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserWtfImpressionFeatures, - default = true - ) - case object EnableCandidateUserFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserFeatures, - default = true - ) - case object EnableCandidateUserAuthorRealTimeAggregateFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserAuthorRealTimeAggregateFeatures, - default = true - ) - case object EnableCandidateUserResurrectionFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserResurrectionFeatures, - default = true - ) - case object EnableCandidateUserTimelinesAuthorAggregateFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserTimelinesAuthorAggregateFeatures, - default = true - ) - case object EnableUserCandidateEdgeFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateEdgeFeatures, - default = true - ) - case object EnableUserCandidateWtfImpressionCandidateFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateWtfImpressionCandidateFeatures, - default = true - ) - case object EnableUserWtfAlgEdgeFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserWtfAlgEdgeFeatures, - default = false - ) - case object EnableSimilarToUserFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableSimilarToUserFeatures, - default = true - ) - - case object EnableCandidatePrecomputedNotificationFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidatePrecomputedNotificationFeatures, - default = false - ) - - case object EnableCandidateClientFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateClientFeatures, - default = false - ) - - case object EnableUserClientFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserClientFeatures, - default = false - ) - - case object EnableSeparateClientForTimelinesAuthors - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForTimelinesAuthor, - default = false - ) - - case object EnableSeparateClientForMetricCenterUserCounting - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientMetricCenterUserCounting, - default = false - ) - - case object EnableSeparateClientForNotifications - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForNotifications, - default = false - ) - - case object EnableSeparateClientForGizmoduck - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForGizmoduck, - default = false - ) - - case object GlobalFetchTimeout - extends FSBoundedParam[Duration]( - name = FeatureHydrationSourcesFeatureSwitchKeys.FeatureHydrationTimeout, - default = 240.millisecond, - min = 100.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.docx new file mode 100644 index 000000000..a83d70c0c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.scala deleted file mode 100644 index 179ae7081..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.scala +++ /dev/null @@ -1,191 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -class FeatureStoreTimelinesAuthorSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreTimelinesAuthorSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_timelines_author") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStoreTimelinesAuthorSourceId - override def featureContext: FeatureContext = getFeatureContext - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - AuthorFeaturesEntityDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userEntityId = UserEntity.withId(UserId(targetUserId)) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val authorTopicEntity = if (topicProof.isDefined) { - backupSourceStats.counter("candidates_with_topic_proof").incr() - Set( - AuthorTopicEntity.withId( - EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) - } else Nil - - val entities = - Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStoreTimelinesAuthorSource { - private val DatasetCacheScope = "feature_store_local_cache_timelines_author" - private val DefaultCacheMaxKeys = 20000 - - import FeatureStoreFeatures._ - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - similarToUserTimelinesAuthorAggregateFeatures ++ - candidateUserTimelinesAuthorAggregateFeatures ++ - authorTopicFeatures - - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(authorTopicFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & - HasParams.paramGate(FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & - HasParams.paramGate( - FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures) - ), - )) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.docx new file mode 100644 index 000000000..66eafd511 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.scala deleted file mode 100644 index 110985c92..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -class FeatureStoreUserMetricCountsSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreUserMetricCountsSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_mc_counting") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStoreUserMetricCountsSourceId - override def featureContext: FeatureContext = getFeatureContext - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - MetricCenterUserCountingFeaturesDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userEntityId = UserEntity.withId(UserId(targetUserId)) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val authorTopicEntity = if (topicProof.isDefined) { - backupSourceStats.counter("candidates_with_topic_proof").incr() - Set( - AuthorTopicEntity.withId( - EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) - } else Nil - - val entities = - Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStoreUserMetricCountsSource { - private val DatasetCacheScope = "feature_store_local_cache_mc_user_counting" - private val DefaultCacheMaxKeys = 20000 - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureStoreFeatures.candidateUserMetricCountFeatures ++ - FeatureStoreFeatures.similarToUserMetricCountFeatures ++ - FeatureStoreFeatures.targetUserMetricCountFeatures - - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.targetUserMetricCountFeatures), - gate = HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.candidateUserMetricCountFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.similarToUserMetricCountFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - )) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.docx new file mode 100644 index 000000000..cb2a80374 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.scala deleted file mode 100644 index 59e3ea186..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.scala +++ /dev/null @@ -1,152 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storage.client.manhattan.kv.ManhattanKVClient -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder -import com.twitter.storage.client.manhattan.kv.impl.Component -import com.twitter.storage.client.manhattan.kv.impl.Component0 -import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.strato.generated.client.ml.featureStore.McUserCountingOnUserClientColumn -import com.twitter.strato.generated.client.ml.featureStore.onboarding.TimelinesAuthorFeaturesOnUserClientColumn -import com.twitter.timelines.author_features.v1.thriftscala.AuthorFeatures -import com.twitter.conversions.DurationOps._ -import com.twitter.onboarding.relevance.features.thriftscala.MCUserCountingFeatures -import java.lang.{Long => JLong} -import scala.util.Random - -object HydrationSourcesModule extends TwitterModule { - - val readFromManhattan = flag( - "feature_hydration_enable_reading_from_manhattan", - false, - "Whether to read the data from Manhattan or Strato") - - val manhattanAppId = - flag("frs_readonly.appId", "ml_features_athena", "RO App Id used by the RO FRS service") - val manhattanDestName = flag( - "frs_readonly.destName", - "/s/manhattan/athena.native-thrift", - "manhattan Dest Name used by the RO FRS service") - - @Provides - @Singleton - def providesAthenaManhattanClient( - serviceIdentifier: ServiceIdentifier - ): ManhattanKVEndpoint = { - val client = ManhattanKVClient( - manhattanAppId(), - manhattanDestName(), - ManhattanKVClientMtlsParams(serviceIdentifier) - ) - ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.Weak) - .build() - } - - val manhattanAuthorDataset = "timelines_author_features" - private val defaultCacheMaxKeys = 60000 - private val cacheTTL = 12.hours - private val earlyExpiration = 0.2 - - val authorKeyDesc = KeyDescriptor(Component(LongInjection), Component0) - val authorDatasetKey = authorKeyDesc.withDataset(manhattanAuthorDataset) - val authorValDesc = ValueDescriptor(BinaryCompactScalaInjection(AuthorFeatures)) - - @Provides - @Singleton - def timelinesAuthorStitchCache( - manhattanReadOnlyEndpoint: ManhattanKVEndpoint, - timelinesAuthorFeaturesColumn: TimelinesAuthorFeaturesOnUserClientColumn, - stats: StatsReceiver - ): StitchCache[JLong, Option[AuthorFeatures]] = { - - val stitchCacheStats = - stats - .scope("direct_ds_source_feature_hydration_module").scope("timelines_author") - - val stStat = stitchCacheStats.counter("readFromStrato-each") - val mhtStat = stitchCacheStats.counter("readFromManhattan-each") - - val timelinesAuthorUnderlyingCall = if (readFromManhattan()) { - stitchCacheStats.counter("readFromManhattan").incr() - val authorCacheUnderlyingManhattanCall: JLong => Stitch[Option[AuthorFeatures]] = id => { - mhtStat.incr() - val key = authorDatasetKey.withPkey(id) - manhattanReadOnlyEndpoint - .get(key = key, valueDesc = authorValDesc).map(_.map(value => - clearUnsedFieldsForAuthorFeature(value.contents))) - } - authorCacheUnderlyingManhattanCall - } else { - stitchCacheStats.counter("readFromStrato").incr() - val authorCacheUnderlyingStratoCall: JLong => Stitch[Option[AuthorFeatures]] = id => { - stStat.incr() - val timelinesAuthorFeaturesFetcher = timelinesAuthorFeaturesColumn.fetcher - timelinesAuthorFeaturesFetcher - .fetch(id).map(result => result.v.map(clearUnsedFieldsForAuthorFeature)) - } - authorCacheUnderlyingStratoCall - } - - StitchCache[JLong, Option[AuthorFeatures]]( - underlyingCall = timelinesAuthorUnderlyingCall, - maxCacheSize = defaultCacheMaxKeys, - ttl = randomizedTTL(cacheTTL.inSeconds).seconds, - statsReceiver = stitchCacheStats - ) - - } - - // Not adding manhattan since it didn't seem useful for Author Data, we can add in another phab - // if deemed helpful - @Provides - @Singleton - def metricCenterUserCountingStitchCache( - mcUserCountingFeaturesColumn: McUserCountingOnUserClientColumn, - stats: StatsReceiver - ): StitchCache[JLong, Option[MCUserCountingFeatures]] = { - - val stitchCacheStats = - stats - .scope("direct_ds_source_feature_hydration_module").scope("mc_user_counting") - - val stStat = stitchCacheStats.counter("readFromStrato-each") - stitchCacheStats.counter("readFromStrato").incr() - - val mcUserCountingCacheUnderlyingCall: JLong => Stitch[Option[MCUserCountingFeatures]] = id => { - stStat.incr() - val mcUserCountingFeaturesFetcher = mcUserCountingFeaturesColumn.fetcher - mcUserCountingFeaturesFetcher.fetch(id).map(_.v) - } - - StitchCache[JLong, Option[MCUserCountingFeatures]]( - underlyingCall = mcUserCountingCacheUnderlyingCall, - maxCacheSize = defaultCacheMaxKeys, - ttl = randomizedTTL(cacheTTL.inSeconds).seconds, - statsReceiver = stitchCacheStats - ) - - } - - // clear out fields we don't need to save cache space - private def clearUnsedFieldsForAuthorFeature(entry: AuthorFeatures): AuthorFeatures = { - entry.unsetUserTopics.unsetUserHealth.unsetAuthorCountryCodeAggregates.unsetOriginalAuthorCountryCodeAggregates - } - - // To avoid a cache stampede. See https://en.wikipedia.org/wiki/Cache_stampede - private def randomizedTTL(ttl: Long): Long = { - (ttl - ttl * earlyExpiration * Random.nextDouble()).toLong - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.docx new file mode 100644 index 000000000..edc5142ee Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.scala deleted file mode 100644 index 51975c487..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Inject -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.feature_hydration.adapters.PreFetchedFeatureAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -@Provides -@Singleton -class PreFetchedFeatureSource @Inject() () extends FeatureSource { - override def id: FeatureSourceId = FeatureSourceId.PreFetchedFeatureSourceId - override def featureContext: FeatureContext = PreFetchedFeatureAdapter.getFeatureContext - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - Stitch.value(candidates.map { candidate => - candidate -> PreFetchedFeatureAdapter.adaptToDataRecord((target, candidate)) - }.toMap) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.docx new file mode 100644 index 000000000..3cbedef19 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.scala deleted file mode 100644 index 155d9e442..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Inject -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This source wraps around the separate sources that we hydrate features from - * @param featureStoreSource gets features that require a RPC call to feature store - * @param stratoFeatureHydrationSource gets features that require a RPC call to strato columns - * @param clientContextSource gets features that are already present in the request context - * @param candidateAlgorithmSource gets features that are already present from candidate generation - * @param preFetchedFeatureSource gets features that were prehydrated (shared in request lifecycle) - */ -@Provides -@Singleton -class UserScoringFeatureSource @Inject() ( - featureStoreSource: FeatureStoreSource, - featureStoreGizmoduckSource: FeatureStoreGizmoduckSource, - featureStorePostNuxAlgorithmSource: FeatureStorePostNuxAlgorithmSource, - featureStoreTimelinesAuthorSource: FeatureStoreTimelinesAuthorSource, - featureStoreUserMetricCountsSource: FeatureStoreUserMetricCountsSource, - clientContextSource: ClientContextSource, - candidateAlgorithmSource: CandidateAlgorithmSource, - preFetchedFeatureSource: PreFetchedFeatureSource) - extends FeatureSource { - - override val id: FeatureSourceId = FeatureSourceId.UserScoringFeatureSourceId - - override val featureContext: FeatureContext = FeatureContext.merge( - featureStoreSource.featureContext, - featureStoreGizmoduckSource.featureContext, - featureStorePostNuxAlgorithmSource.featureContext, - featureStoreTimelinesAuthorSource.featureContext, - featureStoreUserMetricCountsSource.featureContext, - clientContextSource.featureContext, - candidateAlgorithmSource.featureContext, - preFetchedFeatureSource.featureContext, - ) - - val sources = - Seq( - featureStoreSource, - featureStorePostNuxAlgorithmSource, - featureStoreTimelinesAuthorSource, - featureStoreUserMetricCountsSource, - featureStoreGizmoduckSource, - clientContextSource, - candidateAlgorithmSource, - preFetchedFeatureSource - ) - - val dataRecordMerger = new DataRecordMerger - - def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - Stitch.collect(sources.map(_.hydrateFeatures(target, candidates))).map { featureMaps => - (for { - candidate <- candidates - } yield { - val combinedDataRecord = new DataRecord - featureMaps - .flatMap(_.get(candidate).toSeq).foreach(dataRecordMerger.merge(combinedDataRecord, _)) - candidate -> combinedDataRecord - }).toMap - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.docx new file mode 100644 index 000000000..239bd1c82 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.scala deleted file mode 100644 index 99bd71310..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.IRecordOneToOneAdapter -import scala.util.Random - -/** - * Helper functions for FeatureStoreSource operations in FRS are available here. - */ -object Utils { - - private val EarlyExpiration = 0.2 - - private[common] def adaptAdditionalFeaturesToDataRecord( - record: DataRecord, - adapterStats: StatsReceiver, - featureAdapters: Seq[IRecordOneToOneAdapter[DataRecord]] - ): DataRecord = { - featureAdapters.foldRight(record) { (adapter, record) => - adapterStats.counter(adapter.getClass.getSimpleName).incr() - adapter.adaptToDataRecord(record) - } - } - - // To avoid a cache stampede. See https://en.wikipedia.org/wiki/Cache_stampede - private[common] def randomizedTTL(ttl: Long): Long = { - (ttl - ttl * EarlyExpiration * Random.nextDouble()).toLong - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD deleted file mode 100644 index b5ece498a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD.docx new file mode 100644 index 000000000..0c5b7ae88 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.docx new file mode 100644 index 000000000..2a9fdda87 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.scala deleted file mode 100644 index 5325c6b56..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.features - -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object LocationFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, Option[GeohashAndCountryCode]] { - override val defaultValue: Option[GeohashAndCountryCode] = None -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.docx new file mode 100644 index 000000000..3cb3fb704 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.scala deleted file mode 100644 index 23571c8df..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.features - -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object TrackingTokenFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Int]] { - override val defaultValue: Option[Int] = None -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.docx new file mode 100644 index 000000000..9e1b4e3b2 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.scala deleted file mode 100644 index 73072b295..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.features - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object UserStateFeature extends Feature[PipelineQuery, Option[UserState]] {} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.docx new file mode 100644 index 000000000..7dd45ac78 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.scala deleted file mode 100644 index 303417d23..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -/** - * contains information if a candidate is from a candidate source generated using the following signals. - */ -case class AddressBookMetadata( - inForwardPhoneBook: Boolean, - inReversePhoneBook: Boolean, - inForwardEmailBook: Boolean, - inReverseEmailBook: Boolean) - -object AddressBookMetadata { - - val ForwardPhoneBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ForwardPhoneBook.toString) - - val ReversePhoneBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ReversePhoneBook.toString) - - val ForwardEmailBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ForwardEmailBook.toString) - - val ReverseEmailBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ReverseEmailBookIbis.toString) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.docx new file mode 100644 index 000000000..9fad8e460 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.scala deleted file mode 100644 index b60afb8b3..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -/** - * Each candidate source algorithm could be based on one, or more, of the 4 general type of - * information we have on a user: - * 1. Social: the user's connections in Twitter's social graph. - * 2. Geo: the user's geographical information. - * 3. Interest: information on the user's chosen interests. - * 4. Activity: information on the user's past activity. - * - * Note that an algorithm can fall under more than one of these categories. - */ -object AlgorithmType extends Enumeration { - type AlgorithmType = Value - - val Social: Value = Value("social") - val Geo: Value = Value("geo") - val Activity: Value = Value("activity") - val Interest: Value = Value("interest") -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD deleted file mode 100644 index c4916b6d0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "hermit/hermit-ml/src/main/scala/com/twitter/hermit/ml/models", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "scrooge/scrooge-serializer/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/wtf/scalding/jobs/strong_tie_prediction", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "src/thrift/com/twitter/timelines/author_features/user_health:thrift-scala", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - ], - exports = [ - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD.docx new file mode 100644 index 000000000..ddab8f1a0 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.docx new file mode 100644 index 000000000..650402a31 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.scala deleted file mode 100644 index 178f34b30..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.scala +++ /dev/null @@ -1,192 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.hermit.constants.AlgorithmFeedbackTokens -import com.twitter.ml.api.thriftscala.{DataRecord => TDataRecord} -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -trait FollowableEntity extends UniversalNoun[Long] - -trait Recommendation - extends FollowableEntity - with HasReason - with HasAdMetadata - with HasTrackingToken { - val score: Option[Double] - - def toThrift: t.Recommendation - - def toOfflineThrift: offline.OfflineRecommendation -} - -case class CandidateUser( - override val id: Long, - override val score: Option[Double] = None, - override val reason: Option[Reason] = None, - override val userCandidateSourceDetails: Option[UserCandidateSourceDetails] = None, - override val adMetadata: Option[AdMetadata] = None, - override val trackingToken: Option[TrackingToken] = None, - override val dataRecord: Option[RichDataRecord] = None, - override val scores: Option[Scores] = None, - override val infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] = None, - override val params: Params = Params.Invalid, - override val engagements: Seq[EngagementType] = Nil, - override val recommendationFlowIdentifier: Option[String] = None) - extends Recommendation - with HasUserCandidateSourceDetails - with HasDataRecord - with HasScores - with HasParams - with HasEngagements - with HasRecommendationFlowIdentifier - with HasInfoPerRankingStage { - - val rankerIdsStr: Option[Seq[String]] = { - val strs = scores.map(_.scores.flatMap(_.rankerId.map(_.toString))) - if (strs.exists(_.nonEmpty)) strs else None - } - - val thriftDataRecord: Option[TDataRecord] = for { - richDataRecord <- dataRecord - dr <- richDataRecord.dataRecord - } yield { - ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(dr) - } - - val toOfflineUserThrift: offline.OfflineUserRecommendation = { - val scoringDetails = - if (userCandidateSourceDetails.isEmpty && score.isEmpty && thriftDataRecord.isEmpty) { - None - } else { - Some( - offline.ScoringDetails( - candidateSourceDetails = userCandidateSourceDetails.map(_.toOfflineThrift), - score = score, - dataRecord = thriftDataRecord, - rankerIds = rankerIdsStr, - infoPerRankingStage = infoPerRankingStage.map(_.mapValues(_.toOfflineThrift)) - ) - ) - } - offline - .OfflineUserRecommendation( - id, - reason.map(_.toOfflineThrift), - adMetadata.map(_.adImpression), - trackingToken.map(_.toOfflineThrift), - scoringDetails = scoringDetails - ) - } - - override val toOfflineThrift: offline.OfflineRecommendation = - offline.OfflineRecommendation.User(toOfflineUserThrift) - - val toUserThrift: t.UserRecommendation = { - val scoringDetails = - if (userCandidateSourceDetails.isEmpty && score.isEmpty && thriftDataRecord.isEmpty && scores.isEmpty) { - None - } else { - Some( - t.ScoringDetails( - candidateSourceDetails = userCandidateSourceDetails.map(_.toThrift), - score = score, - dataRecord = thriftDataRecord, - rankerIds = rankerIdsStr, - debugDataRecord = dataRecord.flatMap(_.debugDataRecord), - infoPerRankingStage = infoPerRankingStage.map(_.mapValues(_.toThrift)) - ) - ) - } - t.UserRecommendation( - userId = id, - reason = reason.map(_.toThrift), - adImpression = adMetadata.map(_.adImpression), - trackingInfo = trackingToken.map(TrackingToken.serialize), - scoringDetails = scoringDetails, - recommendationFlowIdentifier = recommendationFlowIdentifier - ) - } - - override val toThrift: t.Recommendation = - t.Recommendation.User(toUserThrift) - - def setFollowProof(followProofOpt: Option[FollowProof]): CandidateUser = { - this.copy( - reason = reason - .map { reason => - reason.copy( - accountProof = reason.accountProof - .map { accountProof => - accountProof.copy(followProof = followProofOpt) - }.orElse(Some(AccountProof(followProof = followProofOpt))) - ) - }.orElse(Some(Reason(Some(AccountProof(followProof = followProofOpt))))) - ) - } - - def addScore(score: Score): CandidateUser = { - val newScores = scores match { - case Some(existingScores) => existingScores.copy(scores = existingScores.scores :+ score) - case None => Scores(Seq(score)) - } - this.copy(scores = Some(newScores)) - } -} - -object CandidateUser { - val DefaultCandidateScore = 1.0 - - // for converting candidate in ScoringUserRequest - def fromUserRecommendation(candidate: t.UserRecommendation): CandidateUser = { - // we only use the primary candidate source for now - val userCandidateSourceDetails = for { - scoringDetails <- candidate.scoringDetails - candidateSourceDetails <- scoringDetails.candidateSourceDetails - } yield UserCandidateSourceDetails( - primaryCandidateSource = candidateSourceDetails.primarySource - .flatMap(AlgorithmFeedbackTokens.TokenToAlgorithmMap.get).map { algo => - CandidateSourceIdentifier(algo.toString) - }, - candidateSourceScores = fromThriftScoreMap(candidateSourceDetails.candidateSourceScores), - candidateSourceRanks = fromThriftRankMap(candidateSourceDetails.candidateSourceRanks), - addressBookMetadata = None - ) - CandidateUser( - id = candidate.userId, - score = candidate.scoringDetails.flatMap(_.score), - reason = candidate.reason.map(Reason.fromThrift), - userCandidateSourceDetails = userCandidateSourceDetails, - trackingToken = candidate.trackingInfo.map(TrackingToken.deserialize), - recommendationFlowIdentifier = candidate.recommendationFlowIdentifier, - infoPerRankingStage = candidate.scoringDetails.flatMap( - _.infoPerRankingStage.map(_.mapValues(RankingInfo.fromThrift))) - ) - } - - def fromThriftScoreMap( - thriftMapOpt: Option[scala.collection.Map[String, Double]] - ): Map[CandidateSourceIdentifier, Option[Double]] = { - (for { - thriftMap <- thriftMapOpt.toSeq - (algoName, score) <- thriftMap.toSeq - } yield { - CandidateSourceIdentifier(algoName) -> Some(score) - }).toMap - } - - def fromThriftRankMap( - thriftMapOpt: Option[scala.collection.Map[String, Int]] - ): Map[CandidateSourceIdentifier, Int] = { - (for { - thriftMap <- thriftMapOpt.toSeq - (algoName, rank) <- thriftMap.toSeq - } yield { - CandidateSourceIdentifier(algoName) -> rank - }).toMap - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.docx new file mode 100644 index 000000000..4c41bd4af Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.scala deleted file mode 100644 index ac601371c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => frs} -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext - -object ClientContextConverter { - def toFRSOfflineClientContextThrift( - productMixerClientContext: ClientContext - ): offline.OfflineClientContext = - offline.OfflineClientContext( - productMixerClientContext.userId, - productMixerClientContext.guestId, - productMixerClientContext.appId, - productMixerClientContext.countryCode, - productMixerClientContext.languageCode, - productMixerClientContext.guestIdAds, - productMixerClientContext.guestIdMarketing - ) - - def fromThrift(clientContext: frs.ClientContext): ClientContext = ClientContext( - userId = clientContext.userId, - guestId = clientContext.guestId, - appId = clientContext.appId, - ipAddress = clientContext.ipAddress, - userAgent = clientContext.userAgent, - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - isTwoffice = clientContext.isTwoffice, - userRoles = clientContext.userRoles.map(_.toSet), - deviceId = clientContext.deviceId, - guestIdAds = clientContext.guestIdAds, - guestIdMarketing = clientContext.guestIdMarketing, - mobileDeviceId = None, - mobileDeviceAdId = None, - limitAdTracking = None - ) - - def toThrift(clientContext: ClientContext): frs.ClientContext = frs.ClientContext( - userId = clientContext.userId, - guestId = clientContext.guestIdAds, - appId = clientContext.appId, - ipAddress = clientContext.ipAddress, - userAgent = clientContext.userAgent, - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - isTwoffice = clientContext.isTwoffice, - userRoles = clientContext.userRoles, - deviceId = clientContext.deviceId, - guestIdAds = clientContext.guestIdAds, - guestIdMarketing = clientContext.guestIdMarketing - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.docx new file mode 100644 index 000000000..e1e3c316e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala deleted file mode 100644 index b49baf034..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala +++ /dev/null @@ -1,420 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.adserver.thriftscala.{DisplayLocation => AdDisplayLocation} -import com.twitter.follow_recommendations.logging.thriftscala.{ - OfflineDisplayLocation => TOfflineDisplayLocation -} -import com.twitter.follow_recommendations.thriftscala.{DisplayLocation => TDisplayLocation} - -sealed trait DisplayLocation { - def toThrift: TDisplayLocation - - def toOfflineThrift: TOfflineDisplayLocation - - def toFsName: String - - // corresponding display location in adserver if available - // make sure to be consistent with the definition here - def toAdDisplayLocation: Option[AdDisplayLocation] = None -} - -/** - * Make sure you add the new DL to the following files and redeploy our attribution jobs - * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift - * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift - * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala - */ - -object DisplayLocation { - - case object ProfileSidebar extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileSidebar - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ProfileSidebar - override val toFsName: String = "ProfileSidebar" - - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - AdDisplayLocation.ProfileAccountsSidebar - ) - } - - case object HomeTimeline extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimeline - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HomeTimeline - override val toFsName: String = "HomeTimeline" - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - // it is based on the logic that HTL DL should correspond to Sidebar: - AdDisplayLocation.WtfSidebar - ) - } - - case object ReactiveFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ReactiveFollow - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ReactiveFollow - override val toFsName: String = "ReactiveFollow" - } - - case object ExploreTab extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ExploreTab - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ExploreTab - override val toFsName: String = "ExploreTab" - } - - case object MagicRecs extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.MagicRecs - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.MagicRecs - override val toFsName: String = "MagicRecs" - } - - case object AbUploadInjection extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.AbUploadInjection - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.AbUploadInjection - override val toFsName: String = "AbUploadInjection" - } - - case object RuxLandingPage extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.RuxLandingPage - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.RuxLandingPage - override val toFsName: String = "RuxLandingPage" - } - - case object ProfileBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileBonusFollow - override val toFsName: String = "ProfileBonusFollow" - } - - case object ElectionExploreWtf extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ElectionExploreWtf - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ElectionExploreWtf - override val toFsName: String = "ElectionExploreWtf" - } - - case object ClusterFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ClusterFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ClusterFollow - override val toFsName: String = "ClusterFollow" - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - AdDisplayLocation.ClusterFollow - ) - } - - case object HtlBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HtlBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HtlBonusFollow - override val toFsName: String = "HtlBonusFollow" - } - - case object TopicLandingPageHeader extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TopicLandingPageHeader - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TopicLandingPageHeader - override val toFsName: String = "TopicLandingPageHeader" - } - - case object NewUserSarusBackfill extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NewUserSarusBackfill - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NewUserSarusBackfill - override val toFsName: String = "NewUserSarusBackfill" - } - - case object NuxPymk extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxPymk - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxPymk - override val toFsName: String = "NuxPymk" - } - - case object NuxInterests extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxInterests - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxInterests - override val toFsName: String = "NuxInterests" - } - - case object NuxTopicBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxTopicBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxTopicBonusFollow - override val toFsName: String = "NuxTopicBonusFollow" - } - - case object Sidebar extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.Sidebar - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.Sidebar - override val toFsName: String = "Sidebar" - - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - AdDisplayLocation.WtfSidebar - ) - } - - case object CampaignForm extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.CampaignForm - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.CampaignForm - override val toFsName: String = "CampaignForm" - } - - case object ProfileTopFollowers extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileTopFollowers - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileTopFollowers - override val toFsName: String = "ProfileTopFollowers" - } - - case object ProfileTopFollowing extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileTopFollowing - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileTopFollowing - override val toFsName: String = "ProfileTopFollowing" - } - - case object RuxPymk extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.RuxPymk - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.RuxPymk - override val toFsName: String = "RuxPymk" - } - - case object IndiaCovid19CuratedAccountsWtf extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.IndiaCovid19CuratedAccountsWtf - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.IndiaCovid19CuratedAccountsWtf - override val toFsName: String = "IndiaCovid19CuratedAccountsWtf" - } - - case object PeoplePlusPlus extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.PeoplePlusPlus - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.PeoplePlusPlus - override val toFsName: String = "PeoplePlusPlus" - } - - case object TweetNotificationRecs extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TweetNotificationRecs - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TweetNotificationRecs - override val toFsName: String = "TweetNotificationRecs" - } - - case object ProfileDeviceFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileDeviceFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileDeviceFollow - override val toFsName: String = "ProfileDeviceFollow" - } - - case object RecosBackfill extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.RecosBackfill - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.RecosBackfill - override val toFsName: String = "RecosBackfill" - } - - case object HtlSpaceHosts extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HtlSpaceHosts - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HtlSpaceHosts - override val toFsName: String = "HtlSpaceHosts" - } - - case object PostNuxFollowTask extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.PostNuxFollowTask - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.PostNuxFollowTask - override val toFsName: String = "PostNuxFollowTask" - } - - case object TopicLandingPage extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TopicLandingPage - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TopicLandingPage - override val toFsName: String = "TopicLandingPage" - } - - case object UserTypeaheadPrefetch extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.UserTypeaheadPrefetch - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.UserTypeaheadPrefetch - override val toFsName: String = "UserTypeaheadPrefetch" - } - - case object HomeTimelineRelatableAccounts extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineRelatableAccounts - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HomeTimelineRelatableAccounts - override val toFsName: String = "HomeTimelineRelatableAccounts" - } - - case object NuxGeoCategory extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxGeoCategory - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxGeoCategory - override val toFsName: String = "NuxGeoCategory" - } - - case object NuxInterestsCategory extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxInterestsCategory - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxInterestsCategory - override val toFsName: String = "NuxInterestsCategory" - } - - case object TopArticles extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TopArticles - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TopArticles - override val toFsName: String = "TopArticles" - } - - case object NuxPymkCategory extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxPymkCategory - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxPymkCategory - override val toFsName: String = "NuxPymkCategory" - } - - case object HomeTimelineTweetRecs extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineTweetRecs - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HomeTimelineTweetRecs - override val toFsName: String = "HomeTimelineTweetRecs" - } - - case object HtlBulkFriendFollows extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HtlBulkFriendFollows - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HtlBulkFriendFollows - override val toFsName: String = "HtlBulkFriendFollows" - } - - case object NuxAutoFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxAutoFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxAutoFollow - override val toFsName: String = "NuxAutoFollow" - } - - case object SearchBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.SearchBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.SearchBonusFollow - override val toFsName: String = "SearchBonusFollow" - } - - case object ContentRecommender extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ContentRecommender - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ContentRecommender - override val toFsName: String = "ContentRecommender" - } - - case object HomeTimelineReverseChron extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineReverseChron - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HomeTimelineReverseChron - override val toFsName: String = "HomeTimelineReverseChron" - } - - def fromThrift(displayLocation: TDisplayLocation): DisplayLocation = displayLocation match { - case TDisplayLocation.ProfileSidebar => ProfileSidebar - case TDisplayLocation.HomeTimeline => HomeTimeline - case TDisplayLocation.MagicRecs => MagicRecs - case TDisplayLocation.AbUploadInjection => AbUploadInjection - case TDisplayLocation.RuxLandingPage => RuxLandingPage - case TDisplayLocation.ProfileBonusFollow => ProfileBonusFollow - case TDisplayLocation.ElectionExploreWtf => ElectionExploreWtf - case TDisplayLocation.ClusterFollow => ClusterFollow - case TDisplayLocation.HtlBonusFollow => HtlBonusFollow - case TDisplayLocation.ReactiveFollow => ReactiveFollow - case TDisplayLocation.TopicLandingPageHeader => TopicLandingPageHeader - case TDisplayLocation.NewUserSarusBackfill => NewUserSarusBackfill - case TDisplayLocation.NuxPymk => NuxPymk - case TDisplayLocation.NuxInterests => NuxInterests - case TDisplayLocation.NuxTopicBonusFollow => NuxTopicBonusFollow - case TDisplayLocation.ExploreTab => ExploreTab - case TDisplayLocation.Sidebar => Sidebar - case TDisplayLocation.CampaignForm => CampaignForm - case TDisplayLocation.ProfileTopFollowers => ProfileTopFollowers - case TDisplayLocation.ProfileTopFollowing => ProfileTopFollowing - case TDisplayLocation.RuxPymk => RuxPymk - case TDisplayLocation.IndiaCovid19CuratedAccountsWtf => IndiaCovid19CuratedAccountsWtf - case TDisplayLocation.PeoplePlusPlus => PeoplePlusPlus - case TDisplayLocation.TweetNotificationRecs => TweetNotificationRecs - case TDisplayLocation.ProfileDeviceFollow => ProfileDeviceFollow - case TDisplayLocation.RecosBackfill => RecosBackfill - case TDisplayLocation.HtlSpaceHosts => HtlSpaceHosts - case TDisplayLocation.PostNuxFollowTask => PostNuxFollowTask - case TDisplayLocation.TopicLandingPage => TopicLandingPage - case TDisplayLocation.UserTypeaheadPrefetch => UserTypeaheadPrefetch - case TDisplayLocation.HomeTimelineRelatableAccounts => HomeTimelineRelatableAccounts - case TDisplayLocation.NuxGeoCategory => NuxGeoCategory - case TDisplayLocation.NuxInterestsCategory => NuxInterestsCategory - case TDisplayLocation.TopArticles => TopArticles - case TDisplayLocation.NuxPymkCategory => NuxPymkCategory - case TDisplayLocation.HomeTimelineTweetRecs => HomeTimelineTweetRecs - case TDisplayLocation.HtlBulkFriendFollows => HtlBulkFriendFollows - case TDisplayLocation.NuxAutoFollow => NuxAutoFollow - case TDisplayLocation.SearchBonusFollow => SearchBonusFollow - case TDisplayLocation.ContentRecommender => ContentRecommender - case TDisplayLocation.HomeTimelineReverseChron => HomeTimelineReverseChron - case TDisplayLocation.EnumUnknownDisplayLocation(i) => - throw new UnknownDisplayLocationException( - s"Unknown display location thrift enum with value: ${i}") - } - - def fromOfflineThrift(displayLocation: TOfflineDisplayLocation): DisplayLocation = - displayLocation match { - case TOfflineDisplayLocation.ProfileSidebar => ProfileSidebar - case TOfflineDisplayLocation.HomeTimeline => HomeTimeline - case TOfflineDisplayLocation.MagicRecs => MagicRecs - case TOfflineDisplayLocation.AbUploadInjection => AbUploadInjection - case TOfflineDisplayLocation.RuxLandingPage => RuxLandingPage - case TOfflineDisplayLocation.ProfileBonusFollow => ProfileBonusFollow - case TOfflineDisplayLocation.ElectionExploreWtf => ElectionExploreWtf - case TOfflineDisplayLocation.ClusterFollow => ClusterFollow - case TOfflineDisplayLocation.HtlBonusFollow => HtlBonusFollow - case TOfflineDisplayLocation.TopicLandingPageHeader => TopicLandingPageHeader - case TOfflineDisplayLocation.NewUserSarusBackfill => NewUserSarusBackfill - case TOfflineDisplayLocation.NuxPymk => NuxPymk - case TOfflineDisplayLocation.NuxInterests => NuxInterests - case TOfflineDisplayLocation.NuxTopicBonusFollow => NuxTopicBonusFollow - case TOfflineDisplayLocation.ExploreTab => ExploreTab - case TOfflineDisplayLocation.ReactiveFollow => ReactiveFollow - case TOfflineDisplayLocation.Sidebar => Sidebar - case TOfflineDisplayLocation.CampaignForm => CampaignForm - case TOfflineDisplayLocation.ProfileTopFollowers => ProfileTopFollowers - case TOfflineDisplayLocation.ProfileTopFollowing => ProfileTopFollowing - case TOfflineDisplayLocation.RuxPymk => RuxPymk - case TOfflineDisplayLocation.IndiaCovid19CuratedAccountsWtf => IndiaCovid19CuratedAccountsWtf - case TOfflineDisplayLocation.PeoplePlusPlus => PeoplePlusPlus - case TOfflineDisplayLocation.TweetNotificationRecs => TweetNotificationRecs - case TOfflineDisplayLocation.ProfileDeviceFollow => ProfileDeviceFollow - case TOfflineDisplayLocation.RecosBackfill => RecosBackfill - case TOfflineDisplayLocation.HtlSpaceHosts => HtlSpaceHosts - case TOfflineDisplayLocation.PostNuxFollowTask => PostNuxFollowTask - case TOfflineDisplayLocation.TopicLandingPage => TopicLandingPage - case TOfflineDisplayLocation.UserTypeaheadPrefetch => UserTypeaheadPrefetch - case TOfflineDisplayLocation.HomeTimelineRelatableAccounts => HomeTimelineRelatableAccounts - case TOfflineDisplayLocation.NuxGeoCategory => NuxGeoCategory - case TOfflineDisplayLocation.NuxInterestsCategory => NuxInterestsCategory - case TOfflineDisplayLocation.TopArticles => TopArticles - case TOfflineDisplayLocation.NuxPymkCategory => NuxPymkCategory - case TOfflineDisplayLocation.HomeTimelineTweetRecs => HomeTimelineTweetRecs - case TOfflineDisplayLocation.HtlBulkFriendFollows => HtlBulkFriendFollows - case TOfflineDisplayLocation.NuxAutoFollow => NuxAutoFollow - case TOfflineDisplayLocation.SearchBonusFollow => SearchBonusFollow - case TOfflineDisplayLocation.ContentRecommender => ContentRecommender - case TOfflineDisplayLocation.HomeTimelineReverseChron => HomeTimelineReverseChron - case TOfflineDisplayLocation.EnumUnknownOfflineDisplayLocation(i) => - throw new UnknownDisplayLocationException( - s"Unknown offline display location thrift enum with value: ${i}") - } -} - -class UnknownDisplayLocationException(message: String) extends Exception(message) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.docx new file mode 100644 index 000000000..58995ed2d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.scala deleted file mode 100644 index b12a4404c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.thriftscala.{EngagementType => TEngagementType} -import com.twitter.follow_recommendations.logging.thriftscala.{ - EngagementType => OfflineEngagementType -} -sealed trait EngagementType { - def toThrift: TEngagementType - def toOfflineThrift: OfflineEngagementType -} - -object EngagementType { - object Click extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Click - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Click - } - object Like extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Like - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Like - } - object Mention extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Mention - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Mention - } - object Retweet extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Retweet - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Retweet - } - object ProfileView extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.ProfileView - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.ProfileView - } - - def fromThrift(engagementType: TEngagementType): EngagementType = engagementType match { - case TEngagementType.Click => Click - case TEngagementType.Like => Like - case TEngagementType.Mention => Mention - case TEngagementType.Retweet => Retweet - case TEngagementType.ProfileView => ProfileView - case TEngagementType.EnumUnknownEngagementType(i) => - throw new UnknownEngagementTypeException( - s"Unknown engagement type thrift enum with value: ${i}") - } - - def fromOfflineThrift(engagementType: OfflineEngagementType): EngagementType = - engagementType match { - case OfflineEngagementType.Click => Click - case OfflineEngagementType.Like => Like - case OfflineEngagementType.Mention => Mention - case OfflineEngagementType.Retweet => Retweet - case OfflineEngagementType.ProfileView => ProfileView - case OfflineEngagementType.EnumUnknownEngagementType(i) => - throw new UnknownEngagementTypeException( - s"Unknown engagement type offline thrift enum with value: ${i}") - } -} -class UnknownEngagementTypeException(message: String) extends Exception(message) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.docx new file mode 100644 index 000000000..275f54be6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.scala deleted file mode 100644 index 86b496776..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -sealed trait FilterReason { - def reason: String -} - -object FilterReason { - - case object NoReason extends FilterReason { - override val reason: String = "no_reason" - } - - case class ParamReason(paramName: String) extends FilterReason { - override val reason: String = s"param_$paramName" - } - - case object ExcludedId extends FilterReason { - override val reason: String = "excluded_id_from_request" - } - - case object ProfileSidebarBlacklist extends FilterReason { - override val reason: String = "profile_sidebar_blacklisted_id" - } - - case object CuratedAccountsCompetitorList extends FilterReason { - override val reason: String = "curated_blacklisted_id" - } - - case class InvalidRelationshipTypes(relationshipTypes: String) extends FilterReason { - override val reason: String = s"invalid_relationship_types $relationshipTypes" - } - - case object ProfileId extends FilterReason { - override val reason: String = "candidate_has_same_id_as_profile" - } - - case object DismissedId extends FilterReason { - override val reason: String = s"dismissed_candidate" - } - - case object OptedOutId extends FilterReason { - override val reason: String = s"candidate_opted_out_from_criteria_in_request" - } - - // gizmoduck predicates - case object NoUser extends FilterReason { - override val reason: String = "no_user_result_from_gizmoduck" - } - - case object AddressBookUndiscoverable extends FilterReason { - override val reason: String = "not_discoverable_via_address_book" - } - - case object PhoneBookUndiscoverable extends FilterReason { - override val reason: String = "not_discoverable_via_phone_book" - } - - case object Deactivated extends FilterReason { - override val reason: String = "deactivated" - } - - case object Suspended extends FilterReason { - override val reason: String = "suspended" - } - - case object Restricted extends FilterReason { - override val reason: String = "restricted" - } - - case object NsfwUser extends FilterReason { - override val reason: String = "nsfwUser" - } - - case object NsfwAdmin extends FilterReason { - override val reason: String = "nsfwAdmin" - } - - case object HssSignal extends FilterReason { - override val reason: String = "hssSignal" - } - - case object IsProtected extends FilterReason { - override val reason: String = "isProtected" - } - - case class CountryTakedown(countryCode: String) extends FilterReason { - override val reason: String = s"takedown_in_$countryCode" - } - - case object Blink extends FilterReason { - override val reason: String = "blink" - } - - case object AlreadyFollowed extends FilterReason { - override val reason: String = "already_followed" - } - - case object InvalidRelationship extends FilterReason { - override val reason: String = "invalid_relationship" - } - - case object NotFollowingTargetUser extends FilterReason { - override val reason: String = "not_following_target_user" - } - - case object CandidateSideHoldback extends FilterReason { - override val reason: String = "candidate_side_holdback" - } - - case object Inactive extends FilterReason { - override val reason: String = "inactive" - } - - case object MissingRecommendabilityData extends FilterReason { - override val reason: String = "missing_recommendability_data" - } - - case object HighTweetVelocity extends FilterReason { - override val reason: String = "high_tweet_velocity" - } - - case object AlreadyRecommended extends FilterReason { - override val reason: String = "already_recommended" - } - - case object MinStateNotMet extends FilterReason { - override val reason: String = "min_state_user_not_met" - } - - case object FailOpen extends FilterReason { - override val reason: String = "fail_open" - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.docx new file mode 100644 index 000000000..9e76906f3 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.scala deleted file mode 100644 index 15e36321e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class FlowContext(steps: Seq[RecommendationStep]) { - - def toThrift: t.FlowContext = t.FlowContext(steps = steps.map(_.toThrift)) - - def toOfflineThrift: offline.OfflineFlowContext = - offline.OfflineFlowContext(steps = steps.map(_.toOfflineThrift)) -} - -object FlowContext { - - def fromThrift(flowContext: t.FlowContext): FlowContext = { - FlowContext(steps = flowContext.steps.map(RecommendationStep.fromThrift)) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.docx new file mode 100644 index 000000000..7b85c171b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.scala deleted file mode 100644 index 118ff258d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class FlowRecommendation(userId: Long) { - - def toThrift: t.FlowRecommendation = - t.FlowRecommendation(userId = userId) - - def toOfflineThrift: offline.OfflineFlowRecommendation = - offline.OfflineFlowRecommendation(userId = userId) - -} - -object FlowRecommendation { - def fromThrift(flowRecommendation: t.FlowRecommendation): FlowRecommendation = { - FlowRecommendation( - userId = flowRecommendation.userId - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.docx new file mode 100644 index 000000000..5eb9848f2 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.scala deleted file mode 100644 index 782d3fc9e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -case class GeohashAndCountryCode(geohash: Option[String], countryCode: Option[String]) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.docx new file mode 100644 index 000000000..9e6774e73 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.scala deleted file mode 100644 index 57979e376..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.adserver.{thriftscala => t} - -case class AdMetadata( - insertPosition: Int, - // use original ad impression info to avoid losing data in domain model translations - adImpression: t.AdImpression) - -trait HasAdMetadata { - - def adMetadata: Option[AdMetadata] - - def adImpression: Option[t.AdImpression] = { - adMetadata.map(_.adImpression) - } - - def insertPosition: Option[Int] = { - adMetadata.map(_.insertPosition) - } - - def isPromotedAccount: Boolean = adMetadata.isDefined -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.docx new file mode 100644 index 000000000..41d581d7b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.scala deleted file mode 100644 index d4cbdcee8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasByfSeedUserIds { - def byfSeedUserIds: Option[Seq[Long]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.docx new file mode 100644 index 000000000..04658c02b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.scala deleted file mode 100644 index 4e7047b4e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.thriftscala.DebugDataRecord -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.util.Try -import com.twitter.util.logging.Logging -import scala.collection.convert.ImplicitConversions._ - -// contains the standard dataRecord struct, and the debug version if required -case class RichDataRecord( - dataRecord: Option[DataRecord] = None, - debugDataRecord: Option[DebugDataRecord] = None, -) - -trait HasDataRecord extends Logging { - def dataRecord: Option[RichDataRecord] - - def toDebugDataRecord(dr: DataRecord, featureContext: FeatureContext): DebugDataRecord = { - - val binaryFeatures: Option[Set[String]] = if (dr.isSetBinaryFeatures) { - Some(dr.getBinaryFeatures.flatMap { id => - Try(featureContext.getFeature(id).getFeatureName).toOption - }.toSet) - } else None - - val continuousFeatures: Option[Map[String, Double]] = if (dr.isSetContinuousFeatures) { - Some(dr.getContinuousFeatures.flatMap { - case (id, value) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> value.toDouble - } - }.toMap) - } else None - - val discreteFeatures: Option[Map[String, Long]] = if (dr.isSetDiscreteFeatures) { - Some(dr.getDiscreteFeatures.flatMap { - case (id, value) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> value.toLong - } - }.toMap) - } else None - - val stringFeatures: Option[Map[String, String]] = if (dr.isSetStringFeatures) { - Some(dr.getStringFeatures.flatMap { - case (id, value) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> value - } - }.toMap) - } else None - - val sparseBinaryFeatures: Option[Map[String, Set[String]]] = if (dr.isSetSparseBinaryFeatures) { - Some(dr.getSparseBinaryFeatures.flatMap { - case (id, values) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> values.toSet - } - }.toMap) - } else None - - val sparseContinuousFeatures: Option[Map[String, Map[String, Double]]] = - if (dr.isSetSparseContinuousFeatures) { - Some(dr.getSparseContinuousFeatures.flatMap { - case (id, values) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> values.map { - case (str, value) => - str -> value.toDouble - }.toMap - } - }.toMap) - } else None - - DebugDataRecord( - binaryFeatures = binaryFeatures, - continuousFeatures = continuousFeatures, - discreteFeatures = discreteFeatures, - stringFeatures = stringFeatures, - sparseBinaryFeatures = sparseBinaryFeatures, - sparseContinuousFeatures = sparseContinuousFeatures, - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.docx new file mode 100644 index 000000000..18847da9d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.scala deleted file mode 100644 index 0956ca34e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.thriftscala.DebugParams - -case class DebugOptions( - randomizationSeed: Option[Long] = None, - fetchDebugInfo: Boolean = false, - doNotLog: Boolean = false) - -object DebugOptions { - def fromDebugParamsThrift(debugParams: DebugParams): DebugOptions = { - DebugOptions( - debugParams.randomizationSeed, - debugParams.includeDebugInfoInResults.getOrElse(false), - debugParams.doNotLog.getOrElse(false) - ) - } -} - -trait HasDebugOptions { - def debugOptions: Option[DebugOptions] - - def getRandomizationSeed: Option[Long] = debugOptions.flatMap(_.randomizationSeed) - - def fetchDebugInfo: Option[Boolean] = debugOptions.map(_.fetchDebugInfo) -} - -trait HasFrsDebugOptions { - def frsDebugOptions: Option[DebugOptions] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.docx new file mode 100644 index 000000000..fe5a50e91 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.scala deleted file mode 100644 index 3f2154992..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasDismissedUserIds { - // user ids that are recently followed by the target user - def dismissedUserIds: Option[Seq[Long]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.docx new file mode 100644 index 000000000..168304790 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.scala deleted file mode 100644 index e74ae83e1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasDisplayLocation { - def displayLocation: DisplayLocation -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.docx new file mode 100644 index 000000000..b67d69464 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.scala deleted file mode 100644 index de59e4479..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasEngagements { - - def engagements: Seq[EngagementType] - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.docx new file mode 100644 index 000000000..83242e3e9 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.scala deleted file mode 100644 index 3addcef83..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasExcludedUserIds { - // user ids that are going to be excluded from recommendations - def excludedUserIds: Seq[Long] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.docx new file mode 100644 index 000000000..9b7ff8256 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.scala deleted file mode 100644 index a4364bbf4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasGeohashAndCountryCode { - def geohashAndCountryCode: Option[GeohashAndCountryCode] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.docx new file mode 100644 index 000000000..a235a9b24 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.scala deleted file mode 100644 index c4d277412..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasInfoPerRankingStage { - def infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.docx new file mode 100644 index 000000000..e7bab8917 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.scala deleted file mode 100644 index 69f97b673..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasCustomInterests { - def customInterests: Option[Seq[String]] -} - -trait HasUttInterests { - def uttInterestIds: Option[Seq[Long]] -} - -trait HasInterestIds extends HasCustomInterests with HasUttInterests {} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.docx new file mode 100644 index 000000000..0520ea916 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.scala deleted file mode 100644 index 3cf3f66db..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasInvalidRelationshipUserIds { - // user ids that have invalid relationship with the target user - def invalidRelationshipUserIds: Option[Set[Long]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.docx new file mode 100644 index 000000000..3b863a6e2 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.scala deleted file mode 100644 index 8cf1532ce..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasIsSoftUser { - def isSoftUser: Boolean -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.docx new file mode 100644 index 000000000..154d32e28 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.scala deleted file mode 100644 index c5e1e16cc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -// intersection of recent followers and followed by -trait HasMutualFollowedUserIds extends HasRecentFollowedUserIds with HasRecentFollowedByUserIds { - - lazy val recentMutualFollows: Seq[Long] = - recentFollowedUserIds.getOrElse(Nil).intersect(recentFollowedByUserIds.getOrElse(Nil)) - - lazy val numRecentMutualFollows: Int = recentMutualFollows.size -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.docx new file mode 100644 index 000000000..2aec8c4dc Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.scala deleted file mode 100644 index 2480faaad..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasPreviousRecommendationsContext { - - def previouslyRecommendedUserIDs: Set[Long] - - def previouslyFollowedUserIds: Set[Long] - - def skippedFollows: Set[Long] = { - previouslyRecommendedUserIDs.diff(previouslyFollowedUserIds) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.docx new file mode 100644 index 000000000..66fe4c7de Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.scala deleted file mode 100644 index 10cd7c02f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasProfileId { - def profileId: Long -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.docx new file mode 100644 index 000000000..3ab747548 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.scala deleted file mode 100644 index 96527b2ba..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasQualityFactor { - def qualityFactor: Option[Double] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.docx new file mode 100644 index 000000000..c597f74a6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.scala deleted file mode 100644 index bc15e8bd8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentFollowedByUserIds { - // user ids that have recently followed the target user; target user has been "followed by" them. - def recentFollowedByUserIds: Option[Seq[Long]] - - lazy val numRecentFollowedByUserIds: Int = recentFollowedByUserIds.map(_.size).getOrElse(0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.docx new file mode 100644 index 000000000..89f2bcb1b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.scala deleted file mode 100644 index 67ada7c66..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentFollowedUserIds { - // user ids that are recently followed by the target user - def recentFollowedUserIds: Option[Seq[Long]] - - // user ids that are recently followed by the target user in set data-structure - lazy val recentFollowedUserIdsSet: Option[Set[Long]] = recentFollowedUserIds match { - case Some(users) => Some(users.toSet) - case None => Some(Set.empty) - } - - lazy val numRecentFollowedUserIds: Int = recentFollowedUserIds.map(_.size).getOrElse(0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.docx new file mode 100644 index 000000000..f74eb2ff4 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.scala deleted file mode 100644 index 7e3cba4a7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentFollowedUserIdsWithTime { - // user ids that are recently followed by the target user - def recentFollowedUserIdsWithTime: Option[Seq[UserIdWithTimestamp]] - - lazy val numRecentFollowedUserIdsWithTime: Int = - recentFollowedUserIdsWithTime.map(_.size).getOrElse(0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.docx new file mode 100644 index 000000000..2257e4edd Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.scala deleted file mode 100644 index 44420ec27..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentlyEngagedUserIds { - val recentlyEngagedUserIds: Option[Seq[RecentlyEngagedUserId]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.docx new file mode 100644 index 000000000..e595063a5 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.scala deleted file mode 100644 index 5706c7a29..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecommendationFlowIdentifier { - def recommendationFlowIdentifier: Option[String] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.docx new file mode 100644 index 000000000..e67380de9 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.scala deleted file mode 100644 index e8a6698ee..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasScores { - def scores: Option[Scores] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.docx new file mode 100644 index 000000000..4d2c17db9 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.scala deleted file mode 100644 index bbe2ac258..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasSimilarToContext { - - // user ids that are used to generate similar to recommendations - def similarToUserIds: Seq[Long] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.docx new file mode 100644 index 000000000..9c42e4cc4 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.scala deleted file mode 100644 index 4bd6e63e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasTopicId { - def topicId: Long -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.docx new file mode 100644 index 000000000..87dde8189 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.scala deleted file mode 100644 index e0e363449..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.scala +++ /dev/null @@ -1,162 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.hermit.ml.models.Feature -import com.twitter.hermit.model.Algorithm -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -/** - * Used to keep track of a candidate's source not so much as a feature but for filtering candidate - * from specific sources (eg. GizmoduckPredicate) - */ -trait HasUserCandidateSourceDetails { candidateUser: CandidateUser => - def userCandidateSourceDetails: Option[UserCandidateSourceDetails] - - def getAlgorithm: Algorithm = { - val algorithm = for { - details <- userCandidateSourceDetails - identifier <- details.primaryCandidateSource - algorithm <- Algorithm.withNameOpt(identifier.name) - } yield algorithm - - algorithm.getOrElse(throw new Exception("Algorithm missing on candidate user!")) - } - - def getAllAlgorithms: Seq[Algorithm] = { - getCandidateSources.keys - .flatMap(identifier => Algorithm.withNameOpt(identifier.name)).toSeq - } - - def getAddressBookMetadata: Option[AddressBookMetadata] = { - userCandidateSourceDetails.flatMap(_.addressBookMetadata) - } - - def getCandidateSources: Map[CandidateSourceIdentifier, Option[Double]] = { - userCandidateSourceDetails.map(_.candidateSourceScores).getOrElse(Map.empty) - } - - def getCandidateRanks: Map[CandidateSourceIdentifier, Int] = { - userCandidateSourceDetails.map(_.candidateSourceRanks).getOrElse(Map.empty) - } - - def getCandidateFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = { - userCandidateSourceDetails.map(_.candidateSourceFeatures).getOrElse(Map.empty) - } - - def getPrimaryCandidateSource: Option[CandidateSourceIdentifier] = { - userCandidateSourceDetails.flatMap(_.primaryCandidateSource) - } - - def withCandidateSource(source: CandidateSourceIdentifier): CandidateUser = { - withCandidateSourceAndScore(source, candidateUser.score) - } - - def withCandidateSourceAndScore( - source: CandidateSourceIdentifier, - score: Option[Double] - ): CandidateUser = { - withCandidateSourceScoreAndFeatures(source, score, Nil) - } - - def withCandidateSourceAndFeatures( - source: CandidateSourceIdentifier, - features: Seq[Feature] - ): CandidateUser = { - withCandidateSourceScoreAndFeatures(source, candidateUser.score, features) - } - - def withCandidateSourceScoreAndFeatures( - source: CandidateSourceIdentifier, - score: Option[Double], - features: Seq[Feature] - ): CandidateUser = { - val candidateSourceDetails = - candidateUser.userCandidateSourceDetails - .map { details => - details.copy( - primaryCandidateSource = Some(source), - candidateSourceScores = details.candidateSourceScores + (source -> score), - candidateSourceFeatures = details.candidateSourceFeatures + (source -> features) - ) - }.getOrElse( - UserCandidateSourceDetails( - Some(source), - Map(source -> score), - Map.empty, - None, - Map(source -> features))) - candidateUser.copy( - userCandidateSourceDetails = Some(candidateSourceDetails) - ) - } - - def addCandidateSourceScoresMap( - scoreMap: Map[CandidateSourceIdentifier, Option[Double]] - ): CandidateUser = { - val candidateSourceDetails = candidateUser.userCandidateSourceDetails - .map { details => - details.copy(candidateSourceScores = details.candidateSourceScores ++ scoreMap) - }.getOrElse(UserCandidateSourceDetails(scoreMap.keys.headOption, scoreMap, Map.empty, None)) - candidateUser.copy( - userCandidateSourceDetails = Some(candidateSourceDetails) - ) - } - - def addCandidateSourceRanksMap( - rankMap: Map[CandidateSourceIdentifier, Int] - ): CandidateUser = { - val candidateSourceDetails = candidateUser.userCandidateSourceDetails - .map { details => - details.copy(candidateSourceRanks = details.candidateSourceRanks ++ rankMap) - }.getOrElse(UserCandidateSourceDetails(rankMap.keys.headOption, Map.empty, rankMap, None)) - candidateUser.copy( - userCandidateSourceDetails = Some(candidateSourceDetails) - ) - } - - def addInfoPerRankingStage( - rankingStage: String, - scores: Option[Scores], - rank: Int - ): CandidateUser = { - val scoresOpt: Option[Scores] = scores.orElse(candidateUser.scores) - val originalInfoPerRankingStage = - candidateUser.infoPerRankingStage.getOrElse(Map[String, RankingInfo]()) - candidateUser.copy( - infoPerRankingStage = - Some(originalInfoPerRankingStage + (rankingStage -> RankingInfo(scoresOpt, Some(rank)))) - ) - } - - def addAddressBookMetadataIfAvailable( - candidateSources: Seq[CandidateSourceIdentifier] - ): CandidateUser = { - - val addressBookMetadata = AddressBookMetadata( - inForwardPhoneBook = - candidateSources.contains(AddressBookMetadata.ForwardPhoneBookCandidateSource), - inReversePhoneBook = - candidateSources.contains(AddressBookMetadata.ReversePhoneBookCandidateSource), - inForwardEmailBook = - candidateSources.contains(AddressBookMetadata.ForwardEmailBookCandidateSource), - inReverseEmailBook = - candidateSources.contains(AddressBookMetadata.ReverseEmailBookCandidateSource) - ) - - val newCandidateSourceDetails = candidateUser.userCandidateSourceDetails - .map { details => - details.copy(addressBookMetadata = Some(addressBookMetadata)) - }.getOrElse( - UserCandidateSourceDetails( - None, - Map.empty, - Map.empty, - Some(addressBookMetadata), - Map.empty)) - - candidateUser.copy( - userCandidateSourceDetails = Some(newCandidateSourceDetails) - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.docx new file mode 100644 index 000000000..c1e58a0d1 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.scala deleted file mode 100644 index bf0df46f7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.core_workflows.user_model.thriftscala.UserState - -trait HasUserState { - def userState: Option[UserState] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.docx new file mode 100644 index 000000000..9c2cf26a3 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.scala deleted file mode 100644 index d840ed6fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.util.Time - -trait HasWtfImpressions { - - def wtfImpressions: Option[Seq[WtfImpression]] - - lazy val numWtfImpressions: Int = wtfImpressions.map(_.size).getOrElse(0) - - lazy val candidateImpressions: Map[Long, WtfImpression] = wtfImpressions - .map { imprMap => - imprMap.map { i => - i.candidateId -> i - }.toMap - }.getOrElse(Map.empty) - - lazy val latestImpressionTime: Time = { - if (wtfImpressions.exists(_.nonEmpty)) { - wtfImpressions.get.map(_.latestTime).max - } else Time.Top - } - - def getCandidateImpressionCounts(id: Long): Option[Int] = - candidateImpressions.get(id).map(_.counts) - - def getCandidateLatestTime(id: Long): Option[Time] = { - candidateImpressions.get(id).map(_.latestTime) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.docx new file mode 100644 index 000000000..3cc8231e5 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.scala deleted file mode 100644 index df4c228e1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams - -/** -Convenience trait to group together all traits needed for optimus ranking - */ -trait OptimusRequest - extends HasParams - with HasClientContext - with HasDisplayLocation - with HasInterestIds - with HasDebugOptions - with HasPreviousRecommendationsContext {} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.docx new file mode 100644 index 000000000..d60b3674b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.scala deleted file mode 100644 index f37cff56e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.{Product => ProductMixerProduct} - -object Product { - case object MagicRecs extends ProductMixerProduct { - override val identifier: ProductIdentifier = ProductIdentifier("MagicRecs") - override val stringCenterProject: Option[String] = Some("people-discovery") - } - - case object PlaceholderProductMixerProduct extends ProductMixerProduct { - override val identifier: ProductIdentifier = ProductIdentifier("PlaceholderProductMixerProduct") - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.docx new file mode 100644 index 000000000..7faf08bdb Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.scala deleted file mode 100644 index 02eb46b5a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} - -case class RankingInfo( - scores: Option[Scores], - rank: Option[Int]) { - - def toThrift: t.RankingInfo = { - t.RankingInfo(scores.map(_.toThrift), rank) - } - - def toOfflineThrift: offline.RankingInfo = { - offline.RankingInfo(scores.map(_.toOfflineThrift), rank) - } -} - -object RankingInfo { - - def fromThrift(rankingInfo: t.RankingInfo): RankingInfo = { - RankingInfo( - scores = rankingInfo.scores.map(Scores.fromThrift), - rank = rankingInfo.rank - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.docx new file mode 100644 index 000000000..5a3f4059e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.scala deleted file mode 100644 index b7c9c6c75..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.scala +++ /dev/null @@ -1,206 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} - -case class FollowProof(followedBy: Seq[Long], numIds: Int) { - def toThrift: t.FollowProof = { - t.FollowProof(followedBy, numIds) - } - - def toOfflineThrift: offline.FollowProof = offline.FollowProof(followedBy, numIds) -} - -object FollowProof { - - def fromThrift(proof: t.FollowProof): FollowProof = { - FollowProof(proof.userIds, proof.numIds) - } -} - -case class SimilarToProof(similarTo: Seq[Long]) { - def toThrift: t.SimilarToProof = { - t.SimilarToProof(similarTo) - } - - def toOfflineThrift: offline.SimilarToProof = offline.SimilarToProof(similarTo) -} - -object SimilarToProof { - def fromThrift(proof: t.SimilarToProof): SimilarToProof = { - SimilarToProof(proof.userIds) - } -} - -case class PopularInGeoProof(location: String) { - def toThrift: t.PopularInGeoProof = { - t.PopularInGeoProof(location) - } - - def toOfflineThrift: offline.PopularInGeoProof = offline.PopularInGeoProof(location) -} - -object PopularInGeoProof { - - def fromThrift(proof: t.PopularInGeoProof): PopularInGeoProof = { - PopularInGeoProof(proof.location) - } -} - -case class TttInterestProof(interestId: Long, interestDisplayName: String) { - def toThrift: t.TttInterestProof = { - t.TttInterestProof(interestId, interestDisplayName) - } - - def toOfflineThrift: offline.TttInterestProof = - offline.TttInterestProof(interestId, interestDisplayName) -} - -object TttInterestProof { - - def fromThrift(proof: t.TttInterestProof): TttInterestProof = { - TttInterestProof(proof.interestId, proof.interestDisplayName) - } -} - -case class TopicProof(topicId: Long) { - def toThrift: t.TopicProof = { - t.TopicProof(topicId) - } - - def toOfflineThrift: offline.TopicProof = - offline.TopicProof(topicId) -} - -object TopicProof { - def fromThrift(proof: t.TopicProof): TopicProof = { - TopicProof(proof.topicId) - } -} - -case class CustomInterest(query: String) { - def toThrift: t.CustomInterestProof = { - t.CustomInterestProof(query) - } - - def toOfflineThrift: offline.CustomInterestProof = - offline.CustomInterestProof(query) -} - -object CustomInterest { - def fromThrift(proof: t.CustomInterestProof): CustomInterest = { - CustomInterest(proof.query) - } -} - -case class TweetsAuthorProof(tweetIds: Seq[Long]) { - def toThrift: t.TweetsAuthorProof = { - t.TweetsAuthorProof(tweetIds) - } - - def toOfflineThrift: offline.TweetsAuthorProof = - offline.TweetsAuthorProof(tweetIds) -} - -object TweetsAuthorProof { - def fromThrift(proof: t.TweetsAuthorProof): TweetsAuthorProof = { - TweetsAuthorProof(proof.tweetIds) - } -} - -case class DeviceFollowProof(isDeviceFollow: Boolean) { - def toThrift: t.DeviceFollowProof = { - t.DeviceFollowProof(isDeviceFollow) - } - def toOfflineThrift: offline.DeviceFollowProof = - offline.DeviceFollowProof(isDeviceFollow) -} - -object DeviceFollowProof { - def fromThrift(proof: t.DeviceFollowProof): DeviceFollowProof = { - DeviceFollowProof(proof.isDeviceFollow) - } - -} - -case class AccountProof( - followProof: Option[FollowProof] = None, - similarToProof: Option[SimilarToProof] = None, - popularInGeoProof: Option[PopularInGeoProof] = None, - tttInterestProof: Option[TttInterestProof] = None, - topicProof: Option[TopicProof] = None, - customInterestProof: Option[CustomInterest] = None, - tweetsAuthorProof: Option[TweetsAuthorProof] = None, - deviceFollowProof: Option[DeviceFollowProof] = None) { - def toThrift: t.AccountProof = { - t.AccountProof( - followProof.map(_.toThrift), - similarToProof.map(_.toThrift), - popularInGeoProof.map(_.toThrift), - tttInterestProof.map(_.toThrift), - topicProof.map(_.toThrift), - customInterestProof.map(_.toThrift), - tweetsAuthorProof.map(_.toThrift), - deviceFollowProof.map(_.toThrift) - ) - } - - def toOfflineThrift: offline.AccountProof = { - offline.AccountProof( - followProof.map(_.toOfflineThrift), - similarToProof.map(_.toOfflineThrift), - popularInGeoProof.map(_.toOfflineThrift), - tttInterestProof.map(_.toOfflineThrift), - topicProof.map(_.toOfflineThrift), - customInterestProof.map(_.toOfflineThrift), - tweetsAuthorProof.map(_.toOfflineThrift), - deviceFollowProof.map(_.toOfflineThrift) - ) - } -} - -object AccountProof { - def fromThrift(proof: t.AccountProof): AccountProof = { - AccountProof( - proof.followProof.map(FollowProof.fromThrift), - proof.similarToProof.map(SimilarToProof.fromThrift), - proof.popularInGeoProof.map(PopularInGeoProof.fromThrift), - proof.tttInterestProof.map(TttInterestProof.fromThrift), - proof.topicProof.map(TopicProof.fromThrift), - proof.customInterestProof.map(CustomInterest.fromThrift), - proof.tweetsAuthorProof.map(TweetsAuthorProof.fromThrift), - proof.deviceFollowProof.map(DeviceFollowProof.fromThrift) - ) - } -} - -case class Reason(accountProof: Option[AccountProof]) { - def toThrift: t.Reason = { - t.Reason(accountProof.map(_.toThrift)) - } - - def toOfflineThrift: offline.Reason = { - offline.Reason(accountProof.map(_.toOfflineThrift)) - } -} - -object Reason { - - def fromThrift(reason: t.Reason): Reason = { - Reason(reason.accountProof.map(AccountProof.fromThrift)) - } -} - -trait HasReason { - - def reason: Option[Reason] - // helper methods below - - def followedBy: Option[Seq[Long]] = { - for { - reason <- reason - accountProof <- reason.accountProof - followProof <- accountProof.followProof - } yield { followProof.followedBy } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.docx new file mode 100644 index 000000000..85820d541 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.scala deleted file mode 100644 index b20d5d6fb..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class RecentlyEngagedUserId(id: Long, engagementType: EngagementType) { - def toThrift: t.RecentlyEngagedUserId = - t.RecentlyEngagedUserId(id = id, engagementType = engagementType.toThrift) - - def toOfflineThrift: offline.RecentlyEngagedUserId = - offline.RecentlyEngagedUserId(id = id, engagementType = engagementType.toOfflineThrift) -} - -object RecentlyEngagedUserId { - def fromThrift(recentlyEngagedUserId: t.RecentlyEngagedUserId): RecentlyEngagedUserId = { - RecentlyEngagedUserId( - id = recentlyEngagedUserId.id, - engagementType = EngagementType.fromThrift(recentlyEngagedUserId.engagementType) - ) - } - - def fromOfflineThrift( - recentlyEngagedUserId: offline.RecentlyEngagedUserId - ): RecentlyEngagedUserId = { - RecentlyEngagedUserId( - id = recentlyEngagedUserId.id, - engagementType = EngagementType.fromOfflineThrift(recentlyEngagedUserId.engagementType) - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.docx new file mode 100644 index 000000000..ae01eeba2 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.scala deleted file mode 100644 index 4fd4bc70e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} - -case class RecommendationStep( - recommendations: Seq[FlowRecommendation], - followedUserIds: Set[Long]) { - - def toThrift: t.RecommendationStep = t.RecommendationStep( - recommendations = recommendations.map(_.toThrift), - followedUserIds = followedUserIds - ) - - def toOfflineThrift: offline.OfflineRecommendationStep = - offline.OfflineRecommendationStep( - recommendations = recommendations.map(_.toOfflineThrift), - followedUserIds = followedUserIds) - -} - -object RecommendationStep { - - def fromThrift(recommendationStep: t.RecommendationStep): RecommendationStep = { - RecommendationStep( - recommendations = recommendationStep.recommendations.map(FlowRecommendation.fromThrift), - followedUserIds = recommendationStep.followedUserIds.toSet) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.docx new file mode 100644 index 000000000..448928e7e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.scala deleted file mode 100644 index 2d577e21d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdge -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdgeInfo -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.SecondDegreeEdge - -case class PotentialFirstDegreeEdge( - userId: Long, - connectingId: Long, - algorithm: Algorithm, - score: Double, - edgeInfo: FirstDegreeEdgeInfo) - -case class IntermediateSecondDegreeEdge( - connectingId: Long, - candidateId: Long, - edgeInfo: FirstDegreeEdgeInfo) - -case class STPGraph( - firstDegreeEdgeInfoList: List[FirstDegreeEdge], - secondDegreeEdgeInfoList: List[SecondDegreeEdge]) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.docx new file mode 100644 index 000000000..fe49be455 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.scala deleted file mode 100644 index 10c21704a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} - -sealed trait SafetyLevel { - def toThrift: ThriftSafetyLevel -} - -object SafetyLevel { - case object Recommendations extends SafetyLevel { - override val toThrift = ThriftSafetyLevel.Recommendations - } - - case object TopicsLandingPageTopicRecommendations extends SafetyLevel { - override val toThrift = ThriftSafetyLevel.TopicsLandingPageTopicRecommendations - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.docx new file mode 100644 index 000000000..6c0df40db Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.scala deleted file mode 100644 index 1007c5827..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -/** - * Type of Score. This is used to differentiate scores. - * - * Define it as a trait so it is possible to add more information for different score types. - */ -sealed trait ScoreType { - def getName: String -} - -/** - * Existing Score Types - */ -object ScoreType { - - /** - * the score is calculated based on heuristics and most likely not normalized - */ - case object HeuristicBasedScore extends ScoreType { - override def getName: String = "HeuristicBasedScore" - } - - /** - * probability of follow after the candidate is recommended to the user - */ - case object PFollowGivenReco extends ScoreType { - override def getName: String = "PFollowGivenReco" - } - - /** - * probability of engage after the user follows the candidate - */ - case object PEngagementGivenFollow extends ScoreType { - override def getName: String = "PEngagementGivenFollow" - } - - /** - * probability of engage per tweet impression - */ - case object PEngagementPerImpression extends ScoreType { - override def getName: String = "PEngagementPerImpression" - } - - /** - * probability of engage per tweet impression - */ - case object PEngagementGivenReco extends ScoreType { - override def getName: String = "PEngagementGivenReco" - } - - def fromScoreTypeString(scoreTypeName: String): ScoreType = scoreTypeName match { - case "HeuristicBasedScore" => HeuristicBasedScore - case "PFollowGivenReco" => PFollowGivenReco - case "PEngagementGivenFollow" => PEngagementGivenFollow - case "PEngagementPerImpression" => PEngagementPerImpression - case "PEngagementGivenReco" => PEngagementGivenReco - } -} - -/** - * Represent the output from a certain ranker or scorer. All the fields are optional - * - * @param value value of the score - * @param rankerId ranker id - * @param scoreType score type - */ -final case class Score( - value: Double, - rankerId: Option[RankerId] = None, - scoreType: Option[ScoreType] = None) { - - def toThrift: t.Score = t.Score( - value = value, - rankerId = rankerId.map(_.toString), - scoreType = scoreType.map(_.getName) - ) - - def toOfflineThrift: offline.Score = - offline.Score( - value = value, - rankerId = rankerId.map(_.toString), - scoreType = scoreType.map(_.getName) - ) -} - -object Score { - - val RandomScore = Score(0.0d, Some(RankerId.RandomRanker)) - - def optimusScore(score: Double, scoreType: ScoreType): Score = { - Score(value = score, scoreType = Some(scoreType)) - } - - def predictionScore(score: Double, rankerId: RankerId): Score = { - Score(value = score, rankerId = Some(rankerId)) - } - - def fromThrift(thriftScore: t.Score): Score = - Score( - value = thriftScore.value, - rankerId = thriftScore.rankerId.flatMap(RankerId.getRankerByName), - scoreType = thriftScore.scoreType.map(ScoreType.fromScoreTypeString) - ) -} - -/** - * a list of scores - */ -final case class Scores( - scores: Seq[Score], - selectedRankerId: Option[RankerId] = None, - isInProducerScoringExperiment: Boolean = false) { - - def toThrift: t.Scores = - t.Scores( - scores = scores.map(_.toThrift), - selectedRankerId = selectedRankerId.map(_.toString), - isInProducerScoringExperiment = isInProducerScoringExperiment - ) - - def toOfflineThrift: offline.Scores = - offline.Scores( - scores = scores.map(_.toOfflineThrift), - selectedRankerId = selectedRankerId.map(_.toString), - isInProducerScoringExperiment = isInProducerScoringExperiment - ) -} - -object Scores { - val Empty: Scores = Scores(Nil) - - def fromThrift(thriftScores: t.Scores): Scores = - Scores( - scores = thriftScores.scores.map(Score.fromThrift), - selectedRankerId = thriftScores.selectedRankerId.flatMap(RankerId.getRankerByName), - isInProducerScoringExperiment = thriftScores.isInProducerScoringExperiment - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.docx new file mode 100644 index 000000000..957bfd72a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.scala deleted file mode 100644 index 25ff00b48..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.finagle.tracing.Trace - -object Session { - - /** - * The sessionId in FRS is the finagle trace id which is static within the lifetime of a single - * request. - * - * It is used when generating per-candidate tokens (in TrackingTokenTransform) and is also passed - * in to downstream Optimus ranker requests. - * - */ - def getSessionId: Long = Trace.id.traceId.toLong -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.docx new file mode 100644 index 000000000..dfbc8d03e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.scala deleted file mode 100644 index 3af5c6e1d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.usersignalservice.thriftscala.Signal - -trait SignalData { - val userId: Long - val signalType: SignalType -} - -case class RecentFollowsSignal( - override val userId: Long, - override val signalType: SignalType, - followedUserId: Long, - timestamp: Long) - extends SignalData - -object RecentFollowsSignal { - - def fromUssSignal(targetUserId: Long, signal: Signal): RecentFollowsSignal = { - val InternalId.UserId(followedUserId) = signal.targetInternalId.getOrElse( - throw new IllegalArgumentException("RecentFollow Signal does not have internalId")) - - RecentFollowsSignal( - userId = targetUserId, - followedUserId = followedUserId, - timestamp = signal.timestamp, - signalType = signal.signalType - ) - } - - def getRecentFollowedUserIds( - signalDataMap: Option[Map[SignalType, Seq[SignalData]]] - ): Option[Seq[Long]] = { - signalDataMap.map(_.getOrElse(SignalType.AccountFollow, default = Seq.empty).flatMap { - case RecentFollowsSignal(userId, signalType, followedUserId, timestamp) => - Some(followedUserId) - case _ => None - }) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.docx new file mode 100644 index 000000000..8d72eb9d2 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.scala deleted file mode 100644 index 177e08f65..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.finagle.tracing.Trace -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.scrooge.BinaryThriftStructSerializer -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.util.Base64StringEncoder - -/** - * used for attribution per target-candidate pair - * @param sessionId trace-id of the finagle request - * @param controllerData 64-bit encoded binary attributes of our recommendation - * @param algorithmId id for identifying a candidate source. maintained for backwards compatibility - */ -case class TrackingToken( - sessionId: Long, - displayLocation: Option[DisplayLocation], - controllerData: Option[ControllerData], - algorithmId: Option[Int]) { - - def toThrift: t.TrackingToken = { - Trace.id.traceId.toLong - t.TrackingToken( - sessionId = sessionId, - displayLocation = displayLocation.map(_.toThrift), - controllerData = controllerData, - algoId = algorithmId - ) - } - - def toOfflineThrift: offline.TrackingToken = { - offline.TrackingToken( - sessionId = sessionId, - displayLocation = displayLocation.map(_.toOfflineThrift), - controllerData = controllerData, - algoId = algorithmId - ) - } -} - -object TrackingToken { - val binaryThriftSerializer = BinaryThriftStructSerializer[t.TrackingToken](t.TrackingToken) - def serialize(trackingToken: TrackingToken): String = { - Base64StringEncoder.encode(binaryThriftSerializer.toBytes(trackingToken.toThrift)) - } - def deserialize(trackingTokenStr: String): TrackingToken = { - fromThrift(binaryThriftSerializer.fromBytes(Base64StringEncoder.decode(trackingTokenStr))) - } - def fromThrift(token: t.TrackingToken): TrackingToken = { - TrackingToken( - sessionId = token.sessionId, - displayLocation = token.displayLocation.map(DisplayLocation.fromThrift), - controllerData = token.controllerData, - algorithmId = token.algoId - ) - } -} - -trait HasTrackingToken { - def trackingToken: Option[TrackingToken] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.docx new file mode 100644 index 000000000..f82a360d4 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.scala deleted file mode 100644 index 1957e28cc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -case class TweetCandidate( - tweetId: Long, - authorId: Long, - score: Option[Double]) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.docx new file mode 100644 index 000000000..76240ec3b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.scala deleted file mode 100644 index 73b766232..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.hermit.constants.AlgorithmFeedbackTokens._ -import com.twitter.hermit.ml.models.Feature -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -/** - * primaryCandidateSource param is showing the candidate source that responsible for generating this - * candidate, as the candidate might have gone through multiple candidate sources to get generated - * (for example if it has generated by a composite source). WeightedCandidateSourceRanker uses this - * field to do the sampling over candidate sources. All the sources used for generating this - * candidate (including the primary source) and their corresponding score exist in the - * candidateSourceScores field. - */ -case class UserCandidateSourceDetails( - primaryCandidateSource: Option[CandidateSourceIdentifier], - candidateSourceScores: Map[CandidateSourceIdentifier, Option[Double]] = Map.empty, - candidateSourceRanks: Map[CandidateSourceIdentifier, Int] = Map.empty, - addressBookMetadata: Option[AddressBookMetadata] = None, - candidateSourceFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = Map.empty, -) { - - def toThrift: t.CandidateSourceDetails = { - t.CandidateSourceDetails( - candidateSourceScores = Some(candidateSourceScores.map { - case (identifier, score) => - (identifier.name, score.getOrElse(0.0d)) - }), - primarySource = for { - identifier <- primaryCandidateSource - algo <- Algorithm.withNameOpt(identifier.name) - feedbackToken <- AlgorithmToFeedbackTokenMap.get(algo) - } yield feedbackToken - ) - } - - def toOfflineThrift: offline.CandidateSourceDetails = { - offline.CandidateSourceDetails( - candidateSourceScores = Some(candidateSourceScores.map { - case (identifier, score) => - (identifier.name, score.getOrElse(0.0d)) - }), - primarySource = for { - identifier <- primaryCandidateSource - algo <- Algorithm.withNameOpt(identifier.name) - feedbackToken <- AlgorithmToFeedbackTokenMap.get(algo) - } yield feedbackToken - ) - } -} - -object UserCandidateSourceDetails { - val algorithmNameMap: Map[String, Algorithm.Value] = Algorithm.values.map { - algorithmValue: Algorithm.Value => - (algorithmValue.toString, algorithmValue) - }.toMap - - /** - * This method is used to parse the candidate source of the candidates, which is only passed from - * the scoreUserCandidates endpoint. We create custom candidate source identifiers which - * CandidateAlgorithmSource will read from to hydrate the algorithm id feature. - * candidateSourceScores will not be populated from the endpoint, but we add the conversion for - * completeness. Note that the conversion uses the raw string of the Algorithm rather than the - * assigned strings that we give to our own candidate sources in the FRS. - */ - def fromThrift(details: t.CandidateSourceDetails): UserCandidateSourceDetails = { - val primaryCandidateSource: Option[CandidateSourceIdentifier] = for { - primarySourceToken <- details.primarySource - algo <- TokenToAlgorithmMap.get(primarySourceToken) - } yield CandidateSourceIdentifier(algo.toString) - - val candidateSourceScores = for { - scoreMap <- details.candidateSourceScores.toSeq - (name, score) <- scoreMap - algo <- algorithmNameMap.get(name) - } yield { - CandidateSourceIdentifier(algo.toString) -> Some(score) - } - val candidateSourceRanks = for { - rankMap <- details.candidateSourceRanks.toSeq - (name, rank) <- rankMap - algo <- algorithmNameMap.get(name) - } yield { - CandidateSourceIdentifier(algo.toString) -> rank - } - UserCandidateSourceDetails( - primaryCandidateSource = primaryCandidateSource, - candidateSourceScores = candidateSourceScores.toMap, - candidateSourceRanks = candidateSourceRanks.toMap, - addressBookMetadata = None, - candidateSourceFeatures = Map.empty - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.docx new file mode 100644 index 000000000..79a8ac322 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.scala deleted file mode 100644 index 74f33eb40..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -case class UserIdWithTimestamp(userId: Long, timeInMs: Long) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.docx new file mode 100644 index 000000000..8892f49bf Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.scala deleted file mode 100644 index 39e0561a2..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.util.Time - -/** - * Domain model for representing impressions on wtf recommendations in the past 16 days - */ -case class WtfImpression( - candidateId: Long, - displayLocation: DisplayLocation, - latestTime: Time, - counts: Int) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD deleted file mode 100644 index ffcbe65a7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD.docx new file mode 100644 index 000000000..2481e3d79 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.docx new file mode 100644 index 000000000..59243f934 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.scala deleted file mode 100644 index 0713f728f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Param - -class CandidateParamPredicate[A <: HasParams]( - param: Param[Boolean], - reason: FilterReason) - extends Predicate[A] { - override def apply(candidate: A): Stitch[PredicateResult] = { - if (candidate.params(param)) { - Stitch.value(PredicateResult.Valid) - } else { - Stitch.value(PredicateResult.Invalid(Set(reason))) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.docx new file mode 100644 index 000000000..5b5fc554e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.scala deleted file mode 100644 index cf08f5623..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * This predicate allows us to filter candidates given its source. - * To avoid bucket dilution, we only want to evaluate the param (which would implicitly trigger - * bucketing for FSParams) only if the candidate source fn yields true. - * The param provided should be true when we want to keep the candidate and false otherwise. - */ -class CandidateSourceParamPredicate( - val param: Param[Boolean], - val reason: FilterReason, - candidateSources: Set[CandidateSourceIdentifier]) - extends Predicate[CandidateUser] { - override def apply(candidate: CandidateUser): Stitch[PredicateResult] = { - // we want to avoid evaluating the param if the candidate source fn yields false - if (candidate.getCandidateSources.keys.exists(candidateSources.contains) && !candidate.params( - param)) { - Stitch.value(PredicateResult.Invalid(Set(reason))) - } else { - Stitch.value(PredicateResult.Valid) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.docx new file mode 100644 index 000000000..ffb460b88 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.scala deleted file mode 100644 index 16d76ce44..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.FilterReason.CuratedAccountsCompetitorList -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.escherbird.util.stitchcache.StitchCache - -@Singleton -case class CuratedCompetitorListPredicate @Inject() ( - statsReceiver: StatsReceiver, - @Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) competitorAccountFetcher: Fetcher[ - String, - Unit, - Seq[Long] - ]) extends Predicate[CandidateUser] { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - private val cacheStats = stats.scope("cache") - - private val cache = StitchCache[String, Set[Long]]( - maxCacheSize = CuratedCompetitorListPredicate.CacheNumberOfEntries, - ttl = CuratedCompetitorListPredicate.CacheTTL, - statsReceiver = cacheStats, - underlyingCall = (competitorListPrefix: String) => query(competitorListPrefix) - ) - - private def query(prefix: String): Stitch[Set[Long]] = - competitorAccountFetcher.fetch(prefix).map(_.v.getOrElse(Nil).toSet) - - /** - * Caveat here is that though the similarToUserIds allows for a Seq[Long], in practice we would - * only return 1 userId. Multiple userId's would result in filtering candidates associated with - * a different similarToUserId. For example: - * - similarToUser1 -> candidate1, candidate2 - * - similarToUser2 -> candidate3 - * and in the competitorList store we have: - * - similarToUser1 -> candidate3 - * we'll be filtering candidate3 on account of similarToUser1, even though it was generated - * with similarToUser2. This might still be desirable at a product level (since we don't want - * to show these accounts anyway), but might not achieve what you intend to code-wise. - */ - override def apply(candidate: CandidateUser): Stitch[PredicateResult] = { - cache.readThrough(CuratedCompetitorListPredicate.DefaultKey).map { competitorListAccounts => - if (competitorListAccounts.contains(candidate.id)) { - PredicateResult.Invalid(Set(CuratedAccountsCompetitorList)) - } else { - PredicateResult.Valid - } - } - } -} - -object CuratedCompetitorListPredicate { - val DefaultKey: String = "default_list" - val CacheTTL = 5.minutes - val CacheNumberOfEntries = 5 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.docx new file mode 100644 index 000000000..f86c00131 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.scala deleted file mode 100644 index 01a5f96fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.FilterReason.ExcludedId -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasExcludedUserIds -import com.twitter.stitch.Stitch - -object ExcludedUserIdPredicate extends Predicate[(HasExcludedUserIds, CandidateUser)] { - - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val ExcludedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(ExcludedId))) - - override def apply(pair: (HasExcludedUserIds, CandidateUser)): Stitch[PredicateResult] = { - val (excludedUserIds, candidate) = pair - if (excludedUserIds.excludedUserIds.contains(candidate.id)) { - ExcludedStitch - } else { - ValidStitch - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.docx new file mode 100644 index 000000000..9a21b604b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.scala deleted file mode 100644 index c77538a99..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.google.inject.name.Named -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.predicates.InactivePredicateParams._ -import com.twitter.service.metastore.gen.thriftscala.UserRecommendabilityFeatures -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.follow_recommendations.common.models.HasUserState -import com.twitter.follow_recommendations.common.predicates.InactivePredicateParams.DefaultInactivityThreshold -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -import java.lang.{Long => JLong} - -@Singleton -case class InactivePredicate @Inject() ( - statsReceiver: StatsReceiver, - @Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) userRecommendabilityFetcher: Fetcher[ - Long, - Unit, - UserRecommendabilityFeatures - ]) extends Predicate[(HasParams with HasClientContext with HasUserState, CandidateUser)] { - - private val stats: StatsReceiver = statsReceiver.scope("InactivePredicate") - private val cacheStats = stats.scope("cache") - - private def queryUserRecommendable(userId: Long): Stitch[Option[UserRecommendabilityFeatures]] = - userRecommendabilityFetcher.fetch(userId).map(_.v) - - private val userRecommendableCache = - StitchCache[JLong, Option[UserRecommendabilityFeatures]]( - maxCacheSize = 100000, - ttl = 12.hours, - statsReceiver = cacheStats.scope("UserRecommendable"), - underlyingCall = (userId: JLong) => queryUserRecommendable(userId) - ) - - override def apply( - targetAndCandidate: (HasParams with HasClientContext with HasUserState, CandidateUser) - ): Stitch[PredicateResult] = { - val (target, candidate) = targetAndCandidate - - userRecommendableCache - .readThrough(candidate.id).map { - case recFeaturesFetchResult => - recFeaturesFetchResult match { - case None => - PredicateResult.Invalid(Set(FilterReason.MissingRecommendabilityData)) - case Some(recFeatures) => - if (disableInactivityPredicate(target, target.userState, recFeatures.userState)) { - PredicateResult.Valid - } else { - val defaultInactivityThreshold = target.params(DefaultInactivityThreshold).days - val hasBeenActiveRecently = recFeatures.lastStatusUpdateMs - .map(Time.now - Time.fromMilliseconds(_)).getOrElse( - Duration.Top) < defaultInactivityThreshold - stats - .scope(defaultInactivityThreshold.toString).counter( - if (hasBeenActiveRecently) - "active" - else - "inactive" - ).incr() - if (hasBeenActiveRecently && (!target - .params(UseEggFilter) || recFeatures.isNotEgg.contains(1))) { - PredicateResult.Valid - } else { - PredicateResult.Invalid(Set(FilterReason.Inactive)) - } - } - } - }.rescue { - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FilterReason.FailOpen))) - } - } - - private[this] def disableInactivityPredicate( - target: HasParams, - consumerState: Option[UserState], - candidateState: Option[UserState] - ): Boolean = { - target.params(MightBeDisabled) && - consumerState.exists(InactivePredicate.ValidConsumerStates.contains) && - ( - ( - candidateState.exists(InactivePredicate.ValidCandidateStates.contains) && - !target.params(OnlyDisableForNewUserStateCandidates) - ) || - ( - candidateState.contains(UserState.New) && - target.params(OnlyDisableForNewUserStateCandidates) - ) - ) - } -} - -object InactivePredicate { - val ValidConsumerStates: Set[UserState] = Set( - UserState.HeavyNonTweeter, - UserState.MediumNonTweeter, - UserState.HeavyTweeter, - UserState.MediumTweeter - ) - val ValidCandidateStates: Set[UserState] = - Set(UserState.New, UserState.VeryLight, UserState.Light, UserState.NearZero) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.docx new file mode 100644 index 000000000..941ec4989 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.scala deleted file mode 100644 index 0bb52caa8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object InactivePredicateParams { - case object DefaultInactivityThreshold - extends FSBoundedParam[Int]( - name = "inactive_predicate_default_inactivity_threshold", - default = 60, - min = 1, - max = 500 - ) - case object UseEggFilter extends Param[Boolean](true) - case object MightBeDisabled extends FSParam[Boolean]("inactive_predicate_might_be_disabled", true) - case object OnlyDisableForNewUserStateCandidates - extends FSParam[Boolean]( - "inactive_predicate_only_disable_for_new_user_state_candidates", - false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.docx new file mode 100644 index 000000000..31a55e647 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.scala deleted file mode 100644 index 7879860ac..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.HasPreviousRecommendationsContext -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class PreviouslyRecommendedUserIdsPredicate - extends Predicate[(HasPreviousRecommendationsContext, CandidateUser)] { - override def apply( - pair: (HasPreviousRecommendationsContext, CandidateUser) - ): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - - val previouslyRecommendedUserIDs = targetUser.previouslyRecommendedUserIDs - - if (!previouslyRecommendedUserIDs.contains(candidate.id)) { - PreviouslyRecommendedUserIdsPredicate.ValidStitch - } else { - PreviouslyRecommendedUserIdsPredicate.AlreadyRecommendedStitch - } - } -} - -object PreviouslyRecommendedUserIdsPredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val AlreadyRecommendedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(FilterReason.AlreadyRecommended))) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD deleted file mode 100644 index 9d1ca9b40..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD.docx new file mode 100644 index 000000000..8486c6ace Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.docx new file mode 100644 index 000000000..ebce9f74d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.scala deleted file mode 100644 index 550017b95..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.dismiss - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.FilterReason.DismissedId -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDismissedUserIds -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class DismissedCandidatePredicate extends Predicate[(HasDismissedUserIds, CandidateUser)] { - - override def apply(pair: (HasDismissedUserIds, CandidateUser)): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - targetUser.dismissedUserIds - .map { dismissedUserIds => - if (!dismissedUserIds.contains(candidate.id)) { - DismissedCandidatePredicate.ValidStitch - } else { - DismissedCandidatePredicate.DismissedStitch - } - }.getOrElse(DismissedCandidatePredicate.ValidStitch) - } -} - -object DismissedCandidatePredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val DismissedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(DismissedId))) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.docx new file mode 100644 index 000000000..088390fd3 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.scala deleted file mode 100644 index 7f1d51a07..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.dismiss - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object DismissedCandidatePredicateParams { - case object LookBackDuration extends Param[Duration](180.days) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD deleted file mode 100644 index a154121e6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "stitch/stitch-gizmoduck", - "util/util-slf4j-api/src/main/scala", - "util/util-thrift", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD.docx new file mode 100644 index 000000000..3ee081985 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.docx new file mode 100644 index 000000000..b706c6318 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.scala deleted file mode 100644 index 2ca3e2fc5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.scala +++ /dev/null @@ -1,284 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient -import com.twitter.follow_recommendations.common.clients.cache.ThriftBijection -import com.twitter.follow_recommendations.common.models.FilterReason._ -import com.twitter.follow_recommendations.common.models.AddressBookMetadata -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate._ -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateParams._ -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.gizmoduck.thriftscala.LabelValue.BlinkBad -import com.twitter.gizmoduck.thriftscala.LabelValue.BlinkWorst -import com.twitter.gizmoduck.thriftscala.LabelValue -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserResult -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.scrooge.CompactThriftSerializer -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration -import com.twitter.util.logging.Logging -import java.lang.{Long => JLong} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * In this filter, we want to check 4 categories of conditions: - * - if candidate is discoverable given that it's from an address-book/phone-book based source - * - if candidate is unsuitable based on it's safety sub-fields in gizmoduck - * - if candidate is withheld because of country-specific take-down policies - * - if candidate is marked as bad/worst based on blink labels - * We fail close on the query as this is a product-critical filter - */ -@Singleton -case class GizmoduckPredicate @Inject() ( - gizmoduck: Gizmoduck, - client: Client, - statsReceiver: StatsReceiver, - decider: Decider = Decider.False) - extends Predicate[(HasClientContext with HasParams, CandidateUser)] - with Logging { - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - - // track # of Gizmoduck predicate queries that yielded valid & invalid predicate results - private val validPredicateResultCounter = stats.counter("predicate_valid") - private val invalidPredicateResultCounter = stats.counter("predicate_invalid") - - // track # of cases where no Gizmoduck user was found - private val noGizmoduckUserCounter = stats.counter("no_gizmoduck_user_found") - - private val gizmoduckCache = StitchCache[JLong, UserResult]( - maxCacheSize = MaxCacheSize, - ttl = CacheTTL, - statsReceiver = stats.scope("cache"), - underlyingCall = getByUserId - ) - - // Distributed Twemcache to store UserResult objects keyed on user IDs - val bijection = new ThriftBijection[UserResult] { - override val serializer = CompactThriftSerializer(UserResult) - } - val memcacheClient = MemcacheClient[UserResult]( - client = client, - dest = "/s/cache/frs:twemcaches", - valueBijection = bijection, - ttl = CacheTTL, - statsReceiver = stats.scope("twemcache") - ) - - // main method used to apply GizmoduckPredicate to a candidate user - override def apply( - pair: (HasClientContext with HasParams, CandidateUser) - ): Stitch[PredicateResult] = { - val (request, candidate) = pair - // measure the latency of the getGizmoduckPredicateResult, since this predicate - // check is product-critical and relies on querying a core service (Gizmoduck) - StatsUtil.profileStitch( - getGizmoduckPredicateResult(request, candidate), - stats.scope("getGizmoduckPredicateResult") - ) - } - - private def getGizmoduckPredicateResult( - request: HasClientContext with HasParams, - candidate: CandidateUser - ): Stitch[PredicateResult] = { - val timeout: Duration = request.params(GizmoduckGetTimeout) - - val deciderKey: String = DeciderKey.EnableGizmoduckCaching.toString - val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - - // try getting an existing UserResult from cache if possible - val userResultStitch: Stitch[UserResult] = - enableDistributedCaching match { - // read from memcache - case true => memcacheClient.readThrough( - // add a key prefix to address cache key collisions - key = "GizmoduckPredicate" + candidate.id.toString, - underlyingCall = () => getByUserId(candidate.id) - ) - // read from local cache - case false => gizmoduckCache.readThrough(candidate.id) - } - - val predicateResultStitch = userResultStitch.map { - userResult => { - val predicateResult = getPredicateResult(request, candidate, userResult) - if (enableDistributedCaching) { - predicateResult match { - case PredicateResult.Valid => - stats.scope("twemcache").counter("predicate_valid").incr() - case PredicateResult.Invalid(reasons) => - stats.scope("twemcache").counter("predicate_invalid").incr() - } - // log metrics to check if local cache value matches distributed cache value - logPredicateResultEquality( - request, - candidate, - predicateResult - ) - } else { - predicateResult match { - case PredicateResult.Valid => - stats.scope("cache").counter("predicate_valid").incr() - case PredicateResult.Invalid(reasons) => - stats.scope("cache").counter("predicate_invalid").incr() - } - } - predicateResult - } - } - predicateResultStitch - .within(timeout)(DefaultTimer) - .rescue { // fail-open when timeout or exception - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - invalidPredicateResultCounter.incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - } - } - - private def logPredicateResultEquality( - request: HasClientContext with HasParams, - candidate: CandidateUser, - predicateResult: PredicateResult - ): Unit = { - val localCachedUserResult = Option(gizmoduckCache.cache.getIfPresent(candidate.id)) - if (localCachedUserResult.isDefined) { - val localPredicateResult = getPredicateResult(request, candidate, localCachedUserResult.get) - localPredicateResult.equals(predicateResult) match { - case true => stats.scope("has_equal_predicate_value").counter("true").incr() - case false => stats.scope("has_equal_predicate_value").counter("false").incr() - } - } else { - stats.scope("has_equal_predicate_value").counter("undefined").incr() - } - } - - // method to get PredicateResult from UserResult - def getPredicateResult( - request: HasClientContext with HasParams, - candidate: CandidateUser, - userResult: UserResult, - ): PredicateResult = { - userResult.user match { - case Some(user) => - val abPbReasons = getAbPbReason(user, candidate.getAddressBookMetadata) - val safetyReasons = getSafetyReasons(user) - val countryTakedownReasons = getCountryTakedownReasons(user, request.getCountryCode) - val blinkReasons = getBlinkReasons(user) - val allReasons = - abPbReasons ++ safetyReasons ++ countryTakedownReasons ++ blinkReasons - if (allReasons.nonEmpty) { - invalidPredicateResultCounter.incr() - PredicateResult.Invalid(allReasons) - } else { - validPredicateResultCounter.incr() - PredicateResult.Valid - } - case None => - noGizmoduckUserCounter.incr() - invalidPredicateResultCounter.incr() - PredicateResult.Invalid(Set(NoUser)) - } - } - - private def getByUserId(userId: JLong): Stitch[UserResult] = { - StatsUtil.profileStitch( - gizmoduck.getById(userId = userId, queryFields = queryFields, context = lookupContext), - stats.scope("getByUserId") - ) - } -} - -object GizmoduckPredicate { - - private[gizmoduck] val lookupContext: LookupContext = - LookupContext(`includeDeactivated` = true, `safetyLevel` = Some(SafetyLevel.Recommendations)) - - private[gizmoduck] val queryFields: Set[QueryFields] = - Set( - QueryFields.Discoverability, // needed for Address Book / Phone Book discoverability checks in getAbPbReason - QueryFields.Safety, // needed for user state safety checks in getSafetyReasons, getCountryTakedownReasons - QueryFields.Labels, // needed for user label checks in getBlinkReasons - QueryFields.Takedowns // needed for checking takedown labels for a user in getCountryTakedownReasons - ) - - private[gizmoduck] val BlinkLabels: Set[LabelValue] = Set(BlinkBad, BlinkWorst) - - private[gizmoduck] def getAbPbReason( - user: User, - abMetadataOpt: Option[AddressBookMetadata] - ): Set[FilterReason] = { - (for { - discoverability <- user.discoverability - abMetadata <- abMetadataOpt - } yield { - val AddressBookMetadata(fwdPb, rvPb, fwdAb, rvAb) = abMetadata - val abReason: Set[FilterReason] = - if ((!discoverability.discoverableByEmail) && (fwdAb || rvAb)) - Set(AddressBookUndiscoverable) - else Set.empty - val pbReason: Set[FilterReason] = - if ((!discoverability.discoverableByMobilePhone) && (fwdPb || rvPb)) - Set(PhoneBookUndiscoverable) - else Set.empty - abReason ++ pbReason - }).getOrElse(Set.empty) - } - - private[gizmoduck] def getSafetyReasons(user: User): Set[FilterReason] = { - user.safety - .map { s => - val deactivatedReason: Set[FilterReason] = - if (s.deactivated) Set(Deactivated) else Set.empty - val suspendedReason: Set[FilterReason] = if (s.suspended) Set(Suspended) else Set.empty - val restrictedReason: Set[FilterReason] = if (s.restricted) Set(Restricted) else Set.empty - val nsfwUserReason: Set[FilterReason] = if (s.nsfwUser) Set(NsfwUser) else Set.empty - val nsfwAdminReason: Set[FilterReason] = if (s.nsfwAdmin) Set(NsfwAdmin) else Set.empty - val isProtectedReason: Set[FilterReason] = if (s.isProtected) Set(IsProtected) else Set.empty - deactivatedReason ++ suspendedReason ++ restrictedReason ++ nsfwUserReason ++ nsfwAdminReason ++ isProtectedReason - }.getOrElse(Set.empty) - } - - private[gizmoduck] def getCountryTakedownReasons( - user: User, - countryCodeOpt: Option[String] - ): Set[FilterReason] = { - (for { - safety <- user.safety.toSeq - if safety.hasTakedown - takedowns <- user.takedowns.toSeq - takedownCountry <- takedowns.countryCodes - requestingCountry <- countryCodeOpt - if takedownCountry.toLowerCase == requestingCountry.toLowerCase - } yield Set(CountryTakedown(takedownCountry.toLowerCase))).flatten.toSet - } - - private[gizmoduck] def getBlinkReasons(user: User): Set[FilterReason] = { - user.labels - .map(_.labels.map(_.labelValue)) - .getOrElse(Nil) - .exists(BlinkLabels.contains) - for { - labels <- user.labels.toSeq - label <- labels.labels - if (BlinkLabels.contains(label.labelValue)) - } yield Set(Blink) - }.flatten.toSet -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.docx new file mode 100644 index 000000000..d5edf1d13 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.scala deleted file mode 100644 index 36fb2f20f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import java.util.concurrent.TimeUnit - -import com.google.common.base.Ticker -import com.google.common.cache.CacheBuilder -import com.google.common.cache.Cache -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Time -import com.twitter.util.Duration - -/** - * In-memory cache used for caching GizmoduckPredicate query calls in - * com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate. - * - * References the cache implementation in com.twitter.escherbird.util.stitchcache, - * but without the underlying Stitch call. - */ -object GizmoduckPredicateCache { - - private[GizmoduckPredicateCache] class TimeTicker extends Ticker { - override def read(): Long = Time.now.inNanoseconds - } - - def apply[K, V]( - maxCacheSize: Int, - ttl: Duration, - statsReceiver: StatsReceiver - ): Cache[K, V] = { - - val cache: Cache[K, V] = - CacheBuilder - .newBuilder() - .maximumSize(maxCacheSize) - .asInstanceOf[CacheBuilder[K, V]] - .expireAfterWrite(ttl.inSeconds, TimeUnit.SECONDS) - .recordStats() - .ticker(new TimeTicker()) - .build() - - // metrics for tracking cache usage - statsReceiver.provideGauge("cache_size") { cache.size.toFloat } - statsReceiver.provideGauge("cache_hits") { cache.stats.hitCount.toFloat } - statsReceiver.provideGauge("cache_misses") { cache.stats.missCount.toFloat } - statsReceiver.provideGauge("cache_hit_rate") { cache.stats.hitRate.toFloat } - statsReceiver.provideGauge("cache_evictions") { cache.stats.evictionCount.toFloat } - - cache - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.docx new file mode 100644 index 000000000..0e4f33eea Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.scala deleted file mode 100644 index 447eac835..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateParams._ -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GizmoduckPredicateFSConfig @Inject() () extends FeatureSwitchConfig { - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - GizmoduckGetTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.docx new file mode 100644 index 000000000..61844767c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.scala deleted file mode 100644 index 811897e27..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object GizmoduckPredicateParams { - case object GizmoduckGetTimeout - extends FSBoundedParam[Duration]( - name = "gizmoduck_predicate_timeout_in_millis", - default = 200.millisecond, - min = 1.millisecond, - max = 500.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } - val MaxCacheSize: Int = 250000 - val CacheTTL: Duration = Duration.fromHours(6) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD deleted file mode 100644 index d0e9e1015..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "strato/config/columns/hss/user_signals/api:api-strato-client", - "util/util-slf4j-api/src/main/scala", - "util/util-thrift", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD.docx new file mode 100644 index 000000000..e7a8d0c31 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.docx new file mode 100644 index 000000000..4b5e9632d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.scala deleted file mode 100644 index 7464be7df..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.hss - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.FilterReason.FailOpen -import com.twitter.hss.api.thriftscala.SignalValue -import com.twitter.hss.api.thriftscala.UserHealthSignal.AgathaCseDouble -import com.twitter.hss.api.thriftscala.UserHealthSignal.NsfwAgathaUserScoreDouble -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.hss.user_signals.api.HealthSignalsOnUserClientColumn -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Filter out candidates based on Health Signal Store (HSS) health signals - */ -@Singleton -case class HssPredicate @Inject() ( - healthSignalsOnUserClientColumn: HealthSignalsOnUserClientColumn, - statsReceiver: StatsReceiver) - extends Predicate[(HasClientContext with HasParams, CandidateUser)] - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - - override def apply( - pair: (HasClientContext with HasParams, CandidateUser) - ): Stitch[PredicateResult] = { - val (request, candidate) = pair - StatsUtil.profileStitch( - getHssPredicateResult(request, candidate), - stats.scope("getHssPredicateResult") - ) - } - - private def getHssPredicateResult( - request: HasClientContext with HasParams, - candidate: CandidateUser - ): Stitch[PredicateResult] = { - - val hssCseScoreThreshold: Double = request.params(HssPredicateParams.HssCseScoreThreshold) - val hssNsfwScoreThreshold: Double = request.params(HssPredicateParams.HssNsfwScoreThreshold) - val timeout: Duration = request.params(HssPredicateParams.HssApiTimeout) - - healthSignalsOnUserClientColumn.fetcher - .fetch(candidate.id, Seq(AgathaCseDouble, NsfwAgathaUserScoreDouble)) - .map { fetchResult => - fetchResult.v match { - case Some(response) => - val agathaCseScoreDouble: Double = userHealthSignalValueToDoubleOpt( - response.signalValues.get(AgathaCseDouble)).getOrElse(0d) - val agathaNsfwScoreDouble: Double = userHealthSignalValueToDoubleOpt( - response.signalValues.get(NsfwAgathaUserScoreDouble)).getOrElse(0d) - - stats.stat("agathaCseScoreDistribution").add(agathaCseScoreDouble.toFloat) - stats.stat("agathaNsfwScoreDistribution").add(agathaNsfwScoreDouble.toFloat) - - /** - * Only filter out the candidate when it has both high Agatha CSE score and NSFW score, as the Agatha CSE - * model is an old one that may not be precise or have high recall. - */ - if (agathaCseScoreDouble >= hssCseScoreThreshold && agathaNsfwScoreDouble >= hssNsfwScoreThreshold) { - PredicateResult.Invalid(Set(FilterReason.HssSignal)) - } else { - PredicateResult.Valid - } - case None => - PredicateResult.Valid - } - } - .within(timeout)(DefaultTimer) - .rescue { - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - } - } - - private def userHealthSignalValueToDoubleOpt(signalValue: Option[SignalValue]): Option[Double] = { - signalValue match { - case Some(SignalValue.DoubleValue(value)) => Some(value) - case _ => None - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.docx new file mode 100644 index 000000000..9fac4973a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.scala deleted file mode 100644 index 8cc1620a9..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.hss - -import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateParams._ -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HssPredicateFSConfig @Inject() () extends FeatureSwitchConfig { - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - HssCseScoreThreshold, - HssNsfwScoreThreshold, - ) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - HssApiTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.docx new file mode 100644 index 000000000..3ac71df39 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.scala deleted file mode 100644 index ac6e14bbe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.hss - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -object HssPredicateParams { - object HssCseScoreThreshold - extends FSBoundedParam[Double]( - "hss_predicate_cse_score_threshold", - default = 0.992d, - min = 0.0d, - max = 1.0d) - - object HssNsfwScoreThreshold - extends FSBoundedParam[Double]( - "hss_predicate_nsfw_score_threshold", - default = 1.5d, - min = -100.0d, - max = 100.0d) - - object HssApiTimeout - extends FSBoundedParam[Duration]( - name = "hss_predicate_timeout_in_millis", - default = 200.millisecond, - min = 1.millisecond, - max = 500.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD deleted file mode 100644 index 0df7b245a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD.docx new file mode 100644 index 000000000..5b3198d5a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.docx new file mode 100644 index 000000000..623a7c9a3 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.scala deleted file mode 100644 index 84b8bf7a6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.HasInvalidRelationshipUserIds -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class InvalidRelationshipPredicate - extends Predicate[(HasInvalidRelationshipUserIds, CandidateUser)] { - - override def apply( - pair: (HasInvalidRelationshipUserIds, CandidateUser) - ): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - targetUser.invalidRelationshipUserIds match { - case Some(users) => - if (!users.contains(candidate.id)) { - InvalidRelationshipPredicate.ValidStitch - } else { - Stitch.value(InvalidRelationshipPredicate.InvalidRelationshipStitch) - } - case None => Stitch.value(PredicateResult.Valid) - } - } -} - -object InvalidRelationshipPredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val InvalidRelationshipStitch: PredicateResult.Invalid = - PredicateResult.Invalid(Set(FilterReason.InvalidRelationship)) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.docx new file mode 100644 index 000000000..bc70711c7 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.scala deleted file mode 100644 index 60f0080b8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class RecentFollowingPredicate extends Predicate[(HasRecentFollowedUserIds, CandidateUser)] { - - override def apply(pair: (HasRecentFollowedUserIds, CandidateUser)): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - targetUser.recentFollowedUserIdsSet match { - case Some(users) => - if (!users.contains(candidate.id)) { - RecentFollowingPredicate.ValidStitch - } else { - RecentFollowingPredicate.AlreadyFollowedStitch - } - case None => RecentFollowingPredicate.ValidStitch - } - } -} - -object RecentFollowingPredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val AlreadyFollowedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(FilterReason.AlreadyFollowed))) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.docx new file mode 100644 index 000000000..849cff01e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.scala deleted file mode 100644 index f661dbbab..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SgsPredicateFSConfig @Inject() () extends FeatureSwitchConfig { - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - SgsPredicateParams.SgsRelationshipsPredicateTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.docx new file mode 100644 index 000000000..c5b2cfed6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.scala deleted file mode 100644 index dd615c47d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object SgsPredicateParams { - case object SgsRelationshipsPredicateTimeout - extends FSBoundedParam[Duration]( - name = "sgs_predicate_relationships_timeout_in_millis", - default = 300.millisecond, - min = 1.millisecond, - max = 1000.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.docx new file mode 100644 index 000000000..8c477dd2c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.scala deleted file mode 100644 index dec936e58..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.google.common.annotations.VisibleForTesting -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason.InvalidRelationshipTypes -import com.twitter.socialgraph.thriftscala.ExistsRequest -import com.twitter.socialgraph.thriftscala.ExistsResult -import com.twitter.socialgraph.thriftscala.LookupContext -import com.twitter.socialgraph.thriftscala.Relationship -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -class SgsRelationshipsByUserIdPredicate( - socialGraph: SocialGraph, - relationshipMappings: Seq[RelationshipMapping], - statsReceiver: StatsReceiver) - extends Predicate[(Option[Long], CandidateUser)] - with Logging { - private val InvalidFromPrimaryCandidateSourceName = "invalid_from_primary_candidate_source" - private val InvalidFromCandidateSourceName = "invalid_from_candidate_source" - private val NoPrimaryCandidateSource = "no_primary_candidate_source" - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - - override def apply( - pair: (Option[Long], CandidateUser) - ): Stitch[PredicateResult] = { - val (idOpt, candidate) = pair - val relationships = relationshipMappings.map { relationshipMapping: RelationshipMapping => - Relationship( - relationshipMapping.relationshipType, - relationshipMapping.includeBasedOnRelationship) - } - idOpt - .map { id: Long => - val existsRequest = ExistsRequest( - id, - candidate.id, - relationships = relationships, - context = SgsRelationshipsByUserIdPredicate.UnionLookupContext - ) - socialGraph - .exists(existsRequest).map { existsResult: ExistsResult => - if (existsResult.exists) { - candidate.getPrimaryCandidateSource match { - case Some(candidateSource) => - stats - .scope(InvalidFromPrimaryCandidateSourceName).counter( - candidateSource.name).incr() - case None => - stats - .scope(InvalidFromPrimaryCandidateSourceName).counter( - NoPrimaryCandidateSource).incr() - } - candidate.getCandidateSources.foreach({ - case (candidateSource, _) => - stats - .scope(InvalidFromCandidateSourceName).counter(candidateSource.name).incr() - }) - PredicateResult.Invalid(Set(InvalidRelationshipTypes(relationshipMappings - .map { relationshipMapping: RelationshipMapping => - relationshipMapping.relationshipType - }.mkString(", ")))) - } else { - PredicateResult.Valid - } - } - } - // if no user id is present, return true by default - .getOrElse(Stitch.value(PredicateResult.Valid)) - } -} - -object SgsRelationshipsByUserIdPredicate { - // OR Operation - @VisibleForTesting - private[follow_recommendations] val UnionLookupContext = Some( - LookupContext(performUnion = Some(true))) -} - -@Singleton -class ExcludeNonFollowersSgsPredicate @Inject() ( - socialGraph: SocialGraph, - statsReceiver: StatsReceiver) - extends SgsRelationshipsByUserIdPredicate( - socialGraph, - Seq(RelationshipMapping(RelationshipType.FollowedBy, includeBasedOnRelationship = false)), - statsReceiver) - -@Singleton -class ExcludeNonFollowingSgsPredicate @Inject() ( - socialGraph: SocialGraph, - statsReceiver: StatsReceiver) - extends SgsRelationshipsByUserIdPredicate( - socialGraph, - Seq(RelationshipMapping(RelationshipType.Following, includeBasedOnRelationship = false)), - statsReceiver) - -@Singleton -class ExcludeFollowingSgsPredicate @Inject() ( - socialGraph: SocialGraph, - statsReceiver: StatsReceiver) - extends SgsRelationshipsByUserIdPredicate( - socialGraph, - Seq(RelationshipMapping(RelationshipType.Following, includeBasedOnRelationship = true)), - statsReceiver) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.docx new file mode 100644 index 000000000..f7726b52c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.scala deleted file mode 100644 index fdb88ad58..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.scala +++ /dev/null @@ -1,146 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.google.common.annotations.VisibleForTesting -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasProfileId -import com.twitter.follow_recommendations.common.models.FilterReason.FailOpen -import com.twitter.follow_recommendations.common.models.FilterReason.InvalidRelationshipTypes -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.socialgraph.thriftscala.ExistsRequest -import com.twitter.socialgraph.thriftscala.ExistsResult -import com.twitter.socialgraph.thriftscala.LookupContext -import com.twitter.socialgraph.thriftscala.Relationship -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.TimeoutException -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -case class RelationshipMapping( - relationshipType: RelationshipType, - includeBasedOnRelationship: Boolean) - -class SgsRelationshipsPredicate( - socialGraph: SocialGraph, - relationshipMappings: Seq[RelationshipMapping], - statsReceiver: StatsReceiver = NullStatsReceiver) - extends Predicate[(HasClientContext with HasParams, CandidateUser)] - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - override def apply( - pair: (HasClientContext with HasParams, CandidateUser) - ): Stitch[PredicateResult] = { - val (target, candidate) = pair - val timeout = target.params(SgsPredicateParams.SgsRelationshipsPredicateTimeout) - SgsRelationshipsPredicate - .extractUserId(target) - .map { id => - val relationships = relationshipMappings.map { relationshipMapping: RelationshipMapping => - Relationship( - relationshipMapping.relationshipType, - relationshipMapping.includeBasedOnRelationship) - } - val existsRequest = ExistsRequest( - id, - candidate.id, - relationships = relationships, - context = SgsRelationshipsPredicate.UnionLookupContext - ) - socialGraph - .exists(existsRequest).map { existsResult: ExistsResult => - if (existsResult.exists) { - PredicateResult.Invalid(Set(InvalidRelationshipTypes(relationshipMappings - .map { relationshipMapping: RelationshipMapping => - relationshipMapping.relationshipType - }.mkString(", ")))) - } else { - PredicateResult.Valid - } - } - .within(timeout)(com.twitter.finagle.util.DefaultTimer) - } - // if no user id is present, return true by default - .getOrElse(Stitch.value(PredicateResult.Valid)) - .rescue { - case e: TimeoutException => - stats.counter("timeout").incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - } - - } -} - -object SgsRelationshipsPredicate { - // OR Operation - @VisibleForTesting - private[follow_recommendations] val UnionLookupContext = Some( - LookupContext(performUnion = Some(true))) - - private def extractUserId(target: HasClientContext with HasParams): Option[Long] = target match { - case profRequest: HasProfileId => Some(profRequest.profileId) - case userRequest: HasClientContext with HasParams => userRequest.getOptionalUserId - case _ => None - } -} - -@Singleton -class InvalidTargetCandidateRelationshipTypesPredicate @Inject() ( - socialGraph: SocialGraph) - extends SgsRelationshipsPredicate( - socialGraph, - InvalidRelationshipTypesPredicate.InvalidRelationshipTypes) {} - -@Singleton -class NoteworthyAccountsSgsPredicate @Inject() ( - socialGraph: SocialGraph) - extends SgsRelationshipsPredicate( - socialGraph, - InvalidRelationshipTypesPredicate.NoteworthyAccountsInvalidRelationshipTypes) - -object InvalidRelationshipTypesPredicate { - - val InvalidRelationshipTypesExcludeFollowing: Seq[RelationshipMapping] = Seq( - RelationshipMapping(RelationshipType.HideRecommendations, true), - RelationshipMapping(RelationshipType.Blocking, true), - RelationshipMapping(RelationshipType.BlockedBy, true), - RelationshipMapping(RelationshipType.Muting, true), - RelationshipMapping(RelationshipType.MutedBy, true), - RelationshipMapping(RelationshipType.ReportedAsSpam, true), - RelationshipMapping(RelationshipType.ReportedAsSpamBy, true), - RelationshipMapping(RelationshipType.ReportedAsAbuse, true), - RelationshipMapping(RelationshipType.ReportedAsAbuseBy, true) - ) - - val InvalidRelationshipTypes: Seq[RelationshipMapping] = Seq( - RelationshipMapping(RelationshipType.FollowRequestOutgoing, true), - RelationshipMapping(RelationshipType.Following, true), - RelationshipMapping( - RelationshipType.UsedToFollow, - true - ) // this data is accessible for 90 days. - ) ++ InvalidRelationshipTypesExcludeFollowing - - val NoteworthyAccountsInvalidRelationshipTypes: Seq[RelationshipMapping] = Seq( - RelationshipMapping(RelationshipType.Blocking, true), - RelationshipMapping(RelationshipType.BlockedBy, true), - RelationshipMapping(RelationshipType.Muting, true), - RelationshipMapping(RelationshipType.MutedBy, true), - RelationshipMapping(RelationshipType.ReportedAsSpam, true), - RelationshipMapping(RelationshipType.ReportedAsSpamBy, true), - RelationshipMapping(RelationshipType.ReportedAsAbuse, true), - RelationshipMapping(RelationshipType.ReportedAsAbuseBy, true) - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD deleted file mode 100644 index fe3df2d8b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "strato/config/columns/onboarding:onboarding-strato-client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD.docx new file mode 100644 index 000000000..550d0c5cb Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.docx new file mode 100644 index 000000000..fc648809e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.scala deleted file mode 100644 index e0fd6b42c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.scala +++ /dev/null @@ -1,161 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.user_activity - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient -import com.twitter.follow_recommendations.common.clients.cache.ThriftEnumOptionBijection -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.UserRecommendabilityWithLongKeysOnUserClientColumn -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -abstract case class UserStateActivityPredicate( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - validCandidateStates: Set[UserState], - client: Client, - statsReceiver: StatsReceiver, - decider: Decider = Decider.False) - extends Predicate[(HasParams with HasClientContext, CandidateUser)] { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - // client to memcache cluster - val bijection = new ThriftEnumOptionBijection[UserState](UserState.apply) - val memcacheClient = MemcacheClient[Option[UserState]]( - client = client, - dest = "/s/cache/follow_recos_service:twemcaches", - valueBijection = bijection, - ttl = UserActivityPredicateParams.CacheTTL, - statsReceiver = stats.scope("twemcache") - ) - - override def apply( - targetAndCandidate: (HasParams with HasClientContext, CandidateUser) - ): Stitch[PredicateResult] = { - val userRecommendabilityFetcher = userRecommendabilityClient.fetcher - val (_, candidate) = targetAndCandidate - - val deciderKey: String = DeciderKey.EnableExperimentalCaching.toString - val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - val userStateStitch: Stitch[Option[UserState]] = - enableDistributedCaching match { - case true => { - memcacheClient.readThrough( - // add a key prefix to address cache key collisions - key = "UserActivityPredicate" + candidate.id.toString, - underlyingCall = () => queryUserRecommendable(candidate.id) - ) - } - case false => queryUserRecommendable(candidate.id) - } - val resultStitch: Stitch[PredicateResult] = - userStateStitch.map { userStateOpt => - userStateOpt match { - case Some(userState) => { - if (validCandidateStates.contains(userState)) { - PredicateResult.Valid - } else { - PredicateResult.Invalid(Set(FilterReason.MinStateNotMet)) - } - } - case None => { - PredicateResult.Invalid(Set(FilterReason.MissingRecommendabilityData)) - } - } - } - - StatsUtil.profileStitch(resultStitch, stats.scope("apply")) - .rescue { - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FilterReason.FailOpen))) - } - } - - def queryUserRecommendable( - userId: Long - ): Stitch[Option[UserState]] = { - val userRecommendabilityFetcher = userRecommendabilityClient.fetcher - userRecommendabilityFetcher.fetch(userId).map { userCandidate => - userCandidate.v.flatMap(_.userState) - } - } -} - -@Singleton -class MinStateUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.Light, - UserState.HeavyNonTweeter, - UserState.MediumNonTweeter, - UserState.HeavyTweeter, - UserState.MediumTweeter - ), - client, - statsReceiver - ) - -@Singleton -class AllTweeterUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.HeavyTweeter, - UserState.MediumTweeter - ), - client, - statsReceiver - ) - -@Singleton -class HeavyTweeterUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.HeavyTweeter - ), - client, - statsReceiver - ) - -@Singleton -class NonNearZeroUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.New, - UserState.VeryLight, - UserState.Light, - UserState.MediumNonTweeter, - UserState.MediumTweeter, - UserState.HeavyNonTweeter, - UserState.HeavyTweeter - ), - client, - statsReceiver - ) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.docx new file mode 100644 index 000000000..2e1381847 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.scala deleted file mode 100644 index 57e8d958b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.user_activity - -import com.twitter.timelines.configapi.FSParam -import com.twitter.util.Duration - -object UserActivityPredicateParams { - case object HeavyTweeterEnabled - extends FSParam[Boolean]("user_activity_predicate_heavy_tweeter_enabled", false) - val CacheTTL: Duration = Duration.fromHours(6) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.docx new file mode 100644 index 000000000..41b605cbe Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.scala deleted file mode 100644 index 23ccd6f4f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.common - -/** - * To manage the extent of adhoc score modifications, we set a hard limit that from each of the - * types below *ONLY ONE* adhoc scorer can be applied to candidates' scores. More details about the - * usage is available in [[AdhocRanker]] - */ - -object AdhocScoreModificationType extends Enumeration { - type AdhocScoreModificationType = Value - - // This type of scorer increases the score of a subset of candidates through various policies. - val BoostingScorer: AdhocScoreModificationType = Value("boosting") - - // This type of scorer shuffles candidates randomly according to some distribution. - val WeightedRandomSamplingScorer: AdhocScoreModificationType = Value("weighted_random_sampling") - - // This is added solely for testing purposes and should not be used in production. - val InvalidAdhocScorer: AdhocScoreModificationType = Value("invalid_adhoc_scorer") -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD deleted file mode 100644 index 77d496dcf..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD.docx new file mode 100644 index 000000000..a9ad1963e Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.docx new file mode 100644 index 000000000..fa5b179a1 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.scala deleted file mode 100644 index bbbde2b58..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.common - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import scala.collection.mutable - -object DedupCandidates { - def apply[C <: UniversalNoun[Long]](input: Seq[C]): Seq[C] = { - val seen = mutable.HashSet[Long]() - input.filter { candidate => seen.add(candidate.id) } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.docx new file mode 100644 index 000000000..ee28c2a39 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.scala deleted file mode 100644 index f6fdb905a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.common - -object RankerId extends Enumeration { - type RankerId = Value - - val RandomRanker: RankerId = Value("random") - // The production PostNUX ML warm-start auto-retraining model ranker - val PostNuxProdRanker: RankerId = Value("postnux_prod") - val None: RankerId = Value("none") - - // Sampling from the Placket-Luce distribution. Applied after ranker step. Its ranker id is mainly used for logging. - val PlacketLuceSamplingTransformer: RankerId = Value("placket_luce_sampling_transformer") - - def getRankerByName(name: String): Option[RankerId] = - RankerId.values.toSeq.find(_.equals(Value(name))) - -} - -/** - * ML model based heavy ranker ids. - */ -object ModelBasedHeavyRankerId { - import RankerId._ - val HeavyRankerIds: Set[String] = Set( - PostNuxProdRanker.toString, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD deleted file mode 100644 index 2fce8d77a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD.docx new file mode 100644 index 000000000..c3b98c508 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.docx new file mode 100644 index 000000000..a7cda9728 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.scala deleted file mode 100644 index 18ac0436b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.fatigue_ranker - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasWtfImpressions -import com.twitter.follow_recommendations.common.models.WtfImpression -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Time - -/** - * Ranks candidates based on the given weights for each algorithm while preserving the ranks inside each algorithm. - * Reorders the ranked list based on recent impressions from recentImpressionRepo - * - * Note that the penalty is added to the rank of each candidate. To make producer-side experiments - * with multiple rankers possible, we modify the scores for each candidate and ranker as: - * NewScore(C, R) = -(Rank(C, R) + Impression(C, U) x FatigueFactor), - * where C is a candidate, R a ranker and U the target user. - * Note also that fatigue penalty is independent of any of the rankers. - */ -class ImpressionBasedFatigueRanker[ - Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions -]( - fatigueFactor: Int, - statsReceiver: StatsReceiver) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - val stats = statsReceiver.scope("impression_based_fatigue_ranker") - val droppedStats: MemoizingStatsReceiver = new MemoizingStatsReceiver(stats.scope("hard_drops")) - val impressionStats: StatsReceiver = stats.scope("wtf_impressions") - val noImpressionCounter: Counter = impressionStats.counter("no_impressions") - val oldestImpressionStat: Stat = impressionStats.stat("oldest_sec") - - override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - StatsUtil.profileStitch( - Stitch.value(rankCandidates(target, candidates)), - stats.scope("rank") - ) - } - - private def trackTimeSinceOldestImpression(impressions: Seq[WtfImpression]): Unit = { - val timeSinceOldest = Time.now - impressions.map(_.latestTime).min - oldestImpressionStat.add(timeSinceOldest.inSeconds) - } - - private def rankCandidates( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - target.wtfImpressions - .map { wtfImpressions => - if (wtfImpressions.isEmpty) { - noImpressionCounter.incr() - candidates - } else { - val rankerIds = - candidates.flatMap(_.scores.map(_.scores.flatMap(_.rankerId))).flatten.sorted.distinct - - /** - * In below we create a Map from each CandidateUser's ID to a Map from each Ranker that - * the user has a score for, and candidate's corresponding rank when candidates are sorted - * by that Ranker (Only candidates who have this Ranker are considered for ranking). - */ - val candidateRanks: Map[Long, Map[RankerId, Int]] = rankerIds - .flatMap { rankerId => - // Candidates with no scores from this Ranker is first removed to calculate ranks. - val relatedCandidates = - candidates.filter(_.scores.exists(_.scores.exists(_.rankerId.contains(rankerId)))) - relatedCandidates - .sortBy(-_.scores - .flatMap(_.scores.find(_.rankerId.contains(rankerId)).map(_.value)).getOrElse( - 0.0)).zipWithIndex.map { - case (candidate, rank) => (candidate.id, rankerId, rank) - } - }.groupBy(_._1).map { - case (candidate, ranksForAllRankers) => - ( - candidate, - ranksForAllRankers.map { case (_, rankerId, rank) => (rankerId, rank) }.toMap) - } - - val idFatigueCountMap = - wtfImpressions.groupBy(_.candidateId).mapValues(_.map(_.counts).sum) - trackTimeSinceOldestImpression(wtfImpressions) - val rankedCandidates: Seq[CandidateUser] = candidates - .map { candidate => - val candidateImpressions = idFatigueCountMap.getOrElse(candidate.id, 0) - val fatiguedScores = candidate.scores.map { ss => - ss.copy(scores = ss.scores.map { s => - s.rankerId match { - // We set the new score as -rank after fatigue penalty is applied. - case Some(rankerId) => - // If the candidate's ID is not in the candidate->ranks map, or there is no - // rank for this specific ranker and this candidate, we use maximum possible - // rank instead. Note that this indicates that there is a problem. - s.copy(value = -(candidateRanks - .getOrElse(candidate.id, Map()).getOrElse(rankerId, candidates.length) + - candidateImpressions * fatigueFactor)) - // In case a score exists without a RankerId, we pass on the score as is. - case None => s - } - }) - } - candidate.copy(scores = fatiguedScores) - }.zipWithIndex.map { - // We re-rank candidates with their input ordering (which is done by the request-level - // ranker) and fatigue penalty. - case (candidate, inputRank) => - val candidateImpressions = idFatigueCountMap.getOrElse(candidate.id, 0) - (candidate, inputRank + candidateImpressions * fatigueFactor) - }.sortBy(_._2).map(_._1) - // Only populate ranking info when WTF impression info present - val scribeRankingInfo: Boolean = - target.params(ImpressionBasedFatigueRankerParams.ScribeRankingInfoInFatigueRanker) - if (scribeRankingInfo) Utils.addRankingInfo(rankedCandidates, name) else rankedCandidates - } - }.getOrElse(candidates) // no reranking/filtering when wtf impressions not present - } -} - -object ImpressionBasedFatigueRanker { - val DefaultFatigueFactor = 5 - - def build[ - Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions - ]( - baseStatsReceiver: StatsReceiver, - fatigueFactor: Int = DefaultFatigueFactor - ): ImpressionBasedFatigueRanker[Target] = - new ImpressionBasedFatigueRanker(fatigueFactor, baseStatsReceiver) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.docx new file mode 100644 index 000000000..8aef47287 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.scala deleted file mode 100644 index 34fbbeb46..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.fatigue_ranker - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ImpressionBasedFatigueRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(ImpressionBasedFatigueRankerParams.ScribeRankingInfoInFatigueRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.docx new file mode 100644 index 000000000..704dadd49 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.scala deleted file mode 100644 index 075d78bf6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.fatigue_ranker - -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object ImpressionBasedFatigueRankerParams { - // Whether to enable hard dropping of impressed candidates - object DropImpressedCandidateEnabled extends Param[Boolean](false) - // At what # of impressions to hard drop candidates. - object DropCandidateImpressionThreshold extends Param[Int](default = 10) - // Whether to scribe candidate ranking/scoring info per ranking stage - object ScribeRankingInfoInFatigueRanker - extends FSParam[Boolean]("fatigue_ranker_scribe_ranking_info", true) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD deleted file mode 100644 index 3de5523b1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD.docx new file mode 100644 index 000000000..e385c347f Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.docx new file mode 100644 index 000000000..a68da76cd Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.scala deleted file mode 100644 index 8fbeafa25..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.scala +++ /dev/null @@ -1,115 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasQualityFactor -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This class is meant to filter candidates between stages of our ranker by taking the first N - * candidates, merging any candidate source information for candidates with multiple entries. - * To allow us to chain this truncation operation any number of times sequentially within the main - * ranking builder, we abstract the truncation as a separate Ranker - */ -@Singleton -class FirstNRanker[Target <: HasClientContext with HasParams with HasQualityFactor] @Inject() ( - stats: StatsReceiver) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - private val baseStats = stats.scope("first_n_ranker") - val scaledDownByQualityFactorCounter = - baseStats.counter("scaled_down_by_quality_factor") - private val mergeStat = baseStats.scope("merged_candidates") - private val mergeStat2 = mergeStat.counter("2") - private val mergeStat3 = mergeStat.counter("3") - private val mergeStat4 = mergeStat.counter("4+") - private val candidateSizeStats = baseStats.scope("candidate_size") - - private case class CandidateSourceScore( - candidateId: Long, - sourceId: CandidateSourceIdentifier, - score: Option[Double]) - - /** - * Adds the rank of each candidate based on the primary candidate source's score. - * In the event where the provided ordering of candidates do not align with the score, - * we will respect the score, since the ordering might have been mixed up due to other previous - * steps like the shuffleFn in the `WeightedCandidateSourceRanker`. - * @param candidates ordered list of candidates - * @return same ordered list of candidates, but with the rank information appended - */ - def addRank(candidates: Seq[CandidateUser]): Seq[CandidateUser] = { - val candidateSourceRanks = for { - (sourceIdOpt, sourceCandidates) <- candidates.groupBy(_.getPrimaryCandidateSource) - (candidate, rank) <- sourceCandidates.sortBy(-_.score.getOrElse(0.0)).zipWithIndex - } yield { - (candidate, sourceIdOpt) -> rank - } - candidates.map { c => - c.getPrimaryCandidateSource - .map { sourceId => - val sourceRank = candidateSourceRanks((c, c.getPrimaryCandidateSource)) - c.addCandidateSourceRanksMap(Map(sourceId -> sourceRank)) - }.getOrElse(c) - } - } - - override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - - val scaleDownFactor = Math.max( - target.qualityFactor.getOrElse(1.0d), - target.params(FirstNRankerParams.MinNumCandidatesScoredScaleDownFactor) - ) - - if (scaleDownFactor < 1.0d) - scaledDownByQualityFactorCounter.incr() - - val n = (target.params(FirstNRankerParams.CandidatesToRank) * scaleDownFactor).toInt - val scribeRankingInfo: Boolean = - target.params(FirstNRankerParams.ScribeRankingInfoInFirstNRanker) - candidateSizeStats.counter(s"n$n").incr() - val candidatesWithRank = addRank(candidates) - if (target.params(FirstNRankerParams.GroupDuplicateCandidates)) { - val groupedCandidates: Map[Long, Seq[CandidateUser]] = candidatesWithRank.groupBy(_.id) - val topN = candidates - .map { c => - merge(groupedCandidates(c.id)) - }.distinct.take(n) - Stitch.value(if (scribeRankingInfo) Utils.addRankingInfo(topN, name) else topN) - } else { - Stitch.value( - if (scribeRankingInfo) Utils.addRankingInfo(candidatesWithRank, name).take(n) - else candidatesWithRank.take(n)) - } // for efficiency, if don't need to deduplicate - } - - /** - * we use the primary candidate source of the first entry, and aggregate all of the other entries' - * candidate source scores into the first entry's candidateSourceScores - * @param candidates list of candidates with the same id - * @return a single merged candidate - */ - private[first_n_ranker] def merge(candidates: Seq[CandidateUser]): CandidateUser = { - if (candidates.size == 1) { - candidates.head - } else { - candidates.size match { - case 2 => mergeStat2.incr() - case 3 => mergeStat3.incr() - case i if i >= 4 => mergeStat4.incr() - case _ => - } - val allSources = candidates.flatMap(_.getCandidateSources).toMap - val allRanks = candidates.flatMap(_.getCandidateRanks).toMap - candidates.head.addCandidateSourceScoresMap(allSources).addCandidateSourceRanksMap(allRanks) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.docx new file mode 100644 index 000000000..c4064f8bc Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.scala deleted file mode 100644 index 484738dc1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -@Singleton -class FirstNRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(FirstNRankerParams.ScribeRankingInfoInFirstNRanker) - - override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( - FirstNRankerParams.CandidatesToRank - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - FirstNRankerParams.MinNumCandidatesScoredScaleDownFactor - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.docx new file mode 100644 index 000000000..1807936c8 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.scala deleted file mode 100644 index 682b60fed..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -object FirstNRankerFeatureSwitchKeys { - val CandidatePoolSize = "first_n_ranker_candidate_pool_size" - val ScribeRankingInfo = "first_n_ranker_scribe_ranking_info" - val MinNumCandidatesScoredScaleDownFactor = - "first_n_ranker_min_scale_down_factor" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.docx new file mode 100644 index 000000000..4b7877579 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.scala deleted file mode 100644 index ac65a6dde..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object FirstNRankerParams { - case object CandidatesToRank - extends FSBoundedParam[Int]( - FirstNRankerFeatureSwitchKeys.CandidatePoolSize, - default = 100, - min = 50, - max = 600) - - case object GroupDuplicateCandidates extends Param[Boolean](true) - case object ScribeRankingInfoInFirstNRanker - extends FSParam[Boolean](FirstNRankerFeatureSwitchKeys.ScribeRankingInfo, true) - - // the minimum of candidates to score in each request. - object MinNumCandidatesScoredScaleDownFactor - extends FSBoundedParam[Double]( - name = FirstNRankerFeatureSwitchKeys.MinNumCandidatesScoredScaleDownFactor, - default = 0.3, - min = 0.1, - max = 1.0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD deleted file mode 100644 index e71d8a7ab..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD.docx new file mode 100644 index 000000000..73b4cac81 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.docx new file mode 100644 index 000000000..b4f65c160 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.scala deleted file mode 100644 index 973275f51..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.scala +++ /dev/null @@ -1,204 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.interleave_ranker - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -@Singleton -class InterleaveRanker[Target <: HasParams] @Inject() ( - statsReceiver: StatsReceiver) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - private val stats = statsReceiver.scope("interleave_ranker") - private val inputStats = stats.scope("input") - private val interleavingStats = stats.scope("interleave") - - override def rank( - target: Target, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - StatsUtil.profileStitch( - Stitch.value(rankCandidates(target, candidates)), - stats.scope("rank") - ) - } - - private def rankCandidates( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - - /** - * By this stage, all valid candidates should have: - * 1. Their Scores field populated. - * 2. Their selectedRankerId set. - * 3. Have a score associated to their selectedRankerId. - * If there is any candidate that doesn't meet the conditions above, there is a problem in one - * of the previous rankers. Since no new scoring is done in this ranker, we simply remove them. - */ - val validCandidates = - candidates.filter { c => - c.scores.isDefined && - c.scores.exists(_.selectedRankerId.isDefined) && - getCandidateScoreByRankerId(c, c.scores.flatMap(_.selectedRankerId)).isDefined - } - - // To monitor the percentage of valid candidates, as defined above, we track the following: - inputStats.counter("candidates_with_no_scores").incr(candidates.count(_.scores.isEmpty)) - inputStats - .counter("candidates_with_no_selected_ranker").incr(candidates.count { c => - c.scores.isEmpty || c.scores.exists(_.selectedRankerId.isEmpty) - }) - inputStats - .counter("candidates_with_no_score_for_selected_ranker").incr(candidates.count { c => - c.scores.isEmpty || - c.scores.exists(_.selectedRankerId.isEmpty) || - getCandidateScoreByRankerId(c, c.scores.flatMap(_.selectedRankerId)).isEmpty - }) - inputStats.counter("total_num_candidates").incr(candidates.length) - inputStats.counter("total_valid_candidates").incr(validCandidates.length) - - // We only count rankerIds from those candidates who are valid to exclude those candidates with - // a valid selectedRankerId that don't have an associated score for it. - val rankerIds = validCandidates.flatMap(_.scores.flatMap(_.selectedRankerId)).sorted.distinct - rankerIds.foreach { rankerId => - inputStats - .counter(s"valid_scores_for_${rankerId.toString}").incr( - candidates.count(getCandidateScoreByRankerId(_, Some(rankerId)).isDefined)) - inputStats.counter(s"total_candidates_for_${rankerId.toString}").incr(candidates.length) - } - inputStats.counter(s"num_ranker_ids=${rankerIds.length}").incr() - val scribeRankingInfo: Boolean = - target.params(InterleaveRankerParams.ScribeRankingInfoInInterleaveRanker) - if (rankerIds.length <= 1) - // In the case of "Number of RankerIds = 0", we pass on the candidates even though there is - // a problem in a previous ranker that provided the scores. - if (scribeRankingInfo) Utils.addRankingInfo(candidates, name) else candidates - else - if (scribeRankingInfo) - Utils.addRankingInfo(interleaveCandidates(validCandidates, rankerIds), name) - else interleaveCandidates(validCandidates, rankerIds) - } - - @VisibleForTesting - private[interleave_ranker] def interleaveCandidates( - candidates: Seq[CandidateUser], - rankerIds: Seq[RankerId.RankerId] - ): Seq[CandidateUser] = { - val candidatesWithRank = rankerIds - .flatMap { ranker => - candidates - // We first sort all candidates using this ranker. - .sortBy(-getCandidateScoreByRankerId(_, Some(ranker)).getOrElse(Double.MinValue)) - .zipWithIndex.filter( - // but only hold those candidates whose selected ranker is this ranker. - // These ranks will be forced in the final ordering. - _._1.scores.flatMap(_.selectedRankerId).contains(ranker)) - } - - // Only candidates who have isInProducerScoringExperiment set to true will have their position enforced. We - // separate candidates into two groups: (1) Production and (2) Experiment. - val (expCandidates, prodCandidates) = - candidatesWithRank.partition(_._1.scores.exists(_.isInProducerScoringExperiment)) - - // We resolve (potential) conflicts between the enforced ranks of experimental models. - val expCandidatesFinalPos = resolveConflicts(expCandidates) - - // Retrieve non-occupied positions and assign them to candidates who use production ranker. - val occupiedPos = expCandidatesFinalPos.map(_._2).toSet - val prodCandidatesFinalPos = - prodCandidates - .map(_._1).zip( - candidates.indices.filterNot(occupiedPos.contains).sorted.take(prodCandidates.length)) - - // Merge the two groups and sort them by their corresponding positions. - val finalCandidates = (prodCandidatesFinalPos ++ expCandidatesFinalPos).sortBy(_._2).map(_._1) - - // We count the presence of each ranker in the top-3 final positions. - finalCandidates.zip(0 until 3).foreach { - case (c, r) => - // We only do so for candidates that are in a producer-side experiment. - if (c.scores.exists(_.isInProducerScoringExperiment)) - c.scores.flatMap(_.selectedRankerId).map(_.toString).foreach { rankerName => - interleavingStats - .counter(s"num_final_position_${r}_$rankerName") - .incr() - } - } - - finalCandidates - } - - @VisibleForTesting - private[interleave_ranker] def resolveConflicts( - candidatesWithRank: Seq[(CandidateUser, Int)] - ): Seq[(CandidateUser, Int)] = { - // The following two metrics will allow us to calculate the rate of conflicts occurring. - // Example: If overall there are 10 producers in different bucketing experiments, and 3 of them - // are assigned to the same position. The rate would be 3/10, 30%. - val numCandidatesWithConflicts = interleavingStats.counter("candidates_with_conflict") - val numCandidatesNoConflicts = interleavingStats.counter("candidates_without_conflict") - val candidatesGroupedByRank = candidatesWithRank.groupBy(_._2).toSeq.sortBy(_._1).map { - case (rank, candidatesWithRank) => (rank, candidatesWithRank.map(_._1)) - } - - candidatesGroupedByRank.foldLeft(Seq[(CandidateUser, Int)]()) { (upToHere, nextGroup) => - val (rank, candidates) = nextGroup - if (candidates.length > 1) - numCandidatesWithConflicts.incr(candidates.length) - else - numCandidatesNoConflicts.incr() - - // We use the position after the last-assigned candidate as a starting point, or 0 otherwise. - // If candidates' position is after this "starting point", we enforce that position instead. - val minAvailableIndex = scala.math.max(upToHere.lastOption.map(_._2).getOrElse(-1) + 1, rank) - val enforcedPos = - (minAvailableIndex until minAvailableIndex + candidates.length).toList - val shuffledEnforcedPos = - if (candidates.length > 1) scala.util.Random.shuffle(enforcedPos) else enforcedPos - if (shuffledEnforcedPos.length > 1) { - candidates.zip(shuffledEnforcedPos).sortBy(_._2).map(_._1).zipWithIndex.foreach { - case (c, r) => - c.scores.flatMap(_.selectedRankerId).map(_.toString).foreach { rankerName => - // For each ranker, we count the total number of times it has been in a conflict. - interleavingStats - .counter(s"num_${shuffledEnforcedPos.length}-way_conflicts_$rankerName") - .incr() - // We also count the positions each of the rankers have fallen randomly into. In any - // experiment this should converge to uniform distribution given enough occurrences. - // Note that the position here is relative to the other candidates in the conflict and - // not the overall position of each candidate. - interleavingStats - .counter( - s"num_position_${r}_after_${shuffledEnforcedPos.length}-way_conflict_$rankerName") - .incr() - } - } - } - upToHere ++ candidates.zip(shuffledEnforcedPos).sortBy(_._2) - } - } - - @VisibleForTesting - private[interleave_ranker] def getCandidateScoreByRankerId( - candidate: CandidateUser, - rankerIdOpt: Option[RankerId.RankerId] - ): Option[Double] = { - rankerIdOpt match { - case None => None - case Some(rankerId) => - candidate.scores.flatMap { - _.scores.find(_.rankerId.contains(rankerId)).map(_.value) - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.docx new file mode 100644 index 000000000..24254eca3 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.scala deleted file mode 100644 index 5a8c6de2a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.interleave_ranker - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam - -@Singleton -class InterleaveRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(InterleaveRankerParams.ScribeRankingInfoInInterleaveRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.docx new file mode 100644 index 000000000..18090f5a6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.scala deleted file mode 100644 index 84e6ea314..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.interleave_ranker - -import com.twitter.timelines.configapi.FSParam - -object InterleaveRankerParams { - case object ScribeRankingInfoInInterleaveRanker - extends FSParam[Boolean]("interleave_ranker_scribe_ranking_info", true) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD deleted file mode 100644 index 0c277e005..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "src/java/com/twitter/ml/api:api-base", - "util/util-slf4j-api/src/main/scala", - ], -) - -# This is to import only the params from MlRanker, for instance to get request-level heavy ranker. -scala_library( - name = "ml_ranker_params", - sources = [ - "MlRankerParams.scala", - ], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "timelines/src/main/scala/com/twitter/timelines/config/configapi", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD.docx new file mode 100644 index 000000000..2dfaf8f36 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.docx new file mode 100644 index 000000000..f7cdcfc83 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.scala deleted file mode 100644 index 499a14b67..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.base.StatsUtil.profileStitchMapResults -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.UserScoringFeatureSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.follow_recommendations.common.models.RichDataRecord -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging - -/** - * Hydrate features given target and candidates lists. - * This is a required step before MlRanker. - * If a feature is not hydrated before MlRanker is triggered, a runtime exception will be thrown - */ -@Singleton -class HydrateFeaturesTransform[ - Target <: HasClientContext with HasParams with HasDebugOptions with HasPreFetchedFeature with HasSimilarToContext with HasDisplayLocation] @Inject() ( - userScoringFeatureSource: UserScoringFeatureSource, - stats: StatsReceiver) - extends GatedTransform[Target, CandidateUser] - with Logging { - - private val hydrateFeaturesStats = stats.scope("hydrate_features") - - def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - // get features - val featureMapStitch: Stitch[Map[CandidateUser, DataRecord]] = - profileStitchMapResults( - userScoringFeatureSource.hydrateFeatures(target, candidates), - hydrateFeaturesStats) - - featureMapStitch.map { featureMap => - candidates - .map { candidate => - val dataRecord = featureMap(candidate) - // add debugRecord only when the request parameter is set - val debugDataRecord = if (target.debugOptions.exists(_.fetchDebugInfo)) { - Some(candidate.toDebugDataRecord(dataRecord, userScoringFeatureSource.featureContext)) - } else None - candidate.copy( - dataRecord = Some(RichDataRecord(Some(dataRecord), debugDataRecord)) - ) - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.docx new file mode 100644 index 000000000..369190763 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.scala deleted file mode 100644 index 6344c348e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.scala +++ /dev/null @@ -1,219 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.base.StatsUtil.profileSeqResults -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.Scores -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.AdhocScorer -import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.Scorer -import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.ScorerFactory -import com.twitter.follow_recommendations.common.utils.CollectionUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params -import com.twitter.util.logging.Logging - -/** - * This class has a rank function that will perform 4 steps: - * - choose which scorer to use for each candidate - * - score candidates given their respective features - * - add scoring information to the candidate - * - sort candidates by their respective scores - * The feature source and scorer will depend on the request's params - */ -@Singleton -class MlRanker[ - Target <: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions] @Inject() ( - scorerFactory: ScorerFactory, - statsReceiver: StatsReceiver) - extends Ranker[Target, CandidateUser] - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope("ml_ranker") - - private val inputStat = stats.scope("1_input") - private val selectScorerStat = stats.scope("2_select_scorer") - private val scoreStat = stats.scope("3_score") - - override def rank( - target: Target, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - profileSeqResults(candidates, inputStat) - val requestRankerId = target.params(MlRankerParams.RequestScorerIdParam) - val rankerIds = chooseRankerByCandidate(candidates, requestRankerId) - - val scoreStitch = score(candidates, rankerIds, requestRankerId).map { scoredCandidates => - { - // sort the candidates by score - val sortedCandidates = sort(target, scoredCandidates) - // add scribe field to candidates (if applicable) and return candidates - scribeCandidates(target, sortedCandidates) - } - } - StatsUtil.profileStitch(scoreStitch, stats.scope("rank")) - } - - /** - * @param target: The WTF request for a given consumer. - * @param candidates A list of candidates considered for recommendation. - * @return A map from each candidate to a tuple that includes: - * (1) The selected scorer that should be used to rank this candidate - * (2) a flag determining whether the candidate is in a producer-side experiment. - */ - private[ranking] def chooseRankerByCandidate( - candidates: Seq[CandidateUser], - requestRankerId: RankerId - ): Map[CandidateUser, RankerId] = { - candidates.map { candidate => - val selectedCandidateRankerId = - if (candidate.params == Params.Invalid || candidate.params == Params.Empty) { - selectScorerStat.counter("candidate_params_empty").incr() - requestRankerId - } else { - val candidateRankerId = candidate.params(MlRankerParams.CandidateScorerIdParam) - if (candidateRankerId == RankerId.None) { - // This candidate is a not part of any producer-side experiment. - selectScorerStat.counter("default_to_request_ranker").incr() - requestRankerId - } else { - // This candidate is in a treatment bucket of a producer-side experiment. - selectScorerStat.counter("use_candidate_ranker").incr() - candidateRankerId - } - } - selectScorerStat.scope("selected").counter(selectedCandidateRankerId.toString).incr() - candidate -> selectedCandidateRankerId - }.toMap - } - - @VisibleForTesting - private[ranking] def score( - candidates: Seq[CandidateUser], - rankerIds: Map[CandidateUser, RankerId], - requestRankerId: RankerId - ): Stitch[Seq[CandidateUser]] = { - val features = candidates.map(_.dataRecord.flatMap(_.dataRecord)) - - require(features.forall(_.nonEmpty), "features are not hydrated for all the candidates") - - val scorers = scorerFactory.getScorers(rankerIds.values.toSeq.sorted.distinct) - - // Scorers are split into ML-based and Adhoc (defined as a scorer that does not need to call an - // ML prediction service and scores candidates using locally-available data). - val (adhocScorers, mlScorers) = scorers.partition { - case _: AdhocScorer => true - case _ => false - } - - // score candidates - val scoresStitch = score(features.map(_.get), mlScorers) - val candidatesWithMlScoresStitch = scoresStitch.map { scoresSeq => - candidates - .zip(scoresSeq).map { // copy datarecord and score into candidate object - case (candidate, scores) => - val selectedRankerId = rankerIds(candidate) - val useRequestRanker = - candidate.params == Params.Invalid || - candidate.params == Params.Empty || - candidate.params(MlRankerParams.CandidateScorerIdParam) == RankerId.None - candidate.copy( - score = scores.scores.find(_.rankerId.contains(requestRankerId)).map(_.value), - scores = if (scores.scores.nonEmpty) { - Some( - scores.copy( - scores = scores.scores, - selectedRankerId = Some(selectedRankerId), - isInProducerScoringExperiment = !useRequestRanker - )) - } else None - ) - } - } - - candidatesWithMlScoresStitch.map { candidates => - // The basis for adhoc scores are the "request-level" ML ranker. We add the base score here - // while adhoc scorers are applied in [[AdhocRanker]]. - addMlBaseScoresForAdhocScorers(candidates, requestRankerId, adhocScorers) - } - } - - @VisibleForTesting - private[ranking] def addMlBaseScoresForAdhocScorers( - candidates: Seq[CandidateUser], - requestRankerId: RankerId, - adhocScorers: Seq[Scorer] - ): Seq[CandidateUser] = { - candidates.map { candidate => - candidate.scores match { - case Some(oldScores) => - // 1. We fetch the ML score that is the basis of adhoc scores: - val baseMlScoreOpt = Utils.getCandidateScoreByRankerId(candidate, requestRankerId) - - // 2. For each adhoc scorer, we copy the ML score object, changing only the ID and type. - val newScores = adhocScorers flatMap { adhocScorer => - baseMlScoreOpt.map( - _.copy(rankerId = Some(adhocScorer.id), scoreType = adhocScorer.scoreType)) - } - - // 3. We add the new adhoc score entries to the candidate. - candidate.copy(scores = Some(oldScores.copy(scores = oldScores.scores ++ newScores))) - case _ => - // Since there is no base ML score, there should be no adhoc score modification as well. - candidate - } - } - } - - private[this] def score( - dataRecords: Seq[DataRecord], - scorers: Seq[Scorer] - ): Stitch[Seq[Scores]] = { - val scoredResponse = scorers.map { scorer => - StatsUtil.profileStitch(scorer.score(dataRecords), scoreStat.scope(scorer.id.toString)) - } - // If we could score a candidate with too many rankers, it is likely to blow up the whole system. - // and fail back to default production model - StatsUtil.profileStitch(Stitch.collect(scoredResponse), scoreStat).map { scoresByScorerId => - CollectionUtil.transposeLazy(scoresByScorerId).map { scoresPerCandidate => - Scores(scoresPerCandidate) - } - } - } - - // sort candidates using score in descending order - private[this] def sort( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - candidates.sortBy(c => -c.score.getOrElse(MlRanker.DefaultScore)) - } - - private[this] def scribeCandidates( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - val scribeRankingInfo: Boolean = target.params(MlRankerParams.ScribeRankingInfoInMlRanker) - scribeRankingInfo match { - case true => Utils.addRankingInfo(candidates, "MlRanker") - case false => candidates - } - } -} - -object MlRanker { - // this is to ensure candidates with absent scores are ranked the last - val DefaultScore: Double = Double.MinValue -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.docx new file mode 100644 index 000000000..5b2c4156f Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.scala deleted file mode 100644 index c69a32fc5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam - -@Singleton -class MlRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(MlRankerParams.ScribeRankingInfoInMlRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.docx new file mode 100644 index 000000000..e88c8d04b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.scala deleted file mode 100644 index 8463963a6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam - -/** - * When adding Producer side experiments, make sure to register the FS Key in [[ProducerFeatureFilter]] - * in [[FeatureSwitchesModule]], otherwise, the FS will not work. - */ -object MlRankerParams { - // which ranker to use by default for the given request - case object RequestScorerIdParam - extends FSEnumParam[RankerId.type]( - name = "post_nux_ml_flow_ml_ranker_id", - default = RankerId.PostNuxProdRanker, - enum = RankerId - ) - - // which ranker to use for the given candidate - case object CandidateScorerIdParam - extends FSEnumParam[RankerId.type]( - name = "post_nux_ml_flow_candidate_user_scorer_id", - default = RankerId.None, - enum = RankerId - ) - - case object ScribeRankingInfoInMlRanker - extends FSParam[Boolean]("post_nux_ml_flow_scribe_ranking_info_in_ml_ranker", true) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.docx new file mode 100644 index 000000000..ba942ff13 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.scala deleted file mode 100644 index 39921bb71..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.follow_recommendations.common.rankers.common.AdhocScoreModificationType.AdhocScoreModificationType -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.ml.api.DataRecord -import com.twitter.stitch.Stitch - -trait AdhocScorer extends Scorer { - - /** - * NOTE: For instances of [[AdhocScorer]] this function SHOULD NOT be used. - * Please use: - * [[score(target: HasClientContext with HasParams, candidates: Seq[CandidateUser])]] - * instead. - */ - @Deprecated - override def score(records: Seq[DataRecord]): Stitch[Seq[Score]] = - throw new UnsupportedOperationException( - "For instances of AdhocScorer this operation is not defined. Please use " + - "`def score(target: HasClientContext with HasParams, candidates: Seq[CandidateUser])` " + - "instead.") - - /** - * This helps us manage the extend of adhoc modification on candidates' score. There is a hard - * limit of applying ONLY ONE scorer of each type to a score. - */ - val scoreModificationType: AdhocScoreModificationType -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD deleted file mode 100644 index bbcd3c708..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking:ml_ranker_params", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/pluck/source/core_workflows/user_model:condensed_user_state-scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD.docx new file mode 100644 index 000000000..5d5bff65c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.docx new file mode 100644 index 000000000..7fb676019 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.scala deleted file mode 100644 index d27bc6e37..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.cortex.deepbird.thriftjava.ModelSelector -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.prediction_service.{BatchPredictionRequest => JBatchPredictionRequest} -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Future -import com.twitter.util.TimeoutException -import scala.collection.JavaConversions._ -import scala.collection.JavaConverters._ - -/** - * Generic trait that implements the scoring given a deepbirdClient - * To test out a new model, create a scorer extending this trait, override the modelName and inject the scorer - */ -trait DeepbirdScorer extends Scorer { - def modelName: String - def predictionFeature: Feature.Continuous - // Set a default batchSize of 100 when making model prediction calls to the Deepbird V2 prediction server - def batchSize: Int = 100 - def deepbirdClient: DeepbirdPredictionService.ServiceToClient - def baseStats: StatsReceiver - - def modelSelector: ModelSelector = new ModelSelector().setId(modelName) - def stats: StatsReceiver = baseStats.scope(this.getClass.getSimpleName).scope(modelName) - - private def requestCount = stats.counter("requests") - private def emptyRequestCount = stats.counter("empty_requests") - private def successCount = stats.counter("success") - private def failureCount = stats.counter("failures") - private def inputRecordsStat = stats.stat("input_records") - private def outputRecordsStat = stats.stat("output_records") - - // Counters for tracking batch-prediction statistics when making DBv2 prediction calls - // - // numBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers - private def numBatchRequests = stats.counter("batches") - // numEmptyBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers - // that had an empty input DataRecord - private def numEmptyBatchRequests = stats.counter("empty_batches") - // numTimedOutBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers - // that had timed-out - private def numTimedOutBatchRequests = stats.counter("timeout_batches") - - private def batchPredictionLatency = stats.stat("batch_prediction_latency") - private def predictionLatency = stats.stat("prediction_latency") - - private def numEmptyModelPredictions = stats.counter("empty_model_predictions") - private def numNonEmptyModelPredictions = stats.counter("non_empty_model_predictions") - - private val DefaultPredictionScore = 0.0 - - /** - * NOTE: For instances of [[DeepbirdScorer]] this function SHOULD NOT be used. - * Please use [[score(records: Seq[DataRecord])]] instead. - */ - @Deprecated - def score( - target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions, - candidates: Seq[CandidateUser] - ): Seq[Option[Score]] = - throw new UnsupportedOperationException( - "For instances of DeepbirdScorer this operation is not defined. Please use " + - "`def score(records: Seq[DataRecord]): Stitch[Seq[Score]]` " + - "instead.") - - override def score(records: Seq[DataRecord]): Stitch[Seq[Score]] = { - requestCount.incr() - if (records.isEmpty) { - emptyRequestCount.incr() - Stitch.Nil - } else { - inputRecordsStat.add(records.size) - Stitch.callFuture( - batchPredict(records, batchSize) - .map { recordList => - val scores = recordList.map { record => - Score( - value = record.getOrElse(DefaultPredictionScore), - rankerId = Some(id), - scoreType = scoreType) - } - outputRecordsStat.add(scores.size) - scores - }.onSuccess(_ => successCount.incr()) - .onFailure(_ => failureCount.incr())) - } - } - - def batchPredict( - dataRecords: Seq[DataRecord], - batchSize: Int - ): Future[Seq[Option[Double]]] = { - Stat - .timeFuture(predictionLatency) { - val batchedDataRecords = dataRecords.grouped(batchSize).toSeq - numBatchRequests.incr(batchedDataRecords.size) - Future - .collect(batchedDataRecords.map(batch => predict(batch))) - .map(res => res.reduce(_ ++ _)) - } - } - - def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] = { - Stat - .timeFuture(batchPredictionLatency) { - if (dataRecords.isEmpty) { - numEmptyBatchRequests.incr() - Future.Nil - } else { - deepbirdClient - .batchPredictFromModel(new JBatchPredictionRequest(dataRecords.asJava), modelSelector) - .map { response => - response.predictions.toSeq.map { prediction => - val predictionFeatureOption = Option( - new RichDataRecord(prediction).getFeatureValue(predictionFeature) - ) - predictionFeatureOption match { - case Some(predictionValue) => - numNonEmptyModelPredictions.incr() - Option(predictionValue.toDouble) - case None => - numEmptyModelPredictions.incr() - Option(DefaultPredictionScore) - } - } - } - .rescue { - case e: TimeoutException => // DBv2 prediction calls that timed out - numTimedOutBatchRequests.incr() - stats.counter(e.getClass.getSimpleName).incr() - Future.value(dataRecords.map(_ => Option(DefaultPredictionScore))) - case e: Exception => // other generic DBv2 prediction call failures - stats.counter(e.getClass.getSimpleName).incr() - Future.value(dataRecords.map(_ => Option(DefaultPredictionScore))) - } - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.docx new file mode 100644 index 000000000..bdfc5cb53 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.scala deleted file mode 100644 index 861d02fa7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.ml.api.Feature -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -// This is a standard DeepbirdV2 ML Ranker scoring config that should be extended by all ML scorers -// -// Only modify this trait when adding new fields to DeepbirdV2 scorers which -trait DeepbirdProdScorer extends DeepbirdScorer { - override val batchSize = 20 -} - -// Feature.Continuous("prediction") is specific to ClemNet architecture, we can change it to be more informative in the next iteration -trait PostNuxV1DeepbirdProdScorer extends DeepbirdProdScorer { - override val predictionFeature: Feature.Continuous = - new Feature.Continuous("prediction") -} - -// The current, primary PostNUX DeepbirdV2 scorer used in production -@Singleton -class PostnuxDeepbirdProdScorer @Inject() ( - @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) - override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, - override val baseStats: StatsReceiver) - extends PostNuxV1DeepbirdProdScorer { - override val id = RankerId.PostNuxProdRanker - override val modelName = "PostNUX14531GafClemNetWarmStart" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.docx new file mode 100644 index 000000000..b7acbadf1 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.scala deleted file mode 100644 index 92265cc6b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/** - * This scorer assigns random values between 0 and 1 to each candidate as scores. - */ - -@Singleton -class RandomScorer @Inject() ( - @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) - override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, - override val baseStats: StatsReceiver) - extends DeepbirdScorer { - override val id = RankerId.RandomRanker - private val rnd = new scala.util.Random(System.currentTimeMillis()) - - override def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] = { - if (dataRecords.isEmpty) { - Future.Nil - } else { - // All candidates are assigned a random value between 0 and 1 as score. - Future.value(dataRecords.map(_ => Option(rnd.nextDouble()))) - } - } - - override val modelName = "PostNuxRandomRanker" - - // This is not needed since we are overriding the `predict` function, but we have to override - // `predictionFeature` anyway. - override val predictionFeature: Feature.Continuous = - new Feature.Continuous("prediction.pfollow_pengagement") -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.docx new file mode 100644 index 000000000..d4760d58a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.scala deleted file mode 100644 index 2ca611535..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.follow_recommendations.common.models.ScoreType -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -trait Scorer { - - // unique id of the scorer - def id: RankerId.Value - - // type of the output scores - def scoreType: Option[ScoreType] = None - - // Scoring when an ML model is used. - def score(records: Seq[DataRecord]): Stitch[Seq[Score]] - - /** - * Scoring when a non-ML method is applied. E.g: Boosting, randomized reordering, etc. - * This method assumes that candidates' scores are already retrieved from heavy-ranker models and - * are available for use. - */ - def score( - target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions, - candidates: Seq[CandidateUser] - ): Seq[Option[Score]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.docx new file mode 100644 index 000000000..8359bd4ab Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.scala deleted file mode 100644 index a9ea0a21b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScorerFactory @Inject() ( - postnuxProdScorer: PostnuxDeepbirdProdScorer, - randomScorer: RandomScorer, - stats: StatsReceiver) { - - private val scorerFactoryStats = stats.scope("scorer_factory") - private val scorerStat = scorerFactoryStats.scope("scorer") - - def getScorers( - rankerIds: Seq[RankerId] - ): Seq[Scorer] = { - rankerIds.map { scorerId => - val scorer: Scorer = getScorerById(scorerId) - // count # of times a ranker has been requested - scorerStat.counter(scorer.id.toString).incr() - scorer - } - } - - def getScorerById(scorerId: RankerId): Scorer = scorerId match { - case RankerId.PostNuxProdRanker => - postnuxProdScorer - case RankerId.RandomRanker => - randomScorer - case _ => - scorerStat.counter("invalid_scorer_type").incr() - throw new IllegalArgumentException("unknown_scorer_type") - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD deleted file mode 100644 index 82e9fc7f4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD.docx new file mode 100644 index 000000000..d38d99b6a Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.docx new file mode 100644 index 000000000..675866a78 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.scala deleted file mode 100644 index 29f00b698..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.utils - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId - -object Utils { - - /** - * Add the ranking and scoring info for a list of candidates on a given ranking stage. - * @param candidates A list of CandidateUser - * @param rankingStage Should use `Ranker.name` as the ranking stage. - * @return The list of CandidateUser with ranking/scoring info added. - */ - def addRankingInfo(candidates: Seq[CandidateUser], rankingStage: String): Seq[CandidateUser] = { - candidates.zipWithIndex.map { - case (candidate, rank) => - // 1-based ranking for better readability - candidate.addInfoPerRankingStage(rankingStage, candidate.scores, rank + 1) - } - } - - def getCandidateScoreByRankerId(candidate: CandidateUser, rankerId: RankerId): Option[Score] = - candidate.scores.flatMap { ss => ss.scores.find(_.rankerId.contains(rankerId)) } - - def getAllRankerIds(candidates: Seq[CandidateUser]): Seq[RankerId] = - candidates.flatMap(_.scores.map(_.scores.flatMap(_.rankerId))).flatten.distinct -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD deleted file mode 100644 index 3de5523b1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD.docx new file mode 100644 index 000000000..1f4a21aff Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.docx new file mode 100644 index 000000000..b0ed62249 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.scala deleted file mode 100644 index be281a582..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.follow_recommendations.common.utils.RandomUtil -import scala.util.Random - -sealed trait CandidateShuffler[T] { - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] -} - -class NoShuffle[T]() extends CandidateShuffler[T] { - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = input -} - -class RandomShuffler[T]() extends CandidateShuffler[T] { - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = { - seed.map(new Random(_)).getOrElse(Random).shuffle(input) - } -} - -trait RankWeightedRandomShuffler[T] extends CandidateShuffler[T] { - - def rankToWeight(rank: Int): Double - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = { - val candWeights = input.zipWithIndex.map { - case (candidate, rank) => (candidate, rankToWeight(rank)) - } - RandomUtil.weightedRandomShuffle(candWeights, seed.map(new Random(_))).unzip._1 - } -} - -class ExponentialShuffler[T]() extends RankWeightedRandomShuffler[T] { - def rankToWeight(rank: Int): Double = { - 1 / math - .pow(rank.toDouble, 2.0) // this function was proved to be effective in previous DDGs - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.docx new file mode 100644 index 000000000..f2ee74b73 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.scala deleted file mode 100644 index 54e2ad549..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -object WeightMethod extends Enumeration { - type WeightMethod = Value - val WeightedRandomSampling, WeightedRoundRobin = Value -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.docx new file mode 100644 index 000000000..b44530dea Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.scala deleted file mode 100644 index e348560db..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.follow_recommendations.common.utils.RandomUtil -import com.twitter.follow_recommendations.common.utils.MergeUtil -import com.twitter.follow_recommendations.common.utils.Weighted -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightMethod._ -import scala.util.Random - -/** - * This ranker selects the next candidate source to select a candidate from. It supports - * two kinds of algorithm, WeightedRandomSampling or WeightedRoundRobin. WeightedRandomSampling - * pick the next candidate source randomly, WeightedRoundRobin picked the next candidate source - * sequentially based on the weight of the candidate source. It is default to WeightedRandomSampling - * if no weight method is provided. - * - * Example usage of this class: - * - * When use WeightedRandomSampling: - * Input candidate sources and their weights are: {{CS1: 3}, {CS2: 2}, {CS3: 5}} - * Ranked candidates sequence is not determined because of random sampling. - * One possible output candidate sequence is: (CS1_candidate1, CS2_candidate1, CS2_candidate2, - * CS3_candidate1, CS3_candidates2, CS3_candidate3, CS1_candidate2, CS1_candidate3, - * CS3_candidate4, CS3_candidate5, CS1_candidate4, CS1_candidate5, CS2_candidate6, CS2_candidate3,...) - * - * When use WeightedRoundRobin: - * Input candidate sources and their weights are: {{CS1: 3}, {CS2: 2}, {CS3: 5}} - * Output candidate sequence is: (CS1_candidate1, CS1_candidate2, CS1_candidate3, - * CS2_candidate1, CS2_candidates2, CS3_candidate1, CS3_candidate2, CS3_candidate3, - * CS3_candidate4, CS3_candidate5, CS1_candidate4, CS1_candidate5, CS1_candidate6, CS2_candidate3,...) - * - * Note: CS1_candidate1 means the first candidate in CS1 candidate source. - * - * @tparam A candidate source type - * @tparam Rec Recommendation type - * @param candidateSourceWeights relative weights for different candidate sources - */ -class WeightedCandidateSourceBaseRanker[A, Rec]( - candidateSourceWeights: Map[A, Double], - weightMethod: WeightMethod = WeightedRandomSampling, - randomSeed: Option[Long]) { - - /** - * Creates a iterator over algorithms and calls next to return a Stream of candidates - * - * - * @param candidateSources the set of candidate sources that are being sampled - * @param candidateSourceWeights map of candidate source to weight - * @param candidates the map of candidate source to the iterator of its results - * @param weightMethod a enum to indict which weight method to use. Two values are supported - * currently. When WeightedRandomSampling is set, the next candidate is picked from a candidate - * source that is randomly chosen. When WeightedRoundRobin is set, the next candidate is picked - * from current candidate source until the number of candidates reaches to the assigned weight of - * the candidate source. The next call of this function will return a candidate from the next - * candidate source which is after previous candidate source based on the order input - * candidate source sequence. - - * @return stream of candidates - */ - def stream( - candidateSources: Set[A], - candidateSourceWeights: Map[A, Double], - candidates: Map[A, Iterator[Rec]], - weightMethod: WeightMethod = WeightedRandomSampling, - random: Option[Random] = None - ): Stream[Rec] = { - val weightedCandidateSource: Weighted[A] = new Weighted[A] { - override def apply(a: A): Double = candidateSourceWeights.getOrElse(a, 0) - } - - /** - * Generates a stream of candidates. - * - * @param candidateSourceIter an iterator over candidate sources returned by the sampling procedure - * @return stream of candidates - */ - def next(candidateSourceIter: Iterator[A]): Stream[Rec] = { - val source = candidateSourceIter.next() - val it = candidates(source) - if (it.hasNext) { - val currCand = it.next() - currCand #:: next(candidateSourceIter) - } else { - assert(candidateSources.contains(source), "Selected source is not in candidate sources") - // Remove the depleted candidate source and re-sample - stream(candidateSources - source, candidateSourceWeights, candidates, weightMethod, random) - } - } - if (candidateSources.isEmpty) - Stream.empty - else { - val candidateSourceSeq = candidateSources.toSeq - val candidateSourceIter = - if (weightMethod == WeightMethod.WeightedRoundRobin) { - MergeUtil.weightedRoundRobin(candidateSourceSeq)(weightedCandidateSource).iterator - } else { - //default to weighted random sampling if no other weight method is provided - RandomUtil - .weightedRandomSamplingWithReplacement( - candidateSourceSeq, - random - )(weightedCandidateSource).iterator - } - next(candidateSourceIter) - } - } - - def apply(input: Map[A, TraversableOnce[Rec]]): Stream[Rec] = { - stream( - input.keySet, - candidateSourceWeights, - input.map { - case (k, v) => k -> v.toIterator - }, // cannot do mapValues here, as that only returns a view - weightMethod, - randomSeed.map(new Random(_)) - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.docx new file mode 100644 index 000000000..5427c14bf Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala deleted file mode 100644 index c6f55adbc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.common.DedupCandidates -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * Candidate Ranker that mixes and ranks multiple candidate lists from different candidate sources with the - * following steps: - * 1) generate a ranked candidate list of each candidate source by sorting and shuffling the candidate list - * of the algorithm. - * 2) merge the ranked lists generated in 1) into a single list using weighted randomly sampling. - * 3) If dedup is required, dedup the output from 2) by candidate id. - * - * @param basedRanker base ranker - * @param shuffleFn the shuffle function that will be used to shuffle each algorithm's sorted candidate list. - * @param dedup whether to remove duplicated candidates from the final output. - */ -class WeightedCandidateSourceRanker[Target <: HasParams]( - basedRanker: WeightedCandidateSourceBaseRanker[ - CandidateSourceIdentifier, - CandidateUser - ], - shuffleFn: Seq[CandidateUser] => Seq[CandidateUser], - dedup: Boolean) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - - override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - val scribeRankingInfo: Boolean = - target.params(WeightedCandidateSourceRankerParams.ScribeRankingInfoInWeightedRanker) - val rankedCands = rankCandidates(group(candidates)) - Stitch.value(if (scribeRankingInfo) Utils.addRankingInfo(rankedCands, name) else rankedCands) - } - - private def group( - candidates: Seq[CandidateUser] - ): Map[CandidateSourceIdentifier, Seq[CandidateUser]] = { - val flattened = for { - candidate <- candidates - identifier <- candidate.getPrimaryCandidateSource - } yield (identifier, candidate) - flattened.groupBy(_._1).mapValues(_.map(_._2)) - } - - private def rankCandidates( - input: Map[CandidateSourceIdentifier, Seq[CandidateUser]] - ): Seq[CandidateUser] = { - // Sort and shuffle candidates per candidate source. - // Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream. - val sortAndShuffledCandidates = input.map { - case (source, candidates) => - // Note 2: toList is required here since candidates is a view, and it will result in infinit loop when used as part of Stream. - // Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources - val sortedCandidates = candidates.toList - source -> shuffleFn(sortedCandidates).iterator - } - val rankedCandidates = basedRanker(sortAndShuffledCandidates) - - if (dedup) DedupCandidates(rankedCandidates) else rankedCandidates - } -} - -object WeightedCandidateSourceRanker { - - def build[Target <: HasParams]( - candidateSourceWeight: Map[CandidateSourceIdentifier, Double], - shuffleFn: Seq[CandidateUser] => Seq[CandidateUser] = identity, - dedup: Boolean = false, - randomSeed: Option[Long] = None - ): WeightedCandidateSourceRanker[Target] = { - new WeightedCandidateSourceRanker( - new WeightedCandidateSourceBaseRanker( - candidateSourceWeight, - WeightMethod.WeightedRandomSampling, - randomSeed = randomSeed), - shuffleFn, - dedup - ) - } -} - -object WeightedCandidateSourceRankerWithoutRandomSampling { - def build[Target <: HasParams]( - candidateSourceWeight: Map[CandidateSourceIdentifier, Double] - ): WeightedCandidateSourceRanker[Target] = { - new WeightedCandidateSourceRanker( - new WeightedCandidateSourceBaseRanker( - candidateSourceWeight, - WeightMethod.WeightedRoundRobin, - randomSeed = None), - identity, - false, - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.docx new file mode 100644 index 000000000..950b51d9d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.scala deleted file mode 100644 index 58f0fde3e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class WeightedCandidateSourceRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(WeightedCandidateSourceRankerParams.ScribeRankingInfoInWeightedRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.docx new file mode 100644 index 000000000..febc90a68 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.scala deleted file mode 100644 index ff4ecae4b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.timelines.configapi.FSParam - -object WeightedCandidateSourceRankerParams { - case object ScribeRankingInfoInWeightedRanker - extends FSParam[Boolean]("weighted_ranker_scribe_ranking_info", false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD deleted file mode 100644 index 3b7a46db7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD.docx new file mode 100644 index 000000000..c0cfabb90 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.docx new file mode 100644 index 000000000..f6f098d60 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.scala deleted file mode 100644 index d2f4e035b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.follow_recommendations.common.stores - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.TweepCredOnUserClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -// Not a candidate source since it's a intermediary. -@Singleton -class LowTweepCredFollowStore @Inject() (tweepCredOnUserClientColumn: TweepCredOnUserClientColumn) { - - def getLowTweepCredUsers(target: HasRecentFollowedUserIds): Stitch[Seq[CandidateUser]] = { - val newFollowings = - target.recentFollowedUserIds.getOrElse(Nil).take(LowTweepCredFollowStore.NumFlockToRetrieve) - - val validTweepScoreUserIdsStitch: Stitch[Seq[Long]] = Stitch - .traverse(newFollowings) { newFollowingUserId => - val tweepCredScoreOptStitch = tweepCredOnUserClientColumn.fetcher - .fetch(newFollowingUserId) - .map(_.v) - tweepCredScoreOptStitch.map(_.flatMap(tweepCred => - if (tweepCred < LowTweepCredFollowStore.TweepCredThreshold) { - Some(newFollowingUserId) - } else { - None - })) - }.map(_.flatten) - - validTweepScoreUserIdsStitch - .map(_.map(CandidateUser(_, Some(CandidateUser.DefaultCandidateScore)))) - } -} - -object LowTweepCredFollowStore { - val NumFlockToRetrieve = 500 - val TweepCredThreshold = 40 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD deleted file mode 100644 index 35534b064..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD.docx new file mode 100644 index 000000000..31d38ed1d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.docx new file mode 100644 index 000000000..e66ef399c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.scala deleted file mode 100644 index 64f73d6ae..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.dedup - -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.stitch.Stitch -import scala.collection.mutable - -class DedupTransform[Request, Candidate <: UniversalNoun[Long]]() - extends Transform[Request, Candidate] { - override def transform(target: Request, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { - val seen = mutable.HashSet[Long]() - Stitch.value(candidates.filter(candidate => seen.add(candidate.id))) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD deleted file mode 100644 index 79da9c259..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "configapi/configapi-core", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD.docx new file mode 100644 index 000000000..87fad4f94 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.docx new file mode 100644 index 000000000..7e8ab7e8b Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.scala deleted file mode 100644 index 306578a4d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.scala +++ /dev/null @@ -1,202 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.modify_social_proof - -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureServiceClient -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.graph_feature_service.thriftscala.EdgeType -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -object ModifySocialProof { - val GfsLagDuration: Duration = 14.days - val GfsIntersectionIds: Int = 3 - val SgsIntersectionIds: Int = 10 - val LeftEdgeTypes: Set[EdgeType] = Set(EdgeType.Following) - val RightEdgeTypes: Set[EdgeType] = Set(EdgeType.FollowedBy) - - /** - * Given the intersection ID's for a particular candidate, update the candidate's social proof - * @param candidate candidate object - * @param followProof follow proof to be added (includes id's and count) - * @param stats stats for tracking - * @return updated candidate object - */ - def addIntersectionIdsToCandidate( - candidate: CandidateUser, - followProof: FollowProof, - stats: StatsReceiver - ): CandidateUser = { - // create updated set of social proof - val updatedFollowedByOpt = candidate.followedBy match { - case Some(existingFollowedBy) => Some((followProof.followedBy ++ existingFollowedBy).distinct) - case None if followProof.followedBy.nonEmpty => Some(followProof.followedBy.distinct) - case _ => None - } - - val updatedFollowProof = updatedFollowedByOpt.map { updatedFollowedBy => - val updatedCount = followProof.numIds.max(updatedFollowedBy.size) - // track stats - val numSocialProofAdded = updatedFollowedBy.size - candidate.followedBy.size - addCandidatesWithSocialContextCountStat(stats, numSocialProofAdded) - FollowProof(updatedFollowedBy, updatedCount) - } - - candidate.setFollowProof(updatedFollowProof) - } - - private def addCandidatesWithSocialContextCountStat( - statsReceiver: StatsReceiver, - count: Int - ): Unit = { - if (count > 3) { - statsReceiver.counter("4_and_more").incr() - } else { - statsReceiver.counter(count.toString).incr() - } - } -} - -/** - * This class makes a request to gfs/sgs for hydrating additional social proof on each of the - * provided candidates. - */ -@Singleton -class ModifySocialProof @Inject() ( - gfsClient: GraphFeatureServiceClient, - socialGraphClient: SocialGraphClient, - statsReceiver: StatsReceiver, - decider: Decider = Decider.True) - extends Logging { - import ModifySocialProof._ - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val addedStats = stats.scope("num_social_proof_added_per_candidate") - private val gfsStats = stats.scope("graph_feature_service") - private val sgsStats = stats.scope("social_graph_service") - private val previousProofEmptyCounter = stats.counter("previous_proof_empty") - private val emptyFollowProofCounter = stats.counter("empty_followed_proof") - - /** - * For each candidate provided, we get the intersectionIds between the user and the candidate, - * appending the unique results to the social proof (followedBy field) if not already previously - * seen we query GFS for all users, except for cases specified via the mustCallSgs field or for - * very new users, who would not have any data in GFS, due to the lag duration of the service's - * processing. this is determined by GfsLagDuration - * @param userId id of the target user whom we provide recommendations for - * @param candidates list of candidates - * @param intersectionIdsNum if provided, determines the maximum number of accounts we want to be hydrated for social proof - * @param mustCallSgs Determines if we should query SGS regardless of user age or not. - * @return list of candidates updated with additional social proof - */ - def hydrateSocialProof( - userId: Long, - candidates: Seq[CandidateUser], - intersectionIdsNum: Option[Int] = None, - mustCallSgs: Boolean = false, - callSgsCachedColumn: Boolean = false, - gfsLagDuration: Duration = GfsLagDuration, - gfsIntersectionIds: Int = GfsIntersectionIds, - sgsIntersectionIds: Int = SgsIntersectionIds, - ): Stitch[Seq[CandidateUser]] = { - addCandidatesWithSocialContextCountStat( - stats.scope("social_context_count_before_hydration"), - candidates.count(_.followedBy.isDefined) - ) - val candidateIds = candidates.map(_.id) - val userAgeOpt = SnowflakeId.timeFromIdOpt(userId).map(Time.now - _) - - // this decider gate is used to determine what % of requests is allowed to call - // Graph Feature Service. this is useful for ramping down requests to Graph Feature Service - // when necessary - val deciderKey: String = DeciderKey.EnableGraphFeatureServiceRequests.toString - val enableGfsRequests: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - - // if new query sgs - val (candidateToIntersectionIdsMapFut, isGfs) = - if (!enableGfsRequests || mustCallSgs || userAgeOpt.exists(_ < gfsLagDuration)) { - ( - if (callSgsCachedColumn) - socialGraphClient.getIntersectionsFromCachedColumn( - userId, - candidateIds, - intersectionIdsNum.getOrElse(sgsIntersectionIds) - ) - else - socialGraphClient.getIntersections( - userId, - candidateIds, - intersectionIdsNum.getOrElse(sgsIntersectionIds)), - false) - } else { - ( - gfsClient.getIntersections( - userId, - candidateIds, - intersectionIdsNum.getOrElse(gfsIntersectionIds)), - true) - } - val finalCandidates = candidateToIntersectionIdsMapFut - .map { candidateToIntersectionIdsMap => - { - previousProofEmptyCounter.incr(candidates.count(_.followedBy.exists(_.isEmpty))) - candidates.map { candidate => - addIntersectionIdsToCandidate( - candidate, - candidateToIntersectionIdsMap.getOrElse(candidate.id, FollowProof(Seq.empty, 0)), - addedStats) - } - } - } - .within(250.milliseconds)(DefaultTimer) - .rescue { - case e: Exception => - error(e.getMessage) - if (isGfs) { - gfsStats.scope("rescued").counter(e.getClass.getSimpleName).incr() - } else { - sgsStats.scope("rescued").counter(e.getClass.getSimpleName).incr() - } - Stitch.value(candidates) - } - - finalCandidates.onSuccess { candidatesSeq => - emptyFollowProofCounter.incr(candidatesSeq.count(_.followedBy.exists(_.isEmpty))) - addCandidatesWithSocialContextCountStat( - stats.scope("social_context_count_after_hydration"), - candidatesSeq.count(_.followedBy.isDefined) - ) - } - } -} - -/** - * This transform uses ModifySocialProof (which makes a request to gfs/sgs) for hydrating additional - * social proof on each of the provided candidates. - */ -@Singleton -class ModifySocialProofTransform @Inject() (modifySocialProof: ModifySocialProof) - extends GatedTransform[HasClientContext with HasParams, CandidateUser] - with Logging { - - override def transform( - target: HasClientContext with HasParams, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = - target.getOptionalUserId - .map(modifySocialProof.hydrateSocialProof(_, candidates)).getOrElse(Stitch.value(candidates)) -}